From 21ac955a60c238557ef124942ef2ca1332bd7b0c Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Tue, 5 May 2026 14:10:55 -0700 Subject: [PATCH] fix(prometheus): fall back to v3 limiter additional_headers for per-key remaining gauges When parallel_request_limiter_v3 is in use the per-key remaining counters land in standard_logging_payload.hidden_params.additional_headers under "x-ratelimit-model_per_key-remaining-{requests,tokens}" rather than in metadata under "litellm-key-remaining-{requests,tokens}-{model_group}". Without a fallback the gauges litellm_remaining_api_key_requests_for_model and litellm_remaining_api_key_tokens_for_model always default to sys.maxsize (~9.22e18), which is what users see in DataDog/Prometheus when running on the v3 limiter. This change reads both, preferring metadata for back-compat with the legacy parallel_request_limiter and falling back to the v3 headers otherwise. Test file already added in a previous commit. --- litellm/integrations/prometheus.py | 37 +++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 723b142dfad..cd5aab5e655 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -1171,6 +1171,7 @@ class PrometheusLogger(CustomLogger): kwargs=kwargs, metadata=_metadata, model_id=enum_values.model_id, + standard_logging_payload=standard_logging_payload, ) # set latency metrics @@ -1402,24 +1403,54 @@ class PrometheusLogger(CustomLogger): kwargs: dict, metadata: dict, model_id: Optional[str] = None, + standard_logging_payload: Optional[StandardLoggingPayload] = None, ): from litellm.proxy.common_utils.callback_utils import ( get_model_group_from_litellm_kwargs, ) # Set remaining rpm/tpm for API Key + model - # see parallel_request_limiter.py - variables are set there + # The legacy parallel_request_limiter writes the per-key remaining + # counters into request metadata under + # "litellm-key-remaining-{requests,tokens}-{model_group}". + # parallel_request_limiter_v3 instead publishes them on the response + # under hidden_params.additional_headers as + # "x-ratelimit-model_per_key-remaining-{requests,tokens}". + # Read both, preferring metadata, so the gauges work with either + # limiter implementation. Without this fallback the v3 path always + # falls through to sys.maxsize (~9.22e18) and DataDog/Prometheus + # show that placeholder instead of the real remaining value. model_group = get_model_group_from_litellm_kwargs(kwargs) remaining_requests_variable_name = ( f"litellm-key-remaining-requests-{model_group}" ) remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}" + additional_headers: dict = {} + if standard_logging_payload is not None: + hidden_params = ( + standard_logging_payload.get("hidden_params") or {} + if isinstance(standard_logging_payload, dict) + else {} + ) + additional_headers = hidden_params.get("additional_headers") or {} + + fallback_remaining_requests = additional_headers.get( + "x-ratelimit-model_per_key-remaining-requests" + ) + fallback_remaining_tokens = additional_headers.get( + "x-ratelimit-model_per_key-remaining-tokens" + ) + remaining_requests = ( - metadata.get(remaining_requests_variable_name, sys.maxsize) or sys.maxsize + metadata.get(remaining_requests_variable_name) + or fallback_remaining_requests + or sys.maxsize ) remaining_tokens = ( - metadata.get(remaining_tokens_variable_name, sys.maxsize) or sys.maxsize + metadata.get(remaining_tokens_variable_name) + or fallback_remaining_tokens + or sys.maxsize ) self.litellm_remaining_api_key_requests_for_model.labels(