scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

fix(prometheus): fall back to v3 limiter additional_headers for per-key remaining gauges

Browse source 
When parallel_request_limiter_v3 is in use the per-key remaining counters
land in standard_logging_payload.hidden_params.additional_headers under
"x-ratelimit-model_per_key-remaining-{requests,tokens}" rather than in
metadata under "litellm-key-remaining-{requests,tokens}-{model_group}".

Without a fallback the gauges litellm_remaining_api_key_requests_for_model
and litellm_remaining_api_key_tokens_for_model always default to
sys.maxsize (~9.22e18), which is what users see in DataDog/Prometheus
when running on the v3 limiter.

This change reads both, preferring metadata for back-compat with the
legacy parallel_request_limiter and falling back to the v3 headers
otherwise. Test file already added in a previous commit.
This commit is contained in:
oss-agent-shin 2026-05-05 14:10:55 -07:00
parent fde9158d1a
commit 21ac955a60
1 changed files with 34 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
litellm/integrations/prometheus.py
View file

@ -1171,6 +1171,7 @@ class PrometheusLogger(CustomLogger):
kwargs=kwargs,
metadata=_metadata,
model_id=enum_values.model_id,
standard_logging_payload=standard_logging_payload,
)
# set latency metrics
@ -1402,24 +1403,54 @@ class PrometheusLogger(CustomLogger):
kwargs: dict,
metadata: dict,
model_id: Optional[str] = None,
standard_logging_payload: Optional[StandardLoggingPayload] = None,
):
from litellm.proxy.common_utils.callback_utils import (
get_model_group_from_litellm_kwargs,
)
# Set remaining rpm/tpm for API Key + model
# see parallel_request_limiter.py - variables are set there
# The legacy parallel_request_limiter writes the per-key remaining
# counters into request metadata under
# "litellm-key-remaining-{requests,tokens}-{model_group}".
# parallel_request_limiter_v3 instead publishes them on the response
# under hidden_params.additional_headers as
# "x-ratelimit-model_per_key-remaining-{requests,tokens}".
# Read both, preferring metadata, so the gauges work with either
# limiter implementation. Without this fallback the v3 path always
# falls through to sys.maxsize (~9.22e18) and DataDog/Prometheus
# show that placeholder instead of the real remaining value.
model_group = get_model_group_from_litellm_kwargs(kwargs)
remaining_requests_variable_name = (
f"litellm-key-remaining-requests-{model_group}"
)
remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}"
additional_headers: dict = {}
if standard_logging_payload is not None:
hidden_params = (
standard_logging_payload.get("hidden_params") or {}
if isinstance(standard_logging_payload, dict)
else {}
)
additional_headers = hidden_params.get("additional_headers") or {}
fallback_remaining_requests = additional_headers.get(
"x-ratelimit-model_per_key-remaining-requests"
)
fallback_remaining_tokens = additional_headers.get(
"x-ratelimit-model_per_key-remaining-tokens"
)
remaining_requests = (
metadata.get(remaining_requests_variable_name, sys.maxsize) or sys.maxsize
metadata.get(remaining_requests_variable_name)
or fallback_remaining_requests
or sys.maxsize
)
remaining_tokens = (
metadata.get(remaining_tokens_variable_name, sys.maxsize) or sys.maxsize
metadata.get(remaining_tokens_variable_name)
or fallback_remaining_tokens
or sys.maxsize
)
self.litellm_remaining_api_key_requests_for_model.labels(