scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 152

fix: also fix remaining_requests/remaining_tokens truthiness checks

Browse source 
Same class of bug as the overhead latency metric: using truthiness
checks instead of 'is not None' for numeric values. When a provider
returns 0 remaining requests (rate limit exhausted) or 0 remaining
tokens, the gauge was silently not set — hiding a critical signal.

Co-authored-by: Krish Dholakia <krrishdholakia@gmail.com>
This commit is contained in:
Cursor Agent 2026-03-11 00:14:59 +00:00
parent 15ba27b306
commit 6cd57bbbab
2 changed files with 90 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm/integrations/prometheus.py
View file

@ -2013,13 +2013,7 @@ class PrometheusLogger(CustomLogger):
litellm_overhead_time_ms / 1000
) # set as seconds
if remaining_requests:
"""
"model_group",
"api_provider",
"api_base",
"litellm_model_name"
"""
if remaining_requests is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_remaining_requests_metric"
@ -2030,7 +2024,7 @@ class PrometheusLogger(CustomLogger):
remaining_requests
)
if remaining_tokens:
if remaining_tokens is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_remaining_tokens_metric"

94
tests/test_litellm/integrations/test_prometheus_overhead_latency.py
View file

@ -1,8 +1,9 @@
"""
Unit tests for Prometheus overhead latency metric visibility.
Unit tests for Prometheus metric visibility with zero/falsy values.
Verifies that litellm_overhead_latency_metric is observed for every request
where litellm_overhead_time_ms is present, including when the value is 0.
Verifies that numeric Prometheus metrics are emitted even when their value
is 0 — the original code used Python truthiness checks that silently
dropped observations for any falsy numeric value.
"""
import os
@ -28,8 +29,20 @@ def prometheus_logger():
return PrometheusLogger()
def _build_request_kwargs(overhead_time_ms):
"""Build request_kwargs with the given overhead time."""
def _build_request_kwargs(
overhead_time_ms=None,
remaining_requests=None,
remaining_tokens=None,
):
"""Build request_kwargs with the given metric values."""
additional_headers = None
if remaining_requests is not None or remaining_tokens is not None:
additional_headers = {}
if remaining_requests is not None:
additional_headers["x_ratelimit_remaining_requests"] = remaining_requests
if remaining_tokens is not None:
additional_headers["x_ratelimit_remaining_tokens"] = remaining_tokens
return {
"model": "gpt-3.5-turbo",
"litellm_params": {
@ -43,7 +56,7 @@ def _build_request_kwargs(overhead_time_ms):
"custom_llm_provider": "openai",
"response_cost": 0.001,
"hidden_params": {
"additional_headers": None,
"additional_headers": additional_headers,
"litellm_overhead_time_ms": overhead_time_ms,
},
"metadata": {
@ -170,3 +183,72 @@ class TestOverheadLatencyMetricVisibility:
)
prometheus_logger.litellm_overhead_latency_metric.labels.assert_not_called()
class TestRemainingRequestsMetricVisibility:
"""
The litellm_remaining_requests_metric and litellm_remaining_tokens_metric
gauges must be set even when their value is 0, which means the rate limit
is exhausted — a critical signal to report, not suppress.
"""
def _setup_mocks(self, prometheus_logger):
prometheus_logger.litellm_overhead_latency_metric = MagicMock()
prometheus_logger.litellm_remaining_requests_metric = MagicMock()
prometheus_logger.litellm_remaining_tokens_metric = MagicMock()
prometheus_logger.litellm_deployment_success_responses = MagicMock()
prometheus_logger.litellm_deployment_total_requests = MagicMock()
prometheus_logger.litellm_deployment_latency_per_output_token = MagicMock()
prometheus_logger.set_deployment_healthy = MagicMock()
def test_should_set_remaining_requests_when_zero(self, prometheus_logger):
"""remaining_requests=0 means rate limit exhausted — must be reported."""
self._setup_mocks(prometheus_logger)
request_kwargs = _build_request_kwargs(remaining_requests=0)
prometheus_logger.set_llm_deployment_success_metrics(
request_kwargs=request_kwargs,
start_time=datetime.now(),
end_time=datetime.now() + timedelta(seconds=1),
output_tokens=10.0,
enum_values=_build_enum_values(),
)
prometheus_logger.litellm_remaining_requests_metric.labels.assert_called_once()
prometheus_logger.litellm_remaining_requests_metric.labels().set.assert_called_once_with(
0
)
def test_should_set_remaining_tokens_when_zero(self, prometheus_logger):
"""remaining_tokens=0 means token budget exhausted — must be reported."""
self._setup_mocks(prometheus_logger)
request_kwargs = _build_request_kwargs(remaining_tokens=0)
prometheus_logger.set_llm_deployment_success_metrics(
request_kwargs=request_kwargs,
start_time=datetime.now(),
end_time=datetime.now() + timedelta(seconds=1),
output_tokens=10.0,
enum_values=_build_enum_values(),
)
prometheus_logger.litellm_remaining_tokens_metric.labels.assert_called_once()
prometheus_logger.litellm_remaining_tokens_metric.labels().set.assert_called_once_with(
0
)
def test_should_not_set_remaining_metrics_when_none(self, prometheus_logger):
"""When headers are absent, remaining metrics must not be set."""
self._setup_mocks(prometheus_logger)
request_kwargs = _build_request_kwargs()
prometheus_logger.set_llm_deployment_success_metrics(
request_kwargs=request_kwargs,
start_time=datetime.now(),
end_time=datetime.now() + timedelta(seconds=1),
output_tokens=10.0,
enum_values=_build_enum_values(),
)
prometheus_logger.litellm_remaining_requests_metric.labels.assert_not_called()
prometheus_logger.litellm_remaining_tokens_metric.labels.assert_not_called()