From 8ead29498c08cdbb55e5df79b38fbc0755f8f7d1 Mon Sep 17 00:00:00 2001 From: DanBrima <40828002+DanBrima@users.noreply.github.com> Date: Tue, 18 Aug 2026 04:34:38 +0000 Subject: [PATCH] fix(prometheus): drop stale team rate limit series when a limit is removed Prometheus keeps a child series for the life of the process once emitted. The gauges only wrote a value when the v3 limiter reported one, so a team whose model_rpm_limit / model_tpm_limit was removed kept publishing the last remaining and limit values it ever saw, and headroom alerts kept evaluating against a limit no longer being enforced. Drop the child series instead of returning early when the header is absent. NoOpMetric and the label-excluding wrapper gain remove() so the drop still applies when metrics are disabled or labels are filtered. Always reaching the gauge attributes means async_log_success_event now touches them on every team request, so the partially initialized logger in test_prometheus_client_ip_user_agent stubs the new method alongside the sibling rate limit one it already stubbed. --- litellm/integrations/prometheus.py | 60 ++++++++++++------- litellm/types/integrations/prometheus.py | 3 + .../test_prometheus_client_ip_user_agent.py | 1 + ...test_prometheus_team_rate_limit_metrics.py | 30 ++++++++++ 4 files changed, 72 insertions(+), 22 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 42a9148066f..59b395cceb7 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -95,6 +95,13 @@ class _ExcludedLabelMetric: ) return self._metric.labels(*kept_values) if kept_values else self._metric + def remove(self, *labelvalues: str) -> None: + kept_values: Final = tuple( + value for name, value in zip(self._original_labelnames, labelvalues) if name not in self._excluded_labels + ) + if kept_values: + self._metric.remove(*kept_values) + def _get_budget_metrics_per_request_timeout() -> float: raw: Final = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT") @@ -2071,22 +2078,6 @@ class PrometheusLogger(CustomLogger): if user_api_team is None: return - configured: Final = tuple( - (metric_name, value) - for metric_name, value_type, rate_limit_type in _TEAM_RATE_LIMIT_GAUGE_SPECS - if ( - value := self._get_v3_rate_limit_header( - standard_logging_payload=standard_logging_payload, - descriptor_key="model_per_team", - value_type=value_type, - rate_limit_type=rate_limit_type, - ) - ) - is not None - ) - if not configured: - return - enum_values: Final = UserAPIKeyLabelValues( team=user_api_team, team_alias=user_api_team_alias, @@ -2099,29 +2090,54 @@ class PrometheusLogger(CustomLogger): ) label_context: Final = PrometheusLabelFactoryContext(enum_values) - for metric_name, value in configured: - self._set_team_rate_limit_gauge( + for metric_name, value_type, rate_limit_type in _TEAM_RATE_LIMIT_GAUGE_SPECS: + self._sync_team_rate_limit_gauge( gauge=getattr(self, metric_name), metric_name=metric_name, - value=value, + value=self._get_v3_rate_limit_header( + standard_logging_payload=standard_logging_payload, + descriptor_key="model_per_team", + value_type=value_type, + rate_limit_type=rate_limit_type, + ), enum_values=enum_values, label_context=label_context, ) - def _set_team_rate_limit_gauge( + def _sync_team_rate_limit_gauge( self, gauge: _LabeledGauge, metric_name: DEFINED_PROMETHEUS_METRICS, - value: int, + value: int | None, enum_values: UserAPIKeyLabelValues, label_context: PrometheusLabelFactoryContext, ) -> None: + """ + Set the gauge, or drop its child series when this team has no limit + configured for this model. Prometheus keeps a child series for the + life of the process once emitted, so without the drop a team whose + limit is removed would keep publishing the last remaining/limit + values it ever saw, and alerts would evaluate against a number no + longer being enforced. + """ labels: Final = prometheus_label_factory( supported_enum_labels=self.get_labels_for_metric(metric_name), enum_values=enum_values, label_context=label_context, ) - gauge.labels(**labels).set(value) + if value is not None: + gauge.labels(**labels).set(value) + return + + remove: Final = getattr(gauge, "remove", None) + if remove is None: + return + try: + remove(*(labels[name] for name in self.get_labels_for_metric(metric_name))) + except KeyError: + # No child series for this labelset, which is the common case: + # the team never had a limit for this model. + pass def _set_latency_metrics( self, diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 9fe51f026e9..0b1288de3f1 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -981,6 +981,9 @@ class NoOpMetric: def labels(self, *args, **kwargs): return self + def remove(self, *labelvalues: object) -> None: + pass + def inc(self, *args, **kwargs) -> None: pass diff --git a/tests/test_litellm/integrations/test_prometheus_client_ip_user_agent.py b/tests/test_litellm/integrations/test_prometheus_client_ip_user_agent.py index 029b097cb75..08612110ca5 100644 --- a/tests/test_litellm/integrations/test_prometheus_client_ip_user_agent.py +++ b/tests/test_litellm/integrations/test_prometheus_client_ip_user_agent.py @@ -93,6 +93,7 @@ async def test_async_post_call_success_hook_includes_client_ip_user_agent(): logger._increment_token_metrics = MagicMock() logger._increment_remaining_budget_metrics = AsyncMock() logger._set_virtual_key_rate_limit_metrics = MagicMock() + logger._set_team_rate_limit_metrics = MagicMock() logger._set_latency_metrics = MagicMock() logger.set_llm_deployment_success_metrics = MagicMock() logger._increment_cache_metrics = MagicMock() diff --git a/tests/test_litellm/integrations/test_prometheus_team_rate_limit_metrics.py b/tests/test_litellm/integrations/test_prometheus_team_rate_limit_metrics.py index 820c4b75ea1..84a1a7b7405 100644 --- a/tests/test_litellm/integrations/test_prometheus_team_rate_limit_metrics.py +++ b/tests/test_litellm/integrations/test_prometheus_team_rate_limit_metrics.py @@ -149,6 +149,36 @@ def test_emits_nothing_when_team_has_no_configured_limits(): getattr(logger, metric_name).labels.assert_not_called() +def test_drops_stale_series_when_a_team_limit_is_removed(): + """ + Prometheus keeps a child series for the life of the process once emitted, + so a team whose limit is removed would otherwise keep publishing the last + values it saw and alerts would fire on a limit nobody enforces. + """ + logger = _logger_with_mock_team_gauges(labels_are_real=True) + + _set_team_metrics(logger, _payload_with_headers(dict(ALL_TEAM_HEADERS))) + _assert_set_once(logger, "litellm_remaining_team_requests_for_model", 42) + + _set_team_metrics(logger, _payload_with_headers({})) + + for metric_name in TEAM_RATE_LIMIT_METRICS: + gauge = getattr(logger, metric_name) + gauge.remove.assert_called_once_with("team-abc", "research", "gpt-4o-mini") + + +def test_survives_removing_a_series_that_was_never_emitted(): + """The common case: a team that never had a limit for this model.""" + logger = _logger_with_mock_team_gauges() + for metric_name in TEAM_RATE_LIMIT_METRICS: + getattr(logger, metric_name).remove.side_effect = KeyError("not present") + + _set_team_metrics(logger, _payload_with_headers({})) + + for metric_name in TEAM_RATE_LIMIT_METRICS: + getattr(logger, metric_name).labels.assert_not_called() + + def test_emits_only_the_dimension_the_team_configured(): """A team with only an RPM limit must not get a fabricated TPM series.""" logger = _logger_with_mock_team_gauges()