fix(prometheus): drop stale team rate limit series when a limit is removed

Prometheus keeps a child series for the life of the process once emitted.
The gauges only wrote a value when the v3 limiter reported one, so a team
whose model_rpm_limit / model_tpm_limit was removed kept publishing the
last remaining and limit values it ever saw, and headroom alerts kept
evaluating against a limit no longer being enforced.

Drop the child series instead of returning early when the header is
absent. NoOpMetric and the label-excluding wrapper gain remove() so the
drop still applies when metrics are disabled or labels are filtered.

Always reaching the gauge attributes means async_log_success_event now
touches them on every team request, so the partially initialized logger
in test_prometheus_client_ip_user_agent stubs the new method alongside
the sibling rate limit one it already stubbed.

litellm/integrations/prometheus.py

@@ -95,6 +95,13 @@ class _ExcludedLabelMetric:
         )
         return self._metric.labels(*kept_values) if kept_values else self._metric
 
+    def remove(self, *labelvalues: str) -> None:
+        kept_values: Final = tuple(
+            value for name, value in zip(self._original_labelnames, labelvalues) if name not in self._excluded_labels
+        )
+        if kept_values:
+            self._metric.remove(*kept_values)
+
 
 def _get_budget_metrics_per_request_timeout() -> float:
     raw: Final = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT")
@@ -2071,22 +2078,6 @@ class PrometheusLogger(CustomLogger):
         if user_api_team is None:
             return
 
-        configured: Final = tuple(
-            (metric_name, value)
-            for metric_name, value_type, rate_limit_type in _TEAM_RATE_LIMIT_GAUGE_SPECS
-            if (
-                value := self._get_v3_rate_limit_header(
-                    standard_logging_payload=standard_logging_payload,
-                    descriptor_key="model_per_team",
-                    value_type=value_type,
-                    rate_limit_type=rate_limit_type,
-                )
-            )
-            is not None
-        )
-        if not configured:
-            return
-
         enum_values: Final = UserAPIKeyLabelValues(
             team=user_api_team,
             team_alias=user_api_team_alias,
@@ -2099,29 +2090,54 @@ class PrometheusLogger(CustomLogger):
         )
         label_context: Final = PrometheusLabelFactoryContext(enum_values)
 
-        for metric_name, value in configured:
-            self._set_team_rate_limit_gauge(
+        for metric_name, value_type, rate_limit_type in _TEAM_RATE_LIMIT_GAUGE_SPECS:
+            self._sync_team_rate_limit_gauge(
                 gauge=getattr(self, metric_name),
                 metric_name=metric_name,
-                value=value,
+                value=self._get_v3_rate_limit_header(
+                    standard_logging_payload=standard_logging_payload,
+                    descriptor_key="model_per_team",
+                    value_type=value_type,
+                    rate_limit_type=rate_limit_type,
+                ),
                 enum_values=enum_values,
                 label_context=label_context,
             )
 
-    def _set_team_rate_limit_gauge(
+    def _sync_team_rate_limit_gauge(
         self,
         gauge: _LabeledGauge,
         metric_name: DEFINED_PROMETHEUS_METRICS,
-        value: int,
+        value: int | None,
         enum_values: UserAPIKeyLabelValues,
         label_context: PrometheusLabelFactoryContext,
     ) -> None:
+        """
+        Set the gauge, or drop its child series when this team has no limit
+        configured for this model. Prometheus keeps a child series for the
+        life of the process once emitted, so without the drop a team whose
+        limit is removed would keep publishing the last remaining/limit
+        values it ever saw, and alerts would evaluate against a number no
+        longer being enforced.
+        """
         labels: Final = prometheus_label_factory(
             supported_enum_labels=self.get_labels_for_metric(metric_name),
             enum_values=enum_values,
             label_context=label_context,
         )
-        gauge.labels(**labels).set(value)
+        if value is not None:
+            gauge.labels(**labels).set(value)
+            return
+
+        remove: Final = getattr(gauge, "remove", None)
+        if remove is None:
+            return
+        try:
+            remove(*(labels[name] for name in self.get_labels_for_metric(metric_name)))
+        except KeyError:
+            # No child series for this labelset, which is the common case:
+            # the team never had a limit for this model.
+            pass
 
     def _set_latency_metrics(
         self,

litellm/types/integrations/prometheus.py

@@ -981,6 +981,9 @@ class NoOpMetric:
     def labels(self, *args, **kwargs):
         return self
 
+    def remove(self, *labelvalues: object) -> None:
+        pass
+
     def inc(self, *args, **kwargs) -> None:
         pass
 

tests/test_litellm/integrations/test_prometheus_client_ip_user_agent.py

@@ -93,6 +93,7 @@ async def test_async_post_call_success_hook_includes_client_ip_user_agent():
         logger._increment_token_metrics = MagicMock()
         logger._increment_remaining_budget_metrics = AsyncMock()
         logger._set_virtual_key_rate_limit_metrics = MagicMock()
+        logger._set_team_rate_limit_metrics = MagicMock()
         logger._set_latency_metrics = MagicMock()
         logger.set_llm_deployment_success_metrics = MagicMock()
         logger._increment_cache_metrics = MagicMock()

tests/test_litellm/integrations/test_prometheus_team_rate_limit_metrics.py

@@ -149,6 +149,36 @@ def test_emits_nothing_when_team_has_no_configured_limits():
         getattr(logger, metric_name).labels.assert_not_called()
 
 
+def test_drops_stale_series_when_a_team_limit_is_removed():
+    """
+    Prometheus keeps a child series for the life of the process once emitted,
+    so a team whose limit is removed would otherwise keep publishing the last
+    values it saw and alerts would fire on a limit nobody enforces.
+    """
+    logger = _logger_with_mock_team_gauges(labels_are_real=True)
+
+    _set_team_metrics(logger, _payload_with_headers(dict(ALL_TEAM_HEADERS)))
+    _assert_set_once(logger, "litellm_remaining_team_requests_for_model", 42)
+
+    _set_team_metrics(logger, _payload_with_headers({}))
+
+    for metric_name in TEAM_RATE_LIMIT_METRICS:
+        gauge = getattr(logger, metric_name)
+        gauge.remove.assert_called_once_with("team-abc", "research", "gpt-4o-mini")
+
+
+def test_survives_removing_a_series_that_was_never_emitted():
+    """The common case: a team that never had a limit for this model."""
+    logger = _logger_with_mock_team_gauges()
+    for metric_name in TEAM_RATE_LIMIT_METRICS:
+        getattr(logger, metric_name).remove.side_effect = KeyError("not present")
+
+    _set_team_metrics(logger, _payload_with_headers({}))
+
+    for metric_name in TEAM_RATE_LIMIT_METRICS:
+        getattr(logger, metric_name).labels.assert_not_called()
+
+
 def test_emits_only_the_dimension_the_team_configured():
     """A team with only an RPM limit must not get a fabricated TPM series."""
     logger = _logger_with_mock_team_gauges()