diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index f3a55a5204d..6934b384977 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -184,6 +184,17 @@ _TEAM_RATE_LIMIT_GAUGE_SPECS: Final[ ) +def _series_retirement_supported() -> bool: + """ + ``prometheus_client`` refuses to remove a labelset in multiprocess mode and + warns when asked, because each worker owns its own mmap file and cannot + retire a series another worker wrote. Retirement is therefore a + single-process capability, and attempting it under multiprocess collection + would only emit warnings while leaving the sample in place. + """ + return not ("PROMETHEUS_MULTIPROC_DIR" in os.environ or "prometheus_multiproc_dir" in os.environ) + + class PrometheusLogger(CustomLogger): # Class variables or attributes @@ -2181,13 +2192,18 @@ class PrometheusLogger(CustomLogger): label_context=label_context, ) label_values: Final = tuple(labels.get(name, "") for name in labelnames) - self._drop_superseded_team_series( - gauge=gauge, metric_name=metric_name, labels=labels, label_values=label_values - ) + can_retire: Final = _series_retirement_supported() + if can_retire: + self._drop_superseded_team_series( + gauge=gauge, metric_name=metric_name, labels=labels, label_values=label_values + ) if value is not None: gauge.labels(*label_values).set(value) return + if not can_retire: + return + self._forget_team_series(metric_name=metric_name, labels=labels) try: gauge.remove(*label_values) diff --git a/tests/test_litellm/integrations/test_prometheus_team_rate_limit_metrics.py b/tests/test_litellm/integrations/test_prometheus_team_rate_limit_metrics.py index a653865b6dc..c244651bef6 100644 --- a/tests/test_litellm/integrations/test_prometheus_team_rate_limit_metrics.py +++ b/tests/test_litellm/integrations/test_prometheus_team_rate_limit_metrics.py @@ -42,6 +42,16 @@ TEAM_RATE_LIMIT_METRICS = ( ) +@pytest.fixture(autouse=True) +def _single_process_collection(monkeypatch): + """ + Series retirement is only possible outside multiprocess collection, so pin + the mode rather than depending on whatever the ambient environment has set. + """ + monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False) + monkeypatch.delenv("prometheus_multiproc_dir", raising=False) + + def _logger_with_mock_team_gauges() -> PrometheusLogger: with patch("litellm.integrations.prometheus.PrometheusLogger.__init__", return_value=None): logger = PrometheusLogger() @@ -498,3 +508,33 @@ def test_rename_survives_a_tracked_series_that_is_already_gone(): renamed = {**TEAM_LABELS, "team_alias": "ml-research"} assert registry.get_sample_value("litellm_team_rpm_limit", renamed) == 60 + + +def test_does_not_attempt_retirement_under_multiprocess_collection(monkeypatch): + """ + prometheus_client refuses to remove a labelset when PROMETHEUS_MULTIPROC_DIR + is set, warning instead, because a worker cannot retire a series another + worker wrote. Attempting it on every team request would emit warnings while + leaving the sample in place, so the gauges are set and nothing is retired. + """ + monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", "/tmp/does-not-need-to-exist") + logger = _logger_with_mock_team_gauges() + + _set_team_metrics(logger, _payload_with_headers(dict(ALL_TEAM_HEADERS))) + _set_team_metrics(logger, _payload_with_headers({})) + + _assert_set_once(logger, "litellm_remaining_team_requests_for_model", 42) + for metric_name in TEAM_RATE_LIMIT_METRICS: + getattr(logger, metric_name).remove.assert_not_called() + + +def test_retires_series_when_collection_is_single_process(monkeypatch): + monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False) + monkeypatch.delenv("prometheus_multiproc_dir", raising=False) + logger = _logger_with_mock_team_gauges() + + _set_team_metrics(logger, _payload_with_headers(dict(ALL_TEAM_HEADERS))) + _set_team_metrics(logger, _payload_with_headers({})) + + for metric_name in TEAM_RATE_LIMIT_METRICS: + getattr(logger, metric_name).remove.assert_called_once_with("team-abc", "research", "gpt-4o-mini")