From 413965d1029e1c7ea913f39a3281e62b86bcad02 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:07:16 -0700 Subject: [PATCH] test(prometheus): check emitted labels instead of wrapper types, close the admission match The exclude-labels test now emits through the spend and provider budget metrics and checks the scrape keeps all their labels. The admission match arms end in assert_never so the match is exhaustive. --- litellm/integrations/prometheus.py | 8 ++++++-- .../enterprise/integrations/test_prometheus.py | 16 ++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index ac48a9c954e..ee6ce548ff7 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -15,7 +15,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast from pydantic import BaseModel -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypedDict, assert_never import litellm from litellm._internal_context import with_service_target @@ -229,12 +229,14 @@ class _LabeledMetric: def remove(self, *labelvalues: object) -> None: match self._tracker: case SharedPrometheusSeriesAdmissions(): - return + pass case BoundedPrometheusSeriesTracker(): kept_values: Final = self._kept_values(labelvalues) with self._tracker.lock: self._tracker.forget_series(self._metric_name, kept_values) self._metric.remove(*kept_values) + case _: + assert_never(self._tracker) def _admits(self, kept_values: tuple[str, ...]) -> bool: match self._tracker: @@ -246,6 +248,8 @@ class _LabeledMetric: return self._tracker.admit_series( metric=self._metric, metric_name=self._metric_name, label_values=kept_values, limits=self._limits ) + case _: + assert_never(self._tracker) def _kept_values(self, values: tuple[object, ...]) -> tuple[str, ...]: return tuple( diff --git a/tests/unit/enterprise/integrations/test_prometheus.py b/tests/unit/enterprise/integrations/test_prometheus.py index 792dd977822..987cfa03a5e 100644 --- a/tests/unit/enterprise/integrations/test_prometheus.py +++ b/tests/unit/enterprise/integrations/test_prometheus.py @@ -689,9 +689,8 @@ def test_exclude_only_hardcoded_label_drops_all_labels(reset_prometheus_exclude_ def test_exclude_labels_does_not_touch_unrelated_metrics(reset_prometheus_exclude_settings): - """A metric that never declares the excluded label is left as a plain prometheus metric, - not wrapped, so no behavior changes for it.""" - from litellm.integrations.prometheus import _LabeledMetric + """A metric that never declares the excluded label keeps every one of its own labels in the scrape.""" + from prometheus_client import generate_latest clear_prometheus_registry() litellm.prometheus_metrics_config = None @@ -699,10 +698,15 @@ def test_exclude_labels_does_not_touch_unrelated_metrics(reset_prometheus_exclud litellm.prometheus_exclude_labels = ["guardrail_name"] logger = PrometheusLogger() + spend_labels = {name: f"{name}-value" for name in PrometheusMetricLabels.get_labels("litellm_spend_metric")} + logger.litellm_spend_metric.labels(**spend_labels).inc(1.5) + logger.litellm_provider_remaining_budget_metric.labels("anthropic").set(5.0) - assert not isinstance(logger.litellm_spend_metric, _LabeledMetric) - assert not isinstance(logger.litellm_provider_remaining_budget_metric, _LabeledMetric) - assert isinstance(logger.litellm_guardrail_latency_metric, _LabeledMetric) + scrape = generate_latest(REGISTRY).decode() + spend_line = next(line for line in scrape.splitlines() if line.startswith("litellm_spend_metric_total{")) + assert all(f'{name}="{value}"' in spend_line for name, value in spend_labels.items()) + assert spend_line.endswith(" 1.5") + assert 'litellm_provider_remaining_budget_metric{api_provider="anthropic"} 5.0' in scrape # ==============================================================================