feat(prometheus): add global exclude_metrics and exclude_labels options (#34201)

* feat(prometheus): add global exclude_metrics and exclude_labels options

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(prometheus): apply global exclude_labels to hard-coded metric labels

Metrics built with hard-coded labelnames lists (guardrail, provider budget,
callback, managed file/batch, batch cost) bypassed prometheus_exclude_labels
because only labels resolved via get_labels_for_metric were filtered. Route
every metric through a factory that strips excluded labels at construction and
proxies labels() so excluded labels are dropped at emission too. Add the
non-enum hard-coded labels (guardrail_name, status, error_type, hook_type,
purpose, file_type, result) to exclude-config validation so they are accepted
instead of raising ValueError at logger init.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(prometheus): simplify exclude-label factory to the kwargs labelnames path

All metric definitions pass labelnames as a keyword argument, and the only
metrics that pass it positionally resolve their labels through
get_labels_for_metric, which already drops excluded labels, so they never carry
an excluded label into the factory. Drop the unreachable positional
reconstruction branch.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* chore: re-trigger CI (flaky unrelated bedrock agentcore test)

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(prometheus): use immutable constructions to satisfy LIT002 budget

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: shivam <shivam@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/__init__.py

@@ -449,6 +449,8 @@ enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
 custom_prometheus_metadata_labels: List[str] = []
 custom_prometheus_tags: List[str] = []
 prometheus_metrics_config: Optional[List] = None
+prometheus_exclude_metrics: Optional[List[str]] = None
+prometheus_exclude_labels: Optional[List[str]] = None
 prometheus_emit_stream_label: bool = False
 # Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on
 # `litellm_proxy_failed_requests_metric`. Off by default to preserve the

litellm/integrations/prometheus.py

@@ -64,11 +64,48 @@ from litellm.types.utils import (
 
 if TYPE_CHECKING:
     from apscheduler.schedulers.asyncio import AsyncIOScheduler
+    from prometheus_client.metrics import MetricWrapperBase
 else:
     AsyncIOScheduler = Any
 
 _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT = 5.0
 
+_NON_ENUM_METRIC_LABELS: frozenset[str] = frozenset(
+    (
+        "guardrail_name",
+        "status",
+        "error_type",
+        "hook_type",
+        "purpose",
+        "file_type",
+        "result",
+    )
+)
+
+
+class _ExcludedLabelMetric:
+    """Proxies a prometheus metric whose declared ``labelnames`` had globally
+    excluded labels removed, dropping those labels from every ``labels(...)``
+    call so the emitted arguments always match the metric's real label set."""
+
+    def __init__(
+        self,
+        metric: MetricWrapperBase,
+        original_labelnames: tuple[str, ...],
+        excluded_labels: frozenset[str],
+    ) -> None:
+        self._metric = metric
+        self._original_labelnames = original_labelnames
+        self._excluded_labels = excluded_labels
+
+    def labels(self, *labelvalues: str, **labelkwargs: str) -> MetricWrapperBase:
+        values = labelvalues or tuple(labelkwargs[name] for name in self._original_labelnames)
+        kept_values = tuple(
+            value for name, value in zip(self._original_labelnames, values) if name not in self._excluded_labels
+        )
+        return self._metric.labels(*kept_values) if kept_values else self._metric
+
+
 # Tiers a caller may name in a request, across the providers that accept the
 # parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and
 # Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which
@@ -122,6 +159,8 @@ class PrometheusLogger(CustomLogger):
             # Always initialize label_filters, even for non-premium users
             self.label_filters = self._parse_prometheus_config()
 
+            self.exclude_metrics, self.exclude_labels = self._parse_exclude_config()
+
             # Cache resolved label sets per metric. Several entries in
             # ``PrometheusMetricLabels.get_labels`` read module-level toggles
             # (e.g. ``litellm.prometheus_emit_stream_label``,
@@ -696,6 +735,44 @@ class PrometheusLogger(CustomLogger):
         self._pretty_print_prometheus_config(label_filters)
         return label_filters
 
+    def _parse_exclude_config(self) -> tuple[frozenset[str], frozenset[str]]:
+        """Parse and validate the global ``exclude_metrics`` / ``exclude_labels`` settings."""
+        from typing import get_args
+
+        import litellm
+
+        exclude_metrics = frozenset(litellm.prometheus_exclude_metrics or ())
+        exclude_labels = frozenset(litellm.prometheus_exclude_labels or ())
+
+        valid_metrics = frozenset(get_args(DEFINED_PROMETHEUS_METRICS))
+        invalid_metrics = sorted(exclude_metrics - valid_metrics)
+
+        valid_labels = self._all_defined_labels()
+        invalid_labels = sorted(exclude_labels - valid_labels)
+
+        errors = (
+            *(f"Invalid metric name in prometheus_exclude_metrics: {metric}" for metric in invalid_metrics),
+            *(f"Invalid label name in prometheus_exclude_labels: {label}" for label in invalid_labels),
+        )
+        if errors:
+            raise ValueError("Prometheus exclude configuration validation failed:\n" + "\n".join(errors))
+
+        return exclude_metrics, exclude_labels
+
+    @staticmethod
+    def _all_defined_labels() -> frozenset[str]:
+        """Every label a metric can emit: enum labels, hard-coded labels, and configured custom labels / tags."""
+        import litellm
+
+        builtin_labels = frozenset(label.value for label in UserAPIKeyLabelNames)
+        custom_metadata_labels = frozenset(
+            _sanitize_prometheus_label_name(label) for label in litellm.custom_prometheus_metadata_labels
+        )
+        custom_tag_labels = frozenset(
+            _sanitize_prometheus_label_name(f"tag_{tag}") for tag in litellm.custom_prometheus_tags
+        )
+        return builtin_labels | _NON_ENUM_METRIC_LABELS | custom_metadata_labels | custom_tag_labels
+
     def _validate_all_configurations(self, parsed_configs: List) -> ValidationResults:
         """Validate all metric configurations and return collected errors"""
         metric_errors = []
@@ -1015,6 +1092,9 @@ class PrometheusLogger(CustomLogger):
 
     def _is_metric_enabled(self, metric_name: str) -> bool:
         """Check if a metric is enabled based on configuration"""
+        if metric_name in self.exclude_metrics:
+            return False
+
         # If no specific configuration is provided, enable all metrics (default behavior)
         if not hasattr(self, "enabled_metrics"):
             return True
@@ -1032,11 +1112,18 @@ class PrometheusLogger(CustomLogger):
             # Extract metric name from the first argument or 'name' keyword argument
             metric_name = args[0] if args else kwargs.get("name", "")
 
-            if self._is_metric_enabled(metric_name):
-                return metric_class(*args, **kwargs)
-            else:
+            if not self._is_metric_enabled(metric_name):
                 return NoOpMetric()
 
+            original_labelnames = tuple(kwargs.get("labelnames") or ())
+            if not (frozenset(original_labelnames) & self.exclude_labels):
+                return metric_class(*args, **kwargs)
+
+            kept = tuple(name for name in original_labelnames if name not in self.exclude_labels)
+            kept_kwargs = {**kwargs, "labelnames": kept}  # mutable-ok: ** needs a mapping to override labelnames
+            real_metric = metric_class(*args, **kept_kwargs)
+            return _ExcludedLabelMetric(real_metric, original_labelnames, self.exclude_labels)
+
         return factory
 
     def get_labels_for_metric(self, metric_name: DEFINED_PROMETHEUS_METRICS) -> List[str]:
@@ -1059,19 +1146,15 @@ class PrometheusLogger(CustomLogger):
         # Get default labels for this metric from PrometheusMetricLabels
         default_labels = PrometheusMetricLabels.get_labels(metric_name)
 
-        # If no label filtering is configured for this metric, use default labels
-        if metric_name not in self.label_filters:
-            self._cached_metric_labels[metric_name] = default_labels
-            return default_labels
+        resolved_labels = [
+            label
+            for label in default_labels
+            if label not in self.exclude_labels
+            and (metric_name not in self.label_filters or label in self.label_filters[metric_name])
+        ]
 
-        # Get configured labels for this metric
-        configured_labels = self.label_filters[metric_name]
-
-        # Return intersection of configured and default labels to ensure we only use valid labels
-        filtered_labels = [label for label in default_labels if label in configured_labels]
-
-        self._cached_metric_labels[metric_name] = filtered_labels
-        return filtered_labels
+        self._cached_metric_labels[metric_name] = resolved_labels
+        return resolved_labels
 
     @staticmethod
     def _guardrail_is_additive(info: StandardLoggingGuardrailInformation) -> bool:

tests/enterprise/litellm_enterprise/integrations/test_prometheus.py

@@ -477,6 +477,244 @@ def test_valid_configuration_passes_validation():
 # ==============================================================================
 
 
+# ==============================================================================
+# GLOBAL EXCLUDE TESTS - exclude_metrics / exclude_labels
+# ==============================================================================
+
+
+@pytest.fixture
+def reset_prometheus_exclude_settings():
+    """Restore the global exclude settings after each test so they don't leak."""
+    prev_metrics = litellm.prometheus_exclude_metrics
+    prev_labels = litellm.prometheus_exclude_labels
+    prev_config = litellm.prometheus_metrics_config
+    try:
+        yield
+    finally:
+        litellm.prometheus_exclude_metrics = prev_metrics
+        litellm.prometheus_exclude_labels = prev_labels
+        litellm.prometheus_metrics_config = prev_config
+
+
+def test_exclude_metrics_disables_only_listed_metrics(reset_prometheus_exclude_settings):
+    """A metric named in exclude_metrics becomes a NoOpMetric; others stay real."""
+    from litellm.types.integrations.prometheus import NoOpMetric
+
+    clear_prometheus_registry()
+    litellm.prometheus_metrics_config = None
+    litellm.prometheus_exclude_labels = None
+    litellm.prometheus_exclude_metrics = [
+        "litellm_spend_metric",
+        "litellm_input_tokens_metric",
+    ]
+
+    logger = PrometheusLogger()
+
+    assert isinstance(logger.litellm_spend_metric, NoOpMetric)
+    assert isinstance(logger.litellm_input_tokens_metric, NoOpMetric)
+    # A metric not in the exclude list is still a real prometheus metric
+    assert not isinstance(logger.litellm_output_tokens_metric, NoOpMetric)
+
+
+def test_exclude_metrics_wins_over_include_config(reset_prometheus_exclude_settings):
+    """exclude_metrics removes a metric even if an include-based group enabled it."""
+    from litellm.types.integrations.prometheus import NoOpMetric
+
+    clear_prometheus_registry()
+    litellm.prometheus_exclude_labels = None
+    litellm.prometheus_metrics_config = [
+        {
+            "group": "tokens",
+            "metrics": ["litellm_input_tokens_metric", "litellm_output_tokens_metric"],
+        }
+    ]
+    litellm.prometheus_exclude_metrics = ["litellm_input_tokens_metric"]
+
+    logger = PrometheusLogger()
+
+    assert isinstance(logger.litellm_input_tokens_metric, NoOpMetric)
+    assert not isinstance(logger.litellm_output_tokens_metric, NoOpMetric)
+
+
+def test_exclude_labels_dropped_globally(reset_prometheus_exclude_settings):
+    """exclude_labels removes the label from every metric that would emit it."""
+    clear_prometheus_registry()
+    litellm.prometheus_metrics_config = None
+    litellm.prometheus_exclude_metrics = None
+    litellm.prometheus_exclude_labels = ["hashed_api_key", "api_key_alias"]
+
+    logger = PrometheusLogger()
+
+    for metric_name in ("litellm_spend_metric", "litellm_input_tokens_metric"):
+        labels = logger.get_labels_for_metric(metric_name)
+        assert "hashed_api_key" not in labels
+        assert "api_key_alias" not in labels
+        # Other default labels remain
+        assert "team" in labels
+
+
+def test_exclude_labels_intersect_with_include_labels(reset_prometheus_exclude_settings):
+    """exclude_labels is applied on top of an include-based label filter."""
+    clear_prometheus_registry()
+    litellm.prometheus_exclude_metrics = None
+    litellm.prometheus_metrics_config = [
+        {
+            "group": "spend",
+            "metrics": ["litellm_spend_metric"],
+            "include_labels": ["hashed_api_key", "team", "api_provider"],
+        }
+    ]
+    litellm.prometheus_exclude_labels = ["hashed_api_key"]
+
+    logger = PrometheusLogger()
+
+    labels = logger.get_labels_for_metric("litellm_spend_metric")
+    assert "hashed_api_key" not in labels
+    assert set(labels) == {"team", "api_provider"}
+
+
+def test_no_exclude_settings_is_backward_compatible(reset_prometheus_exclude_settings):
+    """With no exclude settings, all metrics and default labels are preserved."""
+    from litellm.types.integrations.prometheus import NoOpMetric
+
+    clear_prometheus_registry()
+    litellm.prometheus_metrics_config = None
+    litellm.prometheus_exclude_metrics = None
+    litellm.prometheus_exclude_labels = None
+
+    logger = PrometheusLogger()
+
+    assert logger.exclude_metrics == frozenset()
+    assert logger.exclude_labels == frozenset()
+    assert not isinstance(logger.litellm_spend_metric, NoOpMetric)
+    default_labels = PrometheusMetricLabels.get_labels("litellm_spend_metric")
+    assert logger.get_labels_for_metric("litellm_spend_metric") == default_labels
+
+
+def test_invalid_exclude_metric_name_raises(reset_prometheus_exclude_settings):
+    """An unknown metric name in exclude_metrics fails fast at logger init."""
+    clear_prometheus_registry()
+    litellm.prometheus_metrics_config = None
+    litellm.prometheus_exclude_labels = None
+    litellm.prometheus_exclude_metrics = ["not_a_real_metric"]
+
+    with pytest.raises(ValueError) as exc_info:
+        PrometheusLogger()
+
+    assert "not_a_real_metric" in str(exc_info.value)
+    assert "prometheus_exclude_metrics" in str(exc_info.value)
+
+
+def test_invalid_exclude_label_name_raises(reset_prometheus_exclude_settings):
+    """An unknown label name in exclude_labels fails fast at logger init."""
+    clear_prometheus_registry()
+    litellm.prometheus_metrics_config = None
+    litellm.prometheus_exclude_metrics = None
+    litellm.prometheus_exclude_labels = ["not_a_real_label"]
+
+    with pytest.raises(ValueError) as exc_info:
+        PrometheusLogger()
+
+    assert "not_a_real_label" in str(exc_info.value)
+    assert "prometheus_exclude_labels" in str(exc_info.value)
+
+
+@pytest.mark.parametrize(
+    "hardcoded_label",
+    ["guardrail_name", "status", "error_type", "hook_type", "purpose", "file_type", "result"],
+)
+def test_exclude_hardcoded_label_name_is_accepted(reset_prometheus_exclude_settings, hardcoded_label):
+    """Labels that only appear in hard-coded metric definitions (not UserAPIKeyLabelNames)
+    are valid exclude targets and must not fail validation at logger init."""
+    clear_prometheus_registry()
+    litellm.prometheus_metrics_config = None
+    litellm.prometheus_exclude_metrics = None
+    litellm.prometheus_exclude_labels = [hardcoded_label]
+
+    logger = PrometheusLogger()
+
+    assert hardcoded_label in logger.exclude_labels
+
+
+def test_exclude_labels_dropped_from_hardcoded_metric(reset_prometheus_exclude_settings):
+    """A metric built with a hard-coded labelnames list drops excluded labels from its
+    declared label set instead of silently retaining them."""
+    clear_prometheus_registry()
+    litellm.prometheus_metrics_config = None
+    litellm.prometheus_exclude_metrics = None
+    litellm.prometheus_exclude_labels = ["guardrail_name"]
+
+    logger = PrometheusLogger()
+
+    labelnames = logger.litellm_guardrail_latency_metric._metric._labelnames
+    assert "guardrail_name" not in labelnames
+    assert set(labelnames) == {"status", "error_type", "hook_type"}
+
+
+def test_hardcoded_metric_emission_omits_excluded_label(reset_prometheus_exclude_settings):
+    """Emitting a hard-coded metric with the excluded label still passed keeps the emission
+    working and the excluded label never reaches the scrape output."""
+    from prometheus_client import generate_latest
+
+    clear_prometheus_registry()
+    litellm.prometheus_metrics_config = None
+    litellm.prometheus_exclude_metrics = None
+    litellm.prometheus_exclude_labels = ["guardrail_name"]
+
+    logger = PrometheusLogger()
+    logger.litellm_guardrail_latency_metric.labels(
+        guardrail_name="my_guardrail",
+        status="success",
+        error_type="",
+        hook_type="pre_call",
+    ).observe(0.25)
+
+    scrape = generate_latest(REGISTRY).decode()
+    assert "litellm_guardrail_latency_seconds_bucket" in scrape
+    assert "my_guardrail" not in scrape
+    assert 'guardrail_name="' not in scrape
+    assert 'status="success"' in scrape
+
+
+def test_exclude_only_hardcoded_label_drops_all_labels(reset_prometheus_exclude_settings):
+    """Excluding the sole label of a hard-coded metric leaves it label-less and still emittable
+    via both keyword and positional labels() calls."""
+    clear_prometheus_registry()
+    litellm.prometheus_metrics_config = None
+    litellm.prometheus_exclude_metrics = None
+    litellm.prometheus_exclude_labels = ["result", "api_provider"]
+
+    logger = PrometheusLogger()
+
+    assert logger.litellm_managed_file_deleted_total._metric._labelnames == ()
+    assert logger.litellm_provider_remaining_budget_metric._metric._labelnames == ()
+
+    logger.litellm_managed_file_deleted_total.labels(result="blocked").inc()
+    logger.litellm_provider_remaining_budget_metric.labels("anthropic").set(5.0)
+
+
+def test_exclude_labels_does_not_touch_unrelated_metrics(reset_prometheus_exclude_settings):
+    """A metric that never declares the excluded label is left as a plain prometheus metric,
+    not wrapped, so no behavior changes for it."""
+    from litellm.integrations.prometheus import _ExcludedLabelMetric
+
+    clear_prometheus_registry()
+    litellm.prometheus_metrics_config = None
+    litellm.prometheus_exclude_metrics = None
+    litellm.prometheus_exclude_labels = ["guardrail_name"]
+
+    logger = PrometheusLogger()
+
+    assert not isinstance(logger.litellm_spend_metric, _ExcludedLabelMetric)
+    assert not isinstance(logger.litellm_provider_remaining_budget_metric, _ExcludedLabelMetric)
+    assert isinstance(logger.litellm_guardrail_latency_metric, _ExcludedLabelMetric)
+
+
+# ==============================================================================
+# END GLOBAL EXCLUDE TESTS
+# ==============================================================================
+
+
 # ==============================================================================
 # SEMANTIC VALIDATION TESTS - Detect logical errors in metric increments
 # ==============================================================================