diff --git a/litellm/__init__.py b/litellm/__init__.py index 3f8c742c5a2..a9a78846fa1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -449,6 +449,8 @@ enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None custom_prometheus_metadata_labels: List[str] = [] custom_prometheus_tags: List[str] = [] prometheus_metrics_config: Optional[List] = None +prometheus_exclude_metrics: Optional[List[str]] = None +prometheus_exclude_labels: Optional[List[str]] = None prometheus_emit_stream_label: bool = False # Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on # `litellm_proxy_failed_requests_metric`. Off by default to preserve the diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 41a4d026fe1..1bc7308f7df 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -64,11 +64,48 @@ from litellm.types.utils import ( if TYPE_CHECKING: from apscheduler.schedulers.asyncio import AsyncIOScheduler + from prometheus_client.metrics import MetricWrapperBase else: AsyncIOScheduler = Any _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT = 5.0 +_NON_ENUM_METRIC_LABELS: frozenset[str] = frozenset( + ( + "guardrail_name", + "status", + "error_type", + "hook_type", + "purpose", + "file_type", + "result", + ) +) + + +class _ExcludedLabelMetric: + """Proxies a prometheus metric whose declared ``labelnames`` had globally + excluded labels removed, dropping those labels from every ``labels(...)`` + call so the emitted arguments always match the metric's real label set.""" + + def __init__( + self, + metric: MetricWrapperBase, + original_labelnames: tuple[str, ...], + excluded_labels: frozenset[str], + ) -> None: + self._metric = metric + self._original_labelnames = original_labelnames + self._excluded_labels = excluded_labels + + def labels(self, *labelvalues: str, **labelkwargs: str) -> MetricWrapperBase: + values = labelvalues or tuple(labelkwargs[name] for name in self._original_labelnames) + kept_values = tuple( + value for name, value in zip(self._original_labelnames, values) if name not in self._excluded_labels + ) + return self._metric.labels(*kept_values) if kept_values else self._metric + + # Tiers a caller may name in a request, across the providers that accept the # parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and # Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which @@ -122,6 +159,8 @@ class PrometheusLogger(CustomLogger): # Always initialize label_filters, even for non-premium users self.label_filters = self._parse_prometheus_config() + self.exclude_metrics, self.exclude_labels = self._parse_exclude_config() + # Cache resolved label sets per metric. Several entries in # ``PrometheusMetricLabels.get_labels`` read module-level toggles # (e.g. ``litellm.prometheus_emit_stream_label``, @@ -696,6 +735,44 @@ class PrometheusLogger(CustomLogger): self._pretty_print_prometheus_config(label_filters) return label_filters + def _parse_exclude_config(self) -> tuple[frozenset[str], frozenset[str]]: + """Parse and validate the global ``exclude_metrics`` / ``exclude_labels`` settings.""" + from typing import get_args + + import litellm + + exclude_metrics = frozenset(litellm.prometheus_exclude_metrics or ()) + exclude_labels = frozenset(litellm.prometheus_exclude_labels or ()) + + valid_metrics = frozenset(get_args(DEFINED_PROMETHEUS_METRICS)) + invalid_metrics = sorted(exclude_metrics - valid_metrics) + + valid_labels = self._all_defined_labels() + invalid_labels = sorted(exclude_labels - valid_labels) + + errors = ( + *(f"Invalid metric name in prometheus_exclude_metrics: {metric}" for metric in invalid_metrics), + *(f"Invalid label name in prometheus_exclude_labels: {label}" for label in invalid_labels), + ) + if errors: + raise ValueError("Prometheus exclude configuration validation failed:\n" + "\n".join(errors)) + + return exclude_metrics, exclude_labels + + @staticmethod + def _all_defined_labels() -> frozenset[str]: + """Every label a metric can emit: enum labels, hard-coded labels, and configured custom labels / tags.""" + import litellm + + builtin_labels = frozenset(label.value for label in UserAPIKeyLabelNames) + custom_metadata_labels = frozenset( + _sanitize_prometheus_label_name(label) for label in litellm.custom_prometheus_metadata_labels + ) + custom_tag_labels = frozenset( + _sanitize_prometheus_label_name(f"tag_{tag}") for tag in litellm.custom_prometheus_tags + ) + return builtin_labels | _NON_ENUM_METRIC_LABELS | custom_metadata_labels | custom_tag_labels + def _validate_all_configurations(self, parsed_configs: List) -> ValidationResults: """Validate all metric configurations and return collected errors""" metric_errors = [] @@ -1015,6 +1092,9 @@ class PrometheusLogger(CustomLogger): def _is_metric_enabled(self, metric_name: str) -> bool: """Check if a metric is enabled based on configuration""" + if metric_name in self.exclude_metrics: + return False + # If no specific configuration is provided, enable all metrics (default behavior) if not hasattr(self, "enabled_metrics"): return True @@ -1032,11 +1112,18 @@ class PrometheusLogger(CustomLogger): # Extract metric name from the first argument or 'name' keyword argument metric_name = args[0] if args else kwargs.get("name", "") - if self._is_metric_enabled(metric_name): - return metric_class(*args, **kwargs) - else: + if not self._is_metric_enabled(metric_name): return NoOpMetric() + original_labelnames = tuple(kwargs.get("labelnames") or ()) + if not (frozenset(original_labelnames) & self.exclude_labels): + return metric_class(*args, **kwargs) + + kept = tuple(name for name in original_labelnames if name not in self.exclude_labels) + kept_kwargs = {**kwargs, "labelnames": kept} # mutable-ok: ** needs a mapping to override labelnames + real_metric = metric_class(*args, **kept_kwargs) + return _ExcludedLabelMetric(real_metric, original_labelnames, self.exclude_labels) + return factory def get_labels_for_metric(self, metric_name: DEFINED_PROMETHEUS_METRICS) -> List[str]: @@ -1059,19 +1146,15 @@ class PrometheusLogger(CustomLogger): # Get default labels for this metric from PrometheusMetricLabels default_labels = PrometheusMetricLabels.get_labels(metric_name) - # If no label filtering is configured for this metric, use default labels - if metric_name not in self.label_filters: - self._cached_metric_labels[metric_name] = default_labels - return default_labels + resolved_labels = [ + label + for label in default_labels + if label not in self.exclude_labels + and (metric_name not in self.label_filters or label in self.label_filters[metric_name]) + ] - # Get configured labels for this metric - configured_labels = self.label_filters[metric_name] - - # Return intersection of configured and default labels to ensure we only use valid labels - filtered_labels = [label for label in default_labels if label in configured_labels] - - self._cached_metric_labels[metric_name] = filtered_labels - return filtered_labels + self._cached_metric_labels[metric_name] = resolved_labels + return resolved_labels @staticmethod def _guardrail_is_additive(info: StandardLoggingGuardrailInformation) -> bool: diff --git a/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py b/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py index ebea96e2152..0cd6055e09d 100644 --- a/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py +++ b/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py @@ -477,6 +477,244 @@ def test_valid_configuration_passes_validation(): # ============================================================================== +# ============================================================================== +# GLOBAL EXCLUDE TESTS - exclude_metrics / exclude_labels +# ============================================================================== + + +@pytest.fixture +def reset_prometheus_exclude_settings(): + """Restore the global exclude settings after each test so they don't leak.""" + prev_metrics = litellm.prometheus_exclude_metrics + prev_labels = litellm.prometheus_exclude_labels + prev_config = litellm.prometheus_metrics_config + try: + yield + finally: + litellm.prometheus_exclude_metrics = prev_metrics + litellm.prometheus_exclude_labels = prev_labels + litellm.prometheus_metrics_config = prev_config + + +def test_exclude_metrics_disables_only_listed_metrics(reset_prometheus_exclude_settings): + """A metric named in exclude_metrics becomes a NoOpMetric; others stay real.""" + from litellm.types.integrations.prometheus import NoOpMetric + + clear_prometheus_registry() + litellm.prometheus_metrics_config = None + litellm.prometheus_exclude_labels = None + litellm.prometheus_exclude_metrics = [ + "litellm_spend_metric", + "litellm_input_tokens_metric", + ] + + logger = PrometheusLogger() + + assert isinstance(logger.litellm_spend_metric, NoOpMetric) + assert isinstance(logger.litellm_input_tokens_metric, NoOpMetric) + # A metric not in the exclude list is still a real prometheus metric + assert not isinstance(logger.litellm_output_tokens_metric, NoOpMetric) + + +def test_exclude_metrics_wins_over_include_config(reset_prometheus_exclude_settings): + """exclude_metrics removes a metric even if an include-based group enabled it.""" + from litellm.types.integrations.prometheus import NoOpMetric + + clear_prometheus_registry() + litellm.prometheus_exclude_labels = None + litellm.prometheus_metrics_config = [ + { + "group": "tokens", + "metrics": ["litellm_input_tokens_metric", "litellm_output_tokens_metric"], + } + ] + litellm.prometheus_exclude_metrics = ["litellm_input_tokens_metric"] + + logger = PrometheusLogger() + + assert isinstance(logger.litellm_input_tokens_metric, NoOpMetric) + assert not isinstance(logger.litellm_output_tokens_metric, NoOpMetric) + + +def test_exclude_labels_dropped_globally(reset_prometheus_exclude_settings): + """exclude_labels removes the label from every metric that would emit it.""" + clear_prometheus_registry() + litellm.prometheus_metrics_config = None + litellm.prometheus_exclude_metrics = None + litellm.prometheus_exclude_labels = ["hashed_api_key", "api_key_alias"] + + logger = PrometheusLogger() + + for metric_name in ("litellm_spend_metric", "litellm_input_tokens_metric"): + labels = logger.get_labels_for_metric(metric_name) + assert "hashed_api_key" not in labels + assert "api_key_alias" not in labels + # Other default labels remain + assert "team" in labels + + +def test_exclude_labels_intersect_with_include_labels(reset_prometheus_exclude_settings): + """exclude_labels is applied on top of an include-based label filter.""" + clear_prometheus_registry() + litellm.prometheus_exclude_metrics = None + litellm.prometheus_metrics_config = [ + { + "group": "spend", + "metrics": ["litellm_spend_metric"], + "include_labels": ["hashed_api_key", "team", "api_provider"], + } + ] + litellm.prometheus_exclude_labels = ["hashed_api_key"] + + logger = PrometheusLogger() + + labels = logger.get_labels_for_metric("litellm_spend_metric") + assert "hashed_api_key" not in labels + assert set(labels) == {"team", "api_provider"} + + +def test_no_exclude_settings_is_backward_compatible(reset_prometheus_exclude_settings): + """With no exclude settings, all metrics and default labels are preserved.""" + from litellm.types.integrations.prometheus import NoOpMetric + + clear_prometheus_registry() + litellm.prometheus_metrics_config = None + litellm.prometheus_exclude_metrics = None + litellm.prometheus_exclude_labels = None + + logger = PrometheusLogger() + + assert logger.exclude_metrics == frozenset() + assert logger.exclude_labels == frozenset() + assert not isinstance(logger.litellm_spend_metric, NoOpMetric) + default_labels = PrometheusMetricLabels.get_labels("litellm_spend_metric") + assert logger.get_labels_for_metric("litellm_spend_metric") == default_labels + + +def test_invalid_exclude_metric_name_raises(reset_prometheus_exclude_settings): + """An unknown metric name in exclude_metrics fails fast at logger init.""" + clear_prometheus_registry() + litellm.prometheus_metrics_config = None + litellm.prometheus_exclude_labels = None + litellm.prometheus_exclude_metrics = ["not_a_real_metric"] + + with pytest.raises(ValueError) as exc_info: + PrometheusLogger() + + assert "not_a_real_metric" in str(exc_info.value) + assert "prometheus_exclude_metrics" in str(exc_info.value) + + +def test_invalid_exclude_label_name_raises(reset_prometheus_exclude_settings): + """An unknown label name in exclude_labels fails fast at logger init.""" + clear_prometheus_registry() + litellm.prometheus_metrics_config = None + litellm.prometheus_exclude_metrics = None + litellm.prometheus_exclude_labels = ["not_a_real_label"] + + with pytest.raises(ValueError) as exc_info: + PrometheusLogger() + + assert "not_a_real_label" in str(exc_info.value) + assert "prometheus_exclude_labels" in str(exc_info.value) + + +@pytest.mark.parametrize( + "hardcoded_label", + ["guardrail_name", "status", "error_type", "hook_type", "purpose", "file_type", "result"], +) +def test_exclude_hardcoded_label_name_is_accepted(reset_prometheus_exclude_settings, hardcoded_label): + """Labels that only appear in hard-coded metric definitions (not UserAPIKeyLabelNames) + are valid exclude targets and must not fail validation at logger init.""" + clear_prometheus_registry() + litellm.prometheus_metrics_config = None + litellm.prometheus_exclude_metrics = None + litellm.prometheus_exclude_labels = [hardcoded_label] + + logger = PrometheusLogger() + + assert hardcoded_label in logger.exclude_labels + + +def test_exclude_labels_dropped_from_hardcoded_metric(reset_prometheus_exclude_settings): + """A metric built with a hard-coded labelnames list drops excluded labels from its + declared label set instead of silently retaining them.""" + clear_prometheus_registry() + litellm.prometheus_metrics_config = None + litellm.prometheus_exclude_metrics = None + litellm.prometheus_exclude_labels = ["guardrail_name"] + + logger = PrometheusLogger() + + labelnames = logger.litellm_guardrail_latency_metric._metric._labelnames + assert "guardrail_name" not in labelnames + assert set(labelnames) == {"status", "error_type", "hook_type"} + + +def test_hardcoded_metric_emission_omits_excluded_label(reset_prometheus_exclude_settings): + """Emitting a hard-coded metric with the excluded label still passed keeps the emission + working and the excluded label never reaches the scrape output.""" + from prometheus_client import generate_latest + + clear_prometheus_registry() + litellm.prometheus_metrics_config = None + litellm.prometheus_exclude_metrics = None + litellm.prometheus_exclude_labels = ["guardrail_name"] + + logger = PrometheusLogger() + logger.litellm_guardrail_latency_metric.labels( + guardrail_name="my_guardrail", + status="success", + error_type="", + hook_type="pre_call", + ).observe(0.25) + + scrape = generate_latest(REGISTRY).decode() + assert "litellm_guardrail_latency_seconds_bucket" in scrape + assert "my_guardrail" not in scrape + assert 'guardrail_name="' not in scrape + assert 'status="success"' in scrape + + +def test_exclude_only_hardcoded_label_drops_all_labels(reset_prometheus_exclude_settings): + """Excluding the sole label of a hard-coded metric leaves it label-less and still emittable + via both keyword and positional labels() calls.""" + clear_prometheus_registry() + litellm.prometheus_metrics_config = None + litellm.prometheus_exclude_metrics = None + litellm.prometheus_exclude_labels = ["result", "api_provider"] + + logger = PrometheusLogger() + + assert logger.litellm_managed_file_deleted_total._metric._labelnames == () + assert logger.litellm_provider_remaining_budget_metric._metric._labelnames == () + + logger.litellm_managed_file_deleted_total.labels(result="blocked").inc() + logger.litellm_provider_remaining_budget_metric.labels("anthropic").set(5.0) + + +def test_exclude_labels_does_not_touch_unrelated_metrics(reset_prometheus_exclude_settings): + """A metric that never declares the excluded label is left as a plain prometheus metric, + not wrapped, so no behavior changes for it.""" + from litellm.integrations.prometheus import _ExcludedLabelMetric + + clear_prometheus_registry() + litellm.prometheus_metrics_config = None + litellm.prometheus_exclude_metrics = None + litellm.prometheus_exclude_labels = ["guardrail_name"] + + logger = PrometheusLogger() + + assert not isinstance(logger.litellm_spend_metric, _ExcludedLabelMetric) + assert not isinstance(logger.litellm_provider_remaining_budget_metric, _ExcludedLabelMetric) + assert isinstance(logger.litellm_guardrail_latency_metric, _ExcludedLabelMetric) + + +# ============================================================================== +# END GLOBAL EXCLUDE TESTS +# ============================================================================== + + # ============================================================================== # SEMANTIC VALIDATION TESTS - Detect logical errors in metric increments # ==============================================================================