From aa44477008f107cf136c149c4b5793a2da57aeac Mon Sep 17 00:00:00 2001 From: ademicho123 Date: Sat, 25 Jul 2026 21:36:38 +0100 Subject: [PATCH] fix: apply wildcard label-filter logic to PrometheusLogger, fix test config leak --- litellm/integrations/prometheus.py | 72 ++++++++++++++++--- .../test_prometheus_wildcard_labels.py | 49 +++++++------ 2 files changed, 91 insertions(+), 30 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 64d4dd578b2..da2547a7556 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -669,14 +669,21 @@ class PrometheusLogger(CustomLogger): parsed_config = group_config parsed_configs.append(parsed_config) - self.enabled_metrics.update(parsed_config.metrics) + + # A wildcard group only sets a default label filter - it should + # never narrow down which metrics get created. Only groups that + # name real metrics feed into enabled_metrics, same as before. + if parsed_config.metrics != [PROMETHEUS_METRICS_WILDCARD]: + self.enabled_metrics.update(parsed_config.metrics) # Validate all configurations validation_results = self._validate_all_configurations(parsed_configs) if validation_results.has_errors: self._pretty_print_validation_errors(validation_results) - error_message = "Configuration validation failed:\n" + "\n".join(validation_results.all_error_messages) + error_message = "Configuration validation failed:\n" + "\n".join( + validation_results.all_error_messages + ) raise ValueError(error_message) # Build label filters from valid configurations @@ -688,20 +695,53 @@ class PrometheusLogger(CustomLogger): def _validate_all_configurations(self, parsed_configs: List) -> ValidationResults: """Validate all metric configurations and return collected errors""" + from typing import get_args + metric_errors = [] label_errors = [] for config in parsed_configs: + if PROMETHEUS_METRICS_WILDCARD in config.metrics: + if len(config.metrics) > 1: + raise ValueError( + f"Prometheus metrics group '{config.group}' mixes the " + f"wildcard '{PROMETHEUS_METRICS_WILDCARD}' with named " + f"metrics ({config.metrics}). Put named-metric overrides " + "in a separate group - the wildcard group only sets the " + "default label filter, it can't also be scoped." + ) + + if config.include_labels: + all_valid_labels = { + label + for metric_name in get_args(DEFINED_PROMETHEUS_METRICS) + for label in PrometheusMetricLabels.get_labels(metric_name) + } + unknown_labels = [ + label + for label in config.include_labels + if label not in all_valid_labels + ] + if unknown_labels: + label_errors.append( + LabelValidationError( + metric_name=PROMETHEUS_METRICS_WILDCARD, + invalid_labels=unknown_labels, + valid_labels=sorted(all_valid_labels), + ) + ) + continue + for metric_name in config.metrics: - # Validate metric name metric_error = self._validate_single_metric_name(metric_name) if metric_error: metric_errors.append(metric_error) - continue # Skip label validation if metric name is invalid + continue - # Validate labels if provided if config.include_labels: - label_error = self._validate_single_metric_labels(metric_name, config.include_labels) + label_error = self._validate_single_metric_labels( + metric_name, config.include_labels + ) if label_error: label_errors.append(label_error) @@ -738,12 +778,26 @@ class PrometheusLogger(CustomLogger): def _build_label_filters(self, parsed_configs: List) -> Dict[str, List[str]]: """Build label filters from validated configurations""" - label_filters = {} + from typing import get_args - for config in parsed_configs: + label_filters: Dict[str, List[str]] = {} + + wildcard_configs = [ + c for c in parsed_configs if c.metrics == [PROMETHEUS_METRICS_WILDCARD] + ] + named_configs = [ + c for c in parsed_configs if c.metrics != [PROMETHEUS_METRICS_WILDCARD] + ] + + for config in wildcard_configs: + if not config.include_labels: + continue + for metric_name in get_args(DEFINED_PROMETHEUS_METRICS): + label_filters[metric_name] = config.include_labels + + for config in named_configs: for metric_name in config.metrics: if config.include_labels: - # Only add if metric name is valid (validation already passed) if self._validate_single_metric_name(metric_name) is None: label_filters[metric_name] = config.include_labels diff --git a/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py b/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py index 9545fd794a3..b99ca75b1ac 100644 --- a/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py +++ b/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py @@ -63,17 +63,21 @@ def test_named_group_overrides_wildcard_for_that_metric(): assert label_filters["litellm_total_tokens_metric"] == ["hashed_api_key", "team"] -def test_wildcard_group_does_not_disable_other_metrics(): +def test_wildcard_group_does_not_disable_other_metrics(monkeypatch): """A wildcard-only config should leave every metric enabled - it's a label filter, not an allowlist.""" logger = _bare_logger() - litellm.prometheus_metrics_config = [ - { - "group": "defaults", - "metrics": [PROMETHEUS_METRICS_WILDCARD], - "include_labels": ["team"], - } - ] + monkeypatch.setattr( + litellm, + "prometheus_metrics_config", + [ + { + "group": "defaults", + "metrics": [PROMETHEUS_METRICS_WILDCARD], + "include_labels": ["team"], + } + ], + ) logger._parse_prometheus_config() @@ -82,21 +86,25 @@ def test_wildcard_group_does_not_disable_other_metrics(): assert logger._is_metric_enabled("litellm_mcp_tool_calls_total") is True -def test_wildcard_combined_with_named_enable_list(): +def test_wildcard_combined_with_named_enable_list(monkeypatch): """Wildcard for labels + a separate group naming which metrics are enabled - the two concerns are independent.""" logger = _bare_logger() - litellm.prometheus_metrics_config = [ - { - "group": "enabled", - "metrics": ["litellm_spend_metric", "litellm_total_tokens_metric"], - }, - { - "group": "defaults", - "metrics": [PROMETHEUS_METRICS_WILDCARD], - "include_labels": ["team"], - }, - ] + monkeypatch.setattr( + litellm, + "prometheus_metrics_config", + [ + { + "group": "enabled", + "metrics": ["litellm_spend_metric", "litellm_total_tokens_metric"], + }, + { + "group": "defaults", + "metrics": [PROMETHEUS_METRICS_WILDCARD], + "include_labels": ["team"], + }, + ], + ) label_filters = logger._parse_prometheus_config() @@ -108,7 +116,6 @@ def test_wildcard_combined_with_named_enable_list(): assert logger._is_metric_enabled("litellm_cache_hits_metric") is False assert label_filters["litellm_spend_metric"] == ["team"] - def test_wildcard_mixed_with_named_metric_in_same_group_raises(): logger = _bare_logger() configs = [