From cdaaf4065c037bb046b15283b82091f98055500c Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Tue, 25 Aug 2026 22:18:37 -0700 Subject: [PATCH] fix(prometheus): validate caller identity mode before collectors register Fail config load on an invalid prometheus_deployment_and_latency_caller_identity value (including null) and on include_labels entries the selected mode removes from a target metric, instead of booting green with an empty /metrics. Validate the mode at the top of PrometheusLogger.__init__ so an invalid value raises before any collector lands in the process-global registry, keeping retries free of duplicated-timeseries errors. Label-validation errors now name the mode setting alongside the rejected label. --- litellm/integrations/prometheus.py | 6 ++ litellm/proxy/proxy_server.py | 13 +++-- litellm/types/integrations/prometheus.py | 72 ++++++++++++++++++++---- 3 files changed, 73 insertions(+), 18 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 2b0a0658650..9ecd16dac25 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -49,6 +49,7 @@ from litellm.types.integrations.prometheus import * from litellm.types.integrations.prometheus import ( _sanitize_prometheus_label_name, _sanitize_prometheus_label_value, + validate_prometheus_deployment_and_latency_caller_identity, ) from litellm.types.utils import ( StandardLoggingGuardrailInformation, @@ -172,6 +173,11 @@ class PrometheusLogger(CustomLogger): try: from prometheus_client import Counter, Gauge, Histogram + # Validate the caller-identity mode before any collector registers so an + # invalid value cannot leave partially-registered metrics behind in the + # process-global registry. + validate_prometheus_deployment_and_latency_caller_identity() + # Always initialize label_filters, even for non-premium users self.label_filters = self._parse_prometheus_config() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8925e701b97..bbd8e43a878 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4852,12 +4852,13 @@ class ProxyConfig: if litellm_settings is None: litellm_settings = {} if litellm_settings: - # Prometheus collectors have fixed label schemas. Load this setting - # before processing callbacks so YAML key order cannot construct the - # collectors with the default caller-identity mode. - caller_identity_mode: Final = litellm_settings.get("prometheus_deployment_and_latency_caller_identity") - if caller_identity_mode is not None: - litellm.prometheus_deployment_and_latency_caller_identity = caller_identity_mode + # Prometheus collectors have fixed label schemas. Load and validate this + # setting before processing callbacks so YAML key order cannot construct + # the collectors with the default caller-identity mode, and so an invalid + # value fails the boot instead of being swallowed by callback init. + from litellm.types.integrations.prometheus import validate_caller_identity_settings + + validate_caller_identity_settings(litellm_settings) # ANSI escape code for blue text blue_color_code: Final = "\033[94m" diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 2ca299e3520..f6c8a011b86 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -92,7 +92,20 @@ class LabelValidationError: @property def message(self) -> str: - return f"Invalid labels for metric '{self.metric_name}': {self.invalid_labels}" + base_message: Final = f"Invalid labels for metric '{self.metric_name}': {self.invalid_labels}" + if self.metric_name in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS and any( + label in ("api_key_alias", "user_email") for label in self.invalid_labels + ): + mode: Final[object] = getattr( + litellm, + "prometheus_deployment_and_latency_caller_identity", + "api_key_alias", + ) + return ( + f"{base_message} (the caller-identity label on this metric is set by " + f"prometheus_deployment_and_latency_caller_identity={mode!r})" + ) + return base_message @dataclass @@ -297,6 +310,51 @@ PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES: Final[tuple[str, ...]] ) +def validate_prometheus_deployment_and_latency_caller_identity() -> str: + """Return the configured caller-identity mode, raising on an invalid value.""" + caller_identity: Final[object] = getattr( + litellm, + "prometheus_deployment_and_latency_caller_identity", + "api_key_alias", + ) + if isinstance(caller_identity, str) and caller_identity in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES: + return caller_identity + accepted_values: Final = ", ".join(PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES) + raise ValueError( + "Invalid prometheus_deployment_and_latency_caller_identity=" + f"{caller_identity!r}. Accepted values: {accepted_values}." + ) + + +def validate_caller_identity_settings(litellm_settings: Mapping[str, Any]) -> None: + """Store the caller-identity mode from litellm_settings and validate it together + with prometheus_metrics_config, raising on an invalid value or on include_labels + that request a label the selected mode removes.""" + if "prometheus_deployment_and_latency_caller_identity" not in litellm_settings: + return + litellm.prometheus_deployment_and_latency_caller_identity = litellm_settings[ + "prometheus_deployment_and_latency_caller_identity" + ] + caller_identity_mode: Final = validate_prometheus_deployment_and_latency_caller_identity() + if caller_identity_mode != "user_email": + return + conflicting_metrics: Final = tuple( + metric_name + for metric_config in (litellm_settings.get("prometheus_metrics_config") or ()) + if isinstance(metric_config, dict) and "api_key_alias" in (metric_config.get("include_labels") or ()) + for metric_name in (metric_config.get("metrics") or ()) + if metric_name in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS + ) + if conflicting_metrics: + conflicting_names: Final = ", ".join(conflicting_metrics) + raise ValueError( + "prometheus_metrics_config include_labels contains 'api_key_alias' for " + f"{conflicting_names}, but prometheus_deployment_and_latency_caller_identity=" + "'user_email' replaces that label on these metrics. Use 'user_email' in " + "include_labels or change the mode." + ) + + def _resolve_deployment_and_latency_caller_identity_labels( metric_name: str, labels: Sequence[object], @@ -308,17 +366,7 @@ def _resolve_deployment_and_latency_caller_identity_labels( if metric_name not in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS: return resolved_labels - caller_identity: Final[object] = getattr( - litellm, - "prometheus_deployment_and_latency_caller_identity", - "api_key_alias", - ) - if caller_identity not in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES: - accepted_values: Final = ", ".join(PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES) - raise ValueError( - "Invalid prometheus_deployment_and_latency_caller_identity=" - f"{caller_identity!r}. Accepted values: {accepted_values}." - ) + caller_identity: Final = validate_prometheus_deployment_and_latency_caller_identity() alias_index: Final = resolved_labels.index(UserAPIKeyLabelNames.API_KEY_ALIAS.value) if caller_identity == "user_email":