fix(prometheus): validate caller identity mode before collectors register

Fail config load on an invalid prometheus_deployment_and_latency_caller_identity
value (including null) and on include_labels entries the selected mode removes
from a target metric, instead of booting green with an empty /metrics.
Validate the mode at the top of PrometheusLogger.__init__ so an invalid value
raises before any collector lands in the process-global registry, keeping
retries free of duplicated-timeseries errors. Label-validation errors now name
the mode setting alongside the rejected label.

litellm/integrations/prometheus.py

@@ -49,6 +49,7 @@ from litellm.types.integrations.prometheus import *
 from litellm.types.integrations.prometheus import (
     _sanitize_prometheus_label_name,
     _sanitize_prometheus_label_value,
+    validate_prometheus_deployment_and_latency_caller_identity,
 )
 from litellm.types.utils import (
     StandardLoggingGuardrailInformation,
@@ -172,6 +173,11 @@ class PrometheusLogger(CustomLogger):
         try:
             from prometheus_client import Counter, Gauge, Histogram
 
+            # Validate the caller-identity mode before any collector registers so an
+            # invalid value cannot leave partially-registered metrics behind in the
+            # process-global registry.
+            validate_prometheus_deployment_and_latency_caller_identity()
+
             # Always initialize label_filters, even for non-premium users
             self.label_filters = self._parse_prometheus_config()
 

litellm/proxy/proxy_server.py

@@ -4852,12 +4852,13 @@ class ProxyConfig:
         if litellm_settings is None:
             litellm_settings = {}
         if litellm_settings:
-            # Prometheus collectors have fixed label schemas. Load this setting
-            # before processing callbacks so YAML key order cannot construct the
-            # collectors with the default caller-identity mode.
-            caller_identity_mode: Final = litellm_settings.get("prometheus_deployment_and_latency_caller_identity")
-            if caller_identity_mode is not None:
-                litellm.prometheus_deployment_and_latency_caller_identity = caller_identity_mode
+            # Prometheus collectors have fixed label schemas. Load and validate this
+            # setting before processing callbacks so YAML key order cannot construct
+            # the collectors with the default caller-identity mode, and so an invalid
+            # value fails the boot instead of being swallowed by callback init.
+            from litellm.types.integrations.prometheus import validate_caller_identity_settings
+
+            validate_caller_identity_settings(litellm_settings)
 
             # ANSI escape code for blue text
             blue_color_code: Final = "\033[94m"

litellm/types/integrations/prometheus.py

@@ -92,7 +92,20 @@ class LabelValidationError:
 
     @property
     def message(self) -> str:
-        return f"Invalid labels for metric '{self.metric_name}': {self.invalid_labels}"
+        base_message: Final = f"Invalid labels for metric '{self.metric_name}': {self.invalid_labels}"
+        if self.metric_name in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS and any(
+            label in ("api_key_alias", "user_email") for label in self.invalid_labels
+        ):
+            mode: Final[object] = getattr(
+                litellm,
+                "prometheus_deployment_and_latency_caller_identity",
+                "api_key_alias",
+            )
+            return (
+                f"{base_message} (the caller-identity label on this metric is set by "
+                f"prometheus_deployment_and_latency_caller_identity={mode!r})"
+            )
+        return base_message
 
 
 @dataclass
@@ -297,6 +310,51 @@ PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES: Final[tuple[str, ...]]
 )
 
 
+def validate_prometheus_deployment_and_latency_caller_identity() -> str:
+    """Return the configured caller-identity mode, raising on an invalid value."""
+    caller_identity: Final[object] = getattr(
+        litellm,
+        "prometheus_deployment_and_latency_caller_identity",
+        "api_key_alias",
+    )
+    if isinstance(caller_identity, str) and caller_identity in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES:
+        return caller_identity
+    accepted_values: Final = ", ".join(PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES)
+    raise ValueError(
+        "Invalid prometheus_deployment_and_latency_caller_identity="
+        f"{caller_identity!r}. Accepted values: {accepted_values}."
+    )
+
+
+def validate_caller_identity_settings(litellm_settings: Mapping[str, Any]) -> None:
+    """Store the caller-identity mode from litellm_settings and validate it together
+    with prometheus_metrics_config, raising on an invalid value or on include_labels
+    that request a label the selected mode removes."""
+    if "prometheus_deployment_and_latency_caller_identity" not in litellm_settings:
+        return
+    litellm.prometheus_deployment_and_latency_caller_identity = litellm_settings[
+        "prometheus_deployment_and_latency_caller_identity"
+    ]
+    caller_identity_mode: Final = validate_prometheus_deployment_and_latency_caller_identity()
+    if caller_identity_mode != "user_email":
+        return
+    conflicting_metrics: Final = tuple(
+        metric_name
+        for metric_config in (litellm_settings.get("prometheus_metrics_config") or ())
+        if isinstance(metric_config, dict) and "api_key_alias" in (metric_config.get("include_labels") or ())
+        for metric_name in (metric_config.get("metrics") or ())
+        if metric_name in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS
+    )
+    if conflicting_metrics:
+        conflicting_names: Final = ", ".join(conflicting_metrics)
+        raise ValueError(
+            "prometheus_metrics_config include_labels contains 'api_key_alias' for "
+            f"{conflicting_names}, but prometheus_deployment_and_latency_caller_identity="
+            "'user_email' replaces that label on these metrics. Use 'user_email' in "
+            "include_labels or change the mode."
+        )
+
+
 def _resolve_deployment_and_latency_caller_identity_labels(
     metric_name: str,
     labels: Sequence[object],
@@ -308,17 +366,7 @@ def _resolve_deployment_and_latency_caller_identity_labels(
     if metric_name not in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS:
         return resolved_labels
 
-    caller_identity: Final[object] = getattr(
-        litellm,
-        "prometheus_deployment_and_latency_caller_identity",
-        "api_key_alias",
-    )
-    if caller_identity not in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES:
-        accepted_values: Final = ", ".join(PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES)
-        raise ValueError(
-            "Invalid prometheus_deployment_and_latency_caller_identity="
-            f"{caller_identity!r}. Accepted values: {accepted_values}."
-        )
+    caller_identity: Final = validate_prometheus_deployment_and_latency_caller_identity()
 
     alias_index: Final = resolved_labels.index(UserAPIKeyLabelNames.API_KEY_ALIAS.value)
     if caller_identity == "user_email":