Prometheus - tags, fix '[tag]="false"' when tag is set (#12916)

* fix(prometheus.py): sanitize tag-based labels to handle colons (:) and spaces ( )

* fix(prometheus.py): working tag based metrics

* fix(prometheus.py): emit request tags on post call success hook

* fix(prometheus.py): add user agent tags on request failure

* fix(prometheus.py): add request tags to deployment failure metric

s

enterprise/litellm_enterprise/integrations/prometheus.py

@@ -1,4 +1,4 @@
- # used for /metrics endpoint on LiteLLM Proxy
+# used for /metrics endpoint on LiteLLM Proxy
 #### What this does ####
 #    On success, log events to Prometheus
 import sys
@@ -1231,8 +1231,15 @@ class PrometheusLogger(CustomLogger):
                     "team_alias",
                 ] + EXCEPTION_LABELS,
         """
+        from litellm.litellm_core_utils.litellm_logging import (
+            StandardLoggingPayloadSetup,
+        )
+
         try:
-            _tags = cast(List[str], request_data.get("tags") or [])
+            _tags = StandardLoggingPayloadSetup._get_request_tags(
+                request_data.get("metadata", {}),
+                request_data.get("proxy_server_request", {}),
+            )
             enum_values = UserAPIKeyLabelValues(
                 end_user=user_api_key_dict.end_user_id,
                 user=user_api_key_dict.user_id,
@@ -1277,6 +1284,10 @@ class PrometheusLogger(CustomLogger):
         Proxy level tracking - triggered when the proxy responds with a success response to the client
         """
         try:
+            from litellm.litellm_core_utils.litellm_logging import (
+                StandardLoggingPayloadSetup,
+            )
+
             enum_values = UserAPIKeyLabelValues(
                 end_user=user_api_key_dict.end_user_id,
                 hashed_api_key=user_api_key_dict.api_key,
@@ -1288,6 +1299,9 @@ class PrometheusLogger(CustomLogger):
                 user_email=user_api_key_dict.user_email,
                 status_code="200",
                 route=user_api_key_dict.request_route,
+                tags=StandardLoggingPayloadSetup._get_request_tags(
+                    data.get("metadata", {}), data.get("proxy_server_request", {})
+                ),
             )
             _labels = prometheus_label_factory(
                 supported_enum_labels=self.get_labels_for_metric(
@@ -1352,6 +1366,7 @@ class PrometheusLogger(CustomLogger):
                 team_alias=standard_logging_payload["metadata"][
                     "user_api_key_team_alias"
                 ],
+                tags=standard_logging_payload.get("request_tags", []),
             )
 
             """
@@ -2148,9 +2163,11 @@ class PrometheusLogger(CustomLogger):
 
         It emits the current remaining budget metrics for all Keys and Teams.
         """
+        from enterprise.litellm_enterprise.integrations.prometheus import (
+            PrometheusLogger,
+        )
         from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES
         from litellm.integrations.custom_logger import CustomLogger
-        from enterprise.litellm_enterprise.integrations.prometheus import PrometheusLogger
 
         prometheus_loggers: List[CustomLogger] = (
             litellm.logging_callback_manager.get_custom_loggers_for_type(
@@ -2280,6 +2297,8 @@ def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
     """
     Get custom labels from tags based on admin configuration
     """
+    from litellm.types.integrations.prometheus import _sanitize_prometheus_label_name
+
     configured_tags = litellm.custom_prometheus_tags
     if configured_tags is None or len(configured_tags) == 0:
         return {}
@@ -2289,7 +2308,7 @@ def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
     # Map each configured tag to its presence in the request tags
     for configured_tag in configured_tags:
         # Create a safe prometheus label name
-        label_name = f"tag_{configured_tag}".replace("-", "_").replace(".", "_")
+        label_name = _sanitize_prometheus_label_name(f"tag_{configured_tag}")
 
         # Check if this tag is present in the request tags
         if configured_tag in tags:

litellm/proxy/_new_secret_config.yaml

@@ -1,27 +1,9 @@
 model_list:
-  - model_name: gpt-4o-mini-batch
+  - model_name: openai/gpt-4o
     litellm_params:
-      model: azure/gpt-4o-mini
-      api_key: os.environ/AZURE_API_KEY_HIDDEN
-      api_base: os.environ/AZURE_API_BASE_HIDDEN
-    model_info: 
-      version: 2
-  - model_name: gpt-3.5-turbo-disallow
-    litellm_params:
-      model: gpt-3.5-turbo
-    model_info: 
-      version: 2
-  - model_name: zapier-byok-provider/openai/*
-    litellm_params:
-      model: openai/*
-      api_base: http://0.0.0.0:8090
-  - model_name: openai/gpt-4o-mini
-    litellm_params:
-      model: openai/gpt-4o-mini
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
 
 litellm_settings:
-  model_group_alias: {"gpt-3.5-turbo-custom": "gpt-3.5-turbo-disallow"}
-  model_group_settings:
-    forward_client_headers_to_llm_api: 
-      - "gpt-3.5-turbo-allow"
-      - "zapier-byok-provider/openai/*"
+  callbacks: ["prometheus"]
+  custom_prometheus_tags: ["User-Agent: Roo-Code"]

litellm/types/integrations/prometheus.py

@@ -1,3 +1,4 @@
+import re
 from dataclasses import dataclass
 from enum import Enum
 from typing import Dict, List, Literal, Optional, Tuple, Union
@@ -8,6 +9,38 @@ from typing_extensions import Annotated
 import litellm
 
 
+def _sanitize_prometheus_label_name(label: str) -> str:
+    """
+    Sanitize a label name to comply with Prometheus label name requirements.
+
+    Prometheus label names must match: ^[a-zA-Z_][a-zA-Z0-9_]*$
+    - First character: letter (a-z, A-Z) or underscore (_)
+    - Subsequent characters: letters, digits (0-9), or underscores (_)
+
+    Args:
+        label: The label name to sanitize
+
+    Returns:
+        A sanitized label name that complies with Prometheus requirements
+    """
+    if not label:
+        return "_"
+
+    # Replace all invalid characters with underscores
+    # Keep only letters, digits, and underscores
+    sanitized = re.sub(r"[^a-zA-Z0-9_]", "_", label)
+
+    # Ensure first character is valid (letter or underscore)
+    if sanitized and not re.match(r"^[a-zA-Z_]", sanitized[0]):
+        sanitized = "_" + sanitized
+
+    # Handle empty string after sanitization
+    if not sanitized:
+        sanitized = "_"
+
+    return sanitized
+
+
 @dataclass
 class MetricValidationError:
     """Error for invalid metric name"""
@@ -353,7 +386,7 @@ class PrometheusMetricLabels:
         # Add custom metadata labels
         custom_labels.extend(
             [
-                metric.replace(".", "_")
+                _sanitize_prometheus_label_name(metric)
                 for metric in litellm.custom_prometheus_metadata_labels
             ]
         )
@@ -361,7 +394,7 @@ class PrometheusMetricLabels:
         # Add custom tags labels
         custom_labels.extend(
             [
-                f"tag_{tag}".replace("-", "_").replace(".", "_")
+                _sanitize_prometheus_label_name(f"tag_{tag}")
                 for tag in litellm.custom_prometheus_tags
             ]
         )