feat(prometheus): add opt-in stream label to litellm_proxy_total_requests_metric (#22023)

Set prometheus_emit_stream_label: true in litellm_settings to emit a
stream label (True/False/None) on litellm_proxy_total_requests_metric.

Opt-in to avoid breaking cardinality on existing deployments.

docs/my-website/docs/proxy/prometheus.md

@@ -122,7 +122,7 @@ Use this to track overall LiteLLM Proxy usage.
 | Metric Name          | Description                          |
 |----------------------|--------------------------------------|
 | `litellm_proxy_failed_requests_metric`             | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "user_email", "exception_status", "exception_class", "route", "model_id"`          |
-| `litellm_proxy_total_requests_metric`             | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"`          |
+| `litellm_proxy_total_requests_metric`             | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"`. Optionally includes `"stream"` — see [Emit Stream Label](#emit-stream-label).          |
 
 ### Callback Logging Metrics
 
@@ -214,9 +214,31 @@ litellm_settings:
 ```
 
 
+### Emit Stream Label
+
+Add a `stream` label to `litellm_proxy_total_requests_metric` to split requests by streaming vs. non-streaming. Disabled by default.
+
+```yaml title="config.yaml"
+litellm_settings:
+  callbacks: ["prometheus"]
+  prometheus_emit_stream_label: true
+```
+
+When enabled, `litellm_proxy_total_requests_metric` gains a `stream` label with values `"True"`, `"False"`, or `"None"`.
+
+```
+litellm_proxy_total_requests_metric{..., stream="True"} 42
+litellm_proxy_total_requests_metric{..., stream="False"} 100
+```
+
+:::note
+This label is opt-in because adding a new label to an existing metric changes its cardinality and breaks existing Prometheus queries / Grafana dashboards that target this metric. Enable it only on fresh deployments or when you are ready to update your dashboards.
+:::
+
+
 ## [BETA] Custom Metrics
 
-Track custom metrics on prometheus on all events mentioned above. 
+Track custom metrics on prometheus on all events mentioned above.
 
 ### Custom Metadata Labels
 

litellm/__init__.py

@@ -374,6 +374,7 @@ enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
 custom_prometheus_metadata_labels: List[str] = []
 custom_prometheus_tags: List[str] = []
 prometheus_metrics_config: Optional[List] = None
+prometheus_emit_stream_label: bool = False
 disable_add_prefix_to_prompt: bool = (
     False  # used by anthropic, to disable adding prefix to prompt
 )

litellm/integrations/prometheus.py

@@ -974,6 +974,9 @@ class PrometheusLogger(CustomLogger):
             ),
             client_ip=standard_logging_payload["metadata"].get("requester_ip_address"),
             user_agent=standard_logging_payload["metadata"].get("user_agent"),
+            stream=str(standard_logging_payload.get("stream"))
+            if litellm.prometheus_emit_stream_label
+            else None,
         )
 
         if (
@@ -1624,6 +1627,9 @@ class PrometheusLogger(CustomLogger):
                 client_ip=_metadata.get("requester_ip_address"),
                 user_agent=_metadata.get("user_agent"),
                 model_id=model_id,
+                stream=str(request_data.get("stream"))
+                if litellm.prometheus_emit_stream_label
+                else None,
             )
             _labels = prometheus_label_factory(
                 supported_enum_labels=self.get_labels_for_metric(

litellm/types/integrations/prometheus.py

@@ -55,22 +55,21 @@ def _sanitize_prometheus_label_value(value: Optional[Any]) -> Optional[str]:
         return None
 
     # Coerce non-string values (int, bool, etc.) to str before sanitizing
-    if not isinstance(value, str):
-        value = str(value)
+    str_value: str = value if isinstance(value, str) else str(value)
 
     # Remove Unicode line/paragraph separators that break text format
-    value = value.replace("\u2028", "").replace("\u2029", "")
+    str_value = str_value.replace("\u2028", "").replace("\u2029", "")
 
     # Remove carriage returns
-    value = value.replace("\r", "")
+    str_value = str_value.replace("\r", "")
 
     # Replace newlines with spaces
-    value = value.replace("\n", " ")
+    str_value = str_value.replace("\n", " ")
 
     # Escape backslashes and double quotes per Prometheus exposition format
-    value = value.replace("\\", "\\\\").replace('"', '\\"')
+    str_value = str_value.replace("\\", "\\\\").replace('"', '\\"')
 
-    return value
+    return str_value
 
 
 @dataclass
@@ -185,6 +184,7 @@ class UserAPIKeyLabelNames(Enum):
     CLIENT_IP = "client_ip"
     USER_AGENT = "user_agent"
     CALLBACK_NAME = "callback_name"
+    STREAM = "stream"
 
 
 DEFINED_PROMETHEUS_METRICS = Literal[
@@ -638,6 +638,14 @@ class PrometheusMetricLabels:
             ]
         )
 
+        # Conditionally add stream label to litellm_proxy_total_requests_metric
+        if (
+            label_name == "litellm_proxy_total_requests_metric"
+            and litellm.prometheus_emit_stream_label is True
+            and UserAPIKeyLabelNames.STREAM.value not in default_labels
+        ):
+            custom_labels.append(UserAPIKeyLabelNames.STREAM.value)
+
         return default_labels + custom_labels
 
 
@@ -709,6 +717,9 @@ class UserAPIKeyLabelValues(BaseModel):
     user_agent: Annotated[
         Optional[str], Field(..., alias=UserAPIKeyLabelNames.USER_AGENT.value)
     ] = None
+    stream: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.STREAM.value)
+    ] = None
 
 
 class PrometheusMetricsConfig(BaseModel):

tests/test_litellm/integrations/test_prometheus_stream_label.py

@@ -0,0 +1,81 @@
+"""
+Unit tests for prometheus_emit_stream_label opt-in setting.
+
+Tests that:
+- stream label is NOT added to litellm_proxy_total_requests_metric by default
+- stream label IS added when litellm.prometheus_emit_stream_label = True
+- stream value is populated correctly from standard_logging_payload
+"""
+import pytest
+
+import litellm
+from litellm.types.integrations.prometheus import (
+    PrometheusMetricLabels,
+    UserAPIKeyLabelNames,
+)
+
+
+def test_stream_label_not_present_by_default():
+    """stream label should NOT appear in litellm_proxy_total_requests_metric unless opted in"""
+    litellm.prometheus_emit_stream_label = False
+    labels = PrometheusMetricLabels.get_labels("litellm_proxy_total_requests_metric")
+    assert UserAPIKeyLabelNames.STREAM.value not in labels
+
+
+def test_stream_label_present_when_opted_in():
+    """stream label SHOULD appear in litellm_proxy_total_requests_metric when opted in"""
+    litellm.prometheus_emit_stream_label = True
+    try:
+        labels = PrometheusMetricLabels.get_labels("litellm_proxy_total_requests_metric")
+        assert UserAPIKeyLabelNames.STREAM.value in labels
+    finally:
+        litellm.prometheus_emit_stream_label = False
+
+
+def test_stream_label_not_in_other_metrics_when_opted_in():
+    """stream label should NOT be added to other metrics even when opted in"""
+    litellm.prometheus_emit_stream_label = True
+    try:
+        other_metrics = [
+            "litellm_proxy_failed_requests_metric",
+            "litellm_spend_metric",
+            "litellm_input_tokens_metric",
+            "litellm_output_tokens_metric",
+            "litellm_llm_api_latency_metric",
+        ]
+        for metric in other_metrics:
+            labels = PrometheusMetricLabels.get_labels(metric)
+            assert UserAPIKeyLabelNames.STREAM.value not in labels, (
+                f"stream label should not be in {metric}"
+            )
+    finally:
+        litellm.prometheus_emit_stream_label = False
+
+
+def test_stream_label_name():
+    """STREAM label name should be 'stream'"""
+    assert UserAPIKeyLabelNames.STREAM.value == "stream"
+
+
+def test_user_api_key_label_values_has_stream_field():
+    """UserAPIKeyLabelValues should accept stream field"""
+    from litellm.types.integrations.prometheus import UserAPIKeyLabelValues
+
+    values = UserAPIKeyLabelValues(stream="True")
+    assert values.stream == "True"
+
+    values_false = UserAPIKeyLabelValues(stream="False")
+    assert values_false.stream == "False"
+
+    values_none = UserAPIKeyLabelValues()
+    assert values_none.stream is None
+
+
+def test_stream_label_in_model_dump():
+    """stream field appears in model_dump() output for use in prometheus_label_factory"""
+    from litellm.types.integrations.prometheus import UserAPIKeyLabelValues
+
+    values = UserAPIKeyLabelValues(stream="True")
+    dumped = values.model_dump()
+    assert "stream" in dumped
+    assert dumped["stream"] == "True"