From e0818b169660d5fb53558b3c8392da381f512e12 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 24 Feb 2026 11:51:42 -0800 Subject: [PATCH] feat(prometheus): add opt-in stream label to litellm_proxy_total_requests_metric (#22023) Set prometheus_emit_stream_label: true in litellm_settings to emit a stream label (True/False/None) on litellm_proxy_total_requests_metric. Opt-in to avoid breaking cardinality on existing deployments. --- docs/my-website/docs/proxy/prometheus.md | 26 +++++- litellm/__init__.py | 1 + litellm/integrations/prometheus.py | 6 ++ litellm/types/integrations/prometheus.py | 25 ++++-- .../test_prometheus_stream_label.py | 81 +++++++++++++++++++ 5 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_stream_label.py diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 93a0675f097..18a139d1d29 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -122,7 +122,7 @@ Use this to track overall LiteLLM Proxy usage. | Metric Name | Description | |----------------------|--------------------------------------| | `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "user_email", "exception_status", "exception_class", "route", "model_id"` | -| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"` | +| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"`. Optionally includes `"stream"` — see [Emit Stream Label](#emit-stream-label). | ### Callback Logging Metrics @@ -214,9 +214,31 @@ litellm_settings: ``` +### Emit Stream Label + +Add a `stream` label to `litellm_proxy_total_requests_metric` to split requests by streaming vs. non-streaming. Disabled by default. + +```yaml title="config.yaml" +litellm_settings: + callbacks: ["prometheus"] + prometheus_emit_stream_label: true +``` + +When enabled, `litellm_proxy_total_requests_metric` gains a `stream` label with values `"True"`, `"False"`, or `"None"`. + +``` +litellm_proxy_total_requests_metric{..., stream="True"} 42 +litellm_proxy_total_requests_metric{..., stream="False"} 100 +``` + +:::note +This label is opt-in because adding a new label to an existing metric changes its cardinality and breaks existing Prometheus queries / Grafana dashboards that target this metric. Enable it only on fresh deployments or when you are ready to update your dashboards. +::: + + ## [BETA] Custom Metrics -Track custom metrics on prometheus on all events mentioned above. +Track custom metrics on prometheus on all events mentioned above. ### Custom Metadata Labels diff --git a/litellm/__init__.py b/litellm/__init__.py index 1e74b5692e4..6e42f2c1ea5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -374,6 +374,7 @@ enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None custom_prometheus_metadata_labels: List[str] = [] custom_prometheus_tags: List[str] = [] prometheus_metrics_config: Optional[List] = None +prometheus_emit_stream_label: bool = False disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 4c7afd5a57c..08db77e8571 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -974,6 +974,9 @@ class PrometheusLogger(CustomLogger): ), client_ip=standard_logging_payload["metadata"].get("requester_ip_address"), user_agent=standard_logging_payload["metadata"].get("user_agent"), + stream=str(standard_logging_payload.get("stream")) + if litellm.prometheus_emit_stream_label + else None, ) if ( @@ -1624,6 +1627,9 @@ class PrometheusLogger(CustomLogger): client_ip=_metadata.get("requester_ip_address"), user_agent=_metadata.get("user_agent"), model_id=model_id, + stream=str(request_data.get("stream")) + if litellm.prometheus_emit_stream_label + else None, ) _labels = prometheus_label_factory( supported_enum_labels=self.get_labels_for_metric( diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index fd788af9ac1..482b87085dd 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -55,22 +55,21 @@ def _sanitize_prometheus_label_value(value: Optional[Any]) -> Optional[str]: return None # Coerce non-string values (int, bool, etc.) to str before sanitizing - if not isinstance(value, str): - value = str(value) + str_value: str = value if isinstance(value, str) else str(value) # Remove Unicode line/paragraph separators that break text format - value = value.replace("\u2028", "").replace("\u2029", "") + str_value = str_value.replace("\u2028", "").replace("\u2029", "") # Remove carriage returns - value = value.replace("\r", "") + str_value = str_value.replace("\r", "") # Replace newlines with spaces - value = value.replace("\n", " ") + str_value = str_value.replace("\n", " ") # Escape backslashes and double quotes per Prometheus exposition format - value = value.replace("\\", "\\\\").replace('"', '\\"') + str_value = str_value.replace("\\", "\\\\").replace('"', '\\"') - return value + return str_value @dataclass @@ -185,6 +184,7 @@ class UserAPIKeyLabelNames(Enum): CLIENT_IP = "client_ip" USER_AGENT = "user_agent" CALLBACK_NAME = "callback_name" + STREAM = "stream" DEFINED_PROMETHEUS_METRICS = Literal[ @@ -638,6 +638,14 @@ class PrometheusMetricLabels: ] ) + # Conditionally add stream label to litellm_proxy_total_requests_metric + if ( + label_name == "litellm_proxy_total_requests_metric" + and litellm.prometheus_emit_stream_label is True + and UserAPIKeyLabelNames.STREAM.value not in default_labels + ): + custom_labels.append(UserAPIKeyLabelNames.STREAM.value) + return default_labels + custom_labels @@ -709,6 +717,9 @@ class UserAPIKeyLabelValues(BaseModel): user_agent: Annotated[ Optional[str], Field(..., alias=UserAPIKeyLabelNames.USER_AGENT.value) ] = None + stream: Annotated[ + Optional[str], Field(..., alias=UserAPIKeyLabelNames.STREAM.value) + ] = None class PrometheusMetricsConfig(BaseModel): diff --git a/tests/test_litellm/integrations/test_prometheus_stream_label.py b/tests/test_litellm/integrations/test_prometheus_stream_label.py new file mode 100644 index 00000000000..a00a468e0fb --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_stream_label.py @@ -0,0 +1,81 @@ +""" +Unit tests for prometheus_emit_stream_label opt-in setting. + +Tests that: +- stream label is NOT added to litellm_proxy_total_requests_metric by default +- stream label IS added when litellm.prometheus_emit_stream_label = True +- stream value is populated correctly from standard_logging_payload +""" +import pytest + +import litellm +from litellm.types.integrations.prometheus import ( + PrometheusMetricLabels, + UserAPIKeyLabelNames, +) + + +def test_stream_label_not_present_by_default(): + """stream label should NOT appear in litellm_proxy_total_requests_metric unless opted in""" + litellm.prometheus_emit_stream_label = False + labels = PrometheusMetricLabels.get_labels("litellm_proxy_total_requests_metric") + assert UserAPIKeyLabelNames.STREAM.value not in labels + + +def test_stream_label_present_when_opted_in(): + """stream label SHOULD appear in litellm_proxy_total_requests_metric when opted in""" + litellm.prometheus_emit_stream_label = True + try: + labels = PrometheusMetricLabels.get_labels("litellm_proxy_total_requests_metric") + assert UserAPIKeyLabelNames.STREAM.value in labels + finally: + litellm.prometheus_emit_stream_label = False + + +def test_stream_label_not_in_other_metrics_when_opted_in(): + """stream label should NOT be added to other metrics even when opted in""" + litellm.prometheus_emit_stream_label = True + try: + other_metrics = [ + "litellm_proxy_failed_requests_metric", + "litellm_spend_metric", + "litellm_input_tokens_metric", + "litellm_output_tokens_metric", + "litellm_llm_api_latency_metric", + ] + for metric in other_metrics: + labels = PrometheusMetricLabels.get_labels(metric) + assert UserAPIKeyLabelNames.STREAM.value not in labels, ( + f"stream label should not be in {metric}" + ) + finally: + litellm.prometheus_emit_stream_label = False + + +def test_stream_label_name(): + """STREAM label name should be 'stream'""" + assert UserAPIKeyLabelNames.STREAM.value == "stream" + + +def test_user_api_key_label_values_has_stream_field(): + """UserAPIKeyLabelValues should accept stream field""" + from litellm.types.integrations.prometheus import UserAPIKeyLabelValues + + values = UserAPIKeyLabelValues(stream="True") + assert values.stream == "True" + + values_false = UserAPIKeyLabelValues(stream="False") + assert values_false.stream == "False" + + values_none = UserAPIKeyLabelValues() + assert values_none.stream is None + + +def test_stream_label_in_model_dump(): + """stream field appears in model_dump() output for use in prometheus_label_factory""" + from litellm.types.integrations.prometheus import UserAPIKeyLabelValues + + values = UserAPIKeyLabelValues(stream="True") + dumped = values.model_dump() + assert "stream" in dumped + assert dumped["stream"] == "True"