Merge 702a35f05d into 97aae6c65e

litellm/__init__.py

@@ -171,6 +171,7 @@ langsmith_batch_size: Optional[int] = None
 prometheus_initialize_budget_metrics: Optional[bool] = False
 prometheus_latency_buckets: Optional[List[float]] = None
 require_auth_for_metrics_endpoint: Optional[bool] = True
+allow_unauthenticated_metrics: Optional[bool] = None
 argilla_batch_size: Optional[int] = None
 datadog_use_v1: Optional[bool] = False  # if you want to use v1 datadog logged payload.
 gcs_pub_sub_use_v1: Optional[bool] = False  # if you want to use v1 gcs pubsub logged payload
@@ -484,6 +485,7 @@ prometheus_deployment_and_latency_caller_identity: Literal[
 # (RPM/TPM/concurrent/budget).
 prometheus_emit_rate_limit_labels: bool = False
 prometheus_user_budget_label_include_email_alias: bool = False
+prometheus_mask_pii_labels: Optional[bool] = None
 prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
 prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
 prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0

litellm/integrations/prometheus.py

@@ -5029,7 +5029,26 @@ class PrometheusLogger(CustomLogger):
 
         # Mount the metrics app to the app
         app.mount("/metrics", metrics_app)
-        verbose_proxy_logger.debug("Starting Prometheus Metrics on /metrics (no authentication)")
+        is_auth_required = (
+            getattr(litellm, "allow_unauthenticated_metrics", None) is not True
+            and getattr(litellm, "require_auth_for_metrics_endpoint", None) is not False
+        )
+        if is_auth_required:
+            verbose_proxy_logger.debug("Starting Prometheus Metrics on /metrics (authenticated)")
+        else:
+            verbose_proxy_logger.warning(
+                "SECURITY WARNING: /metrics endpoint is exposed WITHOUT authentication. "
+                "Sensitive multi-tenant operational data and metric labels may be visible to unauthorized clients. "
+                "To secure this endpoint, remove allow_unauthenticated_metrics or set require_auth_for_metrics_endpoint: true."
+            )
+            import warnings
+
+            warnings.warn(
+                "/metrics endpoint is exposed without authentication. "
+                "Ensure this endpoint is behind a firewall/private VPC or set require_auth_for_metrics_endpoint: true.",
+                UserWarning,
+                stacklevel=2,
+            )
 
 
 def _label_source(enum_values: UserAPIKeyLabelValues) -> Mapping[str, object]:

litellm/integrations/prometheus_helpers/__init__.py

@@ -6,6 +6,8 @@ Helpers for the Prometheus integration (extracted to keep ``prometheus.py`` smal
 
 from __future__ import annotations
 
+import hashlib
+import re
 from typing import Final, Literal, Protocol, cast
 
 from litellm.types.integrations.prometheus import (
@@ -16,6 +18,61 @@ from litellm.types.integrations.prometheus import (
 
 _get_end_user_id_for_cost_tracking = None
 
+_EMAIL_PATTERN: Final = re.compile(r"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+")
+_UUID_PATTERN: Final = re.compile(
+    r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+)
+
+_PII_LABEL_KEYS: Final = frozenset(
+    {
+        "user",
+        "user_email",
+        "user_alias",
+        "team_alias",
+        "client_ip",
+        "user_agent",
+    }
+)
+
+
+def sanitize_pii_label_value(label_name: str, value: object | None) -> str | None:
+    """
+    Sanitize sensitive tenant PII before Prometheus label emission.
+    Protects user emails, client IPs, user-agent automation IDs, and team aliases.
+    """
+    if value is None:
+        return None
+
+    str_val: Final[str] = value if isinstance(value, str) else str(value)
+    if not str_val:
+        return str_val
+
+    if label_name in ("user", "user_email", "end_user"):
+        if _EMAIL_PATTERN.search(str_val):
+            return f"redacted_user_{hashlib.sha256(str_val.encode('utf-8')).hexdigest()[:8]}"
+        return f"user_{hashlib.sha256(str_val.encode('utf-8')).hexdigest()[:8]}"
+
+    if label_name in ("team_alias", "user_alias", "api_key_alias"):
+        if _EMAIL_PATTERN.search(str_val):
+            return _EMAIL_PATTERN.sub("[REDACTED_EMAIL]", str_val)
+        return str_val
+
+    if label_name == "client_ip":
+        if ":" in str_val:
+            return f"ipv6_{hashlib.sha256(str_val.encode('utf-8')).hexdigest()[:8]}"
+        parts = str_val.split(".")
+        if len(parts) == 4:
+            return f"{parts[0]}.{parts[1]}.0.0/16"
+        return f"ip_{hashlib.sha256(str_val.encode('utf-8')).hexdigest()[:8]}"
+
+    if label_name == "user_agent":
+        sanitized = _UUID_PATTERN.sub("[UUID]", str_val)
+        if len(sanitized) > 64:
+            return sanitized[:64] + "..."
+        return sanitized
+
+    return str_val
+
 
 class _EndUserIdGetter(Protocol):
     def __call__(
@@ -59,11 +116,30 @@ class PrometheusLabelFactoryContext:
     _END_USER_NOT_COMPUTED = object()
 
     def __init__(self, enum_values: UserAPIKeyLabelValues) -> None:
+        import litellm
+
         self.enum_values = enum_values
         enum_dict: Final = enum_values.model_dump()
-        self.sanitized_enum: dict[str, str | None] = {
-            k: sanitize_prometheus_label_value(v) for k, v in enum_dict.items()
-        }
+
+        mask_pii = getattr(litellm, "prometheus_mask_pii_labels", None)
+        if mask_pii is None:
+            # Auto-sanitize PII if metrics endpoint is exposed unauthenticated
+            mask_pii = (
+                getattr(litellm, "allow_unauthenticated_metrics", None) is True
+                or getattr(litellm, "require_auth_for_metrics_endpoint", None) is False
+            )
+
+        if mask_pii:
+            self.sanitized_enum: dict[str, str | None] = {
+                k: sanitize_prometheus_label_value(
+                    sanitize_pii_label_value(k, v) if k in _PII_LABEL_KEYS else v
+                )
+                for k, v in enum_dict.items()
+            }
+        else:
+            self.sanitized_enum: dict[str, str | None] = {
+                k: sanitize_prometheus_label_value(v) for k, v in enum_dict.items()
+            }
         self.custom_by_sanitized_key: dict[str, str | None] = {}
         if enum_values.custom_metadata_labels is not None:
             for key, value in enum_values.custom_metadata_labels.items():
@@ -106,8 +182,19 @@ class PrometheusLabelFactoryContext:
     def get_resolved_end_user(self) -> str | None:
         if self._resolved_end_user is self._END_USER_NOT_COMPUTED:
             fn: Final = get_cached_end_user_id_for_cost_tracking()
-            self._resolved_end_user = fn(
+            resolved = fn(
                 litellm_params={"user_api_key_end_user_id": self.enum_values.end_user},
                 service_type="prometheus",
             )
+            import litellm
+
+            mask_pii = getattr(litellm, "prometheus_mask_pii_labels", None)
+            if mask_pii is None:
+                mask_pii = (
+                    getattr(litellm, "allow_unauthenticated_metrics", None) is True
+                    or getattr(litellm, "require_auth_for_metrics_endpoint", None) is False
+                )
+            if mask_pii and resolved:
+                resolved = sanitize_pii_label_value("end_user", resolved)
+            self._resolved_end_user = resolved
         return cast(str | None, self._resolved_end_user)

litellm/proxy/auth/user_api_key_auth.py

@@ -211,7 +211,10 @@ def _normalize_public_auth_route(route: str) -> str:
 def _route_requires_auth_despite_public(route: str, general_settings: dict | None) -> bool:
     normalized_route: Final = _normalize_public_auth_route(route)
     if normalized_route == "/metrics":
-        return litellm.require_auth_for_metrics_endpoint is not False
+        return (
+            litellm.allow_unauthenticated_metrics is not True
+            and litellm.require_auth_for_metrics_endpoint is not False
+        )
 
     return False
 

litellm/proxy/middleware/prometheus_auth_middleware.py

@@ -34,7 +34,9 @@ class PrometheusAuthMiddleware:
 
     ```yaml
     litellm_settings:
-        require_auth_for_metrics_endpoint: false
+        allow_unauthenticated_metrics: true
+        # or legacy flag:
+        # require_auth_for_metrics_endpoint: false
     ```
     """
 
@@ -47,8 +49,12 @@ class PrometheusAuthMiddleware:
             await self.app(scope, receive, send)
             return
 
-        # Run auth by default; allow legacy public metrics only when explicitly disabled.
-        if litellm.require_auth_for_metrics_endpoint is not False:
+        # Run auth by default; allow public metrics only when explicitly opted out.
+        is_auth_required = (
+            litellm.allow_unauthenticated_metrics is not True
+            and litellm.require_auth_for_metrics_endpoint is not False
+        )
+        if is_auth_required:
             # user_api_key_auth reads the request body, which consumes ASGI `receive`.
             # Buffer those messages and replay them for the inner app; otherwise a
             # successful auth would forward an exhausted receive and /metrics hangs.
@@ -79,7 +85,8 @@ class PrometheusAuthMiddleware:
                 body: Final = json.dumps(
                     f"Unauthorized access to metrics endpoint: {error_message} "
                     f"To allow unauthenticated access, set "
-                    f"`litellm_settings.require_auth_for_metrics_endpoint: false` "
+                    f"`litellm_settings.allow_unauthenticated_metrics: true` "
+                    f"(or `litellm_settings.require_auth_for_metrics_endpoint: false`) "
                     f"in your proxy_config.yaml."
                 ).encode("utf-8")
                 await send(

tests/unit/integrations/test_prometheus_pii_sanitization.py

@@ -0,0 +1,144 @@
+import warnings
+import pytest
+from unittest.mock import MagicMock, patch
+
+import litellm
+from litellm.integrations.prometheus_helpers import (
+    PrometheusLabelFactoryContext,
+    sanitize_pii_label_value,
+)
+from litellm.types.integrations.prometheus import UserAPIKeyLabelValues
+
+
+def test_sanitize_pii_label_value_user_email():
+    """Verify that user identifiers containing email addresses are sanitized to a deterministic hash."""
+    email_user = "employee@internal.company.com"
+    sanitized = sanitize_pii_label_value("user", email_user)
+    assert sanitized is not None
+    assert "internal.company.com" not in sanitized
+    assert "employee" not in sanitized
+    assert sanitized.startswith("redacted_user_")
+
+    sanitized_email = sanitize_pii_label_value("user_email", email_user)
+    assert sanitized_email is not None
+    assert "internal.company.com" not in sanitized_email
+    assert sanitized_email.startswith("redacted_user_")
+
+
+def test_sanitize_pii_label_value_team_alias():
+    """Verify that employee emails in team_alias are masked."""
+    team_alias = "Enterprise Corp (POC: admin@corp.example.com)"
+    sanitized = sanitize_pii_label_value("team_alias", team_alias)
+    assert sanitized is not None
+    assert "admin@corp.example.com" not in sanitized
+    assert "[REDACTED_EMAIL]" in sanitized
+    assert "Enterprise Corp" in sanitized
+
+
+def test_sanitize_pii_label_value_client_ip():
+    """Verify that IPv4 client IPs mask host topology and IPv6 IPs are hashed."""
+    ipv4 = "198.51.100.42"
+    sanitized_ipv4 = sanitize_pii_label_value("client_ip", ipv4)
+    assert sanitized_ipv4 == "198.51.0.0/16"
+
+    ipv6 = "2001:0db8:85a3:0000:0000:8a2e:0370:7334"
+    sanitized_ipv6 = sanitize_pii_label_value("client_ip", ipv6)
+    assert sanitized_ipv6 is not None
+    assert sanitized_ipv6.startswith("ipv6_")
+
+
+def test_sanitize_pii_label_value_user_agent_uuid():
+    """Verify that UUIDs (e.g. Azure Logic Apps workflow IDs) in user-agent strings are redacted."""
+    ua = "AzureLogicApps/1.0 workflow/d9b4b726-1e6f-4f3c-8a5e-2f190e3fbb61"
+    sanitized = sanitize_pii_label_value("user_agent", ua)
+    assert "d9b4b726-1e6f-4f3c-8a5e-2f190e3fbb61" not in sanitized
+    assert "AzureLogicApps/1.0 workflow/[UUID]" in sanitized
+
+
+def test_prometheus_label_context_auto_sanitization_when_unauthenticated(monkeypatch):
+    """
+    When /metrics endpoint is exposed unauthenticated (via allow_unauthenticated_metrics=True
+    or require_auth_for_metrics_endpoint=False), PII fields must automatically be sanitized
+    in PrometheusLabelFactoryContext to protect multi-tenant privacy.
+    """
+    monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", True)
+    monkeypatch.setattr(litellm, "require_auth_for_metrics_endpoint", False)
+    monkeypatch.setattr(litellm, "prometheus_mask_pii_labels", None)
+
+    enum_values = UserAPIKeyLabelValues(
+        user="john.doe@example.com",
+        user_email="john.doe@example.com",
+        team_alias="Finance Dept (lead: jane@example.com)",
+        client_ip="203.0.113.195",
+        user_agent="AzureLogicApps/1.0 workflow/12345678-1234-1234-1234-123456789abc",
+    )
+
+    ctx = PrometheusLabelFactoryContext(enum_values)
+
+    assert "john.doe@example.com" not in str(ctx.sanitized_enum.get("user"))
+    assert ctx.sanitized_enum["user"].startswith("redacted_user_")
+    assert "john.doe@example.com" not in str(ctx.sanitized_enum.get("user_email"))
+    assert "[REDACTED_EMAIL]" in ctx.sanitized_enum["team_alias"]
+    assert ctx.sanitized_enum["client_ip"] == "203.0.0.0/16"
+    assert "12345678-1234-1234-1234-123456789abc" not in ctx.sanitized_enum["user_agent"]
+    assert "[UUID]" in ctx.sanitized_enum["user_agent"]
+
+
+def test_prometheus_label_context_preserves_raw_when_authenticated(monkeypatch):
+    """
+    When /metrics endpoint is properly authenticated, raw label values are preserved
+    for internal dashboard backward-compatibility.
+    """
+    monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", None)
+    monkeypatch.setattr(litellm, "require_auth_for_metrics_endpoint", True)
+    monkeypatch.setattr(litellm, "prometheus_mask_pii_labels", None)
+
+    enum_values = UserAPIKeyLabelValues(
+        user="john.doe@example.com",
+        user_email="john.doe@example.com",
+        team_alias="Finance Dept",
+        client_ip="203.0.113.195",
+    )
+
+    ctx = PrometheusLabelFactoryContext(enum_values)
+
+    assert ctx.sanitized_enum["user"] == "john.doe@example.com"
+    assert ctx.sanitized_enum["user_email"] == "john.doe@example.com"
+    assert ctx.sanitized_enum["team_alias"] == "Finance Dept"
+    assert ctx.sanitized_enum["client_ip"] == "203.0.113.195"
+
+
+def test_prometheus_label_context_explicit_mask_pii_flag(monkeypatch):
+    """
+    When prometheus_mask_pii_labels=True is explicitly set, PII is masked even when authenticated.
+    """
+    monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", None)
+    monkeypatch.setattr(litellm, "require_auth_for_metrics_endpoint", True)
+    monkeypatch.setattr(litellm, "prometheus_mask_pii_labels", True)
+
+    enum_values = UserAPIKeyLabelValues(
+        user="john.doe@example.com",
+        client_ip="203.0.113.195",
+    )
+
+    ctx = PrometheusLabelFactoryContext(enum_values)
+    assert ctx.sanitized_enum["user"].startswith("redacted_user_")
+    assert ctx.sanitized_enum["client_ip"] == "203.0.0.0/16"
+
+
+def test_prometheus_mount_warning_when_unauthenticated(monkeypatch):
+    """
+    Verify that mounting /metrics when unauthenticated emits a UserWarning and logs a warning.
+    """
+    from litellm.integrations.prometheus import PrometheusLogger
+
+    monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", True)
+    monkeypatch.setattr(litellm, "require_auth_for_metrics_endpoint", False)
+
+    mock_app = MagicMock()
+    with patch("litellm.proxy.proxy_server.app", mock_app), \
+         patch("litellm.integrations.prometheus_metrics_endpoint.make_metrics_asgi_app", return_value=MagicMock()):
+        with pytest.warns(UserWarning, match="without authentication"):
+            PrometheusLogger._mount_metrics_endpoint()
+
+        mock_app.mount.assert_called_once()

tests/unit/proxy/middleware/test_prometheus_auth_middleware.py

@@ -207,6 +207,43 @@ def test_no_auth_metrics_when_disabled(app_with_middleware, monkeypatch):
     assert response.json() == {"msg": "metrics OK"}
 
 
+def test_allow_unauthenticated_metrics_opt_out(app_with_middleware, monkeypatch):
+    """
+    Test that when allow_unauthenticated_metrics is True, requests to /metrics
+    bypass the auth check (explicit opt-out).
+    """
+    monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", True)
+
+    def should_not_be_called(*args, **kwargs):
+        raise Exception("Auth should not be called when unauthenticated metrics allowed")
+
+    monkeypatch.setattr(
+        "litellm.proxy.middleware.prometheus_auth_middleware.user_api_key_auth",
+        should_not_be_called,
+    )
+
+    client = TestClient(app_with_middleware)
+    response = client.get("/metrics")
+    assert response.status_code == 200, response.text
+    assert response.json() == {"msg": "metrics OK"}
+
+
+def test_allow_unauthenticated_metrics_false_enforces_auth(app_with_middleware, monkeypatch):
+    """
+    Test that when allow_unauthenticated_metrics is explicitly False, auth is enforced.
+    """
+    monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", False)
+    monkeypatch.setattr(
+        "litellm.proxy.middleware.prometheus_auth_middleware.user_api_key_auth",
+        fake_invalid_auth,
+    )
+
+    client = TestClient(app_with_middleware)
+    response = client.get("/metrics")
+    assert response.status_code == 401, response.text
+    assert "allow_unauthenticated_metrics" in response.text
+
+
 def test_non_metrics_requests_pass_through(app_with_middleware, monkeypatch):
     """
     Test that non-metrics endpoints pass through the middleware unaffected.