diff --git a/litellm/__init__.py b/litellm/__init__.py index 8e5dd68a186..13ca83249b5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -171,6 +171,7 @@ langsmith_batch_size: Optional[int] = None prometheus_initialize_budget_metrics: Optional[bool] = False prometheus_latency_buckets: Optional[List[float]] = None require_auth_for_metrics_endpoint: Optional[bool] = True +allow_unauthenticated_metrics: Optional[bool] = None argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload @@ -484,6 +485,7 @@ prometheus_deployment_and_latency_caller_identity: Literal[ # (RPM/TPM/concurrent/budget). prometheus_emit_rate_limit_labels: bool = False prometheus_user_budget_label_include_email_alias: bool = False +prometheus_mask_pii_labels: Optional[bool] = None prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000 prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0 prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0 diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index fc1a601f9da..580c5a20c33 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -5029,7 +5029,26 @@ class PrometheusLogger(CustomLogger): # Mount the metrics app to the app app.mount("/metrics", metrics_app) - verbose_proxy_logger.debug("Starting Prometheus Metrics on /metrics (no authentication)") + is_auth_required = ( + getattr(litellm, "allow_unauthenticated_metrics", None) is not True + and getattr(litellm, "require_auth_for_metrics_endpoint", None) is not False + ) + if is_auth_required: + verbose_proxy_logger.debug("Starting Prometheus Metrics on /metrics (authenticated)") + else: + verbose_proxy_logger.warning( + "SECURITY WARNING: /metrics endpoint is exposed WITHOUT authentication. " + "Sensitive multi-tenant operational data and metric labels may be visible to unauthorized clients. " + "To secure this endpoint, remove allow_unauthenticated_metrics or set require_auth_for_metrics_endpoint: true." + ) + import warnings + + warnings.warn( + "/metrics endpoint is exposed without authentication. " + "Ensure this endpoint is behind a firewall/private VPC or set require_auth_for_metrics_endpoint: true.", + UserWarning, + stacklevel=2, + ) def _label_source(enum_values: UserAPIKeyLabelValues) -> Mapping[str, object]: diff --git a/litellm/integrations/prometheus_helpers/__init__.py b/litellm/integrations/prometheus_helpers/__init__.py index 5c4cba6a2eb..3d555397eed 100644 --- a/litellm/integrations/prometheus_helpers/__init__.py +++ b/litellm/integrations/prometheus_helpers/__init__.py @@ -6,6 +6,8 @@ Helpers for the Prometheus integration (extracted to keep ``prometheus.py`` smal from __future__ import annotations +import hashlib +import re from typing import Final, Literal, Protocol, cast from litellm.types.integrations.prometheus import ( @@ -16,6 +18,61 @@ from litellm.types.integrations.prometheus import ( _get_end_user_id_for_cost_tracking = None +_EMAIL_PATTERN: Final = re.compile(r"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+") +_UUID_PATTERN: Final = re.compile( + r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" +) + +_PII_LABEL_KEYS: Final = frozenset( + { + "user", + "user_email", + "user_alias", + "team_alias", + "client_ip", + "user_agent", + } +) + + +def sanitize_pii_label_value(label_name: str, value: object | None) -> str | None: + """ + Sanitize sensitive tenant PII before Prometheus label emission. + Protects user emails, client IPs, user-agent automation IDs, and team aliases. + """ + if value is None: + return None + + str_val: Final[str] = value if isinstance(value, str) else str(value) + if not str_val: + return str_val + + if label_name in ("user", "user_email", "end_user"): + if _EMAIL_PATTERN.search(str_val): + return f"redacted_user_{hashlib.sha256(str_val.encode('utf-8')).hexdigest()[:8]}" + return f"user_{hashlib.sha256(str_val.encode('utf-8')).hexdigest()[:8]}" + + if label_name in ("team_alias", "user_alias", "api_key_alias"): + if _EMAIL_PATTERN.search(str_val): + return _EMAIL_PATTERN.sub("[REDACTED_EMAIL]", str_val) + return str_val + + if label_name == "client_ip": + if ":" in str_val: + return f"ipv6_{hashlib.sha256(str_val.encode('utf-8')).hexdigest()[:8]}" + parts = str_val.split(".") + if len(parts) == 4: + return f"{parts[0]}.{parts[1]}.0.0/16" + return f"ip_{hashlib.sha256(str_val.encode('utf-8')).hexdigest()[:8]}" + + if label_name == "user_agent": + sanitized = _UUID_PATTERN.sub("[UUID]", str_val) + if len(sanitized) > 64: + return sanitized[:64] + "..." + return sanitized + + return str_val + class _EndUserIdGetter(Protocol): def __call__( @@ -59,11 +116,30 @@ class PrometheusLabelFactoryContext: _END_USER_NOT_COMPUTED = object() def __init__(self, enum_values: UserAPIKeyLabelValues) -> None: + import litellm + self.enum_values = enum_values enum_dict: Final = enum_values.model_dump() - self.sanitized_enum: dict[str, str | None] = { - k: sanitize_prometheus_label_value(v) for k, v in enum_dict.items() - } + + mask_pii = getattr(litellm, "prometheus_mask_pii_labels", None) + if mask_pii is None: + # Auto-sanitize PII if metrics endpoint is exposed unauthenticated + mask_pii = ( + getattr(litellm, "allow_unauthenticated_metrics", None) is True + or getattr(litellm, "require_auth_for_metrics_endpoint", None) is False + ) + + if mask_pii: + self.sanitized_enum: dict[str, str | None] = { + k: sanitize_prometheus_label_value( + sanitize_pii_label_value(k, v) if k in _PII_LABEL_KEYS else v + ) + for k, v in enum_dict.items() + } + else: + self.sanitized_enum: dict[str, str | None] = { + k: sanitize_prometheus_label_value(v) for k, v in enum_dict.items() + } self.custom_by_sanitized_key: dict[str, str | None] = {} if enum_values.custom_metadata_labels is not None: for key, value in enum_values.custom_metadata_labels.items(): @@ -106,8 +182,19 @@ class PrometheusLabelFactoryContext: def get_resolved_end_user(self) -> str | None: if self._resolved_end_user is self._END_USER_NOT_COMPUTED: fn: Final = get_cached_end_user_id_for_cost_tracking() - self._resolved_end_user = fn( + resolved = fn( litellm_params={"user_api_key_end_user_id": self.enum_values.end_user}, service_type="prometheus", ) + import litellm + + mask_pii = getattr(litellm, "prometheus_mask_pii_labels", None) + if mask_pii is None: + mask_pii = ( + getattr(litellm, "allow_unauthenticated_metrics", None) is True + or getattr(litellm, "require_auth_for_metrics_endpoint", None) is False + ) + if mask_pii and resolved: + resolved = sanitize_pii_label_value("end_user", resolved) + self._resolved_end_user = resolved return cast(str | None, self._resolved_end_user) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 82135cb1b74..4e1c325ad73 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -211,7 +211,10 @@ def _normalize_public_auth_route(route: str) -> str: def _route_requires_auth_despite_public(route: str, general_settings: dict | None) -> bool: normalized_route: Final = _normalize_public_auth_route(route) if normalized_route == "/metrics": - return litellm.require_auth_for_metrics_endpoint is not False + return ( + litellm.allow_unauthenticated_metrics is not True + and litellm.require_auth_for_metrics_endpoint is not False + ) return False diff --git a/litellm/proxy/middleware/prometheus_auth_middleware.py b/litellm/proxy/middleware/prometheus_auth_middleware.py index f5d9eb11dc6..8814aa1aed5 100644 --- a/litellm/proxy/middleware/prometheus_auth_middleware.py +++ b/litellm/proxy/middleware/prometheus_auth_middleware.py @@ -34,7 +34,9 @@ class PrometheusAuthMiddleware: ```yaml litellm_settings: - require_auth_for_metrics_endpoint: false + allow_unauthenticated_metrics: true + # or legacy flag: + # require_auth_for_metrics_endpoint: false ``` """ @@ -47,8 +49,12 @@ class PrometheusAuthMiddleware: await self.app(scope, receive, send) return - # Run auth by default; allow legacy public metrics only when explicitly disabled. - if litellm.require_auth_for_metrics_endpoint is not False: + # Run auth by default; allow public metrics only when explicitly opted out. + is_auth_required = ( + litellm.allow_unauthenticated_metrics is not True + and litellm.require_auth_for_metrics_endpoint is not False + ) + if is_auth_required: # user_api_key_auth reads the request body, which consumes ASGI `receive`. # Buffer those messages and replay them for the inner app; otherwise a # successful auth would forward an exhausted receive and /metrics hangs. @@ -79,7 +85,8 @@ class PrometheusAuthMiddleware: body: Final = json.dumps( f"Unauthorized access to metrics endpoint: {error_message} " f"To allow unauthenticated access, set " - f"`litellm_settings.require_auth_for_metrics_endpoint: false` " + f"`litellm_settings.allow_unauthenticated_metrics: true` " + f"(or `litellm_settings.require_auth_for_metrics_endpoint: false`) " f"in your proxy_config.yaml." ).encode("utf-8") await send( diff --git a/tests/unit/integrations/test_prometheus_pii_sanitization.py b/tests/unit/integrations/test_prometheus_pii_sanitization.py new file mode 100644 index 00000000000..a0b284c150c --- /dev/null +++ b/tests/unit/integrations/test_prometheus_pii_sanitization.py @@ -0,0 +1,144 @@ +import warnings +import pytest +from unittest.mock import MagicMock, patch + +import litellm +from litellm.integrations.prometheus_helpers import ( + PrometheusLabelFactoryContext, + sanitize_pii_label_value, +) +from litellm.types.integrations.prometheus import UserAPIKeyLabelValues + + +def test_sanitize_pii_label_value_user_email(): + """Verify that user identifiers containing email addresses are sanitized to a deterministic hash.""" + email_user = "employee@internal.company.com" + sanitized = sanitize_pii_label_value("user", email_user) + assert sanitized is not None + assert "internal.company.com" not in sanitized + assert "employee" not in sanitized + assert sanitized.startswith("redacted_user_") + + sanitized_email = sanitize_pii_label_value("user_email", email_user) + assert sanitized_email is not None + assert "internal.company.com" not in sanitized_email + assert sanitized_email.startswith("redacted_user_") + + +def test_sanitize_pii_label_value_team_alias(): + """Verify that employee emails in team_alias are masked.""" + team_alias = "Enterprise Corp (POC: admin@corp.example.com)" + sanitized = sanitize_pii_label_value("team_alias", team_alias) + assert sanitized is not None + assert "admin@corp.example.com" not in sanitized + assert "[REDACTED_EMAIL]" in sanitized + assert "Enterprise Corp" in sanitized + + +def test_sanitize_pii_label_value_client_ip(): + """Verify that IPv4 client IPs mask host topology and IPv6 IPs are hashed.""" + ipv4 = "198.51.100.42" + sanitized_ipv4 = sanitize_pii_label_value("client_ip", ipv4) + assert sanitized_ipv4 == "198.51.0.0/16" + + ipv6 = "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + sanitized_ipv6 = sanitize_pii_label_value("client_ip", ipv6) + assert sanitized_ipv6 is not None + assert sanitized_ipv6.startswith("ipv6_") + + +def test_sanitize_pii_label_value_user_agent_uuid(): + """Verify that UUIDs (e.g. Azure Logic Apps workflow IDs) in user-agent strings are redacted.""" + ua = "AzureLogicApps/1.0 workflow/d9b4b726-1e6f-4f3c-8a5e-2f190e3fbb61" + sanitized = sanitize_pii_label_value("user_agent", ua) + assert "d9b4b726-1e6f-4f3c-8a5e-2f190e3fbb61" not in sanitized + assert "AzureLogicApps/1.0 workflow/[UUID]" in sanitized + + +def test_prometheus_label_context_auto_sanitization_when_unauthenticated(monkeypatch): + """ + When /metrics endpoint is exposed unauthenticated (via allow_unauthenticated_metrics=True + or require_auth_for_metrics_endpoint=False), PII fields must automatically be sanitized + in PrometheusLabelFactoryContext to protect multi-tenant privacy. + """ + monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", True) + monkeypatch.setattr(litellm, "require_auth_for_metrics_endpoint", False) + monkeypatch.setattr(litellm, "prometheus_mask_pii_labels", None) + + enum_values = UserAPIKeyLabelValues( + user="john.doe@example.com", + user_email="john.doe@example.com", + team_alias="Finance Dept (lead: jane@example.com)", + client_ip="203.0.113.195", + user_agent="AzureLogicApps/1.0 workflow/12345678-1234-1234-1234-123456789abc", + ) + + ctx = PrometheusLabelFactoryContext(enum_values) + + assert "john.doe@example.com" not in str(ctx.sanitized_enum.get("user")) + assert ctx.sanitized_enum["user"].startswith("redacted_user_") + assert "john.doe@example.com" not in str(ctx.sanitized_enum.get("user_email")) + assert "[REDACTED_EMAIL]" in ctx.sanitized_enum["team_alias"] + assert ctx.sanitized_enum["client_ip"] == "203.0.0.0/16" + assert "12345678-1234-1234-1234-123456789abc" not in ctx.sanitized_enum["user_agent"] + assert "[UUID]" in ctx.sanitized_enum["user_agent"] + + +def test_prometheus_label_context_preserves_raw_when_authenticated(monkeypatch): + """ + When /metrics endpoint is properly authenticated, raw label values are preserved + for internal dashboard backward-compatibility. + """ + monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", None) + monkeypatch.setattr(litellm, "require_auth_for_metrics_endpoint", True) + monkeypatch.setattr(litellm, "prometheus_mask_pii_labels", None) + + enum_values = UserAPIKeyLabelValues( + user="john.doe@example.com", + user_email="john.doe@example.com", + team_alias="Finance Dept", + client_ip="203.0.113.195", + ) + + ctx = PrometheusLabelFactoryContext(enum_values) + + assert ctx.sanitized_enum["user"] == "john.doe@example.com" + assert ctx.sanitized_enum["user_email"] == "john.doe@example.com" + assert ctx.sanitized_enum["team_alias"] == "Finance Dept" + assert ctx.sanitized_enum["client_ip"] == "203.0.113.195" + + +def test_prometheus_label_context_explicit_mask_pii_flag(monkeypatch): + """ + When prometheus_mask_pii_labels=True is explicitly set, PII is masked even when authenticated. + """ + monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", None) + monkeypatch.setattr(litellm, "require_auth_for_metrics_endpoint", True) + monkeypatch.setattr(litellm, "prometheus_mask_pii_labels", True) + + enum_values = UserAPIKeyLabelValues( + user="john.doe@example.com", + client_ip="203.0.113.195", + ) + + ctx = PrometheusLabelFactoryContext(enum_values) + assert ctx.sanitized_enum["user"].startswith("redacted_user_") + assert ctx.sanitized_enum["client_ip"] == "203.0.0.0/16" + + +def test_prometheus_mount_warning_when_unauthenticated(monkeypatch): + """ + Verify that mounting /metrics when unauthenticated emits a UserWarning and logs a warning. + """ + from litellm.integrations.prometheus import PrometheusLogger + + monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", True) + monkeypatch.setattr(litellm, "require_auth_for_metrics_endpoint", False) + + mock_app = MagicMock() + with patch("litellm.proxy.proxy_server.app", mock_app), \ + patch("litellm.integrations.prometheus_metrics_endpoint.make_metrics_asgi_app", return_value=MagicMock()): + with pytest.warns(UserWarning, match="without authentication"): + PrometheusLogger._mount_metrics_endpoint() + + mock_app.mount.assert_called_once() diff --git a/tests/unit/proxy/middleware/test_prometheus_auth_middleware.py b/tests/unit/proxy/middleware/test_prometheus_auth_middleware.py index beb841878d5..7890060340d 100644 --- a/tests/unit/proxy/middleware/test_prometheus_auth_middleware.py +++ b/tests/unit/proxy/middleware/test_prometheus_auth_middleware.py @@ -207,6 +207,43 @@ def test_no_auth_metrics_when_disabled(app_with_middleware, monkeypatch): assert response.json() == {"msg": "metrics OK"} +def test_allow_unauthenticated_metrics_opt_out(app_with_middleware, monkeypatch): + """ + Test that when allow_unauthenticated_metrics is True, requests to /metrics + bypass the auth check (explicit opt-out). + """ + monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", True) + + def should_not_be_called(*args, **kwargs): + raise Exception("Auth should not be called when unauthenticated metrics allowed") + + monkeypatch.setattr( + "litellm.proxy.middleware.prometheus_auth_middleware.user_api_key_auth", + should_not_be_called, + ) + + client = TestClient(app_with_middleware) + response = client.get("/metrics") + assert response.status_code == 200, response.text + assert response.json() == {"msg": "metrics OK"} + + +def test_allow_unauthenticated_metrics_false_enforces_auth(app_with_middleware, monkeypatch): + """ + Test that when allow_unauthenticated_metrics is explicitly False, auth is enforced. + """ + monkeypatch.setattr(litellm, "allow_unauthenticated_metrics", False) + monkeypatch.setattr( + "litellm.proxy.middleware.prometheus_auth_middleware.user_api_key_auth", + fake_invalid_auth, + ) + + client = TestClient(app_with_middleware) + response = client.get("/metrics") + assert response.status_code == 401, response.text + assert "allow_unauthenticated_metrics" in response.text + + def test_non_metrics_requests_pass_through(app_with_middleware, monkeypatch): """ Test that non-metrics endpoints pass through the middleware unaffected.