From d2711ac5592fb777887a7c204b9f73c18bfd115c Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 13 Nov 2026 02:06:07 +0800 Subject: [PATCH] feat(alerts): add management controls and opt-in daily summaries (#51418) * feat(alerts): add alert controls and daily notification summaries * fix(alerts): reserve daily notifications before delivery --- .../types/enterprise_callbacks/send_emails.py | 4 +- .../SlackAlerting/slack_alerting.py | 38 +++- .../integrations/model_deprecation_email.py | 8 +- .../performance_alert_notifications.py | 183 +++++++++++++++ litellm/models/alert.py | 18 ++ litellm/models/performance_alert.py | 8 + litellm/proxy/alerting_endpoints/alerts.py | 10 +- litellm/proxy/proxy_server.py | 29 +++ litellm/repositories/alert_repository.py | 52 +++++ litellm/repositories/config_repository.py | 8 + .../performance_alert_repository.py | 13 +- litellm/types/integrations/slack_alerting.py | 1 + .../Alerts/Alerts.integration.test.tsx | 76 +++++-- .../src/components/Alerts/Alerts.tsx | 7 +- .../src/components/Alerts/ManageAlerts.tsx | 150 ++++++++++++ .../Alerts/PerformanceMonitoring.tsx | 99 -------- .../DashboardHeader.integration.test.tsx | 10 + .../DeprecationEmailSetup.test.tsx | 8 +- .../useDeprecationAlerts.integration.test.tsx | 12 +- .../useDeprecationAlerts.ts | 3 + .../email_events/email_event_settings.tsx | 3 + .../src/components/settings.tsx | 1 + .../src/i18n/locales/en.json | 29 ++- .../src/lib/http/schema.d.ts | 24 +- tests/integration/database/test_alerts.py | 124 +++++++++- .../test_model_deprecation_alert.py | 17 +- .../test_model_deprecation_email.py | 35 +++ .../test_performance_alert_notifications.py | 213 ++++++++++++++++++ .../proxy/alerting_endpoints/test_alerts.py | 39 +++- .../test_performance_alert_repository.py | 8 +- 30 files changed, 1074 insertions(+), 156 deletions(-) create mode 100644 litellm/integrations/performance_alert_notifications.py create mode 100644 litellm/ui/litellm-dashboard/src/components/Alerts/ManageAlerts.tsx delete mode 100644 litellm/ui/litellm-dashboard/src/components/Alerts/PerformanceMonitoring.tsx create mode 100644 tests/unit/integrations/test_performance_alert_notifications.py diff --git a/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py b/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py index bb3b712d50b..678c5c09cf5 100644 --- a/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py +++ b/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py @@ -35,6 +35,7 @@ class SendKeyRotatedEmailEvent(WebhookEvent): class EmailEvent(str, enum.Enum): model_deprecations = MODEL_DEPRECATION_EMAIL_EVENT + performance_alerts = "Performance Alerts" virtual_key_created = "Virtual Key Created" new_user_invitation = "New User Invitation" virtual_key_rotated = "Virtual Key Rotated" @@ -60,7 +61,8 @@ class DefaultEmailSettings(LiteLLMBaseModel): settings: dict[EmailEvent, bool] = Field( default_factory=lambda: { - EmailEvent.model_deprecations: False, # Explicit opt-in for owner retirement reminders. + EmailEvent.model_deprecations: False, + EmailEvent.performance_alerts: False, EmailEvent.virtual_key_created: True, # On by default EmailEvent.new_user_invitation: True, # On by default EmailEvent.virtual_key_rotated: True, # On by default diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index dd611947228..f728d054c0d 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -2,10 +2,11 @@ # Class for sending Slack Alerts # import asyncio import datetime +import json import os import random import time -from collections.abc import Callable +from collections.abc import Awaitable, Callable from datetime import timedelta from typing import TYPE_CHECKING, Any, Final, Literal @@ -38,6 +39,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) +from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.table_repositories import InvitationLinkRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository @@ -87,6 +89,12 @@ def _proxy_llm_router() -> Router | None: _DAILY_REPORT_TARGET: Final = "daily_report_schedule" +async def _model_deprecations_enabled() -> bool: + from litellm.proxy.proxy_server import prisma_client + + return prisma_client is None or await ConfigRepository(prisma_client, use_writer=True).model_deprecations_enabled() + + class SlackAlerting(CustomBatchLogger): """ Class for sending Slack Alerts @@ -97,12 +105,13 @@ class SlackAlerting(CustomBatchLogger): self, internal_usage_cache: DualCache | None = None, alerting_threshold: float | None = None, # threshold for slow / hanging llm responses (in seconds) - alerting: list | None = [], + alerting: list[str] | None = [], alert_types: list[AlertType] = DEFAULT_ALERT_TYPES, alert_to_webhook_url: dict[AlertType, list[str] | str] | None = None, # if user wants to separate alerts to diff channels alerting_args={}, default_webhook_url: str | None = None, + model_deprecations_enabled: Callable[[], Awaitable[bool]] = _model_deprecations_enabled, alert_type_config: dict[str, dict] | None = None, async_http_handler: AsyncHTTPHandler | None = None, **kwargs, @@ -120,6 +129,7 @@ class SlackAlerting(CustomBatchLogger): self.is_running = False self.alerting_args = SlackAlertingArgs(**alerting_args) self.default_webhook_url = default_webhook_url + self.model_deprecations_enabled: Final = model_deprecations_enabled self.flush_lock = asyncio.Lock() self.periodic_started = False self._periodic_flush_task: asyncio.Task[None] | None = None @@ -142,7 +152,7 @@ class SlackAlerting(CustomBatchLogger): def update_values( self, - alerting: list | None = None, + alerting: list[str] | None = None, alerting_threshold: float | None = None, alert_types: list[AlertType] | None = None, alert_to_webhook_url: dict[AlertType, list[str] | str] | None = None, @@ -1114,7 +1124,7 @@ Model Info: The daily lock is claimed only once there is something to say, so an empty pass never blocks a later real one, and a sent alert is stamped in the shared cache for a day so sibling pods stop asking """ - if not self._deprecation_alerts_enabled(): + if not self._deprecation_alerts_enabled() or not await self.model_deprecations_enabled(): return False from litellm.proxy.common_utils.model_deprecation import ( @@ -1194,6 +1204,26 @@ Model Info: continue await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) + def performance_alert_webhooks(self) -> tuple[str, ...]: + if not self.alerting or "slack" not in self.alerting or AlertType.performance_alerts not in self.alert_types: + return () + configured: Final = (self.alert_to_webhook_url or {}).get(AlertType.performance_alerts) + destination: Final = ( + configured + or self.default_webhook_url + or os.getenv("SLACK_WEBHOOK_URL") + or os.getenv("ALERTING_WEBHOOK_URL") + ) + if isinstance(destination, list): + return tuple(dict.fromkeys(destination)) + return (destination,) if destination else () + + async def send_performance_alert_summary(self, webhook: str, message: str) -> bool: + response: Final = await self.async_http_handler.post( + url=webhook, headers={"Content-Type": "application/json"}, data=json.dumps({"text": message}) + ) + return response.status_code == 200 + async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool: """ Sends structured alert to webhook, if set. diff --git a/litellm/integrations/model_deprecation_email.py b/litellm/integrations/model_deprecation_email.py index 4093a108dcc..11988d8ce57 100644 --- a/litellm/integrations/model_deprecation_email.py +++ b/litellm/integrations/model_deprecation_email.py @@ -214,9 +214,13 @@ class ModelDeprecationEmailSender: return True async def run(self, llm_router: Router, *, now: datetime | None = None) -> int: + if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"): + return 0 + if not await self.config.model_deprecations_enabled(): + return 0 current: Final = now or datetime.now(timezone.utc) settings: Final = await self.config.get_param("general_settings") - if settings is None or not _OptIn.model_validate(settings.param_value).enabled: + if not _OptIn.model_validate(settings.param_value if settings else {}).enabled: return 0 base_url: Final = os.getenv("PROXY_BASE_URL", "") if not base_url: @@ -253,7 +257,7 @@ class ModelDeprecationEmailSender: async def run_scheduled_model_deprecation_emails() -> None: - """Poll the email opt-in independently; a large owner mailing never delays Slack checks.""" + """Poll the email settings independently; a large owner mailing never delays Slack checks.""" from litellm._logging import verbose_proxy_logger while True: diff --git a/litellm/integrations/performance_alert_notifications.py b/litellm/integrations/performance_alert_notifications.py new file mode 100644 index 00000000000..1d1c8a8ab00 --- /dev/null +++ b/litellm/integrations/performance_alert_notifications.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import asyncio +import hashlib +import html +import os +from collections.abc import Awaitable, Callable, Sequence +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Final + +from pydantic import AliasPath, BaseModel, ConfigDict, Field + +from litellm._logging import verbose_proxy_logger +from litellm.models.alert import AlertSummary +from litellm.repositories.alert_repository import AlertRepository +from litellm.repositories.config_repository import ConfigRepository + +if TYPE_CHECKING: + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + + +class _EmailSettings(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + enabled: bool = Field(False, strict=True, validation_alias=AliasPath("email_settings", "Performance Alerts")) + + +class _Delivery(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + revision: int = 1 + cursor: int + first_window_start: datetime + next_send_at: datetime + lease_until: datetime + + +def _description(alert: AlertSummary) -> str: + labels: Final = { + "spend": "Spend spike", + "traffic": "Traffic spike", + "latency": "Slower responses", + "reliability": "More failed requests", + } + metric: Final = alert.evidence + + def amount(value: float) -> str: + match metric.unit: + case "usd": + return f"${value:,.2f}" + case "ratio": + return f"{value:.1%}" + case "milliseconds": + return f"{value:,.0f} ms" + case "requests": + return f"{value:,.0f} requests" + + return f"{labels[alert.type]} for {alert.resource_name}: {amount(metric.value)} vs {amount(metric.baseline)}" + + +def slack_text(alerts: Sequence[AlertSummary], base_url: str) -> str: + rows: Final = tuple( + f"<{base_url}/ui/alerts?alert={alert.id}|{html.escape(_description(alert), quote=False)}>" + for alert in alerts[:5] + ) + return "New performance alerts\n\n" + "\n".join(rows) + f"\n\n<{base_url}/ui/alerts|See all alerts>" + + +def email_body(alerts: Sequence[AlertSummary], base_url: str) -> tuple[str, str]: + rows: Final = tuple( + f'
  • {html.escape(_description(alert))}
    • ' + for alert in alerts[:5] + ) + text: Final = ( + "New performance alerts\n\n" + + "\n".join(f"{_description(alert)}\n{base_url}/ui/alerts?alert={alert.id}" for alert in alerts[:5]) + + f"\n\nSee all alerts: {base_url}/ui/alerts" + ) + return ( + f'

    New performance alerts

    See all alerts

    ', + text, + ) + + +class PerformanceAlertNotifier: + def __init__(self, config: ConfigRepository, alerts: AlertRepository) -> None: + self.config: Final = config + self.alerts: Final = alerts + + async def deliver( + self, + destination: str, + send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]], + now: datetime, + ) -> None: + key: Final = "performance_alert_digest:" + hashlib.sha256(destination.encode()).hexdigest() + row: Final = await self.config.get_param(key) + previous: Final = ( + _Delivery.model_validate(row.param_value) + if row + else _Delivery( + revision=0, + cursor=0, + first_window_start=now - timedelta(days=1), + next_send_at=now, + lease_until=now, + ) + ) + if previous.next_send_at > now or previous.lease_until > now: + return + through: Final = await self.alerts.latest_id() + found: Final = await self.alerts.summaries(previous.cursor, through) + alerts: Final = ( + found + if previous.cursor + else tuple(alert for alert in found if alert.created_at >= previous.first_window_start) + ) + claim: Final = previous.model_copy( + update={ + "revision": previous.revision + 1, + "lease_until": now + timedelta(minutes=2), + "next_send_at": now + timedelta(days=1) if alerts else now, + } + ) + if not await self.config.set_param_if_revision(key, claim.model_dump(mode="json"), previous.revision): + return + if alerts and not await send(alerts, f"{key}-{through}"): + return + sent: Final = claim.model_copy( + update={ + "revision": claim.revision + 1, + "cursor": through, + "next_send_at": now + timedelta(days=1) if alerts else now, + "lease_until": now, + } + ) + if not await self.config.set_param_if_revision(key, sent.model_dump(mode="json"), claim.revision): + verbose_proxy_logger.warning("Could not record performance alert summary delivery") + + async def run(self, slack: SlackAlerting | None, *, now: datetime | None = None) -> None: + from litellm.proxy.utils import send_email + + base_url: Final = os.getenv("PROXY_BASE_URL", "").rstrip("/") + if not base_url: + return + if slack is not None: + for webhook in slack.performance_alert_webhooks(): + + async def send_slack(alerts: Sequence[AlertSummary], receipt: str, webhook: str = webhook) -> bool: + return await slack.send_performance_alert_summary(webhook, slack_text(alerts, base_url)) + + await self._try_deliver("slack:" + webhook, send_slack, now or datetime.now(timezone.utc)) + if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"): + return + settings: Final = await self.config.get_param("general_settings") + if not _EmailSettings.model_validate(settings.param_value if settings else {}).enabled: + return + after: str = "" # rebind-ok: advances the bounded recipient page + while recipients := await self.alerts.recipients(after): + for recipient in recipients: + + async def send_mail( + alerts: Sequence[AlertSummary], receipt: str, address: str = recipient.email + ) -> bool: + body, text = email_body(alerts, base_url) + return await send_email( + receiver_email=address, + subject="New performance alerts", + html=body, + text=text, + message_id=f"<{receipt}@alerts.litellm.ai>", + ) + + await self._try_deliver("email:" + recipient.email, send_mail, now or datetime.now(timezone.utc)) + after = recipients[-1].user_id + await asyncio.sleep(0.1) + + async def _try_deliver( + self, destination: str, send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]], now: datetime + ) -> None: + try: + await asyncio.wait_for(self.deliver(destination, send, now), timeout=60) + except Exception as exc: # noqa: BLE001 # one unavailable destination must not stop the other recipients + verbose_proxy_logger.warning("Performance alert summary delivery failed (%s)", type(exc).__name__) diff --git a/litellm/models/alert.py b/litellm/models/alert.py index f96e042ff6b..4db127bf121 100644 --- a/litellm/models/alert.py +++ b/litellm/models/alert.py @@ -41,6 +41,24 @@ class DeprecationEvidence(BaseModel): ) +class AlertSummary(BaseModel): + model_config = ConfigDict(frozen=True) + + id: AlertID + type: AlertType + resource_type: AlertResourceType + resource_name: str + created_at: AwareDatetime + evidence: MetricEvidence + + +class AlertRecipient(BaseModel): + model_config = ConfigDict(frozen=True) + + user_id: str + email: str + + class AlertDetection(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") diff --git a/litellm/models/performance_alert.py b/litellm/models/performance_alert.py index 1773aa460dc..1cae91e5379 100644 --- a/litellm/models/performance_alert.py +++ b/litellm/models/performance_alert.py @@ -85,6 +85,7 @@ class AlertMonitoring(BaseModel): model_config = ConfigDict(frozen=True) enabled: bool = Field(description="Whether performance monitoring is enabled. Defaults to true.") + model_deprecations_enabled: bool = Field(default=True, description="Whether model deprecation alerts are enabled.") alerts_used: int = Field(ge=0, le=FREE_ALERT_LIMIT, description="Lifetime free allowance consumed, capped at five.") alert_limit: int | None = Field(description="Free allowance, or null for Enterprise.") disabled_by_config: bool = Field(description="Whether performance_alerts_enabled is false in proxy settings.") @@ -100,9 +101,16 @@ class AlertMonitoringUpdate(BaseModel): enabled: bool = Field(strict=True, description="Enable or disable performance monitoring for this gateway.") +class ModelDeprecationMonitoringUpdate(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + model_deprecations_enabled: bool = Field(strict=True, description="Enable or disable model deprecation alerts.") + + class AlertMonitoringStore(Protocol): async def monitoring(self) -> AlertMonitoring: ... async def set_enabled(self, enabled: bool) -> AlertMonitoring: ... + async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring: ... class PerformanceAlertStore(Protocol): diff --git a/litellm/proxy/alerting_endpoints/alerts.py b/litellm/proxy/alerting_endpoints/alerts.py index a6ed06c3b96..9c701208e0d 100644 --- a/litellm/proxy/alerting_endpoints/alerts.py +++ b/litellm/proxy/alerting_endpoints/alerts.py @@ -12,7 +12,7 @@ from litellm.models.alert import ( AlertView, AlertViewPage, ) -from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate +from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate, ModelDeprecationMonitoringUpdate from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.deps.repositories import AlertMonitoringRepositoryDep, AlertRepositoryDep @@ -60,7 +60,7 @@ async def list_alerts( return AlertViewPage(data=await store.enrich(page.data), next_cursor=page.next_cursor) -@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read performance monitoring settings") +@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read alert monitoring settings") async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -> AlertMonitoring: """Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin.""" _authorize(user, write=False) @@ -69,14 +69,16 @@ async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) - return await repository.monitoring() -@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable performance monitoring") +@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable alert monitoring") async def update_monitoring( - data: AlertMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep + data: AlertMonitoringUpdate | ModelDeprecationMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep ) -> AlertMonitoring: """Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts.""" _authorize(user, write=True) if repository is None: raise HTTPException(status_code=503, detail="Alerts require a database connection") + if isinstance(data, ModelDeprecationMonitoringUpdate): + return await repository.set_model_deprecations_enabled(data.model_deprecations_enabled) if data.enabled and (await repository.monitoring()).disabled_by_config: raise HTTPException(status_code=409, detail="Performance monitoring is disabled in proxy settings") return await repository.set_enabled(data.enabled) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e33966ceb29..b37eede1745 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11948,6 +11948,35 @@ class ProxyStartupEvent: misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) + async def send_performance_alert_summaries() -> None: + from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier + from litellm.repositories.alert_repository import AlertRepository + + slack: Final = proxy_logging_obj.slack_alerting_instance + email_connected: Final = bool(os.getenv("SMTP_HOST") and os.getenv("SMTP_SENDER_EMAIL")) + if not email_connected and not slack.performance_alert_webhooks(): + return + if _current_general_settings().get("performance_alerts_enabled", True) is not True: + return + if not ( + await PerformanceAlertRepository(prisma_client, enterprise=premium_user is True).monitoring() + ).enabled: + return + await PerformanceAlertNotifier( + ConfigRepository(prisma_client, use_writer=True), AlertRepository(prisma_client) + ).run(slack) + + scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # APScheduler does not provide typed job methods + send_performance_alert_summaries, + "interval", + minutes=5, + next_run_time=datetime.now(timezone.utc), + id="performance_alert_summaries", + replace_existing=True, + max_instances=1, + coalesce=True, + ) + ### PTU DAILY ROLLUP ### from litellm.proxy.spend_tracking.ptu_feature_flag import ( is_ptu_cost_attribution_enabled, diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py index eccdf16263d..6b878fa6afc 100644 --- a/litellm/repositories/alert_repository.py +++ b/litellm/repositories/alert_repository.py @@ -21,8 +21,10 @@ from litellm.models.alert import ( AlertFilter, AlertHistoryPoint, AlertPage, + AlertRecipient, AlertResourceType, AlertStatus, + AlertSummary, AlertView, ) from litellm.repositories.prisma_protocols import PrismaRecord, TableActions @@ -64,6 +66,17 @@ class _HistoryRow(BaseModel): _HISTORY_ROWS: Final = TypeAdapter(tuple[_HistoryRow, ...]) +_SUMMARIES: Final = TypeAdapter(tuple[AlertSummary, ...]) +_RECIPIENTS: Final = TypeAdapter(tuple[AlertRecipient, ...]) + + +class _LatestAlert(BaseModel): + model_config = ConfigDict(frozen=True) + + id: int + + +_LATEST_ALERT: Final = TypeAdapter(tuple[_LatestAlert, ...]) _HISTORY_SOURCES: Final[ Mapping[AlertResourceType, tuple[LiteralString, LiteralString, LiteralString, LiteralString, LiteralString]] ] = { @@ -158,6 +171,45 @@ class AlertRepository: TableActions[PrismaRecord], prisma_client.writer_db.litellm_alert ) + async def latest_id(self) -> int: + async with self._database.tx(timeout=timedelta(seconds=5)) as transaction: + await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)") + rows: Final = _LATEST_ALERT.validate_python( + await transaction.query_raw('SELECT COALESCE(MAX(id), 0) AS id FROM "LiteLLM_Alert"') + ) + return rows[0].id + + async def recipients(self, after: str) -> tuple[AlertRecipient, ...]: + async with self._database.tx(timeout=timedelta(seconds=5)) as transaction: + await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)") + rows: Final = await transaction.query_raw( + """SELECT user_id, LOWER(TRIM(user_email)) AS email FROM "LiteLLM_UserTable" + WHERE user_role = 'proxy_admin' AND user_id > $1 AND LENGTH(TRIM(user_email)) BETWEEN 3 AND 320 + ORDER BY user_id LIMIT 100""", + after, + ) + return _RECIPIENTS.validate_python(rows) + + async def summaries(self, after: int, through: int) -> tuple[AlertSummary, ...]: + async with self._database.tx(timeout=timedelta(seconds=5)) as transaction: + await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)") + rows: Final = await transaction.query_raw( + """SELECT a.id::text, a.type, a.resource_type, a.created_at AT TIME ZONE 'UTC' AS created_at, + LEFT(COALESCE(k.key_alias, t.team_alias, u.user_alias, + CASE WHEN a.resource_type = 'model' THEN a.resource_id ELSE a.resource_type END), 120) AS resource_name, + jsonb_build_object('value', a.evidence->'value', 'baseline', a.evidence->'baseline', + 'unit', a.evidence->'unit', 'sample_size', a.evidence->'sample_size') AS evidence + FROM "LiteLLM_Alert" a + LEFT JOIN "LiteLLM_VerificationToken" k ON a.resource_type = 'key' AND k.token = a.resource_id + LEFT JOIN "LiteLLM_TeamTable" t ON a.resource_type = 'team' AND t.team_id = a.resource_id + LEFT JOIN "LiteLLM_UserTable" u ON a.resource_type = 'user' AND u.user_id = a.resource_id + WHERE a.id > $1 AND a.id <= $2 AND a.status = 'open' AND a.type != 'model_deprecation' + ORDER BY a.id DESC LIMIT 5""", + after, + through, + ) + return _SUMMARIES.validate_python(rows) + async def record(self, detection: AlertDetection) -> Alert: from prisma import Json from prisma.errors import UniqueViolationError diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py index e9973e1f674..744c33cd9f8 100644 --- a/litellm/repositories/config_repository.py +++ b/litellm/repositories/config_repository.py @@ -60,6 +60,7 @@ ROUTER_SETTINGS_PARAM: Final = "router_settings" _JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) _JSON_RULES: Final = TypeAdapter(list[JsonValue]) _ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue]) +_BOOL: Final = TypeAdapter(bool) def _stored_router_settings(param_value: object) -> Mapping[str, JsonValue]: @@ -138,6 +139,13 @@ class ConfigRepository: ) return ConfigParam(param_name=param_name, param_value=param_value) + async def model_deprecations_enabled(self) -> bool: + record: Final = await self.get_param("model_deprecation_alerts_enabled") + return True if record is None else _BOOL.validate_python(record.param_value, strict=True) + + async def set_model_deprecations_enabled(self, enabled: bool) -> None: + await self.set_param("model_deprecation_alerts_enabled", enabled) + async def set_param_if_revision(self, param_name: str, param_value: object, revision: int) -> bool: rows: Final = await self._writer.query_raw( """INSERT INTO "LiteLLM_Config" (param_name, param_value, last_run_at) diff --git a/litellm/repositories/performance_alert_repository.py b/litellm/repositories/performance_alert_repository.py index 6d55670e0b8..9eaaed77d3c 100644 --- a/litellm/repositories/performance_alert_repository.py +++ b/litellm/repositories/performance_alert_repository.py @@ -24,6 +24,7 @@ from litellm.models.performance_alert import ( ) from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.repositories.alert_repository import record_alerts +from litellm.repositories.config_repository import ConfigRepository if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -161,6 +162,7 @@ class PerformanceAlertRepository: ) self._enterprise: Final = enterprise self._configured: Final = configured + self._config: Final = ConfigRepository(prisma_client, use_writer=True) async def _monitoring(self, transaction: _Transaction, *, lock: bool = False) -> AlertMonitoring: query: Final = 'SELECT enabled, alerts_used FROM "LiteLLM_PerformanceAlertState" WHERE id = 1' @@ -190,7 +192,14 @@ class PerformanceAlertRepository: async def monitoring(self) -> AlertMonitoring: async with self._transaction() as transaction: - return await self._monitoring(transaction) + monitoring: Final = await self._monitoring(transaction) + return monitoring.model_copy( + update={"model_deprecations_enabled": await self._config.model_deprecations_enabled()} + ) + + async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring: + await self._config.set_model_deprecations_enabled(enabled) + return await self.monitoring() async def set_enabled(self, enabled: bool) -> AlertMonitoring: async with self._database.tx(timeout=timedelta(seconds=10)) as transaction: @@ -203,7 +212,7 @@ class PerformanceAlertRepository: updated_at = now() AT TIME ZONE 'UTC' WHERE id = 1""", enabled, ) - return await self._monitoring(transaction) + return await self.monitoring() @asynccontextmanager async def _transaction(self) -> AsyncGenerator[_Transaction]: diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py index 89c2ad841e8..77322a4850a 100644 --- a/litellm/types/integrations/slack_alerting.py +++ b/litellm/types/integrations/slack_alerting.py @@ -188,6 +188,7 @@ class AlertType(str, Enum): cooldown_deployment = "cooldown_deployment" new_model_added = "new_model_added" model_deprecation_warnings = "model_deprecation_warnings" + performance_alerts = "performance_alerts" # Outage alerts outage_alerts = "outage_alerts" diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx index 073d22e52f5..31902008f2a 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx @@ -27,7 +27,13 @@ const calls: { path: string; query: URLSearchParams; method: string; body: unkno let status = "open"; let rejectWrite = false; let rejectHistory = false; -let monitoring = { enabled: true, alerts_used: 0, alert_limit: 5 as number | null, disabled_by_config: false }; +let monitoring = { + model_deprecations_enabled: true, + enabled: true, + alerts_used: 0, + alert_limit: 5 as number | null, + disabled_by_config: false, +}; let rejectMonitoring = false; function response(data: unknown, code = 200) { return new Response(JSON.stringify(data), { status: code, headers: { "Content-Type": "application/json" } }); @@ -41,7 +47,13 @@ describe("Alerts", () => { rejectWrite = false; rejectHistory = false; rejectMonitoring = false; - monitoring = { enabled: true, alerts_used: 0, alert_limit: 5, disabled_by_config: false }; + monitoring = { + model_deprecations_enabled: true, + enabled: true, + alerts_used: 0, + alert_limit: 5, + disabled_by_config: false, + }; role = "proxy_admin"; document.cookie = `token=${btoa(JSON.stringify({ alg: "none" }))}.${btoa(JSON.stringify({ key: "example-session", exp: 4102444800 }))}.example; path=/`; vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { @@ -57,7 +69,7 @@ describe("Alerts", () => { if (rejectMonitoring) return response({ detail: "Unavailable" }, 503); if (request.method === "PATCH") { if (rejectWrite) return response({ detail: "Unavailable" }, 503); - monitoring = { ...monitoring, enabled: body.enabled }; + monitoring = { ...monitoring, ...body }; } return response(monitoring); } @@ -144,7 +156,7 @@ describe("Alerts", () => { expect(await screen.findByText("No resolved alerts.")).toBeInTheDocument(); }); - it("keeps deprecation email setup accessible from the inbox", async () => { + it("keeps alert email setup accessible from the inbox", async () => { const user = userEvent.setup(); renderWithProviders(); await user.click(await screen.findByRole("button", { name: "Set up email reminders" })); @@ -161,12 +173,36 @@ describe("Alerts", () => { expect(screen.queryByRole("button", { name: "Dismiss" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Mark resolved" })).not.toBeInTheDocument(); }); + it("keeps notification opt-in separate from the model deprecation toggle", async () => { + const user = userEvent.setup(); + renderWithProviders(); + expect( + await screen.findByText( + "Model deprecation alerts are unlimited. Performance monitoring includes 5 free alerts, then requires Enterprise.", + ), + ).toBeInTheDocument(); + expect(screen.queryByRole("switch")).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Manage alerts" })); + const toggle = await screen.findByRole("switch", { name: "Model deprecations" }); + expect(toggle).toBeChecked(); + await user.click(toggle); + await waitFor(() => expect(toggle).not.toBeChecked()); + expect(calls.filter((call) => call.method === "PATCH")).toEqual([ + expect.objectContaining({ path: "/alerts/monitoring", body: { model_deprecations_enabled: false } }), + ]); + expect(screen.getByRole("switch", { name: "Performance monitoring" })).toBeChecked(); + expect(screen.getByText(/Opt in to Performance Alerts in email or Slack settings/)).toBeInTheDocument(); + await user.keyboard("{Escape}"); + expect(await screen.findByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument(); + }); + it("defaults on and persists opting out through the monitoring API", async () => { const user = userEvent.setup(); renderWithProviders(); + await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" })); const toggle = await screen.findByRole("switch", { name: "Performance monitoring" }); expect(toggle).toBeChecked(); - expect(screen.getByText("0 of 5 free alerts used.")).toBeInTheDocument(); + expect(screen.getByText("0 of 5 free alerts generated.")).toBeInTheDocument(); await user.click(toggle); expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument(); expect(toggle).not.toBeChecked(); @@ -174,15 +210,15 @@ describe("Alerts", () => { path: "/alerts/monitoring", body: { enabled: false }, }); - expect(screen.getByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument(); await user.click(toggle); - expect(await screen.findByText("0 of 5 free alerts used.")).toBeInTheDocument(); + expect(await screen.findByText("0 of 5 free alerts generated.")).toBeInTheDocument(); }); it("explains exhaustion without blocking saved alerts or model deprecations", async () => { monitoring.alerts_used = 5; const user = userEvent.setup(); renderWithProviders(); + await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" })); expect( await screen.findByText("You’ve used all 5 free alerts. New performance alerts are paused."), ).toBeInTheDocument(); @@ -190,8 +226,6 @@ describe("Alerts", () => { "href", "https://www.litellm.ai/enterprise", ); - expect(screen.getByRole("button", { name: "Mark resolved" })).toBeEnabled(); - expect(screen.getByRole("button", { name: "Set up email reminders" })).toBeInTheDocument(); await user.click(screen.getByRole("switch", { name: "Performance monitoring" })); expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument(); expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument(); @@ -200,6 +234,7 @@ describe("Alerts", () => { it("keeps the saved toggle value when an older refresh finishes later", async () => { const user = userEvent.setup(); renderWithProviders(); + await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" })); const toggle = await screen.findByRole("switch", { name: "Performance monitoring" }); const originalFetch = vi.mocked(globalThis.fetch).getMockImplementation()!; const staleResponse = response(monitoring); @@ -211,7 +246,9 @@ describe("Alerts", () => { } return originalFetch(input, init); }); - await user.click(screen.getByRole("button", { name: "Refresh" })); + await act(async () => { + void testQueryClient.invalidateQueries({ queryKey: ["alert-monitoring"] }); + }); await user.click(toggle); expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument(); await act(async () => { @@ -227,6 +264,7 @@ describe("Alerts", () => { monitoring.alert_limit = null; monitoring.alerts_used = 5; renderWithProviders(); + await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" })); expect(await screen.findByText("On with Enterprise.")).toBeInTheDocument(); expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument(); }); @@ -236,22 +274,26 @@ describe("Alerts", () => { monitoring.alerts_used = 5; monitoring.enabled = false; renderWithProviders(); + await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" })); expect(await screen.findByText("Disabled in proxy settings.")).toBeInTheDocument(); expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument(); expect(screen.getByRole("switch", { name: "Performance monitoring" })).toHaveAttribute("aria-disabled", "true"); }); - it("lets viewers see monitoring status without a toggle", async () => { + it("lets viewers inspect alert settings without changing them", async () => { role = "proxy_admin_viewer"; renderWithProviders(); - expect(await screen.findByText("0 of 5 free alerts used.")).toBeInTheDocument(); - expect(screen.queryByRole("switch")).not.toBeInTheDocument(); + await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" })); + expect(await screen.findByText("0 of 5 free alerts generated.")).toBeInTheDocument(); + expect(screen.getByRole("switch", { name: "Performance monitoring" })).toHaveAttribute("aria-disabled", "true"); + expect(screen.getByRole("switch", { name: "Model deprecations" })).toHaveAttribute("aria-disabled", "true"); }); it("keeps the saved preference when a monitoring change fails", async () => { rejectWrite = true; const user = userEvent.setup(); renderWithProviders(); + await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" })); await user.click(await screen.findByRole("switch", { name: "Performance monitoring" })); expect(await screen.findByRole("alert")).toHaveTextContent("Could not save monitoring settings"); expect(screen.getByRole("switch", { name: "Performance monitoring" })).toBeChecked(); @@ -261,10 +303,12 @@ describe("Alerts", () => { rejectMonitoring = true; const user = userEvent.setup(); renderWithProviders(); - expect(await screen.findByRole("alert")).toHaveTextContent("Could not load monitoring settings"); - expect(screen.getByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument(); + await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" })); + expect(await within(screen.getByRole("dialog")).findByRole("alert")).toHaveTextContent( + "Could not load monitoring settings", + ); rejectMonitoring = false; - await user.click(screen.getByRole("button", { name: "Retry" })); + await user.click(within(screen.getByRole("dialog")).getByRole("button", { name: "Retry" })); expect(await screen.findByRole("switch", { name: "Performance monitoring" })).toBeChecked(); }); }); diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx index 3a4d8127bea..dfaa12318de 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx @@ -18,7 +18,7 @@ import { ALERT_TYPES, AlertFilter, AlertItem, TYPE_LABEL_KEYS, presentAlert } fr import AlertActions from "./AlertActions"; import AlertChart from "./AlertChart"; import AlertDetail from "./AlertDetail"; -import PerformanceMonitoring from "./PerformanceMonitoring"; +import ManageAlerts from "./ManageAlerts"; const parsers = { alert: parseAsString, @@ -123,8 +123,7 @@ function AlertsInbox({ auth }: { auth: ReturnType }) { } return ( <> -

    {t("alerts.unified.title")}

    - + void setParams({ state: value as AlertStatus })} @@ -205,7 +204,7 @@ function AlertsInbox({ auth }: { auth: ReturnType }) { {list.isFetchingNextPage ? t("alerts.unified.loadingMore") : t("alerts.unified.loadMore")} )} - {showDeprecations && } + ); } diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/ManageAlerts.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/ManageAlerts.tsx new file mode 100644 index 00000000000..9959ce19050 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/ManageAlerts.tsx @@ -0,0 +1,150 @@ +import { Settings } from "lucide-react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { uiHref } from "@/utils/uiHref"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/components/ui/button"; +import { Switch } from "@/components/ui/switch"; +import { backgroundFetchClient } from "@/lib/http/api"; +import { getProxyBaseUrl } from "@/components/networking"; +import type { components } from "@/lib/http/schema"; + +function statusMessage(data: components["schemas"]["AlertMonitoring"]) { + if (data.disabled_by_config) return "alerts.monitoring.disabledByConfig"; + if (!data.enabled) return "alerts.monitoring.off"; + if (data.alert_limit === null) return "alerts.monitoring.enterprise"; + if (data.alerts_used >= data.alert_limit) return "alerts.monitoring.exhausted"; + return "alerts.monitoring.allowance"; +} + +export default function ManageAlerts({ scope, readOnly }: { scope: string; readOnly: boolean }) { + const { t } = useTranslation(); + const client = useQueryClient(); + const queryKey = ["alert-monitoring", getProxyBaseUrl(), scope]; + const queryOptions = { + queryKey, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const { data } = await backgroundFetchClient.GET("/alerts/monitoring", { signal }); + if (!data) throw new Error("Could not load monitoring settings"); + return data; + }, + staleTime: 60_000, + retry: false, + }; + const query = useQuery(queryOptions); + const mutation = useMutation({ + mutationFn: async ( + body: components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"], + ) => { + const { data } = await backgroundFetchClient.PATCH("/alerts/monitoring", { body }); + if (!data) throw new Error("Could not save monitoring settings"); + return data; + }, + onSuccess: async (data) => { + await client.cancelQueries({ queryKey }); + client.setQueryData(queryKey, data); + await client.invalidateQueries({ queryKey: ["deprecationAlerts"] }); + }, + }); + const data = query.data; + const messageKey = data ? statusMessage(data) : undefined; + return ( +
    +
    +

    {t("alerts.unified.title")}

    + + }> + + {t("alerts.monitoring.manage")} + + + + {t("alerts.monitoring.manage")} + {t("alerts.monitoring.description")} + + {query.isError && ( +
    +

    {t("alerts.monitoring.loadError")}

    + +
    + )} + {!query.isError && !data && ( +

    + {t("alerts.monitoring.loading")} +

    + )} + {!query.isError && data && ( +
    +
    +
    + +

    {t("alerts.monitoring.deprecationsDescription")}

    +
    + mutation.mutate({ model_deprecations_enabled: enabled })} + /> +
    +
    +
    + +

    + {messageKey && t(messageKey, { used: data.alerts_used, limit: data.alert_limit })} + {messageKey === "alerts.monitoring.exhausted" && ( + + {t("alerts.monitoring.upgrade")} + + )} +

    +
    + mutation.mutate({ enabled })} + aria-describedby="performance-monitoring-status" + /> +
    +
    + )} + {mutation.isError && ( +

    + {t("alerts.monitoring.saveError")} +

    + )} + {readOnly &&

    {t("alerts.monitoring.adminOnly")}

    } +

    {t("alerts.monitoring.notifications")}

    + + {t("alerts.monitoring.connections")} + +     +     +
    + {data && ( +

    + {t(data.alert_limit === null ? "alerts.monitoring.enterpriseNote" : "alerts.monitoring.freeNote")} +

    + )} +
    + ); +} diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/PerformanceMonitoring.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/PerformanceMonitoring.tsx deleted file mode 100644 index 1839bf00abd..00000000000 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/PerformanceMonitoring.tsx +++ /dev/null @@ -1,99 +0,0 @@ -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { useTranslation } from "react-i18next"; -import { Button } from "@/components/ui/button"; -import { Switch } from "@/components/ui/switch"; -import { backgroundFetchClient } from "@/lib/http/api"; -import { getProxyBaseUrl } from "@/components/networking"; -import type { components } from "@/lib/http/schema"; - -function statusMessage(data: components["schemas"]["AlertMonitoring"]) { - if (data.disabled_by_config) return "alerts.monitoring.disabledByConfig"; - if (!data.enabled) return "alerts.monitoring.off"; - if (data.alert_limit === null) return "alerts.monitoring.enterprise"; - if (data.alerts_used >= data.alert_limit) return "alerts.monitoring.exhausted"; - return "alerts.monitoring.allowance"; -} - -export default function PerformanceMonitoring({ scope, readOnly }: { scope: string; readOnly: boolean }) { - const { t } = useTranslation(); - const client = useQueryClient(); - const queryKey = ["alert-monitoring", getProxyBaseUrl(), scope]; - const queryOptions = { - queryKey, - queryFn: async ({ signal }: { signal: AbortSignal }) => { - const { data } = await backgroundFetchClient.GET("/alerts/monitoring", { signal }); - if (!data) throw new Error("Could not load monitoring settings"); - return data; - }, - staleTime: 60_000, - retry: false, - }; - const query = useQuery(queryOptions); - const mutation = useMutation({ - mutationFn: async (enabled: boolean) => { - const { data } = await backgroundFetchClient.PATCH("/alerts/monitoring", { body: { enabled } }); - if (!data) throw new Error("Could not save monitoring settings"); - return data; - }, - onSuccess: async (data) => { - await client.cancelQueries({ queryKey }); - client.setQueryData(queryKey, data); - }, - }); - if (query.isError) - return ( -
    -

    {t("alerts.monitoring.loadError")}

    - -
    - ); - if (!query.data) - return ( -

    - {t("alerts.monitoring.loading")} -

    - ); - const data = query.data; - const messageKey = statusMessage(data); - const message = t(messageKey, { used: data.alerts_used, limit: data.alert_limit }); - return ( -
    -
    -
    - - {!readOnly && ( - mutation.mutate(enabled)} - aria-describedby="performance-monitoring-status" - /> - )} -
    -

    - {message} - {messageKey === "alerts.monitoring.exhausted" && ( - - {t("alerts.monitoring.upgrade")} - - )} -

    -
    - {mutation.isError && ( -

    - {t("alerts.monitoring.saveError")} -

    - )} -
    - ); -} diff --git a/litellm/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx index 61fcfc308ad..d6468c0c243 100644 --- a/litellm/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx @@ -45,6 +45,16 @@ beforeEach(() => { if (url.endsWith("/management/v1/users/me")) { return Response.json({ user_id: "example-admin", user_role: role }); } + if (url.endsWith("/alerts/monitoring")) { + const monitoring = { + enabled: true, + model_deprecations_enabled: true, + alerts_used: 0, + alert_limit: 5, + disabled_by_config: false, + }; + return Response.json(monitoring); + } if (url.endsWith("/model/deprecations?warn_within_days=30")) { if (rejectDeprecations) return Response.json({ detail: "Unavailable" }, { status: 503 }); return Response.json({ deprecated: [], imminent: [] }); diff --git a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.test.tsx b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.test.tsx index 6058e59d9a1..a7e9c37d477 100644 --- a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.test.tsx @@ -13,11 +13,11 @@ describe("deprecation email setup", () => { const user = userEvent.setup(); render(); await user.click(screen.getByRole("button", { name: "Set up email reminders" })); - expect(await screen.findByRole("dialog", { name: "Set up model deprecation emails" })).toBeInTheDocument(); + expect(await screen.findByRole("dialog", { name: "Set up alert emails" })).toBeInTheDocument(); expect(screen.getByText("Connect email delivery")).toBeInTheDocument(); - expect(screen.getByText("Enable model deprecation emails")).toBeInTheDocument(); - expect(screen.getByText("Owners get two reminders")).toBeInTheDocument(); - expect(screen.getByText(/This is off by default/)).toBeInTheDocument(); + expect(screen.getByText("Choose your email alerts")).toBeInTheDocument(); + expect(screen.getByText("Know what gets sent")).toBeInTheDocument(); + expect(screen.getByText(/Both are off by default/)).toBeInTheDocument(); expect(screen.getByRole("link", { name: "Open Email Settings" })).toHaveAttribute( "href", "/ui/logging-and-alerts?tab=email-alerts", diff --git a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx index 897850393e2..d65b9e94969 100644 --- a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx @@ -19,6 +19,15 @@ describe("deprecation alert API wiring", () => { vi.clearAllMocks(); testQueryClient.clear(); }); + it("stops loading usage and acknowledgements when model deprecations are disabled", async () => { + vi.mocked(useCurrentUser).mockReturnValue({ data: { user_id: "admin", user_role: "proxy_admin" } } as ReturnType< + typeof useCurrentUser + >); + vi.mocked(backgroundFetchClient.GET).mockResolvedValue({ data: { model_deprecations_enabled: false } } as never); + renderWithProviders(); + expect(await screen.findByText("0 alerts")).toBeInTheDocument(); + expect(backgroundFetchClient.GET).toHaveBeenCalledExactlyOnceWith("/alerts/monitoring", expect.anything()); + }); it("should avoid querying gateway-wide usage for non-admins", () => { vi.mocked(useCurrentUser).mockReturnValue({ data: { user_id: "internal", user_role: "internal_user" }, @@ -36,7 +45,7 @@ describe("deprecation alert API wiring", () => { } as never); renderWithProviders(); expect(await screen.findByText("0 alerts")).toBeInTheDocument(); - expect(backgroundFetchClient.GET).toHaveBeenCalledTimes(1); + expect(backgroundFetchClient.GET).toHaveBeenCalledTimes(2); expect(backgroundFetchClient.GET).toHaveBeenCalledWith( "/model/deprecations", expect.objectContaining({ params: { query: { warn_within_days: 30 } } }), @@ -59,6 +68,7 @@ describe("deprecation alert API wiring", () => { ], }; vi.mocked(backgroundFetchClient.GET) + .mockResolvedValueOnce({ data: { model_deprecations_enabled: true } } as never) .mockResolvedValueOnce({ data: schedule } as never) .mockResolvedValueOnce({ data: activity } as never) .mockResolvedValueOnce({ data: { status: "open" } } as never); diff --git a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts index ce126d5cd4f..36515d09202 100644 --- a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts +++ b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts @@ -17,6 +17,9 @@ export function useDeprecationAlerts(accessToken: string | null) { queryKey: ["deprecationAlerts", scope, range], enabled, queryFn: async ({ signal }: { signal: AbortSignal }) => { + const { data: monitoring } = await backgroundFetchClient.GET("/alerts/monitoring", { signal }); + if (!monitoring) throw new Error("Alert settings are unavailable"); + if (monitoring.model_deprecations_enabled === false) return []; const { data: schedule } = await backgroundFetchClient.GET("/model/deprecations", { params: { query: { warn_within_days: 30 } }, signal, diff --git a/litellm/ui/litellm-dashboard/src/components/email_events/email_event_settings.tsx b/litellm/ui/litellm-dashboard/src/components/email_events/email_event_settings.tsx index 5ccadf939a9..7f21b8370c5 100644 --- a/litellm/ui/litellm-dashboard/src/components/email_events/email_event_settings.tsx +++ b/litellm/ui/litellm-dashboard/src/components/email_events/email_event_settings.tsx @@ -74,6 +74,9 @@ const EmailEventSettings: React.FC = ({ accessToken }) // Helper function to get a description for each event type const getEventDescription = (event: EmailEvent): string => { + if (event === "Performance Alerts") { + return t("loggingAndAlerts.emailEvents.descriptions.performanceAlerts"); + } if (event === "Model Deprecations") { return t("loggingAndAlerts.emailEvents.descriptions.modelDeprecations"); } diff --git a/litellm/ui/litellm-dashboard/src/components/settings.tsx b/litellm/ui/litellm-dashboard/src/components/settings.tsx index b9aea32c99d..51922ab616d 100644 --- a/litellm/ui/litellm-dashboard/src/components/settings.tsx +++ b/litellm/ui/litellm-dashboard/src/components/settings.tsx @@ -421,6 +421,7 @@ const Settings: React.FC = ({ accessToken, userRole, userID, outage_alerts: "loggingAndAlerts.alertTypes.labels.outageAlerts", region_outage_alerts: "loggingAndAlerts.alertTypes.labels.regionOutageAlerts", model_deprecation_warnings: "loggingAndAlerts.alertTypes.labels.modelDeprecationWarnings", + performance_alerts: "loggingAndAlerts.alertTypes.labels.performanceAlerts", }; useEffect(() => { diff --git a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json index 30439a3fb25..de72ec1410e 100644 --- a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json @@ -5938,18 +5938,18 @@ "description": "Add your SMTP settings and public gateway URL in Email Settings. Send a test email to verify delivery." }, "enable": { - "title": "Enable model deprecation emails", - "description": "In Email events, turn on Model Deprecations and save. This is off by default." + "title": "Choose your email alerts", + "description": "In Email events, enable Model Deprecations, Performance Alerts, or both, then save. Both are off by default." }, "reminders": { - "title": "Owners get two reminders", - "description": "Affected key owners and team admins receive a reminder 30 days before retirement, and one final reminder 1 day before. Recipients are based on actual usage in the last 30 days. Dates are checked in UTC." + "title": "Know what gets sent", + "description": "Model deprecations notify affected key owners and team admins 30 days and 1 day before retirement. Performance alerts send gateway admins one daily summary when there are new findings, with up to five alerts and a link to the rest." } }, - "summary": "Give affected owners time to migrate. Email reminders at 30 days and 1 day.", + "summary": "Get model retirement reminders and daily performance summaries by email.", "trigger": "Set up email reminders", - "title": "Set up model deprecation emails", - "description": "Help key owners and team admins act before a provider retires a model. Only their affected usage is included.", + "title": "Set up alert emails", + "description": "Choose which alerts to send through your existing email connection.", "openSettings": "Open Email Settings" }, "migrationDetail": { @@ -6017,9 +6017,18 @@ "more": "+{{hidden}} more" }, "monitoring": { + "manage": "Manage alerts", + "description": "These settings apply to everyone using this gateway.", + "deprecations": "Model deprecations", + "deprecationsDescription": "Show retirement alerts and send configured email and Slack reminders.", + "adminOnly": "Only gateway admins can change these settings.", + "notifications": "Opt in to Performance Alerts in email or Slack settings to receive a daily summary of new findings. Email summaries go to gateway admins.", + "connections": "Manage email and Slack connections", + "freeNote": "Model deprecation alerts are unlimited. Performance monitoring includes 5 free alerts, then requires Enterprise.", + "enterpriseNote": "Unlimited model deprecation and performance alerts with Enterprise.", "title": "Performance monitoring", "loading": "Loading monitoring settings…", - "allowance": "{{used}} of {{limit}} free alerts used.", + "allowance": "{{used}} of {{limit}} free alerts generated.", "enterprise": "On with Enterprise.", "off": "Off. Saved alerts remain available.", "exhausted": "You’ve used all 5 free alerts. New performance alerts are paused.", @@ -6518,7 +6527,8 @@ "dailyReports": "Weekly/Monthly Spend Reports", "outageAlerts": "Outage Alerts", "regionOutageAlerts": "Region Outage Alerts", - "modelDeprecationWarnings": "Model Deprecation Warnings" + "modelDeprecationWarnings": "Model Deprecation Warnings", + "performanceAlerts": "Performance Alerts" }, "enterpriseFeature": "✨ Enterprise Feature", "saveChanges": "Save Changes", @@ -6625,6 +6635,7 @@ "description": "Select which events should trigger email notifications.", "descriptions": { "modelDeprecations": "Email affected key owners and team admins 30 days and 1 day before model deprecation. Uses requests from the last 30 days. Off by default.", + "performanceAlerts": "Email gateway admins a daily summary when there are new performance alerts. Shows up to five alerts with a link to see all.", "virtualKeyCreated": "An email will be sent to the user when a new virtual key is created with their user ID", "newUserInvitation": "An email will be sent to the email address of the user when a new user is created", "generic": "Receive an email notification when {{event}}" diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts index d643e191bb0..3053ec54592 100644 --- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1030,7 +1030,7 @@ export interface paths { cookie?: never; }; /** - * Read performance monitoring settings + * Read alert monitoring settings * @description Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin. */ get: operations["get_monitoring_alerts_monitoring_get"]; @@ -1040,7 +1040,7 @@ export interface paths { options?: never; head?: never; /** - * Enable or disable performance monitoring + * Enable or disable alert monitoring * @description Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts. */ patch: operations["update_monitoring_alerts_monitoring_patch"]; @@ -27339,6 +27339,12 @@ export interface components { * @description Whether performance monitoring is enabled. Defaults to true. */ enabled: boolean; + /** + * Model Deprecations Enabled + * @description Whether model deprecation alerts are enabled. + * @default true + */ + model_deprecations_enabled: boolean; }; /** AlertMonitoringUpdate */ AlertMonitoringUpdate: { @@ -27353,7 +27359,7 @@ export interface components { * @description Enum for alert types and management event types * @enum {string} */ - AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted" | "mcp_tool_description_blocked" | "mcp_pinned_tools_changed"; + AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "performance_alerts" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted" | "mcp_tool_description_blocked" | "mcp_pinned_tools_changed"; /** AlertUpdate */ AlertUpdate: { /** @@ -33628,7 +33634,7 @@ export interface components { * EmailEvent * @enum {string} */ - EmailEvent: "Model Deprecations" | "Virtual Key Created" | "New User Invitation" | "Virtual Key Rotated" | "Soft Budget Crossed" | "Max Budget Alert"; + EmailEvent: "Model Deprecations" | "Performance Alerts" | "Virtual Key Created" | "New User Invitation" | "Virtual Key Rotated" | "Soft Budget Crossed" | "Max Budget Alert"; /** EmailEventSettings */ EmailEventSettings: { /** Enabled */ @@ -40207,6 +40213,14 @@ export interface components { */ team_public_model_name?: string | null; }; + /** ModelDeprecationMonitoringUpdate */ + ModelDeprecationMonitoringUpdate: { + /** + * Model Deprecations Enabled + * @description Enable or disable model deprecation alerts. + */ + model_deprecations_enabled: boolean; + }; /** ModelDeprecationResponse */ ModelDeprecationResponse: { /** @@ -55078,7 +55092,7 @@ export interface operations { }; requestBody: { content: { - "application/json": components["schemas"]["AlertMonitoringUpdate"]; + "application/json": components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"]; }; }; responses: { diff --git a/tests/integration/database/test_alerts.py b/tests/integration/database/test_alerts.py index aaf36dd7ef2..06908c3a569 100644 --- a/tests/integration/database/test_alerts.py +++ b/tests/integration/database/test_alerts.py @@ -1,7 +1,8 @@ import asyncio +import hashlib import os import uuid -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Final @@ -22,6 +23,7 @@ from litellm.models.alert import ( from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.alert_repository import AlertRepository +from litellm.repositories.config_repository import ConfigRepository from tests.integration._support.client import Gateway from tests.integration._support.database import read_rows, write_rows @@ -41,6 +43,7 @@ def saved_page(content: bytes) -> AlertPage: @dataclass(frozen=True, slots=True) class Alerts: repository: AlertRepository + config: ConfigRepository resource_id: str def detection(self, kind: AlertType = "spend", day: int = 0) -> AlertDetection: @@ -75,10 +78,12 @@ async def alerts() -> AsyncIterator[Alerts]: client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache())) await client.connect() try: - yield Alerts(AlertRepository(client), resource_id) + yield Alerts(AlertRepository(client), ConfigRepository(client, use_writer=True), resource_id) finally: try: write_rows('DELETE FROM "LiteLLM_Alert" WHERE resource_id=%s', (resource_id,)) + receipt: Final = "performance_alert_digest:" + hashlib.sha256(f"test:{resource_id}".encode()).hexdigest() + write_rows('DELETE FROM "LiteLLM_Config" WHERE param_name=%s', (receipt,)) finally: await client.disconnect() @@ -372,3 +377,118 @@ async def test_deleted_resource_preserves_history_and_history_timeout_preserves_ finally: write_rows('DELETE FROM "LiteLLM_TeamTable" WHERE team_id=%s', (team,)) write_rows('DELETE FROM "LiteLLM_DailyTeamSpend" WHERE team_id=%s', (team,)) + + +@pytest.mark.asyncio +async def test_daily_performance_summary_deduplicates_workers_and_keeps_failed_deliveries(alerts: Alerts) -> None: + from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier, slack_text + from litellm.models.alert import AlertSummary + + destination: Final = f"test:{alerts.resource_id}" + notifier: Final = PerformanceAlertNotifier(alerts.config, alerts.repository) + first: Final = await alerts.repository.record(alerts.detection()) + now: Final = first.created_at + timedelta(minutes=1) + delivered: Final[list[str]] = [] # mutable-ok: captures the external delivery boundary + + async def send(rows: Sequence[AlertSummary], receipt: str) -> bool: + delivered.append(slack_text(rows, "https://gateway.example")) + return True + + async def unavailable(rows: Sequence[AlertSummary], receipt: str) -> bool: + return False + + await notifier.deliver(destination, send, now) + assert len(delivered) == 1 + for day in range(1, 8): + await alerts.repository.record(alerts.detection(day=day)) + restarted: Final = PerformanceAlertNotifier(alerts.config, alerts.repository) + await restarted.deliver(destination, send, now + timedelta(hours=23)) + assert len(delivered) == 1 + tomorrow: Final = now + timedelta(days=1) + await restarted.deliver(destination, unavailable, tomorrow) + assert len(delivered) == 1 + await restarted.deliver(destination, send, tomorrow + timedelta(minutes=3)) + assert len(delivered) == 1 + retry: Final = tomorrow + timedelta(days=1, minutes=3) + await asyncio.gather( + notifier.deliver(destination, send, retry), + restarted.deliver(destination, send, retry), + ) + assert len(delivered) == 2 + assert delivered[-1].count("?alert=") == 5 + assert f"?alert={first.id}|" not in delivered[-1] + await restarted.deliver(destination, send, retry + timedelta(days=1, minutes=1)) + assert len(delivered) == 2 + + +@pytest.mark.asyncio +async def test_first_summary_does_not_replay_old_alerts(alerts: Alerts) -> None: + from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier + from litellm.models.alert import AlertSummary + + saved: Final = await alerts.repository.record(alerts.detection()) + delivered: Final[list[tuple[AlertSummary, ...]]] = [] # mutable-ok: captures the external delivery boundary + + async def send(rows: Sequence[AlertSummary], receipt: str) -> bool: + delivered.append(tuple(rows)) + return True + + notifier: Final = PerformanceAlertNotifier(alerts.config, alerts.repository) + now: Final = saved.created_at + timedelta(days=2) + await notifier.deliver(f"test:{alerts.resource_id}", send, now) + assert delivered == [] + recent: Final = await alerts.repository.record(alerts.detection(day=1)) + await notifier.deliver(f"test:{alerts.resource_id}", send, now + timedelta(minutes=5)) + assert tuple(row.id for row in delivered[0]) == (recent.id,) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("crash", (False, True)) +async def test_failed_first_summary_does_not_replay_old_alerts_on_retry(alerts: Alerts, crash: bool) -> None: + from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier + from litellm.models.alert import AlertSummary + + old: Final = await alerts.repository.record(alerts.detection()) + recent: Final = await alerts.repository.record(alerts.detection(day=1)) + now: Final = recent.created_at + timedelta(days=2) + write_rows('UPDATE "LiteLLM_Alert" SET created_at=%s WHERE id=%s', (now, int(recent.id))) + write_rows('UPDATE "LiteLLM_Alert" SET created_at=%s WHERE id=%s', (now - timedelta(days=2), int(old.id))) + delivered: Final[list[tuple[str, ...]]] = [] # mutable-ok: captures the external delivery boundary + + async def unavailable(rows: Sequence[AlertSummary], receipt: str) -> bool: + if crash: + raise RuntimeError("Connection lost after sending") + return False + + async def send(rows: Sequence[AlertSummary], receipt: str) -> bool: + delivered.append(tuple(row.id for row in rows)) + return True + + notifier: Final = PerformanceAlertNotifier(alerts.config, alerts.repository) + if crash: + with pytest.raises(RuntimeError, match="Connection lost after sending"): + await notifier.deliver(f"test:{alerts.resource_id}", unavailable, now) + else: + await notifier.deliver(f"test:{alerts.resource_id}", unavailable, now) + await notifier.deliver(f"test:{alerts.resource_id}", send, now + timedelta(minutes=3)) + assert delivered == [] + await notifier.deliver(f"test:{alerts.resource_id}", send, now + timedelta(days=1, minutes=3)) + assert delivered == [(recent.id,)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enabled", (False, True)) +async def test_model_deprecation_preference_persists_without_changing_performance( + alerts: Alerts, gateway: Gateway, enabled: bool +) -> None: + previous: Final = gateway.request("GET", "/alerts/monitoring") + assert previous.status_code == 200, previous.text + before: Final = previous.json() + try: + updated: Final = gateway.request("PATCH", "/alerts/monitoring", {"model_deprecations_enabled": enabled}) + assert updated.status_code == 200, updated.text + assert updated.json() == {**before, "model_deprecations_enabled": enabled} + refreshed: Final = gateway.request("GET", "/alerts/monitoring") + assert refreshed.json() == updated.json() + finally: + await alerts.config.set_model_deprecations_enabled(before["model_deprecations_enabled"]) diff --git a/tests/unit/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/unit/integrations/SlackAlerting/test_model_deprecation_alert.py index 2416d54579f..218df62fe2d 100644 --- a/tests/unit/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/unit/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -2,11 +2,11 @@ import asyncio from itertools import chain, repeat +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest - import litellm from litellm.constants import SLACK_MODEL_DEPRECATION_LOCK_ID from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting @@ -53,6 +53,21 @@ async def test_should_skip_when_no_alerting_configured(): assert sent is False +@pytest.mark.asyncio +async def test_gateway_opt_out_stops_deprecation_checks() -> None: + async def disabled() -> bool: + return False + + alerting: Final = SlackAlerting( + alerting=["slack"], + alert_types=[AlertType.model_deprecation_warnings], + model_deprecations_enabled=disabled, + ) + router: Final = MagicMock() + assert not await alerting.send_model_deprecation_alert(llm_router=router) + router.registry.assert_not_called() + + @pytest.mark.asyncio async def test_should_skip_when_no_deprecations_found(monkeypatch): monkeypatch.setattr(litellm, "model_cost", {}) diff --git a/tests/unit/integrations/email_alerting/test_model_deprecation_email.py b/tests/unit/integrations/email_alerting/test_model_deprecation_email.py index 54dbe342148..4d64aed7db7 100644 --- a/tests/unit/integrations/email_alerting/test_model_deprecation_email.py +++ b/tests/unit/integrations/email_alerting/test_model_deprecation_email.py @@ -4,6 +4,7 @@ import asyncio from dataclasses import replace from datetime import date, datetime, timedelta, timezone from types import MappingProxyType +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -46,6 +47,9 @@ class MemoryConfig: def __init__(self): self.rows = {"general_settings": {"email_settings": {"Model Deprecations": True}}} + async def model_deprecations_enabled(self): + return self.rows.get("model_deprecation_alerts_enabled", True) + async def get_param(self, key): return ConfigParam(key, self.rows[key]) if key in self.rows else None @@ -63,6 +67,8 @@ class MemoryConfig: @pytest.fixture() def sender(monkeypatch): monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example") + monkeypatch.setenv("SMTP_HOST", "smtp.example.com") + monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com") instance = module.ModelDeprecationEmailSender(MagicMock()) instance.config = MemoryConfig() instance._impacts = AsyncMock( @@ -296,3 +302,32 @@ async def test_should_keep_the_email_loop_running_after_a_delivery_error(monkeyp await module.run_scheduled_model_deprecation_emails() assert runner.run.await_count == 2 assert all(call.args == (router,) for call in runner.run.await_args_list) + + +@pytest.mark.asyncio +async def test_gateway_opt_out_stops_owner_usage_scans_and_email(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SMTP_HOST", "smtp.example.com") + monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com") + client: Final = MagicMock() + client.writer_db.litellm_config.find_unique = AsyncMock(return_value=MagicMock(param_value=False)) + client.db.litellm_dailyuserspend.group_by = AsyncMock() + instance: Final = module.ModelDeprecationEmailSender(client) + assert await instance.run(MagicMock(), now=NOW) == 0 + client.db.litellm_dailyuserspend.group_by.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("missing", ("SMTP_HOST", "SMTP_SENDER_EMAIL")) +async def test_missing_email_connection_does_not_scan_owner_usage( + monkeypatch: pytest.MonkeyPatch, missing: str +) -> None: + monkeypatch.setenv("SMTP_HOST", "smtp.example.com") + monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com") + monkeypatch.delenv(missing) + client: Final = MagicMock() + client.writer_db.litellm_config.find_unique = AsyncMock() + client.db.litellm_dailyuserspend.group_by = AsyncMock() + instance: Final = module.ModelDeprecationEmailSender(client) + assert await instance.run(MagicMock(), now=NOW) == 0 + client.writer_db.litellm_config.find_unique.assert_not_awaited() + client.db.litellm_dailyuserspend.group_by.assert_not_awaited() diff --git a/tests/unit/integrations/test_performance_alert_notifications.py b/tests/unit/integrations/test_performance_alert_notifications.py new file mode 100644 index 00000000000..37a6b6728a1 --- /dev/null +++ b/tests/unit/integrations/test_performance_alert_notifications.py @@ -0,0 +1,213 @@ +import smtplib +from datetime import datetime, timezone +from typing import Final, Literal +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from pydantic import TypeAdapter + +from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier, email_body, slack_text +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.models.alert import AlertSummary, MetricEvidence +from litellm.repositories.alert_repository import AlertRepository +from litellm.repositories.config_repository import ConfigRepository +from litellm.types.integrations.slack_alerting import AlertType + +BASE_URL: Final = "https://gateway.example" +WEBHOOK: Final = "https://hooks.slack.example/alerts" +_TEXT: Final = TypeAdapter(dict[str, str]) + + +def summary(index: int) -> AlertSummary: + return AlertSummary( + id=str(index), + type="spend", + resource_type="team", + resource_name=f"Research {index} ", + created_at=datetime(2026, 1, 9, tzinfo=timezone.utc), + evidence=MetricEvidence(value=1200, baseline=200, unit="usd", sample_size=2000), + ) + + +def test_daily_messages_include_five_findings_and_escape_resource_names() -> None: + alerts: Final = tuple(summary(index) for index in range(1, 7)) + body, text = email_body(alerts, BASE_URL) + slack: Final = slack_text(alerts, BASE_URL) + assert body.count("
  • ") == 5 + assert slack.count("?alert=") == 5 + assert text.count("?alert=") == 5 + assert "$1,200.00 vs $200.00" in text + assert "Research 6" not in body + text + slack + assert "Research 1 <team>" in body and "Research 1 <team>" in slack + assert f'See all alerts' in body + assert f"<{BASE_URL}/ui/alerts|See all alerts>" in slack + + +@pytest.mark.parametrize( + ("kind", "metric", "expected"), + ( + ("reliability", MetricEvidence(value=0.25, baseline=0.02, unit="ratio", sample_size=2000), "25.0% vs 2.0%"), + ( + "latency", + MetricEvidence(value=4000, baseline=1500, unit="milliseconds", sample_size=2000), + "4,000 ms vs 1,500 ms", + ), + ( + "traffic", + MetricEvidence(value=12000, baseline=2000, unit="requests", sample_size=2000), + "12,000 requests vs 2,000 requests", + ), + ), +) +def test_daily_messages_preserve_metric_units( + kind: Literal["reliability", "latency", "traffic"], metric: MetricEvidence, expected: str +) -> None: + alert: Final = summary(1).model_copy(update={"type": kind, "evidence": metric}) + body, text = email_body((alert,), BASE_URL) + assert expected in body and expected in text + assert expected in slack_text((alert,), BASE_URL) + + +@pytest.mark.asyncio +async def test_connected_channels_do_not_subscribe_to_performance_summaries(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PROXY_BASE_URL", BASE_URL) + monkeypatch.setenv("SMTP_HOST", "smtp.example.com") + monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com") + client: Final = MagicMock() + client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None) + client.writer_db.tx = MagicMock() + slack: Final = SlackAlerting(alerting=["slack"], default_webhook_url=WEBHOOK) + notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client)) + await notifier.run(slack) + assert slack.performance_alert_webhooks() == () + client.writer_db.tx.assert_not_called() + + +@pytest.mark.asyncio +async def test_slack_summary_reuses_the_configured_connection_and_reports_delivery() -> None: + posted: Final = MagicMock(return_value=httpx.Response(200, text="ok")) + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(posted)) + try: + slack: Final = SlackAlerting( + alerting=["slack"], + alert_types=[AlertType.performance_alerts], + default_webhook_url=WEBHOOK, + async_http_handler=handler, + ) + assert slack.performance_alert_webhooks() == (WEBHOOK,) + message: Final = slack_text((summary(1),), BASE_URL) + assert await slack.send_performance_alert_summary(WEBHOOK, message) + request: Final = posted.call_args.args[0] + assert str(request.url) == WEBHOOK + assert _TEXT.validate_json(request.content) == {"text": message} + finally: + await handler.close() + + +@pytest.mark.asyncio +async def test_opted_in_summary_reaches_slack_from_the_notification_job(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PROXY_BASE_URL", BASE_URL) + monkeypatch.delenv("SMTP_HOST", raising=False) + alert: Final = summary(1) + client: Final = MagicMock() + client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None) + client.writer_db.query_raw = AsyncMock(return_value=[{"param_name": "receipt"}]) + client.writer_db.tx.return_value.__aenter__.return_value.query_raw = AsyncMock( + side_effect=((), ({"id": 1},), (), (alert.model_dump(mode="json"),)) + ) + posted: Final = MagicMock(return_value=httpx.Response(200, text="ok")) + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(posted)) + try: + slack: Final = SlackAlerting( + alerting=["slack"], + alert_types=[AlertType.performance_alerts], + default_webhook_url=WEBHOOK, + async_http_handler=handler, + ) + notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client)) + await notifier.run(slack, now=alert.created_at) + posted.assert_called_once() + assert _TEXT.validate_json(posted.call_args.args[0].content) == {"text": slack_text((alert,), BASE_URL)} + finally: + await handler.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", (httpx.ReadTimeout("Webhook timed out"), httpx.Response(503))) +async def test_unavailable_webhook_does_not_block_other_destinations( + monkeypatch: pytest.MonkeyPatch, failure: Exception | httpx.Response +) -> None: + monkeypatch.setenv("PROXY_BASE_URL", BASE_URL) + monkeypatch.delenv("SMTP_HOST", raising=False) + alert: Final = summary(1) + client: Final = MagicMock() + client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None) + client.writer_db.query_raw = AsyncMock(return_value=[{"param_name": "receipt"}]) + client.writer_db.tx.return_value.__aenter__.return_value.query_raw = AsyncMock( + side_effect=((), ({"id": 1},), (), (alert.model_dump(mode="json"),)) * 2 + ) + posted: Final = MagicMock(side_effect=(failure, httpx.Response(200))) + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(posted)) + try: + slack: Final = SlackAlerting( + alerting=["slack"], + alert_types=[AlertType.performance_alerts], + alert_to_webhook_url={AlertType.performance_alerts: [WEBHOOK, WEBHOOK + "/second", WEBHOOK]}, + async_http_handler=handler, + ) + notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client)) + await notifier.run(slack, now=alert.created_at) + assert tuple(str(call.args[0].url) for call in posted.call_args_list) == (WEBHOOK, WEBHOOK + "/second") + assert _TEXT.validate_json(posted.call_args.args[0].content) == {"text": slack_text((alert,), BASE_URL)} + finally: + await handler.close() + + +@pytest.mark.asyncio +async def test_opted_in_email_reaches_recipients_across_pages(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PROXY_BASE_URL", BASE_URL) + monkeypatch.setenv("SMTP_HOST", "smtp.example.com") + monkeypatch.setenv("SMTP_PORT", "25") + monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com") + monkeypatch.setenv("SMTP_TLS", "False") + monkeypatch.setenv("SMTP_USE_SSL", "False") + monkeypatch.delenv("SMTP_USERNAME", raising=False) + monkeypatch.delenv("SMTP_PASSWORD", raising=False) + smtp: Final = MagicMock() + monkeypatch.setattr(smtplib, "SMTP", MagicMock(return_value=smtp)) + alert: Final = summary(1) + client: Final = MagicMock() + client.writer_db.litellm_config.find_unique = AsyncMock( + side_effect=(MagicMock(param_value={"email_settings": {"Performance Alerts": True}}), None, None) + ) + client.writer_db.query_raw = AsyncMock(return_value=[{"param_name": "receipt"}]) + client.writer_db.tx.return_value.__aenter__.return_value.query_raw = AsyncMock( + side_effect=( + (), + ({"user_id": "a", "email": "first@example.com"},), + (), + ({"id": 1},), + (), + (alert.model_dump(mode="json"),), + (), + ({"user_id": "b", "email": "second@example.com"},), + (), + ({"id": 1},), + (), + (alert.model_dump(mode="json"),), + (), + (), + ) + ) + notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client)) + await notifier.run(None, now=alert.created_at) + sends: Final = smtp.__enter__.return_value.send_message.call_args_list + assert tuple(call.kwargs["to_addrs"] for call in sends) == ("first@example.com", "second@example.com") + for call in sends: + message: Final = call.kwargs["msg"] + assert message["Subject"] == "New performance alerts" + assert f"{BASE_URL}/ui/alerts?alert=1" in message.get_payload(0).get_payload(decode=True).decode() + assert message["Message-ID"] + assert sends[0].kwargs["msg"]["Message-ID"] != sends[1].kwargs["msg"]["Message-ID"] diff --git a/tests/unit/proxy/alerting_endpoints/test_alerts.py b/tests/unit/proxy/alerting_endpoints/test_alerts.py index 97d13c176ff..332d62b4e04 100644 --- a/tests/unit/proxy/alerting_endpoints/test_alerts.py +++ b/tests/unit/proxy/alerting_endpoints/test_alerts.py @@ -55,6 +55,7 @@ def app(table: MagicMock, history_query: AsyncMock) -> FastAPI: app.include_router(router) prisma: Final = MagicMock() prisma.writer_db.litellm_alert = table + prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=None) prisma.writer_db.tx.return_value.__aenter__.return_value.query_raw = history_query app.dependency_overrides[get_database] = lambda: Database(prisma, native_database()) app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( @@ -123,8 +124,38 @@ def test_monitoring_only_accepts_a_boolean_opt_out( history_query.assert_not_awaited() +@pytest.mark.parametrize("enabled", (False, True)) +def test_deprecation_preference_updates_without_changing_performance(app: FastAPI, enabled: bool) -> None: + prisma: Final = MagicMock() + prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=MagicMock(param_value=enabled)) + prisma.writer_db.litellm_config.upsert = AsyncMock() + transaction: Final = prisma.writer_db.tx.return_value.__aenter__.return_value + transaction.query_raw = AsyncMock(side_effect=((), ({"enabled": False, "alerts_used": 3},))) + transaction.execute_raw = AsyncMock() + app.dependency_overrides[get_alert_monitoring_repository] = lambda: PerformanceAlertRepository(prisma) + with TestClient(app) as client: + response: Final = client.patch("/alerts/monitoring", json={"model_deprecations_enabled": enabled}) + assert response.status_code == 200, response.text + assert response.json() == { + "enabled": False, + "model_deprecations_enabled": enabled, + "alerts_used": 3, + "alert_limit": 5, + "disabled_by_config": False, + } + prisma.writer_db.litellm_config.upsert.assert_awaited_once_with( + where={"param_name": "model_deprecation_alerts_enabled"}, + data={ + "create": {"param_name": "model_deprecation_alerts_enabled", "param_value": str(enabled).lower()}, + "update": {"param_value": str(enabled).lower()}, + }, + ) + transaction.execute_raw.assert_not_awaited() + + def test_config_opt_out_cannot_be_enabled_from_the_api(app: FastAPI) -> None: prisma: Final = MagicMock() + prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=None) transaction: Final = prisma.writer_db.tx.return_value.__aenter__.return_value transaction.query_raw = AsyncMock(return_value=[{"enabled": True, "alerts_used": 0}]) transaction.execute_raw = AsyncMock() @@ -134,7 +165,13 @@ def test_config_opt_out_cannot_be_enabled_from_the_api(app: FastAPI) -> None: with TestClient(app) as client: status: Final = client.get("/alerts/monitoring") response: Final = client.patch("/alerts/monitoring", json={"enabled": True}) - assert status.json() == {"enabled": False, "alerts_used": 0, "alert_limit": 5, "disabled_by_config": True} + assert status.json() == { + "enabled": False, + "alerts_used": 0, + "alert_limit": 5, + "disabled_by_config": True, + "model_deprecations_enabled": True, + } assert response.status_code == 409, response.text transaction.execute_raw.assert_not_awaited() diff --git a/tests/unit/repositories/test_performance_alert_repository.py b/tests/unit/repositories/test_performance_alert_repository.py index f3f818a215e..a7c85c770eb 100644 --- a/tests/unit/repositories/test_performance_alert_repository.py +++ b/tests/unit/repositories/test_performance_alert_repository.py @@ -49,6 +49,7 @@ def _repository( transaction: Final = MagicMock(query_raw=query, execute_raw=execute) context: Final = MagicMock(__aenter__=AsyncMock(return_value=transaction), __aexit__=AsyncMock(return_value=False)) client: Final = MagicMock(writer_db=MagicMock(tx=MagicMock(return_value=context))) + client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None) return PerformanceAlertRepository(client, enterprise=enterprise), query, execute @@ -191,7 +192,12 @@ async def test_first_read_initializes_the_allowance_once(enterprise: bool) -> No @pytest.mark.parametrize("enabled", (False, True)) async def test_admin_toggle_persists_the_requested_value(enabled: bool) -> None: repository, query, execute = _repository(()) - query.side_effect = ((), ({"enabled": not enabled, "alerts_used": 2},), ({"enabled": enabled, "alerts_used": 2},)) + query.side_effect = ( + (), + ({"enabled": not enabled, "alerts_used": 2},), + (), + ({"enabled": enabled, "alerts_used": 2},), + ) result: Final = await repository.set_enabled(enabled) assert result.enabled is enabled assert result.active is enabled