diff --git a/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py b/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py
index bb3b712d50b..678c5c09cf5 100644
--- a/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py
+++ b/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py
@@ -35,6 +35,7 @@ class SendKeyRotatedEmailEvent(WebhookEvent):
class EmailEvent(str, enum.Enum):
model_deprecations = MODEL_DEPRECATION_EMAIL_EVENT
+ performance_alerts = "Performance Alerts"
virtual_key_created = "Virtual Key Created"
new_user_invitation = "New User Invitation"
virtual_key_rotated = "Virtual Key Rotated"
@@ -60,7 +61,8 @@ class DefaultEmailSettings(LiteLLMBaseModel):
settings: dict[EmailEvent, bool] = Field(
default_factory=lambda: {
- EmailEvent.model_deprecations: False, # Explicit opt-in for owner retirement reminders.
+ EmailEvent.model_deprecations: False,
+ EmailEvent.performance_alerts: False,
EmailEvent.virtual_key_created: True, # On by default
EmailEvent.new_user_invitation: True, # On by default
EmailEvent.virtual_key_rotated: True, # On by default
diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py
index dd611947228..f728d054c0d 100644
--- a/litellm/integrations/SlackAlerting/slack_alerting.py
+++ b/litellm/integrations/SlackAlerting/slack_alerting.py
@@ -2,10 +2,11 @@
# Class for sending Slack Alerts #
import asyncio
import datetime
+import json
import os
import random
import time
-from collections.abc import Callable
+from collections.abc import Awaitable, Callable
from datetime import timedelta
from typing import TYPE_CHECKING, Any, Final, Literal
@@ -38,6 +39,7 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
+from litellm.repositories.config_repository import ConfigRepository
from litellm.repositories.table_repositories import InvitationLinkRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
@@ -87,6 +89,12 @@ def _proxy_llm_router() -> Router | None:
_DAILY_REPORT_TARGET: Final = "daily_report_schedule"
+async def _model_deprecations_enabled() -> bool:
+ from litellm.proxy.proxy_server import prisma_client
+
+ return prisma_client is None or await ConfigRepository(prisma_client, use_writer=True).model_deprecations_enabled()
+
+
class SlackAlerting(CustomBatchLogger):
"""
Class for sending Slack Alerts
@@ -97,12 +105,13 @@ class SlackAlerting(CustomBatchLogger):
self,
internal_usage_cache: DualCache | None = None,
alerting_threshold: float | None = None, # threshold for slow / hanging llm responses (in seconds)
- alerting: list | None = [],
+ alerting: list[str] | None = [],
alert_types: list[AlertType] = DEFAULT_ALERT_TYPES,
alert_to_webhook_url: dict[AlertType, list[str] | str]
| None = None, # if user wants to separate alerts to diff channels
alerting_args={},
default_webhook_url: str | None = None,
+ model_deprecations_enabled: Callable[[], Awaitable[bool]] = _model_deprecations_enabled,
alert_type_config: dict[str, dict] | None = None,
async_http_handler: AsyncHTTPHandler | None = None,
**kwargs,
@@ -120,6 +129,7 @@ class SlackAlerting(CustomBatchLogger):
self.is_running = False
self.alerting_args = SlackAlertingArgs(**alerting_args)
self.default_webhook_url = default_webhook_url
+ self.model_deprecations_enabled: Final = model_deprecations_enabled
self.flush_lock = asyncio.Lock()
self.periodic_started = False
self._periodic_flush_task: asyncio.Task[None] | None = None
@@ -142,7 +152,7 @@ class SlackAlerting(CustomBatchLogger):
def update_values(
self,
- alerting: list | None = None,
+ alerting: list[str] | None = None,
alerting_threshold: float | None = None,
alert_types: list[AlertType] | None = None,
alert_to_webhook_url: dict[AlertType, list[str] | str] | None = None,
@@ -1114,7 +1124,7 @@ Model Info:
The daily lock is claimed only once there is something to say, so an empty pass never blocks a
later real one, and a sent alert is stamped in the shared cache for a day so sibling pods stop asking
"""
- if not self._deprecation_alerts_enabled():
+ if not self._deprecation_alerts_enabled() or not await self.model_deprecations_enabled():
return False
from litellm.proxy.common_utils.model_deprecation import (
@@ -1194,6 +1204,26 @@ Model Info:
continue
await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS)
+ def performance_alert_webhooks(self) -> tuple[str, ...]:
+ if not self.alerting or "slack" not in self.alerting or AlertType.performance_alerts not in self.alert_types:
+ return ()
+ configured: Final = (self.alert_to_webhook_url or {}).get(AlertType.performance_alerts)
+ destination: Final = (
+ configured
+ or self.default_webhook_url
+ or os.getenv("SLACK_WEBHOOK_URL")
+ or os.getenv("ALERTING_WEBHOOK_URL")
+ )
+ if isinstance(destination, list):
+ return tuple(dict.fromkeys(destination))
+ return (destination,) if destination else ()
+
+ async def send_performance_alert_summary(self, webhook: str, message: str) -> bool:
+ response: Final = await self.async_http_handler.post(
+ url=webhook, headers={"Content-Type": "application/json"}, data=json.dumps({"text": message})
+ )
+ return response.status_code == 200
+
async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool:
"""
Sends structured alert to webhook, if set.
diff --git a/litellm/integrations/model_deprecation_email.py b/litellm/integrations/model_deprecation_email.py
index 4093a108dcc..11988d8ce57 100644
--- a/litellm/integrations/model_deprecation_email.py
+++ b/litellm/integrations/model_deprecation_email.py
@@ -214,9 +214,13 @@ class ModelDeprecationEmailSender:
return True
async def run(self, llm_router: Router, *, now: datetime | None = None) -> int:
+ if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"):
+ return 0
+ if not await self.config.model_deprecations_enabled():
+ return 0
current: Final = now or datetime.now(timezone.utc)
settings: Final = await self.config.get_param("general_settings")
- if settings is None or not _OptIn.model_validate(settings.param_value).enabled:
+ if not _OptIn.model_validate(settings.param_value if settings else {}).enabled:
return 0
base_url: Final = os.getenv("PROXY_BASE_URL", "")
if not base_url:
@@ -253,7 +257,7 @@ class ModelDeprecationEmailSender:
async def run_scheduled_model_deprecation_emails() -> None:
- """Poll the email opt-in independently; a large owner mailing never delays Slack checks."""
+ """Poll the email settings independently; a large owner mailing never delays Slack checks."""
from litellm._logging import verbose_proxy_logger
while True:
diff --git a/litellm/integrations/performance_alert_notifications.py b/litellm/integrations/performance_alert_notifications.py
new file mode 100644
index 00000000000..1d1c8a8ab00
--- /dev/null
+++ b/litellm/integrations/performance_alert_notifications.py
@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import html
+import os
+from collections.abc import Awaitable, Callable, Sequence
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, Final
+
+from pydantic import AliasPath, BaseModel, ConfigDict, Field
+
+from litellm._logging import verbose_proxy_logger
+from litellm.models.alert import AlertSummary
+from litellm.repositories.alert_repository import AlertRepository
+from litellm.repositories.config_repository import ConfigRepository
+
+if TYPE_CHECKING:
+ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
+
+
+class _EmailSettings(BaseModel):
+ model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+ enabled: bool = Field(False, strict=True, validation_alias=AliasPath("email_settings", "Performance Alerts"))
+
+
+class _Delivery(BaseModel):
+ model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+ revision: int = 1
+ cursor: int
+ first_window_start: datetime
+ next_send_at: datetime
+ lease_until: datetime
+
+
+def _description(alert: AlertSummary) -> str:
+ labels: Final = {
+ "spend": "Spend spike",
+ "traffic": "Traffic spike",
+ "latency": "Slower responses",
+ "reliability": "More failed requests",
+ }
+ metric: Final = alert.evidence
+
+ def amount(value: float) -> str:
+ match metric.unit:
+ case "usd":
+ return f"${value:,.2f}"
+ case "ratio":
+ return f"{value:.1%}"
+ case "milliseconds":
+ return f"{value:,.0f} ms"
+ case "requests":
+ return f"{value:,.0f} requests"
+
+ return f"{labels[alert.type]} for {alert.resource_name}: {amount(metric.value)} vs {amount(metric.baseline)}"
+
+
+def slack_text(alerts: Sequence[AlertSummary], base_url: str) -> str:
+ rows: Final = tuple(
+ f"<{base_url}/ui/alerts?alert={alert.id}|{html.escape(_description(alert), quote=False)}>"
+ for alert in alerts[:5]
+ )
+ return "New performance alerts\n\n" + "\n".join(rows) + f"\n\n<{base_url}/ui/alerts|See all alerts>"
+
+
+def email_body(alerts: Sequence[AlertSummary], base_url: str) -> tuple[str, str]:
+ rows: Final = tuple(
+ f'
{html.escape(_description(alert))} '
+ for alert in alerts[:5]
+ )
+ text: Final = (
+ "New performance alerts\n\n"
+ + "\n".join(f"{_description(alert)}\n{base_url}/ui/alerts?alert={alert.id}" for alert in alerts[:5])
+ + f"\n\nSee all alerts: {base_url}/ui/alerts"
+ )
+ return (
+ f'New performance alerts See all alerts
',
+ text,
+ )
+
+
+class PerformanceAlertNotifier:
+ def __init__(self, config: ConfigRepository, alerts: AlertRepository) -> None:
+ self.config: Final = config
+ self.alerts: Final = alerts
+
+ async def deliver(
+ self,
+ destination: str,
+ send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]],
+ now: datetime,
+ ) -> None:
+ key: Final = "performance_alert_digest:" + hashlib.sha256(destination.encode()).hexdigest()
+ row: Final = await self.config.get_param(key)
+ previous: Final = (
+ _Delivery.model_validate(row.param_value)
+ if row
+ else _Delivery(
+ revision=0,
+ cursor=0,
+ first_window_start=now - timedelta(days=1),
+ next_send_at=now,
+ lease_until=now,
+ )
+ )
+ if previous.next_send_at > now or previous.lease_until > now:
+ return
+ through: Final = await self.alerts.latest_id()
+ found: Final = await self.alerts.summaries(previous.cursor, through)
+ alerts: Final = (
+ found
+ if previous.cursor
+ else tuple(alert for alert in found if alert.created_at >= previous.first_window_start)
+ )
+ claim: Final = previous.model_copy(
+ update={
+ "revision": previous.revision + 1,
+ "lease_until": now + timedelta(minutes=2),
+ "next_send_at": now + timedelta(days=1) if alerts else now,
+ }
+ )
+ if not await self.config.set_param_if_revision(key, claim.model_dump(mode="json"), previous.revision):
+ return
+ if alerts and not await send(alerts, f"{key}-{through}"):
+ return
+ sent: Final = claim.model_copy(
+ update={
+ "revision": claim.revision + 1,
+ "cursor": through,
+ "next_send_at": now + timedelta(days=1) if alerts else now,
+ "lease_until": now,
+ }
+ )
+ if not await self.config.set_param_if_revision(key, sent.model_dump(mode="json"), claim.revision):
+ verbose_proxy_logger.warning("Could not record performance alert summary delivery")
+
+ async def run(self, slack: SlackAlerting | None, *, now: datetime | None = None) -> None:
+ from litellm.proxy.utils import send_email
+
+ base_url: Final = os.getenv("PROXY_BASE_URL", "").rstrip("/")
+ if not base_url:
+ return
+ if slack is not None:
+ for webhook in slack.performance_alert_webhooks():
+
+ async def send_slack(alerts: Sequence[AlertSummary], receipt: str, webhook: str = webhook) -> bool:
+ return await slack.send_performance_alert_summary(webhook, slack_text(alerts, base_url))
+
+ await self._try_deliver("slack:" + webhook, send_slack, now or datetime.now(timezone.utc))
+ if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"):
+ return
+ settings: Final = await self.config.get_param("general_settings")
+ if not _EmailSettings.model_validate(settings.param_value if settings else {}).enabled:
+ return
+ after: str = "" # rebind-ok: advances the bounded recipient page
+ while recipients := await self.alerts.recipients(after):
+ for recipient in recipients:
+
+ async def send_mail(
+ alerts: Sequence[AlertSummary], receipt: str, address: str = recipient.email
+ ) -> bool:
+ body, text = email_body(alerts, base_url)
+ return await send_email(
+ receiver_email=address,
+ subject="New performance alerts",
+ html=body,
+ text=text,
+ message_id=f"<{receipt}@alerts.litellm.ai>",
+ )
+
+ await self._try_deliver("email:" + recipient.email, send_mail, now or datetime.now(timezone.utc))
+ after = recipients[-1].user_id
+ await asyncio.sleep(0.1)
+
+ async def _try_deliver(
+ self, destination: str, send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]], now: datetime
+ ) -> None:
+ try:
+ await asyncio.wait_for(self.deliver(destination, send, now), timeout=60)
+ except Exception as exc: # noqa: BLE001 # one unavailable destination must not stop the other recipients
+ verbose_proxy_logger.warning("Performance alert summary delivery failed (%s)", type(exc).__name__)
diff --git a/litellm/models/alert.py b/litellm/models/alert.py
index f96e042ff6b..4db127bf121 100644
--- a/litellm/models/alert.py
+++ b/litellm/models/alert.py
@@ -41,6 +41,24 @@ class DeprecationEvidence(BaseModel):
)
+class AlertSummary(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ id: AlertID
+ type: AlertType
+ resource_type: AlertResourceType
+ resource_name: str
+ created_at: AwareDatetime
+ evidence: MetricEvidence
+
+
+class AlertRecipient(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ user_id: str
+ email: str
+
+
class AlertDetection(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
diff --git a/litellm/models/performance_alert.py b/litellm/models/performance_alert.py
index 1773aa460dc..1cae91e5379 100644
--- a/litellm/models/performance_alert.py
+++ b/litellm/models/performance_alert.py
@@ -85,6 +85,7 @@ class AlertMonitoring(BaseModel):
model_config = ConfigDict(frozen=True)
enabled: bool = Field(description="Whether performance monitoring is enabled. Defaults to true.")
+ model_deprecations_enabled: bool = Field(default=True, description="Whether model deprecation alerts are enabled.")
alerts_used: int = Field(ge=0, le=FREE_ALERT_LIMIT, description="Lifetime free allowance consumed, capped at five.")
alert_limit: int | None = Field(description="Free allowance, or null for Enterprise.")
disabled_by_config: bool = Field(description="Whether performance_alerts_enabled is false in proxy settings.")
@@ -100,9 +101,16 @@ class AlertMonitoringUpdate(BaseModel):
enabled: bool = Field(strict=True, description="Enable or disable performance monitoring for this gateway.")
+class ModelDeprecationMonitoringUpdate(BaseModel):
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ model_deprecations_enabled: bool = Field(strict=True, description="Enable or disable model deprecation alerts.")
+
+
class AlertMonitoringStore(Protocol):
async def monitoring(self) -> AlertMonitoring: ...
async def set_enabled(self, enabled: bool) -> AlertMonitoring: ...
+ async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring: ...
class PerformanceAlertStore(Protocol):
diff --git a/litellm/proxy/alerting_endpoints/alerts.py b/litellm/proxy/alerting_endpoints/alerts.py
index a6ed06c3b96..9c701208e0d 100644
--- a/litellm/proxy/alerting_endpoints/alerts.py
+++ b/litellm/proxy/alerting_endpoints/alerts.py
@@ -12,7 +12,7 @@ from litellm.models.alert import (
AlertView,
AlertViewPage,
)
-from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate
+from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate, ModelDeprecationMonitoringUpdate
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.deps.repositories import AlertMonitoringRepositoryDep, AlertRepositoryDep
@@ -60,7 +60,7 @@ async def list_alerts(
return AlertViewPage(data=await store.enrich(page.data), next_cursor=page.next_cursor)
-@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read performance monitoring settings")
+@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read alert monitoring settings")
async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -> AlertMonitoring:
"""Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin."""
_authorize(user, write=False)
@@ -69,14 +69,16 @@ async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -
return await repository.monitoring()
-@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable performance monitoring")
+@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable alert monitoring")
async def update_monitoring(
- data: AlertMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep
+ data: AlertMonitoringUpdate | ModelDeprecationMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep
) -> AlertMonitoring:
"""Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts."""
_authorize(user, write=True)
if repository is None:
raise HTTPException(status_code=503, detail="Alerts require a database connection")
+ if isinstance(data, ModelDeprecationMonitoringUpdate):
+ return await repository.set_model_deprecations_enabled(data.model_deprecations_enabled)
if data.enabled and (await repository.monitoring()).disabled_by_config:
raise HTTPException(status_code=409, detail="Performance monitoring is disabled in proxy settings")
return await repository.set_enabled(data.enabled)
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index e33966ceb29..b37eede1745 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -11948,6 +11948,35 @@ class ProxyStartupEvent:
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
)
+ async def send_performance_alert_summaries() -> None:
+ from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier
+ from litellm.repositories.alert_repository import AlertRepository
+
+ slack: Final = proxy_logging_obj.slack_alerting_instance
+ email_connected: Final = bool(os.getenv("SMTP_HOST") and os.getenv("SMTP_SENDER_EMAIL"))
+ if not email_connected and not slack.performance_alert_webhooks():
+ return
+ if _current_general_settings().get("performance_alerts_enabled", True) is not True:
+ return
+ if not (
+ await PerformanceAlertRepository(prisma_client, enterprise=premium_user is True).monitoring()
+ ).enabled:
+ return
+ await PerformanceAlertNotifier(
+ ConfigRepository(prisma_client, use_writer=True), AlertRepository(prisma_client)
+ ).run(slack)
+
+ scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # APScheduler does not provide typed job methods
+ send_performance_alert_summaries,
+ "interval",
+ minutes=5,
+ next_run_time=datetime.now(timezone.utc),
+ id="performance_alert_summaries",
+ replace_existing=True,
+ max_instances=1,
+ coalesce=True,
+ )
+
### PTU DAILY ROLLUP ###
from litellm.proxy.spend_tracking.ptu_feature_flag import (
is_ptu_cost_attribution_enabled,
diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py
index eccdf16263d..6b878fa6afc 100644
--- a/litellm/repositories/alert_repository.py
+++ b/litellm/repositories/alert_repository.py
@@ -21,8 +21,10 @@ from litellm.models.alert import (
AlertFilter,
AlertHistoryPoint,
AlertPage,
+ AlertRecipient,
AlertResourceType,
AlertStatus,
+ AlertSummary,
AlertView,
)
from litellm.repositories.prisma_protocols import PrismaRecord, TableActions
@@ -64,6 +66,17 @@ class _HistoryRow(BaseModel):
_HISTORY_ROWS: Final = TypeAdapter(tuple[_HistoryRow, ...])
+_SUMMARIES: Final = TypeAdapter(tuple[AlertSummary, ...])
+_RECIPIENTS: Final = TypeAdapter(tuple[AlertRecipient, ...])
+
+
+class _LatestAlert(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ id: int
+
+
+_LATEST_ALERT: Final = TypeAdapter(tuple[_LatestAlert, ...])
_HISTORY_SOURCES: Final[
Mapping[AlertResourceType, tuple[LiteralString, LiteralString, LiteralString, LiteralString, LiteralString]]
] = {
@@ -158,6 +171,45 @@ class AlertRepository:
TableActions[PrismaRecord], prisma_client.writer_db.litellm_alert
)
+ async def latest_id(self) -> int:
+ async with self._database.tx(timeout=timedelta(seconds=5)) as transaction:
+ await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)")
+ rows: Final = _LATEST_ALERT.validate_python(
+ await transaction.query_raw('SELECT COALESCE(MAX(id), 0) AS id FROM "LiteLLM_Alert"')
+ )
+ return rows[0].id
+
+ async def recipients(self, after: str) -> tuple[AlertRecipient, ...]:
+ async with self._database.tx(timeout=timedelta(seconds=5)) as transaction:
+ await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)")
+ rows: Final = await transaction.query_raw(
+ """SELECT user_id, LOWER(TRIM(user_email)) AS email FROM "LiteLLM_UserTable"
+ WHERE user_role = 'proxy_admin' AND user_id > $1 AND LENGTH(TRIM(user_email)) BETWEEN 3 AND 320
+ ORDER BY user_id LIMIT 100""",
+ after,
+ )
+ return _RECIPIENTS.validate_python(rows)
+
+ async def summaries(self, after: int, through: int) -> tuple[AlertSummary, ...]:
+ async with self._database.tx(timeout=timedelta(seconds=5)) as transaction:
+ await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)")
+ rows: Final = await transaction.query_raw(
+ """SELECT a.id::text, a.type, a.resource_type, a.created_at AT TIME ZONE 'UTC' AS created_at,
+ LEFT(COALESCE(k.key_alias, t.team_alias, u.user_alias,
+ CASE WHEN a.resource_type = 'model' THEN a.resource_id ELSE a.resource_type END), 120) AS resource_name,
+ jsonb_build_object('value', a.evidence->'value', 'baseline', a.evidence->'baseline',
+ 'unit', a.evidence->'unit', 'sample_size', a.evidence->'sample_size') AS evidence
+ FROM "LiteLLM_Alert" a
+ LEFT JOIN "LiteLLM_VerificationToken" k ON a.resource_type = 'key' AND k.token = a.resource_id
+ LEFT JOIN "LiteLLM_TeamTable" t ON a.resource_type = 'team' AND t.team_id = a.resource_id
+ LEFT JOIN "LiteLLM_UserTable" u ON a.resource_type = 'user' AND u.user_id = a.resource_id
+ WHERE a.id > $1 AND a.id <= $2 AND a.status = 'open' AND a.type != 'model_deprecation'
+ ORDER BY a.id DESC LIMIT 5""",
+ after,
+ through,
+ )
+ return _SUMMARIES.validate_python(rows)
+
async def record(self, detection: AlertDetection) -> Alert:
from prisma import Json
from prisma.errors import UniqueViolationError
diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py
index e9973e1f674..744c33cd9f8 100644
--- a/litellm/repositories/config_repository.py
+++ b/litellm/repositories/config_repository.py
@@ -60,6 +60,7 @@ ROUTER_SETTINGS_PARAM: Final = "router_settings"
_JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
_JSON_RULES: Final = TypeAdapter(list[JsonValue])
_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue])
+_BOOL: Final = TypeAdapter(bool)
def _stored_router_settings(param_value: object) -> Mapping[str, JsonValue]:
@@ -138,6 +139,13 @@ class ConfigRepository:
)
return ConfigParam(param_name=param_name, param_value=param_value)
+ async def model_deprecations_enabled(self) -> bool:
+ record: Final = await self.get_param("model_deprecation_alerts_enabled")
+ return True if record is None else _BOOL.validate_python(record.param_value, strict=True)
+
+ async def set_model_deprecations_enabled(self, enabled: bool) -> None:
+ await self.set_param("model_deprecation_alerts_enabled", enabled)
+
async def set_param_if_revision(self, param_name: str, param_value: object, revision: int) -> bool:
rows: Final = await self._writer.query_raw(
"""INSERT INTO "LiteLLM_Config" (param_name, param_value, last_run_at)
diff --git a/litellm/repositories/performance_alert_repository.py b/litellm/repositories/performance_alert_repository.py
index 6d55670e0b8..9eaaed77d3c 100644
--- a/litellm/repositories/performance_alert_repository.py
+++ b/litellm/repositories/performance_alert_repository.py
@@ -24,6 +24,7 @@ from litellm.models.performance_alert import (
)
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.repositories.alert_repository import record_alerts
+from litellm.repositories.config_repository import ConfigRepository
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
@@ -161,6 +162,7 @@ class PerformanceAlertRepository:
)
self._enterprise: Final = enterprise
self._configured: Final = configured
+ self._config: Final = ConfigRepository(prisma_client, use_writer=True)
async def _monitoring(self, transaction: _Transaction, *, lock: bool = False) -> AlertMonitoring:
query: Final = 'SELECT enabled, alerts_used FROM "LiteLLM_PerformanceAlertState" WHERE id = 1'
@@ -190,7 +192,14 @@ class PerformanceAlertRepository:
async def monitoring(self) -> AlertMonitoring:
async with self._transaction() as transaction:
- return await self._monitoring(transaction)
+ monitoring: Final = await self._monitoring(transaction)
+ return monitoring.model_copy(
+ update={"model_deprecations_enabled": await self._config.model_deprecations_enabled()}
+ )
+
+ async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring:
+ await self._config.set_model_deprecations_enabled(enabled)
+ return await self.monitoring()
async def set_enabled(self, enabled: bool) -> AlertMonitoring:
async with self._database.tx(timeout=timedelta(seconds=10)) as transaction:
@@ -203,7 +212,7 @@ class PerformanceAlertRepository:
updated_at = now() AT TIME ZONE 'UTC' WHERE id = 1""",
enabled,
)
- return await self._monitoring(transaction)
+ return await self.monitoring()
@asynccontextmanager
async def _transaction(self) -> AsyncGenerator[_Transaction]:
diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py
index 89c2ad841e8..77322a4850a 100644
--- a/litellm/types/integrations/slack_alerting.py
+++ b/litellm/types/integrations/slack_alerting.py
@@ -188,6 +188,7 @@ class AlertType(str, Enum):
cooldown_deployment = "cooldown_deployment"
new_model_added = "new_model_added"
model_deprecation_warnings = "model_deprecation_warnings"
+ performance_alerts = "performance_alerts"
# Outage alerts
outage_alerts = "outage_alerts"
diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx
index 073d22e52f5..31902008f2a 100644
--- a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx
+++ b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx
@@ -27,7 +27,13 @@ const calls: { path: string; query: URLSearchParams; method: string; body: unkno
let status = "open";
let rejectWrite = false;
let rejectHistory = false;
-let monitoring = { enabled: true, alerts_used: 0, alert_limit: 5 as number | null, disabled_by_config: false };
+let monitoring = {
+ model_deprecations_enabled: true,
+ enabled: true,
+ alerts_used: 0,
+ alert_limit: 5 as number | null,
+ disabled_by_config: false,
+};
let rejectMonitoring = false;
function response(data: unknown, code = 200) {
return new Response(JSON.stringify(data), { status: code, headers: { "Content-Type": "application/json" } });
@@ -41,7 +47,13 @@ describe("Alerts", () => {
rejectWrite = false;
rejectHistory = false;
rejectMonitoring = false;
- monitoring = { enabled: true, alerts_used: 0, alert_limit: 5, disabled_by_config: false };
+ monitoring = {
+ model_deprecations_enabled: true,
+ enabled: true,
+ alerts_used: 0,
+ alert_limit: 5,
+ disabled_by_config: false,
+ };
role = "proxy_admin";
document.cookie = `token=${btoa(JSON.stringify({ alg: "none" }))}.${btoa(JSON.stringify({ key: "example-session", exp: 4102444800 }))}.example; path=/`;
vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
@@ -57,7 +69,7 @@ describe("Alerts", () => {
if (rejectMonitoring) return response({ detail: "Unavailable" }, 503);
if (request.method === "PATCH") {
if (rejectWrite) return response({ detail: "Unavailable" }, 503);
- monitoring = { ...monitoring, enabled: body.enabled };
+ monitoring = { ...monitoring, ...body };
}
return response(monitoring);
}
@@ -144,7 +156,7 @@ describe("Alerts", () => {
expect(await screen.findByText("No resolved alerts.")).toBeInTheDocument();
});
- it("keeps deprecation email setup accessible from the inbox", async () => {
+ it("keeps alert email setup accessible from the inbox", async () => {
const user = userEvent.setup();
renderWithProviders( );
await user.click(await screen.findByRole("button", { name: "Set up email reminders" }));
@@ -161,12 +173,36 @@ describe("Alerts", () => {
expect(screen.queryByRole("button", { name: "Dismiss" })).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Mark resolved" })).not.toBeInTheDocument();
});
+ it("keeps notification opt-in separate from the model deprecation toggle", async () => {
+ const user = userEvent.setup();
+ renderWithProviders( );
+ expect(
+ await screen.findByText(
+ "Model deprecation alerts are unlimited. Performance monitoring includes 5 free alerts, then requires Enterprise.",
+ ),
+ ).toBeInTheDocument();
+ expect(screen.queryByRole("switch")).not.toBeInTheDocument();
+ await user.click(screen.getByRole("button", { name: "Manage alerts" }));
+ const toggle = await screen.findByRole("switch", { name: "Model deprecations" });
+ expect(toggle).toBeChecked();
+ await user.click(toggle);
+ await waitFor(() => expect(toggle).not.toBeChecked());
+ expect(calls.filter((call) => call.method === "PATCH")).toEqual([
+ expect.objectContaining({ path: "/alerts/monitoring", body: { model_deprecations_enabled: false } }),
+ ]);
+ expect(screen.getByRole("switch", { name: "Performance monitoring" })).toBeChecked();
+ expect(screen.getByText(/Opt in to Performance Alerts in email or Slack settings/)).toBeInTheDocument();
+ await user.keyboard("{Escape}");
+ expect(await screen.findByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument();
+ });
+
it("defaults on and persists opting out through the monitoring API", async () => {
const user = userEvent.setup();
renderWithProviders( );
+ await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
const toggle = await screen.findByRole("switch", { name: "Performance monitoring" });
expect(toggle).toBeChecked();
- expect(screen.getByText("0 of 5 free alerts used.")).toBeInTheDocument();
+ expect(screen.getByText("0 of 5 free alerts generated.")).toBeInTheDocument();
await user.click(toggle);
expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument();
expect(toggle).not.toBeChecked();
@@ -174,15 +210,15 @@ describe("Alerts", () => {
path: "/alerts/monitoring",
body: { enabled: false },
});
- expect(screen.getByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument();
await user.click(toggle);
- expect(await screen.findByText("0 of 5 free alerts used.")).toBeInTheDocument();
+ expect(await screen.findByText("0 of 5 free alerts generated.")).toBeInTheDocument();
});
it("explains exhaustion without blocking saved alerts or model deprecations", async () => {
monitoring.alerts_used = 5;
const user = userEvent.setup();
renderWithProviders( );
+ await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
expect(
await screen.findByText("You’ve used all 5 free alerts. New performance alerts are paused."),
).toBeInTheDocument();
@@ -190,8 +226,6 @@ describe("Alerts", () => {
"href",
"https://www.litellm.ai/enterprise",
);
- expect(screen.getByRole("button", { name: "Mark resolved" })).toBeEnabled();
- expect(screen.getByRole("button", { name: "Set up email reminders" })).toBeInTheDocument();
await user.click(screen.getByRole("switch", { name: "Performance monitoring" }));
expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument();
expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument();
@@ -200,6 +234,7 @@ describe("Alerts", () => {
it("keeps the saved toggle value when an older refresh finishes later", async () => {
const user = userEvent.setup();
renderWithProviders( );
+ await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
const toggle = await screen.findByRole("switch", { name: "Performance monitoring" });
const originalFetch = vi.mocked(globalThis.fetch).getMockImplementation()!;
const staleResponse = response(monitoring);
@@ -211,7 +246,9 @@ describe("Alerts", () => {
}
return originalFetch(input, init);
});
- await user.click(screen.getByRole("button", { name: "Refresh" }));
+ await act(async () => {
+ void testQueryClient.invalidateQueries({ queryKey: ["alert-monitoring"] });
+ });
await user.click(toggle);
expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument();
await act(async () => {
@@ -227,6 +264,7 @@ describe("Alerts", () => {
monitoring.alert_limit = null;
monitoring.alerts_used = 5;
renderWithProviders( );
+ await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
expect(await screen.findByText("On with Enterprise.")).toBeInTheDocument();
expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument();
});
@@ -236,22 +274,26 @@ describe("Alerts", () => {
monitoring.alerts_used = 5;
monitoring.enabled = false;
renderWithProviders( );
+ await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
expect(await screen.findByText("Disabled in proxy settings.")).toBeInTheDocument();
expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument();
expect(screen.getByRole("switch", { name: "Performance monitoring" })).toHaveAttribute("aria-disabled", "true");
});
- it("lets viewers see monitoring status without a toggle", async () => {
+ it("lets viewers inspect alert settings without changing them", async () => {
role = "proxy_admin_viewer";
renderWithProviders( );
- expect(await screen.findByText("0 of 5 free alerts used.")).toBeInTheDocument();
- expect(screen.queryByRole("switch")).not.toBeInTheDocument();
+ await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
+ expect(await screen.findByText("0 of 5 free alerts generated.")).toBeInTheDocument();
+ expect(screen.getByRole("switch", { name: "Performance monitoring" })).toHaveAttribute("aria-disabled", "true");
+ expect(screen.getByRole("switch", { name: "Model deprecations" })).toHaveAttribute("aria-disabled", "true");
});
it("keeps the saved preference when a monitoring change fails", async () => {
rejectWrite = true;
const user = userEvent.setup();
renderWithProviders( );
+ await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
await user.click(await screen.findByRole("switch", { name: "Performance monitoring" }));
expect(await screen.findByRole("alert")).toHaveTextContent("Could not save monitoring settings");
expect(screen.getByRole("switch", { name: "Performance monitoring" })).toBeChecked();
@@ -261,10 +303,12 @@ describe("Alerts", () => {
rejectMonitoring = true;
const user = userEvent.setup();
renderWithProviders( );
- expect(await screen.findByRole("alert")).toHaveTextContent("Could not load monitoring settings");
- expect(screen.getByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument();
+ await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
+ expect(await within(screen.getByRole("dialog")).findByRole("alert")).toHaveTextContent(
+ "Could not load monitoring settings",
+ );
rejectMonitoring = false;
- await user.click(screen.getByRole("button", { name: "Retry" }));
+ await user.click(within(screen.getByRole("dialog")).getByRole("button", { name: "Retry" }));
expect(await screen.findByRole("switch", { name: "Performance monitoring" })).toBeChecked();
});
});
diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx
index 3a4d8127bea..dfaa12318de 100644
--- a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx
+++ b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx
@@ -18,7 +18,7 @@ import { ALERT_TYPES, AlertFilter, AlertItem, TYPE_LABEL_KEYS, presentAlert } fr
import AlertActions from "./AlertActions";
import AlertChart from "./AlertChart";
import AlertDetail from "./AlertDetail";
-import PerformanceMonitoring from "./PerformanceMonitoring";
+import ManageAlerts from "./ManageAlerts";
const parsers = {
alert: parseAsString,
@@ -123,8 +123,7 @@ function AlertsInbox({ auth }: { auth: ReturnType }) {
}
return (
<>
- {t("alerts.unified.title")}
-
+
void setParams({ state: value as AlertStatus })}
@@ -205,7 +204,7 @@ function AlertsInbox({ auth }: { auth: ReturnType }) {
{list.isFetchingNextPage ? t("alerts.unified.loadingMore") : t("alerts.unified.loadMore")}
)}
- {showDeprecations && }
+
>
);
}
diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/ManageAlerts.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/ManageAlerts.tsx
new file mode 100644
index 00000000000..9959ce19050
--- /dev/null
+++ b/litellm/ui/litellm-dashboard/src/components/Alerts/ManageAlerts.tsx
@@ -0,0 +1,150 @@
+import { Settings } from "lucide-react";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogHeader,
+ DialogTitle,
+ DialogTrigger,
+} from "@/components/ui/dialog";
+import { uiHref } from "@/utils/uiHref";
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import { useTranslation } from "react-i18next";
+import { Button } from "@/components/ui/button";
+import { Switch } from "@/components/ui/switch";
+import { backgroundFetchClient } from "@/lib/http/api";
+import { getProxyBaseUrl } from "@/components/networking";
+import type { components } from "@/lib/http/schema";
+
+function statusMessage(data: components["schemas"]["AlertMonitoring"]) {
+ if (data.disabled_by_config) return "alerts.monitoring.disabledByConfig";
+ if (!data.enabled) return "alerts.monitoring.off";
+ if (data.alert_limit === null) return "alerts.monitoring.enterprise";
+ if (data.alerts_used >= data.alert_limit) return "alerts.monitoring.exhausted";
+ return "alerts.monitoring.allowance";
+}
+
+export default function ManageAlerts({ scope, readOnly }: { scope: string; readOnly: boolean }) {
+ const { t } = useTranslation();
+ const client = useQueryClient();
+ const queryKey = ["alert-monitoring", getProxyBaseUrl(), scope];
+ const queryOptions = {
+ queryKey,
+ queryFn: async ({ signal }: { signal: AbortSignal }) => {
+ const { data } = await backgroundFetchClient.GET("/alerts/monitoring", { signal });
+ if (!data) throw new Error("Could not load monitoring settings");
+ return data;
+ },
+ staleTime: 60_000,
+ retry: false,
+ };
+ const query = useQuery(queryOptions);
+ const mutation = useMutation({
+ mutationFn: async (
+ body: components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"],
+ ) => {
+ const { data } = await backgroundFetchClient.PATCH("/alerts/monitoring", { body });
+ if (!data) throw new Error("Could not save monitoring settings");
+ return data;
+ },
+ onSuccess: async (data) => {
+ await client.cancelQueries({ queryKey });
+ client.setQueryData(queryKey, data);
+ await client.invalidateQueries({ queryKey: ["deprecationAlerts"] });
+ },
+ });
+ const data = query.data;
+ const messageKey = data ? statusMessage(data) : undefined;
+ return (
+
+
+
{t("alerts.unified.title")}
+
+ }>
+
+ {t("alerts.monitoring.manage")}
+
+
+
+ {t("alerts.monitoring.manage")}
+ {t("alerts.monitoring.description")}
+
+ {query.isError && (
+
+
{t("alerts.monitoring.loadError")}
+
void query.refetch()}>
+ {t("alerts.panel.retry")}
+
+
+ )}
+ {!query.isError && !data && (
+
+ {t("alerts.monitoring.loading")}
+
+ )}
+ {!query.isError && data && (
+
+
+
+
+ {t("alerts.monitoring.deprecations")}
+
+
{t("alerts.monitoring.deprecationsDescription")}
+
+
mutation.mutate({ model_deprecations_enabled: enabled })}
+ />
+
+
+
+
+ {t("alerts.monitoring.title")}
+
+
+ {messageKey && t(messageKey, { used: data.alerts_used, limit: data.alert_limit })}
+ {messageKey === "alerts.monitoring.exhausted" && (
+
+ {t("alerts.monitoring.upgrade")}
+
+ )}
+
+
+
mutation.mutate({ enabled })}
+ aria-describedby="performance-monitoring-status"
+ />
+
+
+ )}
+ {mutation.isError && (
+
+ {t("alerts.monitoring.saveError")}
+
+ )}
+ {readOnly && {t("alerts.monitoring.adminOnly")}
}
+ {t("alerts.monitoring.notifications")}
+
+ {t("alerts.monitoring.connections")}
+
+
+
+
+ {data && (
+
+ {t(data.alert_limit === null ? "alerts.monitoring.enterpriseNote" : "alerts.monitoring.freeNote")}
+
+ )}
+
+ );
+}
diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/PerformanceMonitoring.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/PerformanceMonitoring.tsx
deleted file mode 100644
index 1839bf00abd..00000000000
--- a/litellm/ui/litellm-dashboard/src/components/Alerts/PerformanceMonitoring.tsx
+++ /dev/null
@@ -1,99 +0,0 @@
-import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
-import { useTranslation } from "react-i18next";
-import { Button } from "@/components/ui/button";
-import { Switch } from "@/components/ui/switch";
-import { backgroundFetchClient } from "@/lib/http/api";
-import { getProxyBaseUrl } from "@/components/networking";
-import type { components } from "@/lib/http/schema";
-
-function statusMessage(data: components["schemas"]["AlertMonitoring"]) {
- if (data.disabled_by_config) return "alerts.monitoring.disabledByConfig";
- if (!data.enabled) return "alerts.monitoring.off";
- if (data.alert_limit === null) return "alerts.monitoring.enterprise";
- if (data.alerts_used >= data.alert_limit) return "alerts.monitoring.exhausted";
- return "alerts.monitoring.allowance";
-}
-
-export default function PerformanceMonitoring({ scope, readOnly }: { scope: string; readOnly: boolean }) {
- const { t } = useTranslation();
- const client = useQueryClient();
- const queryKey = ["alert-monitoring", getProxyBaseUrl(), scope];
- const queryOptions = {
- queryKey,
- queryFn: async ({ signal }: { signal: AbortSignal }) => {
- const { data } = await backgroundFetchClient.GET("/alerts/monitoring", { signal });
- if (!data) throw new Error("Could not load monitoring settings");
- return data;
- },
- staleTime: 60_000,
- retry: false,
- };
- const query = useQuery(queryOptions);
- const mutation = useMutation({
- mutationFn: async (enabled: boolean) => {
- const { data } = await backgroundFetchClient.PATCH("/alerts/monitoring", { body: { enabled } });
- if (!data) throw new Error("Could not save monitoring settings");
- return data;
- },
- onSuccess: async (data) => {
- await client.cancelQueries({ queryKey });
- client.setQueryData(queryKey, data);
- },
- });
- if (query.isError)
- return (
-
-
{t("alerts.monitoring.loadError")}
-
void query.refetch()}>
- {t("alerts.panel.retry")}
-
-
- );
- if (!query.data)
- return (
-
- {t("alerts.monitoring.loading")}
-
- );
- const data = query.data;
- const messageKey = statusMessage(data);
- const message = t(messageKey, { used: data.alerts_used, limit: data.alert_limit });
- return (
-
-
-
-
- {t("alerts.monitoring.title")}
-
- {!readOnly && (
- mutation.mutate(enabled)}
- aria-describedby="performance-monitoring-status"
- />
- )}
-
-
- {message}
- {messageKey === "alerts.monitoring.exhausted" && (
-
- {t("alerts.monitoring.upgrade")}
-
- )}
-
-
- {mutation.isError && (
-
- {t("alerts.monitoring.saveError")}
-
- )}
-
- );
-}
diff --git a/litellm/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx
index 61fcfc308ad..d6468c0c243 100644
--- a/litellm/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx
+++ b/litellm/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx
@@ -45,6 +45,16 @@ beforeEach(() => {
if (url.endsWith("/management/v1/users/me")) {
return Response.json({ user_id: "example-admin", user_role: role });
}
+ if (url.endsWith("/alerts/monitoring")) {
+ const monitoring = {
+ enabled: true,
+ model_deprecations_enabled: true,
+ alerts_used: 0,
+ alert_limit: 5,
+ disabled_by_config: false,
+ };
+ return Response.json(monitoring);
+ }
if (url.endsWith("/model/deprecations?warn_within_days=30")) {
if (rejectDeprecations) return Response.json({ detail: "Unavailable" }, { status: 503 });
return Response.json({ deprecated: [], imminent: [] });
diff --git a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.test.tsx b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.test.tsx
index 6058e59d9a1..a7e9c37d477 100644
--- a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.test.tsx
+++ b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.test.tsx
@@ -13,11 +13,11 @@ describe("deprecation email setup", () => {
const user = userEvent.setup();
render( );
await user.click(screen.getByRole("button", { name: "Set up email reminders" }));
- expect(await screen.findByRole("dialog", { name: "Set up model deprecation emails" })).toBeInTheDocument();
+ expect(await screen.findByRole("dialog", { name: "Set up alert emails" })).toBeInTheDocument();
expect(screen.getByText("Connect email delivery")).toBeInTheDocument();
- expect(screen.getByText("Enable model deprecation emails")).toBeInTheDocument();
- expect(screen.getByText("Owners get two reminders")).toBeInTheDocument();
- expect(screen.getByText(/This is off by default/)).toBeInTheDocument();
+ expect(screen.getByText("Choose your email alerts")).toBeInTheDocument();
+ expect(screen.getByText("Know what gets sent")).toBeInTheDocument();
+ expect(screen.getByText(/Both are off by default/)).toBeInTheDocument();
expect(screen.getByRole("link", { name: "Open Email Settings" })).toHaveAttribute(
"href",
"/ui/logging-and-alerts?tab=email-alerts",
diff --git a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx
index 897850393e2..d65b9e94969 100644
--- a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx
+++ b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx
@@ -19,6 +19,15 @@ describe("deprecation alert API wiring", () => {
vi.clearAllMocks();
testQueryClient.clear();
});
+ it("stops loading usage and acknowledgements when model deprecations are disabled", async () => {
+ vi.mocked(useCurrentUser).mockReturnValue({ data: { user_id: "admin", user_role: "proxy_admin" } } as ReturnType<
+ typeof useCurrentUser
+ >);
+ vi.mocked(backgroundFetchClient.GET).mockResolvedValue({ data: { model_deprecations_enabled: false } } as never);
+ renderWithProviders( );
+ expect(await screen.findByText("0 alerts")).toBeInTheDocument();
+ expect(backgroundFetchClient.GET).toHaveBeenCalledExactlyOnceWith("/alerts/monitoring", expect.anything());
+ });
it("should avoid querying gateway-wide usage for non-admins", () => {
vi.mocked(useCurrentUser).mockReturnValue({
data: { user_id: "internal", user_role: "internal_user" },
@@ -36,7 +45,7 @@ describe("deprecation alert API wiring", () => {
} as never);
renderWithProviders( );
expect(await screen.findByText("0 alerts")).toBeInTheDocument();
- expect(backgroundFetchClient.GET).toHaveBeenCalledTimes(1);
+ expect(backgroundFetchClient.GET).toHaveBeenCalledTimes(2);
expect(backgroundFetchClient.GET).toHaveBeenCalledWith(
"/model/deprecations",
expect.objectContaining({ params: { query: { warn_within_days: 30 } } }),
@@ -59,6 +68,7 @@ describe("deprecation alert API wiring", () => {
],
};
vi.mocked(backgroundFetchClient.GET)
+ .mockResolvedValueOnce({ data: { model_deprecations_enabled: true } } as never)
.mockResolvedValueOnce({ data: schedule } as never)
.mockResolvedValueOnce({ data: activity } as never)
.mockResolvedValueOnce({ data: { status: "open" } } as never);
diff --git a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts
index ce126d5cd4f..36515d09202 100644
--- a/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts
+++ b/litellm/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts
@@ -17,6 +17,9 @@ export function useDeprecationAlerts(accessToken: string | null) {
queryKey: ["deprecationAlerts", scope, range],
enabled,
queryFn: async ({ signal }: { signal: AbortSignal }) => {
+ const { data: monitoring } = await backgroundFetchClient.GET("/alerts/monitoring", { signal });
+ if (!monitoring) throw new Error("Alert settings are unavailable");
+ if (monitoring.model_deprecations_enabled === false) return [];
const { data: schedule } = await backgroundFetchClient.GET("/model/deprecations", {
params: { query: { warn_within_days: 30 } },
signal,
diff --git a/litellm/ui/litellm-dashboard/src/components/email_events/email_event_settings.tsx b/litellm/ui/litellm-dashboard/src/components/email_events/email_event_settings.tsx
index 5ccadf939a9..7f21b8370c5 100644
--- a/litellm/ui/litellm-dashboard/src/components/email_events/email_event_settings.tsx
+++ b/litellm/ui/litellm-dashboard/src/components/email_events/email_event_settings.tsx
@@ -74,6 +74,9 @@ const EmailEventSettings: React.FC = ({ accessToken })
// Helper function to get a description for each event type
const getEventDescription = (event: EmailEvent): string => {
+ if (event === "Performance Alerts") {
+ return t("loggingAndAlerts.emailEvents.descriptions.performanceAlerts");
+ }
if (event === "Model Deprecations") {
return t("loggingAndAlerts.emailEvents.descriptions.modelDeprecations");
}
diff --git a/litellm/ui/litellm-dashboard/src/components/settings.tsx b/litellm/ui/litellm-dashboard/src/components/settings.tsx
index b9aea32c99d..51922ab616d 100644
--- a/litellm/ui/litellm-dashboard/src/components/settings.tsx
+++ b/litellm/ui/litellm-dashboard/src/components/settings.tsx
@@ -421,6 +421,7 @@ const Settings: React.FC = ({ accessToken, userRole, userID,
outage_alerts: "loggingAndAlerts.alertTypes.labels.outageAlerts",
region_outage_alerts: "loggingAndAlerts.alertTypes.labels.regionOutageAlerts",
model_deprecation_warnings: "loggingAndAlerts.alertTypes.labels.modelDeprecationWarnings",
+ performance_alerts: "loggingAndAlerts.alertTypes.labels.performanceAlerts",
};
useEffect(() => {
diff --git a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json
index 30439a3fb25..de72ec1410e 100644
--- a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json
+++ b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json
@@ -5938,18 +5938,18 @@
"description": "Add your SMTP settings and public gateway URL in Email Settings. Send a test email to verify delivery."
},
"enable": {
- "title": "Enable model deprecation emails",
- "description": "In Email events, turn on Model Deprecations and save. This is off by default."
+ "title": "Choose your email alerts",
+ "description": "In Email events, enable Model Deprecations, Performance Alerts, or both, then save. Both are off by default."
},
"reminders": {
- "title": "Owners get two reminders",
- "description": "Affected key owners and team admins receive a reminder 30 days before retirement, and one final reminder 1 day before. Recipients are based on actual usage in the last 30 days. Dates are checked in UTC."
+ "title": "Know what gets sent",
+ "description": "Model deprecations notify affected key owners and team admins 30 days and 1 day before retirement. Performance alerts send gateway admins one daily summary when there are new findings, with up to five alerts and a link to the rest."
}
},
- "summary": "Give affected owners time to migrate. Email reminders at 30 days and 1 day.",
+ "summary": "Get model retirement reminders and daily performance summaries by email.",
"trigger": "Set up email reminders",
- "title": "Set up model deprecation emails",
- "description": "Help key owners and team admins act before a provider retires a model. Only their affected usage is included.",
+ "title": "Set up alert emails",
+ "description": "Choose which alerts to send through your existing email connection.",
"openSettings": "Open Email Settings"
},
"migrationDetail": {
@@ -6017,9 +6017,18 @@
"more": "+{{hidden}} more"
},
"monitoring": {
+ "manage": "Manage alerts",
+ "description": "These settings apply to everyone using this gateway.",
+ "deprecations": "Model deprecations",
+ "deprecationsDescription": "Show retirement alerts and send configured email and Slack reminders.",
+ "adminOnly": "Only gateway admins can change these settings.",
+ "notifications": "Opt in to Performance Alerts in email or Slack settings to receive a daily summary of new findings. Email summaries go to gateway admins.",
+ "connections": "Manage email and Slack connections",
+ "freeNote": "Model deprecation alerts are unlimited. Performance monitoring includes 5 free alerts, then requires Enterprise.",
+ "enterpriseNote": "Unlimited model deprecation and performance alerts with Enterprise.",
"title": "Performance monitoring",
"loading": "Loading monitoring settings…",
- "allowance": "{{used}} of {{limit}} free alerts used.",
+ "allowance": "{{used}} of {{limit}} free alerts generated.",
"enterprise": "On with Enterprise.",
"off": "Off. Saved alerts remain available.",
"exhausted": "You’ve used all 5 free alerts. New performance alerts are paused.",
@@ -6518,7 +6527,8 @@
"dailyReports": "Weekly/Monthly Spend Reports",
"outageAlerts": "Outage Alerts",
"regionOutageAlerts": "Region Outage Alerts",
- "modelDeprecationWarnings": "Model Deprecation Warnings"
+ "modelDeprecationWarnings": "Model Deprecation Warnings",
+ "performanceAlerts": "Performance Alerts"
},
"enterpriseFeature": "✨ Enterprise Feature",
"saveChanges": "Save Changes",
@@ -6625,6 +6635,7 @@
"description": "Select which events should trigger email notifications.",
"descriptions": {
"modelDeprecations": "Email affected key owners and team admins 30 days and 1 day before model deprecation. Uses requests from the last 30 days. Off by default.",
+ "performanceAlerts": "Email gateway admins a daily summary when there are new performance alerts. Shows up to five alerts with a link to see all.",
"virtualKeyCreated": "An email will be sent to the user when a new virtual key is created with their user ID",
"newUserInvitation": "An email will be sent to the email address of the user when a new user is created",
"generic": "Receive an email notification when {{event}}"
diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts
index d643e191bb0..3053ec54592 100644
--- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -1030,7 +1030,7 @@ export interface paths {
cookie?: never;
};
/**
- * Read performance monitoring settings
+ * Read alert monitoring settings
* @description Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin.
*/
get: operations["get_monitoring_alerts_monitoring_get"];
@@ -1040,7 +1040,7 @@ export interface paths {
options?: never;
head?: never;
/**
- * Enable or disable performance monitoring
+ * Enable or disable alert monitoring
* @description Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts.
*/
patch: operations["update_monitoring_alerts_monitoring_patch"];
@@ -27339,6 +27339,12 @@ export interface components {
* @description Whether performance monitoring is enabled. Defaults to true.
*/
enabled: boolean;
+ /**
+ * Model Deprecations Enabled
+ * @description Whether model deprecation alerts are enabled.
+ * @default true
+ */
+ model_deprecations_enabled: boolean;
};
/** AlertMonitoringUpdate */
AlertMonitoringUpdate: {
@@ -27353,7 +27359,7 @@ export interface components {
* @description Enum for alert types and management event types
* @enum {string}
*/
- AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted" | "mcp_tool_description_blocked" | "mcp_pinned_tools_changed";
+ AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "performance_alerts" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted" | "mcp_tool_description_blocked" | "mcp_pinned_tools_changed";
/** AlertUpdate */
AlertUpdate: {
/**
@@ -33628,7 +33634,7 @@ export interface components {
* EmailEvent
* @enum {string}
*/
- EmailEvent: "Model Deprecations" | "Virtual Key Created" | "New User Invitation" | "Virtual Key Rotated" | "Soft Budget Crossed" | "Max Budget Alert";
+ EmailEvent: "Model Deprecations" | "Performance Alerts" | "Virtual Key Created" | "New User Invitation" | "Virtual Key Rotated" | "Soft Budget Crossed" | "Max Budget Alert";
/** EmailEventSettings */
EmailEventSettings: {
/** Enabled */
@@ -40207,6 +40213,14 @@ export interface components {
*/
team_public_model_name?: string | null;
};
+ /** ModelDeprecationMonitoringUpdate */
+ ModelDeprecationMonitoringUpdate: {
+ /**
+ * Model Deprecations Enabled
+ * @description Enable or disable model deprecation alerts.
+ */
+ model_deprecations_enabled: boolean;
+ };
/** ModelDeprecationResponse */
ModelDeprecationResponse: {
/**
@@ -55078,7 +55092,7 @@ export interface operations {
};
requestBody: {
content: {
- "application/json": components["schemas"]["AlertMonitoringUpdate"];
+ "application/json": components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"];
};
};
responses: {
diff --git a/tests/integration/database/test_alerts.py b/tests/integration/database/test_alerts.py
index aaf36dd7ef2..06908c3a569 100644
--- a/tests/integration/database/test_alerts.py
+++ b/tests/integration/database/test_alerts.py
@@ -1,7 +1,8 @@
import asyncio
+import hashlib
import os
import uuid
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Final
@@ -22,6 +23,7 @@ from litellm.models.alert import (
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.repositories.alert_repository import AlertRepository
+from litellm.repositories.config_repository import ConfigRepository
from tests.integration._support.client import Gateway
from tests.integration._support.database import read_rows, write_rows
@@ -41,6 +43,7 @@ def saved_page(content: bytes) -> AlertPage:
@dataclass(frozen=True, slots=True)
class Alerts:
repository: AlertRepository
+ config: ConfigRepository
resource_id: str
def detection(self, kind: AlertType = "spend", day: int = 0) -> AlertDetection:
@@ -75,10 +78,12 @@ async def alerts() -> AsyncIterator[Alerts]:
client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache()))
await client.connect()
try:
- yield Alerts(AlertRepository(client), resource_id)
+ yield Alerts(AlertRepository(client), ConfigRepository(client, use_writer=True), resource_id)
finally:
try:
write_rows('DELETE FROM "LiteLLM_Alert" WHERE resource_id=%s', (resource_id,))
+ receipt: Final = "performance_alert_digest:" + hashlib.sha256(f"test:{resource_id}".encode()).hexdigest()
+ write_rows('DELETE FROM "LiteLLM_Config" WHERE param_name=%s', (receipt,))
finally:
await client.disconnect()
@@ -372,3 +377,118 @@ async def test_deleted_resource_preserves_history_and_history_timeout_preserves_
finally:
write_rows('DELETE FROM "LiteLLM_TeamTable" WHERE team_id=%s', (team,))
write_rows('DELETE FROM "LiteLLM_DailyTeamSpend" WHERE team_id=%s', (team,))
+
+
+@pytest.mark.asyncio
+async def test_daily_performance_summary_deduplicates_workers_and_keeps_failed_deliveries(alerts: Alerts) -> None:
+ from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier, slack_text
+ from litellm.models.alert import AlertSummary
+
+ destination: Final = f"test:{alerts.resource_id}"
+ notifier: Final = PerformanceAlertNotifier(alerts.config, alerts.repository)
+ first: Final = await alerts.repository.record(alerts.detection())
+ now: Final = first.created_at + timedelta(minutes=1)
+ delivered: Final[list[str]] = [] # mutable-ok: captures the external delivery boundary
+
+ async def send(rows: Sequence[AlertSummary], receipt: str) -> bool:
+ delivered.append(slack_text(rows, "https://gateway.example"))
+ return True
+
+ async def unavailable(rows: Sequence[AlertSummary], receipt: str) -> bool:
+ return False
+
+ await notifier.deliver(destination, send, now)
+ assert len(delivered) == 1
+ for day in range(1, 8):
+ await alerts.repository.record(alerts.detection(day=day))
+ restarted: Final = PerformanceAlertNotifier(alerts.config, alerts.repository)
+ await restarted.deliver(destination, send, now + timedelta(hours=23))
+ assert len(delivered) == 1
+ tomorrow: Final = now + timedelta(days=1)
+ await restarted.deliver(destination, unavailable, tomorrow)
+ assert len(delivered) == 1
+ await restarted.deliver(destination, send, tomorrow + timedelta(minutes=3))
+ assert len(delivered) == 1
+ retry: Final = tomorrow + timedelta(days=1, minutes=3)
+ await asyncio.gather(
+ notifier.deliver(destination, send, retry),
+ restarted.deliver(destination, send, retry),
+ )
+ assert len(delivered) == 2
+ assert delivered[-1].count("?alert=") == 5
+ assert f"?alert={first.id}|" not in delivered[-1]
+ await restarted.deliver(destination, send, retry + timedelta(days=1, minutes=1))
+ assert len(delivered) == 2
+
+
+@pytest.mark.asyncio
+async def test_first_summary_does_not_replay_old_alerts(alerts: Alerts) -> None:
+ from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier
+ from litellm.models.alert import AlertSummary
+
+ saved: Final = await alerts.repository.record(alerts.detection())
+ delivered: Final[list[tuple[AlertSummary, ...]]] = [] # mutable-ok: captures the external delivery boundary
+
+ async def send(rows: Sequence[AlertSummary], receipt: str) -> bool:
+ delivered.append(tuple(rows))
+ return True
+
+ notifier: Final = PerformanceAlertNotifier(alerts.config, alerts.repository)
+ now: Final = saved.created_at + timedelta(days=2)
+ await notifier.deliver(f"test:{alerts.resource_id}", send, now)
+ assert delivered == []
+ recent: Final = await alerts.repository.record(alerts.detection(day=1))
+ await notifier.deliver(f"test:{alerts.resource_id}", send, now + timedelta(minutes=5))
+ assert tuple(row.id for row in delivered[0]) == (recent.id,)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("crash", (False, True))
+async def test_failed_first_summary_does_not_replay_old_alerts_on_retry(alerts: Alerts, crash: bool) -> None:
+ from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier
+ from litellm.models.alert import AlertSummary
+
+ old: Final = await alerts.repository.record(alerts.detection())
+ recent: Final = await alerts.repository.record(alerts.detection(day=1))
+ now: Final = recent.created_at + timedelta(days=2)
+ write_rows('UPDATE "LiteLLM_Alert" SET created_at=%s WHERE id=%s', (now, int(recent.id)))
+ write_rows('UPDATE "LiteLLM_Alert" SET created_at=%s WHERE id=%s', (now - timedelta(days=2), int(old.id)))
+ delivered: Final[list[tuple[str, ...]]] = [] # mutable-ok: captures the external delivery boundary
+
+ async def unavailable(rows: Sequence[AlertSummary], receipt: str) -> bool:
+ if crash:
+ raise RuntimeError("Connection lost after sending")
+ return False
+
+ async def send(rows: Sequence[AlertSummary], receipt: str) -> bool:
+ delivered.append(tuple(row.id for row in rows))
+ return True
+
+ notifier: Final = PerformanceAlertNotifier(alerts.config, alerts.repository)
+ if crash:
+ with pytest.raises(RuntimeError, match="Connection lost after sending"):
+ await notifier.deliver(f"test:{alerts.resource_id}", unavailable, now)
+ else:
+ await notifier.deliver(f"test:{alerts.resource_id}", unavailable, now)
+ await notifier.deliver(f"test:{alerts.resource_id}", send, now + timedelta(minutes=3))
+ assert delivered == []
+ await notifier.deliver(f"test:{alerts.resource_id}", send, now + timedelta(days=1, minutes=3))
+ assert delivered == [(recent.id,)]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("enabled", (False, True))
+async def test_model_deprecation_preference_persists_without_changing_performance(
+ alerts: Alerts, gateway: Gateway, enabled: bool
+) -> None:
+ previous: Final = gateway.request("GET", "/alerts/monitoring")
+ assert previous.status_code == 200, previous.text
+ before: Final = previous.json()
+ try:
+ updated: Final = gateway.request("PATCH", "/alerts/monitoring", {"model_deprecations_enabled": enabled})
+ assert updated.status_code == 200, updated.text
+ assert updated.json() == {**before, "model_deprecations_enabled": enabled}
+ refreshed: Final = gateway.request("GET", "/alerts/monitoring")
+ assert refreshed.json() == updated.json()
+ finally:
+ await alerts.config.set_model_deprecations_enabled(before["model_deprecations_enabled"])
diff --git a/tests/unit/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/unit/integrations/SlackAlerting/test_model_deprecation_alert.py
index 2416d54579f..218df62fe2d 100644
--- a/tests/unit/integrations/SlackAlerting/test_model_deprecation_alert.py
+++ b/tests/unit/integrations/SlackAlerting/test_model_deprecation_alert.py
@@ -2,11 +2,11 @@
import asyncio
from itertools import chain, repeat
+from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-
import litellm
from litellm.constants import SLACK_MODEL_DEPRECATION_LOCK_ID
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
@@ -53,6 +53,21 @@ async def test_should_skip_when_no_alerting_configured():
assert sent is False
+@pytest.mark.asyncio
+async def test_gateway_opt_out_stops_deprecation_checks() -> None:
+ async def disabled() -> bool:
+ return False
+
+ alerting: Final = SlackAlerting(
+ alerting=["slack"],
+ alert_types=[AlertType.model_deprecation_warnings],
+ model_deprecations_enabled=disabled,
+ )
+ router: Final = MagicMock()
+ assert not await alerting.send_model_deprecation_alert(llm_router=router)
+ router.registry.assert_not_called()
+
+
@pytest.mark.asyncio
async def test_should_skip_when_no_deprecations_found(monkeypatch):
monkeypatch.setattr(litellm, "model_cost", {})
diff --git a/tests/unit/integrations/email_alerting/test_model_deprecation_email.py b/tests/unit/integrations/email_alerting/test_model_deprecation_email.py
index 54dbe342148..4d64aed7db7 100644
--- a/tests/unit/integrations/email_alerting/test_model_deprecation_email.py
+++ b/tests/unit/integrations/email_alerting/test_model_deprecation_email.py
@@ -4,6 +4,7 @@ import asyncio
from dataclasses import replace
from datetime import date, datetime, timedelta, timezone
from types import MappingProxyType
+from typing import Final
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -46,6 +47,9 @@ class MemoryConfig:
def __init__(self):
self.rows = {"general_settings": {"email_settings": {"Model Deprecations": True}}}
+ async def model_deprecations_enabled(self):
+ return self.rows.get("model_deprecation_alerts_enabled", True)
+
async def get_param(self, key):
return ConfigParam(key, self.rows[key]) if key in self.rows else None
@@ -63,6 +67,8 @@ class MemoryConfig:
@pytest.fixture()
def sender(monkeypatch):
monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example")
+ monkeypatch.setenv("SMTP_HOST", "smtp.example.com")
+ monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com")
instance = module.ModelDeprecationEmailSender(MagicMock())
instance.config = MemoryConfig()
instance._impacts = AsyncMock(
@@ -296,3 +302,32 @@ async def test_should_keep_the_email_loop_running_after_a_delivery_error(monkeyp
await module.run_scheduled_model_deprecation_emails()
assert runner.run.await_count == 2
assert all(call.args == (router,) for call in runner.run.await_args_list)
+
+
+@pytest.mark.asyncio
+async def test_gateway_opt_out_stops_owner_usage_scans_and_email(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("SMTP_HOST", "smtp.example.com")
+ monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com")
+ client: Final = MagicMock()
+ client.writer_db.litellm_config.find_unique = AsyncMock(return_value=MagicMock(param_value=False))
+ client.db.litellm_dailyuserspend.group_by = AsyncMock()
+ instance: Final = module.ModelDeprecationEmailSender(client)
+ assert await instance.run(MagicMock(), now=NOW) == 0
+ client.db.litellm_dailyuserspend.group_by.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("missing", ("SMTP_HOST", "SMTP_SENDER_EMAIL"))
+async def test_missing_email_connection_does_not_scan_owner_usage(
+ monkeypatch: pytest.MonkeyPatch, missing: str
+) -> None:
+ monkeypatch.setenv("SMTP_HOST", "smtp.example.com")
+ monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com")
+ monkeypatch.delenv(missing)
+ client: Final = MagicMock()
+ client.writer_db.litellm_config.find_unique = AsyncMock()
+ client.db.litellm_dailyuserspend.group_by = AsyncMock()
+ instance: Final = module.ModelDeprecationEmailSender(client)
+ assert await instance.run(MagicMock(), now=NOW) == 0
+ client.writer_db.litellm_config.find_unique.assert_not_awaited()
+ client.db.litellm_dailyuserspend.group_by.assert_not_awaited()
diff --git a/tests/unit/integrations/test_performance_alert_notifications.py b/tests/unit/integrations/test_performance_alert_notifications.py
new file mode 100644
index 00000000000..37a6b6728a1
--- /dev/null
+++ b/tests/unit/integrations/test_performance_alert_notifications.py
@@ -0,0 +1,213 @@
+import smtplib
+from datetime import datetime, timezone
+from typing import Final, Literal
+from unittest.mock import AsyncMock, MagicMock
+
+import httpx
+import pytest
+from pydantic import TypeAdapter
+
+from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier, email_body, slack_text
+from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.models.alert import AlertSummary, MetricEvidence
+from litellm.repositories.alert_repository import AlertRepository
+from litellm.repositories.config_repository import ConfigRepository
+from litellm.types.integrations.slack_alerting import AlertType
+
+BASE_URL: Final = "https://gateway.example"
+WEBHOOK: Final = "https://hooks.slack.example/alerts"
+_TEXT: Final = TypeAdapter(dict[str, str])
+
+
+def summary(index: int) -> AlertSummary:
+ return AlertSummary(
+ id=str(index),
+ type="spend",
+ resource_type="team",
+ resource_name=f"Research {index} ",
+ created_at=datetime(2026, 1, 9, tzinfo=timezone.utc),
+ evidence=MetricEvidence(value=1200, baseline=200, unit="usd", sample_size=2000),
+ )
+
+
+def test_daily_messages_include_five_findings_and_escape_resource_names() -> None:
+ alerts: Final = tuple(summary(index) for index in range(1, 7))
+ body, text = email_body(alerts, BASE_URL)
+ slack: Final = slack_text(alerts, BASE_URL)
+ assert body.count("") == 5
+ assert slack.count("?alert=") == 5
+ assert text.count("?alert=") == 5
+ assert "$1,200.00 vs $200.00" in text
+ assert "Research 6" not in body + text + slack
+ assert "Research 1 <team>" in body and "Research 1 <team>" in slack
+ assert f'See all alerts ' in body
+ assert f"<{BASE_URL}/ui/alerts|See all alerts>" in slack
+
+
+@pytest.mark.parametrize(
+ ("kind", "metric", "expected"),
+ (
+ ("reliability", MetricEvidence(value=0.25, baseline=0.02, unit="ratio", sample_size=2000), "25.0% vs 2.0%"),
+ (
+ "latency",
+ MetricEvidence(value=4000, baseline=1500, unit="milliseconds", sample_size=2000),
+ "4,000 ms vs 1,500 ms",
+ ),
+ (
+ "traffic",
+ MetricEvidence(value=12000, baseline=2000, unit="requests", sample_size=2000),
+ "12,000 requests vs 2,000 requests",
+ ),
+ ),
+)
+def test_daily_messages_preserve_metric_units(
+ kind: Literal["reliability", "latency", "traffic"], metric: MetricEvidence, expected: str
+) -> None:
+ alert: Final = summary(1).model_copy(update={"type": kind, "evidence": metric})
+ body, text = email_body((alert,), BASE_URL)
+ assert expected in body and expected in text
+ assert expected in slack_text((alert,), BASE_URL)
+
+
+@pytest.mark.asyncio
+async def test_connected_channels_do_not_subscribe_to_performance_summaries(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("PROXY_BASE_URL", BASE_URL)
+ monkeypatch.setenv("SMTP_HOST", "smtp.example.com")
+ monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com")
+ client: Final = MagicMock()
+ client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
+ client.writer_db.tx = MagicMock()
+ slack: Final = SlackAlerting(alerting=["slack"], default_webhook_url=WEBHOOK)
+ notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client))
+ await notifier.run(slack)
+ assert slack.performance_alert_webhooks() == ()
+ client.writer_db.tx.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_slack_summary_reuses_the_configured_connection_and_reports_delivery() -> None:
+ posted: Final = MagicMock(return_value=httpx.Response(200, text="ok"))
+ handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(posted))
+ try:
+ slack: Final = SlackAlerting(
+ alerting=["slack"],
+ alert_types=[AlertType.performance_alerts],
+ default_webhook_url=WEBHOOK,
+ async_http_handler=handler,
+ )
+ assert slack.performance_alert_webhooks() == (WEBHOOK,)
+ message: Final = slack_text((summary(1),), BASE_URL)
+ assert await slack.send_performance_alert_summary(WEBHOOK, message)
+ request: Final = posted.call_args.args[0]
+ assert str(request.url) == WEBHOOK
+ assert _TEXT.validate_json(request.content) == {"text": message}
+ finally:
+ await handler.close()
+
+
+@pytest.mark.asyncio
+async def test_opted_in_summary_reaches_slack_from_the_notification_job(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("PROXY_BASE_URL", BASE_URL)
+ monkeypatch.delenv("SMTP_HOST", raising=False)
+ alert: Final = summary(1)
+ client: Final = MagicMock()
+ client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
+ client.writer_db.query_raw = AsyncMock(return_value=[{"param_name": "receipt"}])
+ client.writer_db.tx.return_value.__aenter__.return_value.query_raw = AsyncMock(
+ side_effect=((), ({"id": 1},), (), (alert.model_dump(mode="json"),))
+ )
+ posted: Final = MagicMock(return_value=httpx.Response(200, text="ok"))
+ handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(posted))
+ try:
+ slack: Final = SlackAlerting(
+ alerting=["slack"],
+ alert_types=[AlertType.performance_alerts],
+ default_webhook_url=WEBHOOK,
+ async_http_handler=handler,
+ )
+ notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client))
+ await notifier.run(slack, now=alert.created_at)
+ posted.assert_called_once()
+ assert _TEXT.validate_json(posted.call_args.args[0].content) == {"text": slack_text((alert,), BASE_URL)}
+ finally:
+ await handler.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("failure", (httpx.ReadTimeout("Webhook timed out"), httpx.Response(503)))
+async def test_unavailable_webhook_does_not_block_other_destinations(
+ monkeypatch: pytest.MonkeyPatch, failure: Exception | httpx.Response
+) -> None:
+ monkeypatch.setenv("PROXY_BASE_URL", BASE_URL)
+ monkeypatch.delenv("SMTP_HOST", raising=False)
+ alert: Final = summary(1)
+ client: Final = MagicMock()
+ client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
+ client.writer_db.query_raw = AsyncMock(return_value=[{"param_name": "receipt"}])
+ client.writer_db.tx.return_value.__aenter__.return_value.query_raw = AsyncMock(
+ side_effect=((), ({"id": 1},), (), (alert.model_dump(mode="json"),)) * 2
+ )
+ posted: Final = MagicMock(side_effect=(failure, httpx.Response(200)))
+ handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(posted))
+ try:
+ slack: Final = SlackAlerting(
+ alerting=["slack"],
+ alert_types=[AlertType.performance_alerts],
+ alert_to_webhook_url={AlertType.performance_alerts: [WEBHOOK, WEBHOOK + "/second", WEBHOOK]},
+ async_http_handler=handler,
+ )
+ notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client))
+ await notifier.run(slack, now=alert.created_at)
+ assert tuple(str(call.args[0].url) for call in posted.call_args_list) == (WEBHOOK, WEBHOOK + "/second")
+ assert _TEXT.validate_json(posted.call_args.args[0].content) == {"text": slack_text((alert,), BASE_URL)}
+ finally:
+ await handler.close()
+
+
+@pytest.mark.asyncio
+async def test_opted_in_email_reaches_recipients_across_pages(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("PROXY_BASE_URL", BASE_URL)
+ monkeypatch.setenv("SMTP_HOST", "smtp.example.com")
+ monkeypatch.setenv("SMTP_PORT", "25")
+ monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com")
+ monkeypatch.setenv("SMTP_TLS", "False")
+ monkeypatch.setenv("SMTP_USE_SSL", "False")
+ monkeypatch.delenv("SMTP_USERNAME", raising=False)
+ monkeypatch.delenv("SMTP_PASSWORD", raising=False)
+ smtp: Final = MagicMock()
+ monkeypatch.setattr(smtplib, "SMTP", MagicMock(return_value=smtp))
+ alert: Final = summary(1)
+ client: Final = MagicMock()
+ client.writer_db.litellm_config.find_unique = AsyncMock(
+ side_effect=(MagicMock(param_value={"email_settings": {"Performance Alerts": True}}), None, None)
+ )
+ client.writer_db.query_raw = AsyncMock(return_value=[{"param_name": "receipt"}])
+ client.writer_db.tx.return_value.__aenter__.return_value.query_raw = AsyncMock(
+ side_effect=(
+ (),
+ ({"user_id": "a", "email": "first@example.com"},),
+ (),
+ ({"id": 1},),
+ (),
+ (alert.model_dump(mode="json"),),
+ (),
+ ({"user_id": "b", "email": "second@example.com"},),
+ (),
+ ({"id": 1},),
+ (),
+ (alert.model_dump(mode="json"),),
+ (),
+ (),
+ )
+ )
+ notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client))
+ await notifier.run(None, now=alert.created_at)
+ sends: Final = smtp.__enter__.return_value.send_message.call_args_list
+ assert tuple(call.kwargs["to_addrs"] for call in sends) == ("first@example.com", "second@example.com")
+ for call in sends:
+ message: Final = call.kwargs["msg"]
+ assert message["Subject"] == "New performance alerts"
+ assert f"{BASE_URL}/ui/alerts?alert=1" in message.get_payload(0).get_payload(decode=True).decode()
+ assert message["Message-ID"]
+ assert sends[0].kwargs["msg"]["Message-ID"] != sends[1].kwargs["msg"]["Message-ID"]
diff --git a/tests/unit/proxy/alerting_endpoints/test_alerts.py b/tests/unit/proxy/alerting_endpoints/test_alerts.py
index 97d13c176ff..332d62b4e04 100644
--- a/tests/unit/proxy/alerting_endpoints/test_alerts.py
+++ b/tests/unit/proxy/alerting_endpoints/test_alerts.py
@@ -55,6 +55,7 @@ def app(table: MagicMock, history_query: AsyncMock) -> FastAPI:
app.include_router(router)
prisma: Final = MagicMock()
prisma.writer_db.litellm_alert = table
+ prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
prisma.writer_db.tx.return_value.__aenter__.return_value.query_raw = history_query
app.dependency_overrides[get_database] = lambda: Database(prisma, native_database())
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
@@ -123,8 +124,38 @@ def test_monitoring_only_accepts_a_boolean_opt_out(
history_query.assert_not_awaited()
+@pytest.mark.parametrize("enabled", (False, True))
+def test_deprecation_preference_updates_without_changing_performance(app: FastAPI, enabled: bool) -> None:
+ prisma: Final = MagicMock()
+ prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=MagicMock(param_value=enabled))
+ prisma.writer_db.litellm_config.upsert = AsyncMock()
+ transaction: Final = prisma.writer_db.tx.return_value.__aenter__.return_value
+ transaction.query_raw = AsyncMock(side_effect=((), ({"enabled": False, "alerts_used": 3},)))
+ transaction.execute_raw = AsyncMock()
+ app.dependency_overrides[get_alert_monitoring_repository] = lambda: PerformanceAlertRepository(prisma)
+ with TestClient(app) as client:
+ response: Final = client.patch("/alerts/monitoring", json={"model_deprecations_enabled": enabled})
+ assert response.status_code == 200, response.text
+ assert response.json() == {
+ "enabled": False,
+ "model_deprecations_enabled": enabled,
+ "alerts_used": 3,
+ "alert_limit": 5,
+ "disabled_by_config": False,
+ }
+ prisma.writer_db.litellm_config.upsert.assert_awaited_once_with(
+ where={"param_name": "model_deprecation_alerts_enabled"},
+ data={
+ "create": {"param_name": "model_deprecation_alerts_enabled", "param_value": str(enabled).lower()},
+ "update": {"param_value": str(enabled).lower()},
+ },
+ )
+ transaction.execute_raw.assert_not_awaited()
+
+
def test_config_opt_out_cannot_be_enabled_from_the_api(app: FastAPI) -> None:
prisma: Final = MagicMock()
+ prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
transaction: Final = prisma.writer_db.tx.return_value.__aenter__.return_value
transaction.query_raw = AsyncMock(return_value=[{"enabled": True, "alerts_used": 0}])
transaction.execute_raw = AsyncMock()
@@ -134,7 +165,13 @@ def test_config_opt_out_cannot_be_enabled_from_the_api(app: FastAPI) -> None:
with TestClient(app) as client:
status: Final = client.get("/alerts/monitoring")
response: Final = client.patch("/alerts/monitoring", json={"enabled": True})
- assert status.json() == {"enabled": False, "alerts_used": 0, "alert_limit": 5, "disabled_by_config": True}
+ assert status.json() == {
+ "enabled": False,
+ "alerts_used": 0,
+ "alert_limit": 5,
+ "disabled_by_config": True,
+ "model_deprecations_enabled": True,
+ }
assert response.status_code == 409, response.text
transaction.execute_raw.assert_not_awaited()
diff --git a/tests/unit/repositories/test_performance_alert_repository.py b/tests/unit/repositories/test_performance_alert_repository.py
index f3f818a215e..a7c85c770eb 100644
--- a/tests/unit/repositories/test_performance_alert_repository.py
+++ b/tests/unit/repositories/test_performance_alert_repository.py
@@ -49,6 +49,7 @@ def _repository(
transaction: Final = MagicMock(query_raw=query, execute_raw=execute)
context: Final = MagicMock(__aenter__=AsyncMock(return_value=transaction), __aexit__=AsyncMock(return_value=False))
client: Final = MagicMock(writer_db=MagicMock(tx=MagicMock(return_value=context)))
+ client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
return PerformanceAlertRepository(client, enterprise=enterprise), query, execute
@@ -191,7 +192,12 @@ async def test_first_read_initializes_the_allowance_once(enterprise: bool) -> No
@pytest.mark.parametrize("enabled", (False, True))
async def test_admin_toggle_persists_the_requested_value(enabled: bool) -> None:
repository, query, execute = _repository(())
- query.side_effect = ((), ({"enabled": not enabled, "alerts_used": 2},), ({"enabled": enabled, "alerts_used": 2},))
+ query.side_effect = (
+ (),
+ ({"enabled": not enabled, "alerts_used": 2},),
+ (),
+ ({"enabled": enabled, "alerts_used": 2},),
+ )
result: Final = await repository.set_enabled(enabled)
assert result.enabled is enabled
assert result.active is enabled