mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
feat(alerts): add management controls and opt-in daily summaries (#51418)
* feat(alerts): add alert controls and daily notification summaries * fix(alerts): reserve daily notifications before delivery
This commit is contained in:
moe-berri
GitHub
parent
4ee55a28f7
commit
d2711ac559
30 changed files with 1074 additions and 156 deletions
|
|
@ -35,6 +35,7 @@ class SendKeyRotatedEmailEvent(WebhookEvent):
|
|||
|
||||
class EmailEvent(str, enum.Enum):
|
||||
model_deprecations = MODEL_DEPRECATION_EMAIL_EVENT
|
||||
performance_alerts = "Performance Alerts"
|
||||
virtual_key_created = "Virtual Key Created"
|
||||
new_user_invitation = "New User Invitation"
|
||||
virtual_key_rotated = "Virtual Key Rotated"
|
||||
|
|
@ -60,7 +61,8 @@ class DefaultEmailSettings(LiteLLMBaseModel):
|
|||
|
||||
settings: dict[EmailEvent, bool] = Field(
|
||||
default_factory=lambda: {
|
||||
EmailEvent.model_deprecations: False, # Explicit opt-in for owner retirement reminders.
|
||||
EmailEvent.model_deprecations: False,
|
||||
EmailEvent.performance_alerts: False,
|
||||
EmailEvent.virtual_key_created: True, # On by default
|
||||
EmailEvent.new_user_invitation: True, # On by default
|
||||
EmailEvent.virtual_key_rotated: True, # On by default
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@
|
|||
# Class for sending Slack Alerts #
|
||||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal
|
||||
|
||||
|
|
@ -38,6 +39,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
from litellm.repositories.table_repositories import InvitationLinkRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
|
|
@ -87,6 +89,12 @@ def _proxy_llm_router() -> Router | None:
|
|||
_DAILY_REPORT_TARGET: Final = "daily_report_schedule"
|
||||
|
||||
|
||||
async def _model_deprecations_enabled() -> bool:
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
return prisma_client is None or await ConfigRepository(prisma_client, use_writer=True).model_deprecations_enabled()
|
||||
|
||||
|
||||
class SlackAlerting(CustomBatchLogger):
|
||||
"""
|
||||
Class for sending Slack Alerts
|
||||
|
|
@ -97,12 +105,13 @@ class SlackAlerting(CustomBatchLogger):
|
|||
self,
|
||||
internal_usage_cache: DualCache | None = None,
|
||||
alerting_threshold: float | None = None, # threshold for slow / hanging llm responses (in seconds)
|
||||
alerting: list | None = [],
|
||||
alerting: list[str] | None = [],
|
||||
alert_types: list[AlertType] = DEFAULT_ALERT_TYPES,
|
||||
alert_to_webhook_url: dict[AlertType, list[str] | str]
|
||||
| None = None, # if user wants to separate alerts to diff channels
|
||||
alerting_args={},
|
||||
default_webhook_url: str | None = None,
|
||||
model_deprecations_enabled: Callable[[], Awaitable[bool]] = _model_deprecations_enabled,
|
||||
alert_type_config: dict[str, dict] | None = None,
|
||||
async_http_handler: AsyncHTTPHandler | None = None,
|
||||
**kwargs,
|
||||
|
|
@ -120,6 +129,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
self.is_running = False
|
||||
self.alerting_args = SlackAlertingArgs(**alerting_args)
|
||||
self.default_webhook_url = default_webhook_url
|
||||
self.model_deprecations_enabled: Final = model_deprecations_enabled
|
||||
self.flush_lock = asyncio.Lock()
|
||||
self.periodic_started = False
|
||||
self._periodic_flush_task: asyncio.Task[None] | None = None
|
||||
|
|
@ -142,7 +152,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
|
||||
def update_values(
|
||||
self,
|
||||
alerting: list | None = None,
|
||||
alerting: list[str] | None = None,
|
||||
alerting_threshold: float | None = None,
|
||||
alert_types: list[AlertType] | None = None,
|
||||
alert_to_webhook_url: dict[AlertType, list[str] | str] | None = None,
|
||||
|
|
@ -1114,7 +1124,7 @@ Model Info:
|
|||
The daily lock is claimed only once there is something to say, so an empty pass never blocks a
|
||||
later real one, and a sent alert is stamped in the shared cache for a day so sibling pods stop asking
|
||||
"""
|
||||
if not self._deprecation_alerts_enabled():
|
||||
if not self._deprecation_alerts_enabled() or not await self.model_deprecations_enabled():
|
||||
return False
|
||||
|
||||
from litellm.proxy.common_utils.model_deprecation import (
|
||||
|
|
@ -1194,6 +1204,26 @@ Model Info:
|
|||
continue
|
||||
await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS)
|
||||
|
||||
def performance_alert_webhooks(self) -> tuple[str, ...]:
|
||||
if not self.alerting or "slack" not in self.alerting or AlertType.performance_alerts not in self.alert_types:
|
||||
return ()
|
||||
configured: Final = (self.alert_to_webhook_url or {}).get(AlertType.performance_alerts)
|
||||
destination: Final = (
|
||||
configured
|
||||
or self.default_webhook_url
|
||||
or os.getenv("SLACK_WEBHOOK_URL")
|
||||
or os.getenv("ALERTING_WEBHOOK_URL")
|
||||
)
|
||||
if isinstance(destination, list):
|
||||
return tuple(dict.fromkeys(destination))
|
||||
return (destination,) if destination else ()
|
||||
|
||||
async def send_performance_alert_summary(self, webhook: str, message: str) -> bool:
|
||||
response: Final = await self.async_http_handler.post(
|
||||
url=webhook, headers={"Content-Type": "application/json"}, data=json.dumps({"text": message})
|
||||
)
|
||||
return response.status_code == 200
|
||||
|
||||
async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool:
|
||||
"""
|
||||
Sends structured alert to webhook, if set.
|
||||
|
|
|
|||
|
|
@ -214,9 +214,13 @@ class ModelDeprecationEmailSender:
|
|||
return True
|
||||
|
||||
async def run(self, llm_router: Router, *, now: datetime | None = None) -> int:
|
||||
if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"):
|
||||
return 0
|
||||
if not await self.config.model_deprecations_enabled():
|
||||
return 0
|
||||
current: Final = now or datetime.now(timezone.utc)
|
||||
settings: Final = await self.config.get_param("general_settings")
|
||||
if settings is None or not _OptIn.model_validate(settings.param_value).enabled:
|
||||
if not _OptIn.model_validate(settings.param_value if settings else {}).enabled:
|
||||
return 0
|
||||
base_url: Final = os.getenv("PROXY_BASE_URL", "")
|
||||
if not base_url:
|
||||
|
|
@ -253,7 +257,7 @@ class ModelDeprecationEmailSender:
|
|||
|
||||
|
||||
async def run_scheduled_model_deprecation_emails() -> None:
|
||||
"""Poll the email opt-in independently; a large owner mailing never delays Slack checks."""
|
||||
"""Poll the email settings independently; a large owner mailing never delays Slack checks."""
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
while True:
|
||||
|
|
|
|||
183
litellm/integrations/performance_alert_notifications.py
Normal file
183
litellm/integrations/performance_alert_notifications.py
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import html
|
||||
import os
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import AliasPath, BaseModel, ConfigDict, Field
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.models.alert import AlertSummary
|
||||
from litellm.repositories.alert_repository import AlertRepository
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
|
||||
|
||||
class _EmailSettings(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
|
||||
enabled: bool = Field(False, strict=True, validation_alias=AliasPath("email_settings", "Performance Alerts"))
|
||||
|
||||
|
||||
class _Delivery(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
|
||||
|
||||
revision: int = 1
|
||||
cursor: int
|
||||
first_window_start: datetime
|
||||
next_send_at: datetime
|
||||
lease_until: datetime
|
||||
|
||||
|
||||
def _description(alert: AlertSummary) -> str:
|
||||
labels: Final = {
|
||||
"spend": "Spend spike",
|
||||
"traffic": "Traffic spike",
|
||||
"latency": "Slower responses",
|
||||
"reliability": "More failed requests",
|
||||
}
|
||||
metric: Final = alert.evidence
|
||||
|
||||
def amount(value: float) -> str:
|
||||
match metric.unit:
|
||||
case "usd":
|
||||
return f"${value:,.2f}"
|
||||
case "ratio":
|
||||
return f"{value:.1%}"
|
||||
case "milliseconds":
|
||||
return f"{value:,.0f} ms"
|
||||
case "requests":
|
||||
return f"{value:,.0f} requests"
|
||||
|
||||
return f"{labels[alert.type]} for {alert.resource_name}: {amount(metric.value)} vs {amount(metric.baseline)}"
|
||||
|
||||
|
||||
def slack_text(alerts: Sequence[AlertSummary], base_url: str) -> str:
|
||||
rows: Final = tuple(
|
||||
f"<{base_url}/ui/alerts?alert={alert.id}|{html.escape(_description(alert), quote=False)}>"
|
||||
for alert in alerts[:5]
|
||||
)
|
||||
return "New performance alerts\n\n" + "\n".join(rows) + f"\n\n<{base_url}/ui/alerts|See all alerts>"
|
||||
|
||||
|
||||
def email_body(alerts: Sequence[AlertSummary], base_url: str) -> tuple[str, str]:
|
||||
rows: Final = tuple(
|
||||
f'<li><a href="{html.escape(base_url, quote=True)}/ui/alerts?alert={alert.id}">{html.escape(_description(alert))}</a></li>'
|
||||
for alert in alerts[:5]
|
||||
)
|
||||
text: Final = (
|
||||
"New performance alerts\n\n"
|
||||
+ "\n".join(f"{_description(alert)}\n{base_url}/ui/alerts?alert={alert.id}" for alert in alerts[:5])
|
||||
+ f"\n\nSee all alerts: {base_url}/ui/alerts"
|
||||
)
|
||||
return (
|
||||
f'<h2>New performance alerts</h2><ul>{"".join(rows)}</ul><p><a href="{html.escape(base_url, quote=True)}/ui/alerts">See all alerts</a></p>',
|
||||
text,
|
||||
)
|
||||
|
||||
|
||||
class PerformanceAlertNotifier:
|
||||
def __init__(self, config: ConfigRepository, alerts: AlertRepository) -> None:
|
||||
self.config: Final = config
|
||||
self.alerts: Final = alerts
|
||||
|
||||
async def deliver(
|
||||
self,
|
||||
destination: str,
|
||||
send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]],
|
||||
now: datetime,
|
||||
) -> None:
|
||||
key: Final = "performance_alert_digest:" + hashlib.sha256(destination.encode()).hexdigest()
|
||||
row: Final = await self.config.get_param(key)
|
||||
previous: Final = (
|
||||
_Delivery.model_validate(row.param_value)
|
||||
if row
|
||||
else _Delivery(
|
||||
revision=0,
|
||||
cursor=0,
|
||||
first_window_start=now - timedelta(days=1),
|
||||
next_send_at=now,
|
||||
lease_until=now,
|
||||
)
|
||||
)
|
||||
if previous.next_send_at > now or previous.lease_until > now:
|
||||
return
|
||||
through: Final = await self.alerts.latest_id()
|
||||
found: Final = await self.alerts.summaries(previous.cursor, through)
|
||||
alerts: Final = (
|
||||
found
|
||||
if previous.cursor
|
||||
else tuple(alert for alert in found if alert.created_at >= previous.first_window_start)
|
||||
)
|
||||
claim: Final = previous.model_copy(
|
||||
update={
|
||||
"revision": previous.revision + 1,
|
||||
"lease_until": now + timedelta(minutes=2),
|
||||
"next_send_at": now + timedelta(days=1) if alerts else now,
|
||||
}
|
||||
)
|
||||
if not await self.config.set_param_if_revision(key, claim.model_dump(mode="json"), previous.revision):
|
||||
return
|
||||
if alerts and not await send(alerts, f"{key}-{through}"):
|
||||
return
|
||||
sent: Final = claim.model_copy(
|
||||
update={
|
||||
"revision": claim.revision + 1,
|
||||
"cursor": through,
|
||||
"next_send_at": now + timedelta(days=1) if alerts else now,
|
||||
"lease_until": now,
|
||||
}
|
||||
)
|
||||
if not await self.config.set_param_if_revision(key, sent.model_dump(mode="json"), claim.revision):
|
||||
verbose_proxy_logger.warning("Could not record performance alert summary delivery")
|
||||
|
||||
async def run(self, slack: SlackAlerting | None, *, now: datetime | None = None) -> None:
|
||||
from litellm.proxy.utils import send_email
|
||||
|
||||
base_url: Final = os.getenv("PROXY_BASE_URL", "").rstrip("/")
|
||||
if not base_url:
|
||||
return
|
||||
if slack is not None:
|
||||
for webhook in slack.performance_alert_webhooks():
|
||||
|
||||
async def send_slack(alerts: Sequence[AlertSummary], receipt: str, webhook: str = webhook) -> bool:
|
||||
return await slack.send_performance_alert_summary(webhook, slack_text(alerts, base_url))
|
||||
|
||||
await self._try_deliver("slack:" + webhook, send_slack, now or datetime.now(timezone.utc))
|
||||
if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"):
|
||||
return
|
||||
settings: Final = await self.config.get_param("general_settings")
|
||||
if not _EmailSettings.model_validate(settings.param_value if settings else {}).enabled:
|
||||
return
|
||||
after: str = "" # rebind-ok: advances the bounded recipient page
|
||||
while recipients := await self.alerts.recipients(after):
|
||||
for recipient in recipients:
|
||||
|
||||
async def send_mail(
|
||||
alerts: Sequence[AlertSummary], receipt: str, address: str = recipient.email
|
||||
) -> bool:
|
||||
body, text = email_body(alerts, base_url)
|
||||
return await send_email(
|
||||
receiver_email=address,
|
||||
subject="New performance alerts",
|
||||
html=body,
|
||||
text=text,
|
||||
message_id=f"<{receipt}@alerts.litellm.ai>",
|
||||
)
|
||||
|
||||
await self._try_deliver("email:" + recipient.email, send_mail, now or datetime.now(timezone.utc))
|
||||
after = recipients[-1].user_id
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
async def _try_deliver(
|
||||
self, destination: str, send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]], now: datetime
|
||||
) -> None:
|
||||
try:
|
||||
await asyncio.wait_for(self.deliver(destination, send, now), timeout=60)
|
||||
except Exception as exc: # noqa: BLE001 # one unavailable destination must not stop the other recipients
|
||||
verbose_proxy_logger.warning("Performance alert summary delivery failed (%s)", type(exc).__name__)
|
||||
|
|
@ -41,6 +41,24 @@ class DeprecationEvidence(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class AlertSummary(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: AlertID
|
||||
type: AlertType
|
||||
resource_type: AlertResourceType
|
||||
resource_name: str
|
||||
created_at: AwareDatetime
|
||||
evidence: MetricEvidence
|
||||
|
||||
|
||||
class AlertRecipient(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
user_id: str
|
||||
email: str
|
||||
|
||||
|
||||
class AlertDetection(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ class AlertMonitoring(BaseModel):
|
|||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
enabled: bool = Field(description="Whether performance monitoring is enabled. Defaults to true.")
|
||||
model_deprecations_enabled: bool = Field(default=True, description="Whether model deprecation alerts are enabled.")
|
||||
alerts_used: int = Field(ge=0, le=FREE_ALERT_LIMIT, description="Lifetime free allowance consumed, capped at five.")
|
||||
alert_limit: int | None = Field(description="Free allowance, or null for Enterprise.")
|
||||
disabled_by_config: bool = Field(description="Whether performance_alerts_enabled is false in proxy settings.")
|
||||
|
|
@ -100,9 +101,16 @@ class AlertMonitoringUpdate(BaseModel):
|
|||
enabled: bool = Field(strict=True, description="Enable or disable performance monitoring for this gateway.")
|
||||
|
||||
|
||||
class ModelDeprecationMonitoringUpdate(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
model_deprecations_enabled: bool = Field(strict=True, description="Enable or disable model deprecation alerts.")
|
||||
|
||||
|
||||
class AlertMonitoringStore(Protocol):
|
||||
async def monitoring(self) -> AlertMonitoring: ...
|
||||
async def set_enabled(self, enabled: bool) -> AlertMonitoring: ...
|
||||
async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring: ...
|
||||
|
||||
|
||||
class PerformanceAlertStore(Protocol):
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from litellm.models.alert import (
|
|||
AlertView,
|
||||
AlertViewPage,
|
||||
)
|
||||
from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate
|
||||
from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate, ModelDeprecationMonitoringUpdate
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.deps.repositories import AlertMonitoringRepositoryDep, AlertRepositoryDep
|
||||
|
|
@ -60,7 +60,7 @@ async def list_alerts(
|
|||
return AlertViewPage(data=await store.enrich(page.data), next_cursor=page.next_cursor)
|
||||
|
||||
|
||||
@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read performance monitoring settings")
|
||||
@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read alert monitoring settings")
|
||||
async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -> AlertMonitoring:
|
||||
"""Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin."""
|
||||
_authorize(user, write=False)
|
||||
|
|
@ -69,14 +69,16 @@ async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -
|
|||
return await repository.monitoring()
|
||||
|
||||
|
||||
@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable performance monitoring")
|
||||
@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable alert monitoring")
|
||||
async def update_monitoring(
|
||||
data: AlertMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep
|
||||
data: AlertMonitoringUpdate | ModelDeprecationMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep
|
||||
) -> AlertMonitoring:
|
||||
"""Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts."""
|
||||
_authorize(user, write=True)
|
||||
if repository is None:
|
||||
raise HTTPException(status_code=503, detail="Alerts require a database connection")
|
||||
if isinstance(data, ModelDeprecationMonitoringUpdate):
|
||||
return await repository.set_model_deprecations_enabled(data.model_deprecations_enabled)
|
||||
if data.enabled and (await repository.monitoring()).disabled_by_config:
|
||||
raise HTTPException(status_code=409, detail="Performance monitoring is disabled in proxy settings")
|
||||
return await repository.set_enabled(data.enabled)
|
||||
|
|
|
|||
|
|
@ -11948,6 +11948,35 @@ class ProxyStartupEvent:
|
|||
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
|
||||
)
|
||||
|
||||
async def send_performance_alert_summaries() -> None:
|
||||
from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier
|
||||
from litellm.repositories.alert_repository import AlertRepository
|
||||
|
||||
slack: Final = proxy_logging_obj.slack_alerting_instance
|
||||
email_connected: Final = bool(os.getenv("SMTP_HOST") and os.getenv("SMTP_SENDER_EMAIL"))
|
||||
if not email_connected and not slack.performance_alert_webhooks():
|
||||
return
|
||||
if _current_general_settings().get("performance_alerts_enabled", True) is not True:
|
||||
return
|
||||
if not (
|
||||
await PerformanceAlertRepository(prisma_client, enterprise=premium_user is True).monitoring()
|
||||
).enabled:
|
||||
return
|
||||
await PerformanceAlertNotifier(
|
||||
ConfigRepository(prisma_client, use_writer=True), AlertRepository(prisma_client)
|
||||
).run(slack)
|
||||
|
||||
scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # APScheduler does not provide typed job methods
|
||||
send_performance_alert_summaries,
|
||||
"interval",
|
||||
minutes=5,
|
||||
next_run_time=datetime.now(timezone.utc),
|
||||
id="performance_alert_summaries",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
)
|
||||
|
||||
### PTU DAILY ROLLUP ###
|
||||
from litellm.proxy.spend_tracking.ptu_feature_flag import (
|
||||
is_ptu_cost_attribution_enabled,
|
||||
|
|
|
|||
|
|
@ -21,8 +21,10 @@ from litellm.models.alert import (
|
|||
AlertFilter,
|
||||
AlertHistoryPoint,
|
||||
AlertPage,
|
||||
AlertRecipient,
|
||||
AlertResourceType,
|
||||
AlertStatus,
|
||||
AlertSummary,
|
||||
AlertView,
|
||||
)
|
||||
from litellm.repositories.prisma_protocols import PrismaRecord, TableActions
|
||||
|
|
@ -64,6 +66,17 @@ class _HistoryRow(BaseModel):
|
|||
|
||||
|
||||
_HISTORY_ROWS: Final = TypeAdapter(tuple[_HistoryRow, ...])
|
||||
_SUMMARIES: Final = TypeAdapter(tuple[AlertSummary, ...])
|
||||
_RECIPIENTS: Final = TypeAdapter(tuple[AlertRecipient, ...])
|
||||
|
||||
|
||||
class _LatestAlert(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: int
|
||||
|
||||
|
||||
_LATEST_ALERT: Final = TypeAdapter(tuple[_LatestAlert, ...])
|
||||
_HISTORY_SOURCES: Final[
|
||||
Mapping[AlertResourceType, tuple[LiteralString, LiteralString, LiteralString, LiteralString, LiteralString]]
|
||||
] = {
|
||||
|
|
@ -158,6 +171,45 @@ class AlertRepository:
|
|||
TableActions[PrismaRecord], prisma_client.writer_db.litellm_alert
|
||||
)
|
||||
|
||||
async def latest_id(self) -> int:
|
||||
async with self._database.tx(timeout=timedelta(seconds=5)) as transaction:
|
||||
await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)")
|
||||
rows: Final = _LATEST_ALERT.validate_python(
|
||||
await transaction.query_raw('SELECT COALESCE(MAX(id), 0) AS id FROM "LiteLLM_Alert"')
|
||||
)
|
||||
return rows[0].id
|
||||
|
||||
async def recipients(self, after: str) -> tuple[AlertRecipient, ...]:
|
||||
async with self._database.tx(timeout=timedelta(seconds=5)) as transaction:
|
||||
await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)")
|
||||
rows: Final = await transaction.query_raw(
|
||||
"""SELECT user_id, LOWER(TRIM(user_email)) AS email FROM "LiteLLM_UserTable"
|
||||
WHERE user_role = 'proxy_admin' AND user_id > $1 AND LENGTH(TRIM(user_email)) BETWEEN 3 AND 320
|
||||
ORDER BY user_id LIMIT 100""",
|
||||
after,
|
||||
)
|
||||
return _RECIPIENTS.validate_python(rows)
|
||||
|
||||
async def summaries(self, after: int, through: int) -> tuple[AlertSummary, ...]:
|
||||
async with self._database.tx(timeout=timedelta(seconds=5)) as transaction:
|
||||
await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)")
|
||||
rows: Final = await transaction.query_raw(
|
||||
"""SELECT a.id::text, a.type, a.resource_type, a.created_at AT TIME ZONE 'UTC' AS created_at,
|
||||
LEFT(COALESCE(k.key_alias, t.team_alias, u.user_alias,
|
||||
CASE WHEN a.resource_type = 'model' THEN a.resource_id ELSE a.resource_type END), 120) AS resource_name,
|
||||
jsonb_build_object('value', a.evidence->'value', 'baseline', a.evidence->'baseline',
|
||||
'unit', a.evidence->'unit', 'sample_size', a.evidence->'sample_size') AS evidence
|
||||
FROM "LiteLLM_Alert" a
|
||||
LEFT JOIN "LiteLLM_VerificationToken" k ON a.resource_type = 'key' AND k.token = a.resource_id
|
||||
LEFT JOIN "LiteLLM_TeamTable" t ON a.resource_type = 'team' AND t.team_id = a.resource_id
|
||||
LEFT JOIN "LiteLLM_UserTable" u ON a.resource_type = 'user' AND u.user_id = a.resource_id
|
||||
WHERE a.id > $1 AND a.id <= $2 AND a.status = 'open' AND a.type != 'model_deprecation'
|
||||
ORDER BY a.id DESC LIMIT 5""",
|
||||
after,
|
||||
through,
|
||||
)
|
||||
return _SUMMARIES.validate_python(rows)
|
||||
|
||||
async def record(self, detection: AlertDetection) -> Alert:
|
||||
from prisma import Json
|
||||
from prisma.errors import UniqueViolationError
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ ROUTER_SETTINGS_PARAM: Final = "router_settings"
|
|||
_JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
_JSON_RULES: Final = TypeAdapter(list[JsonValue])
|
||||
_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_BOOL: Final = TypeAdapter(bool)
|
||||
|
||||
|
||||
def _stored_router_settings(param_value: object) -> Mapping[str, JsonValue]:
|
||||
|
|
@ -138,6 +139,13 @@ class ConfigRepository:
|
|||
)
|
||||
return ConfigParam(param_name=param_name, param_value=param_value)
|
||||
|
||||
async def model_deprecations_enabled(self) -> bool:
|
||||
record: Final = await self.get_param("model_deprecation_alerts_enabled")
|
||||
return True if record is None else _BOOL.validate_python(record.param_value, strict=True)
|
||||
|
||||
async def set_model_deprecations_enabled(self, enabled: bool) -> None:
|
||||
await self.set_param("model_deprecation_alerts_enabled", enabled)
|
||||
|
||||
async def set_param_if_revision(self, param_name: str, param_value: object, revision: int) -> bool:
|
||||
rows: Final = await self._writer.query_raw(
|
||||
"""INSERT INTO "LiteLLM_Config" (param_name, param_value, last_run_at)
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm.models.performance_alert import (
|
|||
)
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
from litellm.repositories.alert_repository import record_alerts
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
|
@ -161,6 +162,7 @@ class PerformanceAlertRepository:
|
|||
)
|
||||
self._enterprise: Final = enterprise
|
||||
self._configured: Final = configured
|
||||
self._config: Final = ConfigRepository(prisma_client, use_writer=True)
|
||||
|
||||
async def _monitoring(self, transaction: _Transaction, *, lock: bool = False) -> AlertMonitoring:
|
||||
query: Final = 'SELECT enabled, alerts_used FROM "LiteLLM_PerformanceAlertState" WHERE id = 1'
|
||||
|
|
@ -190,7 +192,14 @@ class PerformanceAlertRepository:
|
|||
|
||||
async def monitoring(self) -> AlertMonitoring:
|
||||
async with self._transaction() as transaction:
|
||||
return await self._monitoring(transaction)
|
||||
monitoring: Final = await self._monitoring(transaction)
|
||||
return monitoring.model_copy(
|
||||
update={"model_deprecations_enabled": await self._config.model_deprecations_enabled()}
|
||||
)
|
||||
|
||||
async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring:
|
||||
await self._config.set_model_deprecations_enabled(enabled)
|
||||
return await self.monitoring()
|
||||
|
||||
async def set_enabled(self, enabled: bool) -> AlertMonitoring:
|
||||
async with self._database.tx(timeout=timedelta(seconds=10)) as transaction:
|
||||
|
|
@ -203,7 +212,7 @@ class PerformanceAlertRepository:
|
|||
updated_at = now() AT TIME ZONE 'UTC' WHERE id = 1""",
|
||||
enabled,
|
||||
)
|
||||
return await self._monitoring(transaction)
|
||||
return await self.monitoring()
|
||||
|
||||
@asynccontextmanager
|
||||
async def _transaction(self) -> AsyncGenerator[_Transaction]:
|
||||
|
|
|
|||
|
|
@ -188,6 +188,7 @@ class AlertType(str, Enum):
|
|||
cooldown_deployment = "cooldown_deployment"
|
||||
new_model_added = "new_model_added"
|
||||
model_deprecation_warnings = "model_deprecation_warnings"
|
||||
performance_alerts = "performance_alerts"
|
||||
|
||||
# Outage alerts
|
||||
outage_alerts = "outage_alerts"
|
||||
|
|
|
|||
|
|
@ -27,7 +27,13 @@ const calls: { path: string; query: URLSearchParams; method: string; body: unkno
|
|||
let status = "open";
|
||||
let rejectWrite = false;
|
||||
let rejectHistory = false;
|
||||
let monitoring = { enabled: true, alerts_used: 0, alert_limit: 5 as number | null, disabled_by_config: false };
|
||||
let monitoring = {
|
||||
model_deprecations_enabled: true,
|
||||
enabled: true,
|
||||
alerts_used: 0,
|
||||
alert_limit: 5 as number | null,
|
||||
disabled_by_config: false,
|
||||
};
|
||||
let rejectMonitoring = false;
|
||||
function response(data: unknown, code = 200) {
|
||||
return new Response(JSON.stringify(data), { status: code, headers: { "Content-Type": "application/json" } });
|
||||
|
|
@ -41,7 +47,13 @@ describe("Alerts", () => {
|
|||
rejectWrite = false;
|
||||
rejectHistory = false;
|
||||
rejectMonitoring = false;
|
||||
monitoring = { enabled: true, alerts_used: 0, alert_limit: 5, disabled_by_config: false };
|
||||
monitoring = {
|
||||
model_deprecations_enabled: true,
|
||||
enabled: true,
|
||||
alerts_used: 0,
|
||||
alert_limit: 5,
|
||||
disabled_by_config: false,
|
||||
};
|
||||
role = "proxy_admin";
|
||||
document.cookie = `token=${btoa(JSON.stringify({ alg: "none" }))}.${btoa(JSON.stringify({ key: "example-session", exp: 4102444800 }))}.example; path=/`;
|
||||
vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
|
||||
|
|
@ -57,7 +69,7 @@ describe("Alerts", () => {
|
|||
if (rejectMonitoring) return response({ detail: "Unavailable" }, 503);
|
||||
if (request.method === "PATCH") {
|
||||
if (rejectWrite) return response({ detail: "Unavailable" }, 503);
|
||||
monitoring = { ...monitoring, enabled: body.enabled };
|
||||
monitoring = { ...monitoring, ...body };
|
||||
}
|
||||
return response(monitoring);
|
||||
}
|
||||
|
|
@ -144,7 +156,7 @@ describe("Alerts", () => {
|
|||
expect(await screen.findByText("No resolved alerts.")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps deprecation email setup accessible from the inbox", async () => {
|
||||
it("keeps alert email setup accessible from the inbox", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
await user.click(await screen.findByRole("button", { name: "Set up email reminders" }));
|
||||
|
|
@ -161,12 +173,36 @@ describe("Alerts", () => {
|
|||
expect(screen.queryByRole("button", { name: "Dismiss" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Mark resolved" })).not.toBeInTheDocument();
|
||||
});
|
||||
it("keeps notification opt-in separate from the model deprecation toggle", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
expect(
|
||||
await screen.findByText(
|
||||
"Model deprecation alerts are unlimited. Performance monitoring includes 5 free alerts, then requires Enterprise.",
|
||||
),
|
||||
).toBeInTheDocument();
|
||||
expect(screen.queryByRole("switch")).not.toBeInTheDocument();
|
||||
await user.click(screen.getByRole("button", { name: "Manage alerts" }));
|
||||
const toggle = await screen.findByRole("switch", { name: "Model deprecations" });
|
||||
expect(toggle).toBeChecked();
|
||||
await user.click(toggle);
|
||||
await waitFor(() => expect(toggle).not.toBeChecked());
|
||||
expect(calls.filter((call) => call.method === "PATCH")).toEqual([
|
||||
expect.objectContaining({ path: "/alerts/monitoring", body: { model_deprecations_enabled: false } }),
|
||||
]);
|
||||
expect(screen.getByRole("switch", { name: "Performance monitoring" })).toBeChecked();
|
||||
expect(screen.getByText(/Opt in to Performance Alerts in email or Slack settings/)).toBeInTheDocument();
|
||||
await user.keyboard("{Escape}");
|
||||
expect(await screen.findByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("defaults on and persists opting out through the monitoring API", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
|
||||
const toggle = await screen.findByRole("switch", { name: "Performance monitoring" });
|
||||
expect(toggle).toBeChecked();
|
||||
expect(screen.getByText("0 of 5 free alerts used.")).toBeInTheDocument();
|
||||
expect(screen.getByText("0 of 5 free alerts generated.")).toBeInTheDocument();
|
||||
await user.click(toggle);
|
||||
expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument();
|
||||
expect(toggle).not.toBeChecked();
|
||||
|
|
@ -174,15 +210,15 @@ describe("Alerts", () => {
|
|||
path: "/alerts/monitoring",
|
||||
body: { enabled: false },
|
||||
});
|
||||
expect(screen.getByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument();
|
||||
await user.click(toggle);
|
||||
expect(await screen.findByText("0 of 5 free alerts used.")).toBeInTheDocument();
|
||||
expect(await screen.findByText("0 of 5 free alerts generated.")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("explains exhaustion without blocking saved alerts or model deprecations", async () => {
|
||||
monitoring.alerts_used = 5;
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
|
||||
expect(
|
||||
await screen.findByText("You’ve used all 5 free alerts. New performance alerts are paused."),
|
||||
).toBeInTheDocument();
|
||||
|
|
@ -190,8 +226,6 @@ describe("Alerts", () => {
|
|||
"href",
|
||||
"https://www.litellm.ai/enterprise",
|
||||
);
|
||||
expect(screen.getByRole("button", { name: "Mark resolved" })).toBeEnabled();
|
||||
expect(screen.getByRole("button", { name: "Set up email reminders" })).toBeInTheDocument();
|
||||
await user.click(screen.getByRole("switch", { name: "Performance monitoring" }));
|
||||
expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument();
|
||||
|
|
@ -200,6 +234,7 @@ describe("Alerts", () => {
|
|||
it("keeps the saved toggle value when an older refresh finishes later", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
|
||||
const toggle = await screen.findByRole("switch", { name: "Performance monitoring" });
|
||||
const originalFetch = vi.mocked(globalThis.fetch).getMockImplementation()!;
|
||||
const staleResponse = response(monitoring);
|
||||
|
|
@ -211,7 +246,9 @@ describe("Alerts", () => {
|
|||
}
|
||||
return originalFetch(input, init);
|
||||
});
|
||||
await user.click(screen.getByRole("button", { name: "Refresh" }));
|
||||
await act(async () => {
|
||||
void testQueryClient.invalidateQueries({ queryKey: ["alert-monitoring"] });
|
||||
});
|
||||
await user.click(toggle);
|
||||
expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument();
|
||||
await act(async () => {
|
||||
|
|
@ -227,6 +264,7 @@ describe("Alerts", () => {
|
|||
monitoring.alert_limit = null;
|
||||
monitoring.alerts_used = 5;
|
||||
renderWithProviders(<Alerts />);
|
||||
await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
|
||||
expect(await screen.findByText("On with Enterprise.")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
|
@ -236,22 +274,26 @@ describe("Alerts", () => {
|
|||
monitoring.alerts_used = 5;
|
||||
monitoring.enabled = false;
|
||||
renderWithProviders(<Alerts />);
|
||||
await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
|
||||
expect(await screen.findByText("Disabled in proxy settings.")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("switch", { name: "Performance monitoring" })).toHaveAttribute("aria-disabled", "true");
|
||||
});
|
||||
|
||||
it("lets viewers see monitoring status without a toggle", async () => {
|
||||
it("lets viewers inspect alert settings without changing them", async () => {
|
||||
role = "proxy_admin_viewer";
|
||||
renderWithProviders(<Alerts />);
|
||||
expect(await screen.findByText("0 of 5 free alerts used.")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("switch")).not.toBeInTheDocument();
|
||||
await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
|
||||
expect(await screen.findByText("0 of 5 free alerts generated.")).toBeInTheDocument();
|
||||
expect(screen.getByRole("switch", { name: "Performance monitoring" })).toHaveAttribute("aria-disabled", "true");
|
||||
expect(screen.getByRole("switch", { name: "Model deprecations" })).toHaveAttribute("aria-disabled", "true");
|
||||
});
|
||||
|
||||
it("keeps the saved preference when a monitoring change fails", async () => {
|
||||
rejectWrite = true;
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
|
||||
await user.click(await screen.findByRole("switch", { name: "Performance monitoring" }));
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("Could not save monitoring settings");
|
||||
expect(screen.getByRole("switch", { name: "Performance monitoring" })).toBeChecked();
|
||||
|
|
@ -261,10 +303,12 @@ describe("Alerts", () => {
|
|||
rejectMonitoring = true;
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("Could not load monitoring settings");
|
||||
expect(screen.getByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument();
|
||||
await userEvent.click(await screen.findByRole("button", { name: "Manage alerts" }));
|
||||
expect(await within(screen.getByRole("dialog")).findByRole("alert")).toHaveTextContent(
|
||||
"Could not load monitoring settings",
|
||||
);
|
||||
rejectMonitoring = false;
|
||||
await user.click(screen.getByRole("button", { name: "Retry" }));
|
||||
await user.click(within(screen.getByRole("dialog")).getByRole("button", { name: "Retry" }));
|
||||
expect(await screen.findByRole("switch", { name: "Performance monitoring" })).toBeChecked();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import { ALERT_TYPES, AlertFilter, AlertItem, TYPE_LABEL_KEYS, presentAlert } fr
|
|||
import AlertActions from "./AlertActions";
|
||||
import AlertChart from "./AlertChart";
|
||||
import AlertDetail from "./AlertDetail";
|
||||
import PerformanceMonitoring from "./PerformanceMonitoring";
|
||||
import ManageAlerts from "./ManageAlerts";
|
||||
|
||||
const parsers = {
|
||||
alert: parseAsString,
|
||||
|
|
@ -123,8 +123,7 @@ function AlertsInbox({ auth }: { auth: ReturnType<typeof useAuthorized> }) {
|
|||
}
|
||||
return (
|
||||
<>
|
||||
<h1 className="text-2xl font-semibold">{t("alerts.unified.title")}</h1>
|
||||
<PerformanceMonitoring scope={scope} readOnly={Boolean(auth.isViewOnly)} />
|
||||
<ManageAlerts scope={scope} readOnly={Boolean(auth.isViewOnly)} />
|
||||
<PageTabs
|
||||
value={params.state}
|
||||
onValueChange={(value) => void setParams({ state: value as AlertStatus })}
|
||||
|
|
@ -205,7 +204,7 @@ function AlertsInbox({ auth }: { auth: ReturnType<typeof useAuthorized> }) {
|
|||
{list.isFetchingNextPage ? t("alerts.unified.loadingMore") : t("alerts.unified.loadMore")}
|
||||
</Button>
|
||||
)}
|
||||
{showDeprecations && <DeprecationEmailSetup />}
|
||||
<DeprecationEmailSetup />
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,150 @@
|
|||
import { Settings } from "lucide-react";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
DialogTrigger,
|
||||
} from "@/components/ui/dialog";
|
||||
import { uiHref } from "@/utils/uiHref";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { backgroundFetchClient } from "@/lib/http/api";
|
||||
import { getProxyBaseUrl } from "@/components/networking";
|
||||
import type { components } from "@/lib/http/schema";
|
||||
|
||||
function statusMessage(data: components["schemas"]["AlertMonitoring"]) {
|
||||
if (data.disabled_by_config) return "alerts.monitoring.disabledByConfig";
|
||||
if (!data.enabled) return "alerts.monitoring.off";
|
||||
if (data.alert_limit === null) return "alerts.monitoring.enterprise";
|
||||
if (data.alerts_used >= data.alert_limit) return "alerts.monitoring.exhausted";
|
||||
return "alerts.monitoring.allowance";
|
||||
}
|
||||
|
||||
export default function ManageAlerts({ scope, readOnly }: { scope: string; readOnly: boolean }) {
|
||||
const { t } = useTranslation();
|
||||
const client = useQueryClient();
|
||||
const queryKey = ["alert-monitoring", getProxyBaseUrl(), scope];
|
||||
const queryOptions = {
|
||||
queryKey,
|
||||
queryFn: async ({ signal }: { signal: AbortSignal }) => {
|
||||
const { data } = await backgroundFetchClient.GET("/alerts/monitoring", { signal });
|
||||
if (!data) throw new Error("Could not load monitoring settings");
|
||||
return data;
|
||||
},
|
||||
staleTime: 60_000,
|
||||
retry: false,
|
||||
};
|
||||
const query = useQuery(queryOptions);
|
||||
const mutation = useMutation({
|
||||
mutationFn: async (
|
||||
body: components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"],
|
||||
) => {
|
||||
const { data } = await backgroundFetchClient.PATCH("/alerts/monitoring", { body });
|
||||
if (!data) throw new Error("Could not save monitoring settings");
|
||||
return data;
|
||||
},
|
||||
onSuccess: async (data) => {
|
||||
await client.cancelQueries({ queryKey });
|
||||
client.setQueryData(queryKey, data);
|
||||
await client.invalidateQueries({ queryKey: ["deprecationAlerts"] });
|
||||
},
|
||||
});
|
||||
const data = query.data;
|
||||
const messageKey = data ? statusMessage(data) : undefined;
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<h1 className="text-2xl font-semibold">{t("alerts.unified.title")}</h1>
|
||||
<Dialog>
|
||||
<DialogTrigger render={<Button variant="outline" size="sm" />}>
|
||||
<Settings className="size-4" aria-hidden />
|
||||
{t("alerts.monitoring.manage")}
|
||||
</DialogTrigger>
|
||||
<DialogContent>
|
||||
<DialogHeader>
|
||||
<DialogTitle>{t("alerts.monitoring.manage")}</DialogTitle>
|
||||
<DialogDescription>{t("alerts.monitoring.description")}</DialogDescription>
|
||||
</DialogHeader>
|
||||
{query.isError && (
|
||||
<div role="alert" className="space-y-2 text-sm">
|
||||
<p>{t("alerts.monitoring.loadError")}</p>
|
||||
<Button variant="outline" size="sm" onClick={() => void query.refetch()}>
|
||||
{t("alerts.panel.retry")}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
{!query.isError && !data && (
|
||||
<p role="status" className="text-sm text-muted-foreground">
|
||||
{t("alerts.monitoring.loading")}
|
||||
</p>
|
||||
)}
|
||||
{!query.isError && data && (
|
||||
<div className="divide-y divide-border">
|
||||
<div className="flex items-start justify-between gap-6 py-4">
|
||||
<div className="space-y-1">
|
||||
<label htmlFor="model-deprecations-monitoring" className="text-sm font-medium">
|
||||
{t("alerts.monitoring.deprecations")}
|
||||
</label>
|
||||
<p className="text-sm text-muted-foreground">{t("alerts.monitoring.deprecationsDescription")}</p>
|
||||
</div>
|
||||
<Switch
|
||||
id="model-deprecations-monitoring"
|
||||
checked={data.model_deprecations_enabled}
|
||||
disabled={readOnly || mutation.isPending}
|
||||
onCheckedChange={(enabled) => mutation.mutate({ model_deprecations_enabled: enabled })}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-start justify-between gap-6 py-4">
|
||||
<div className="space-y-1">
|
||||
<label htmlFor="performance-monitoring" className="text-sm font-medium">
|
||||
{t("alerts.monitoring.title")}
|
||||
</label>
|
||||
<p id="performance-monitoring-status" className="text-sm text-muted-foreground" role="status">
|
||||
{messageKey && t(messageKey, { used: data.alerts_used, limit: data.alert_limit })}
|
||||
{messageKey === "alerts.monitoring.exhausted" && (
|
||||
<a
|
||||
className="ml-2 text-foreground underline underline-offset-4"
|
||||
href="https://www.litellm.ai/enterprise"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
{t("alerts.monitoring.upgrade")}
|
||||
</a>
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
<Switch
|
||||
id="performance-monitoring"
|
||||
checked={data.enabled}
|
||||
disabled={readOnly || mutation.isPending || data.disabled_by_config}
|
||||
onCheckedChange={(enabled) => mutation.mutate({ enabled })}
|
||||
aria-describedby="performance-monitoring-status"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{mutation.isError && (
|
||||
<p role="alert" className="text-sm text-destructive">
|
||||
{t("alerts.monitoring.saveError")}
|
||||
</p>
|
||||
)}
|
||||
{readOnly && <p className="text-sm text-muted-foreground">{t("alerts.monitoring.adminOnly")}</p>}
|
||||
<p className="text-sm text-muted-foreground">{t("alerts.monitoring.notifications")}</p>
|
||||
<a href={uiHref("logging-and-alerts")} className="text-sm underline underline-offset-4">
|
||||
{t("alerts.monitoring.connections")}
|
||||
</a>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</div>
|
||||
{data && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{t(data.alert_limit === null ? "alerts.monitoring.enterpriseNote" : "alerts.monitoring.freeNote")}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { backgroundFetchClient } from "@/lib/http/api";
|
||||
import { getProxyBaseUrl } from "@/components/networking";
|
||||
import type { components } from "@/lib/http/schema";
|
||||
|
||||
function statusMessage(data: components["schemas"]["AlertMonitoring"]) {
|
||||
if (data.disabled_by_config) return "alerts.monitoring.disabledByConfig";
|
||||
if (!data.enabled) return "alerts.monitoring.off";
|
||||
if (data.alert_limit === null) return "alerts.monitoring.enterprise";
|
||||
if (data.alerts_used >= data.alert_limit) return "alerts.monitoring.exhausted";
|
||||
return "alerts.monitoring.allowance";
|
||||
}
|
||||
|
||||
export default function PerformanceMonitoring({ scope, readOnly }: { scope: string; readOnly: boolean }) {
|
||||
const { t } = useTranslation();
|
||||
const client = useQueryClient();
|
||||
const queryKey = ["alert-monitoring", getProxyBaseUrl(), scope];
|
||||
const queryOptions = {
|
||||
queryKey,
|
||||
queryFn: async ({ signal }: { signal: AbortSignal }) => {
|
||||
const { data } = await backgroundFetchClient.GET("/alerts/monitoring", { signal });
|
||||
if (!data) throw new Error("Could not load monitoring settings");
|
||||
return data;
|
||||
},
|
||||
staleTime: 60_000,
|
||||
retry: false,
|
||||
};
|
||||
const query = useQuery(queryOptions);
|
||||
const mutation = useMutation({
|
||||
mutationFn: async (enabled: boolean) => {
|
||||
const { data } = await backgroundFetchClient.PATCH("/alerts/monitoring", { body: { enabled } });
|
||||
if (!data) throw new Error("Could not save monitoring settings");
|
||||
return data;
|
||||
},
|
||||
onSuccess: async (data) => {
|
||||
await client.cancelQueries({ queryKey });
|
||||
client.setQueryData(queryKey, data);
|
||||
},
|
||||
});
|
||||
if (query.isError)
|
||||
return (
|
||||
<div role="alert" className="flex flex-wrap items-center gap-2 text-sm">
|
||||
<p>{t("alerts.monitoring.loadError")}</p>
|
||||
<Button variant="outline" size="sm" onClick={() => void query.refetch()}>
|
||||
{t("alerts.panel.retry")}
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
if (!query.data)
|
||||
return (
|
||||
<p role="status" className="text-sm text-muted-foreground">
|
||||
{t("alerts.monitoring.loading")}
|
||||
</p>
|
||||
);
|
||||
const data = query.data;
|
||||
const messageKey = statusMessage(data);
|
||||
const message = t(messageKey, { used: data.alerts_used, limit: data.alert_limit });
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<div className="space-y-2 text-sm">
|
||||
<div className="flex items-center gap-3">
|
||||
<label htmlFor="performance-monitoring" className="font-medium">
|
||||
{t("alerts.monitoring.title")}
|
||||
</label>
|
||||
{!readOnly && (
|
||||
<Switch
|
||||
id="performance-monitoring"
|
||||
checked={data.enabled}
|
||||
disabled={mutation.isPending || data.disabled_by_config}
|
||||
onCheckedChange={(enabled) => mutation.mutate(enabled)}
|
||||
aria-describedby="performance-monitoring-status"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
<p id="performance-monitoring-status" className="text-muted-foreground" role="status">
|
||||
{message}
|
||||
{messageKey === "alerts.monitoring.exhausted" && (
|
||||
<a
|
||||
className="ml-2 text-foreground underline underline-offset-4"
|
||||
href="https://www.litellm.ai/enterprise"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
{t("alerts.monitoring.upgrade")}
|
||||
</a>
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
{mutation.isError && (
|
||||
<p role="alert" className="text-sm text-destructive">
|
||||
{t("alerts.monitoring.saveError")}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -45,6 +45,16 @@ beforeEach(() => {
|
|||
if (url.endsWith("/management/v1/users/me")) {
|
||||
return Response.json({ user_id: "example-admin", user_role: role });
|
||||
}
|
||||
if (url.endsWith("/alerts/monitoring")) {
|
||||
const monitoring = {
|
||||
enabled: true,
|
||||
model_deprecations_enabled: true,
|
||||
alerts_used: 0,
|
||||
alert_limit: 5,
|
||||
disabled_by_config: false,
|
||||
};
|
||||
return Response.json(monitoring);
|
||||
}
|
||||
if (url.endsWith("/model/deprecations?warn_within_days=30")) {
|
||||
if (rejectDeprecations) return Response.json({ detail: "Unavailable" }, { status: 503 });
|
||||
return Response.json({ deprecated: [], imminent: [] });
|
||||
|
|
|
|||
|
|
@ -13,11 +13,11 @@ describe("deprecation email setup", () => {
|
|||
const user = userEvent.setup();
|
||||
render(<DeprecationEmailSetup />);
|
||||
await user.click(screen.getByRole("button", { name: "Set up email reminders" }));
|
||||
expect(await screen.findByRole("dialog", { name: "Set up model deprecation emails" })).toBeInTheDocument();
|
||||
expect(await screen.findByRole("dialog", { name: "Set up alert emails" })).toBeInTheDocument();
|
||||
expect(screen.getByText("Connect email delivery")).toBeInTheDocument();
|
||||
expect(screen.getByText("Enable model deprecation emails")).toBeInTheDocument();
|
||||
expect(screen.getByText("Owners get two reminders")).toBeInTheDocument();
|
||||
expect(screen.getByText(/This is off by default/)).toBeInTheDocument();
|
||||
expect(screen.getByText("Choose your email alerts")).toBeInTheDocument();
|
||||
expect(screen.getByText("Know what gets sent")).toBeInTheDocument();
|
||||
expect(screen.getByText(/Both are off by default/)).toBeInTheDocument();
|
||||
expect(screen.getByRole("link", { name: "Open Email Settings" })).toHaveAttribute(
|
||||
"href",
|
||||
"/ui/logging-and-alerts?tab=email-alerts",
|
||||
|
|
|
|||
|
|
@ -19,6 +19,15 @@ describe("deprecation alert API wiring", () => {
|
|||
vi.clearAllMocks();
|
||||
testQueryClient.clear();
|
||||
});
|
||||
it("stops loading usage and acknowledgements when model deprecations are disabled", async () => {
|
||||
vi.mocked(useCurrentUser).mockReturnValue({ data: { user_id: "admin", user_role: "proxy_admin" } } as ReturnType<
|
||||
typeof useCurrentUser
|
||||
>);
|
||||
vi.mocked(backgroundFetchClient.GET).mockResolvedValue({ data: { model_deprecations_enabled: false } } as never);
|
||||
renderWithProviders(<AlertsHost />);
|
||||
expect(await screen.findByText("0 alerts")).toBeInTheDocument();
|
||||
expect(backgroundFetchClient.GET).toHaveBeenCalledExactlyOnceWith("/alerts/monitoring", expect.anything());
|
||||
});
|
||||
it("should avoid querying gateway-wide usage for non-admins", () => {
|
||||
vi.mocked(useCurrentUser).mockReturnValue({
|
||||
data: { user_id: "internal", user_role: "internal_user" },
|
||||
|
|
@ -36,7 +45,7 @@ describe("deprecation alert API wiring", () => {
|
|||
} as never);
|
||||
renderWithProviders(<AlertsHost />);
|
||||
expect(await screen.findByText("0 alerts")).toBeInTheDocument();
|
||||
expect(backgroundFetchClient.GET).toHaveBeenCalledTimes(1);
|
||||
expect(backgroundFetchClient.GET).toHaveBeenCalledTimes(2);
|
||||
expect(backgroundFetchClient.GET).toHaveBeenCalledWith(
|
||||
"/model/deprecations",
|
||||
expect.objectContaining({ params: { query: { warn_within_days: 30 } } }),
|
||||
|
|
@ -59,6 +68,7 @@ describe("deprecation alert API wiring", () => {
|
|||
],
|
||||
};
|
||||
vi.mocked(backgroundFetchClient.GET)
|
||||
.mockResolvedValueOnce({ data: { model_deprecations_enabled: true } } as never)
|
||||
.mockResolvedValueOnce({ data: schedule } as never)
|
||||
.mockResolvedValueOnce({ data: activity } as never)
|
||||
.mockResolvedValueOnce({ data: { status: "open" } } as never);
|
||||
|
|
|
|||
|
|
@ -17,6 +17,9 @@ export function useDeprecationAlerts(accessToken: string | null) {
|
|||
queryKey: ["deprecationAlerts", scope, range],
|
||||
enabled,
|
||||
queryFn: async ({ signal }: { signal: AbortSignal }) => {
|
||||
const { data: monitoring } = await backgroundFetchClient.GET("/alerts/monitoring", { signal });
|
||||
if (!monitoring) throw new Error("Alert settings are unavailable");
|
||||
if (monitoring.model_deprecations_enabled === false) return [];
|
||||
const { data: schedule } = await backgroundFetchClient.GET("/model/deprecations", {
|
||||
params: { query: { warn_within_days: 30 } },
|
||||
signal,
|
||||
|
|
|
|||
|
|
@ -74,6 +74,9 @@ const EmailEventSettings: React.FC<EmailEventSettingsProps> = ({ accessToken })
|
|||
|
||||
// Helper function to get a description for each event type
|
||||
const getEventDescription = (event: EmailEvent): string => {
|
||||
if (event === "Performance Alerts") {
|
||||
return t("loggingAndAlerts.emailEvents.descriptions.performanceAlerts");
|
||||
}
|
||||
if (event === "Model Deprecations") {
|
||||
return t("loggingAndAlerts.emailEvents.descriptions.modelDeprecations");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -421,6 +421,7 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
|
|||
outage_alerts: "loggingAndAlerts.alertTypes.labels.outageAlerts",
|
||||
region_outage_alerts: "loggingAndAlerts.alertTypes.labels.regionOutageAlerts",
|
||||
model_deprecation_warnings: "loggingAndAlerts.alertTypes.labels.modelDeprecationWarnings",
|
||||
performance_alerts: "loggingAndAlerts.alertTypes.labels.performanceAlerts",
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
|
|
|
|||
|
|
@ -5938,18 +5938,18 @@
|
|||
"description": "Add your SMTP settings and public gateway URL in Email Settings. Send a test email to verify delivery."
|
||||
},
|
||||
"enable": {
|
||||
"title": "Enable model deprecation emails",
|
||||
"description": "In Email events, turn on Model Deprecations and save. This is off by default."
|
||||
"title": "Choose your email alerts",
|
||||
"description": "In Email events, enable Model Deprecations, Performance Alerts, or both, then save. Both are off by default."
|
||||
},
|
||||
"reminders": {
|
||||
"title": "Owners get two reminders",
|
||||
"description": "Affected key owners and team admins receive a reminder 30 days before retirement, and one final reminder 1 day before. Recipients are based on actual usage in the last 30 days. Dates are checked in UTC."
|
||||
"title": "Know what gets sent",
|
||||
"description": "Model deprecations notify affected key owners and team admins 30 days and 1 day before retirement. Performance alerts send gateway admins one daily summary when there are new findings, with up to five alerts and a link to the rest."
|
||||
}
|
||||
},
|
||||
"summary": "Give affected owners time to migrate. Email reminders at 30 days and 1 day.",
|
||||
"summary": "Get model retirement reminders and daily performance summaries by email.",
|
||||
"trigger": "Set up email reminders",
|
||||
"title": "Set up model deprecation emails",
|
||||
"description": "Help key owners and team admins act before a provider retires a model. Only their affected usage is included.",
|
||||
"title": "Set up alert emails",
|
||||
"description": "Choose which alerts to send through your existing email connection.",
|
||||
"openSettings": "Open Email Settings"
|
||||
},
|
||||
"migrationDetail": {
|
||||
|
|
@ -6017,9 +6017,18 @@
|
|||
"more": "+{{hidden}} more"
|
||||
},
|
||||
"monitoring": {
|
||||
"manage": "Manage alerts",
|
||||
"description": "These settings apply to everyone using this gateway.",
|
||||
"deprecations": "Model deprecations",
|
||||
"deprecationsDescription": "Show retirement alerts and send configured email and Slack reminders.",
|
||||
"adminOnly": "Only gateway admins can change these settings.",
|
||||
"notifications": "Opt in to Performance Alerts in email or Slack settings to receive a daily summary of new findings. Email summaries go to gateway admins.",
|
||||
"connections": "Manage email and Slack connections",
|
||||
"freeNote": "Model deprecation alerts are unlimited. Performance monitoring includes 5 free alerts, then requires Enterprise.",
|
||||
"enterpriseNote": "Unlimited model deprecation and performance alerts with Enterprise.",
|
||||
"title": "Performance monitoring",
|
||||
"loading": "Loading monitoring settings…",
|
||||
"allowance": "{{used}} of {{limit}} free alerts used.",
|
||||
"allowance": "{{used}} of {{limit}} free alerts generated.",
|
||||
"enterprise": "On with Enterprise.",
|
||||
"off": "Off. Saved alerts remain available.",
|
||||
"exhausted": "You’ve used all 5 free alerts. New performance alerts are paused.",
|
||||
|
|
@ -6518,7 +6527,8 @@
|
|||
"dailyReports": "Weekly/Monthly Spend Reports",
|
||||
"outageAlerts": "Outage Alerts",
|
||||
"regionOutageAlerts": "Region Outage Alerts",
|
||||
"modelDeprecationWarnings": "Model Deprecation Warnings"
|
||||
"modelDeprecationWarnings": "Model Deprecation Warnings",
|
||||
"performanceAlerts": "Performance Alerts"
|
||||
},
|
||||
"enterpriseFeature": "✨ Enterprise Feature",
|
||||
"saveChanges": "Save Changes",
|
||||
|
|
@ -6625,6 +6635,7 @@
|
|||
"description": "Select which events should trigger email notifications.",
|
||||
"descriptions": {
|
||||
"modelDeprecations": "Email affected key owners and team admins 30 days and 1 day before model deprecation. Uses requests from the last 30 days. Off by default.",
|
||||
"performanceAlerts": "Email gateway admins a daily summary when there are new performance alerts. Shows up to five alerts with a link to see all.",
|
||||
"virtualKeyCreated": "An email will be sent to the user when a new virtual key is created with their user ID",
|
||||
"newUserInvitation": "An email will be sent to the email address of the user when a new user is created",
|
||||
"generic": "Receive an email notification when {{event}}"
|
||||
|
|
|
|||
24
litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
24
litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -1030,7 +1030,7 @@ export interface paths {
|
|||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* Read performance monitoring settings
|
||||
* Read alert monitoring settings
|
||||
* @description Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin.
|
||||
*/
|
||||
get: operations["get_monitoring_alerts_monitoring_get"];
|
||||
|
|
@ -1040,7 +1040,7 @@ export interface paths {
|
|||
options?: never;
|
||||
head?: never;
|
||||
/**
|
||||
* Enable or disable performance monitoring
|
||||
* Enable or disable alert monitoring
|
||||
* @description Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts.
|
||||
*/
|
||||
patch: operations["update_monitoring_alerts_monitoring_patch"];
|
||||
|
|
@ -27339,6 +27339,12 @@ export interface components {
|
|||
* @description Whether performance monitoring is enabled. Defaults to true.
|
||||
*/
|
||||
enabled: boolean;
|
||||
/**
|
||||
* Model Deprecations Enabled
|
||||
* @description Whether model deprecation alerts are enabled.
|
||||
* @default true
|
||||
*/
|
||||
model_deprecations_enabled: boolean;
|
||||
};
|
||||
/** AlertMonitoringUpdate */
|
||||
AlertMonitoringUpdate: {
|
||||
|
|
@ -27353,7 +27359,7 @@ export interface components {
|
|||
* @description Enum for alert types and management event types
|
||||
* @enum {string}
|
||||
*/
|
||||
AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted" | "mcp_tool_description_blocked" | "mcp_pinned_tools_changed";
|
||||
AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "performance_alerts" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted" | "mcp_tool_description_blocked" | "mcp_pinned_tools_changed";
|
||||
/** AlertUpdate */
|
||||
AlertUpdate: {
|
||||
/**
|
||||
|
|
@ -33628,7 +33634,7 @@ export interface components {
|
|||
* EmailEvent
|
||||
* @enum {string}
|
||||
*/
|
||||
EmailEvent: "Model Deprecations" | "Virtual Key Created" | "New User Invitation" | "Virtual Key Rotated" | "Soft Budget Crossed" | "Max Budget Alert";
|
||||
EmailEvent: "Model Deprecations" | "Performance Alerts" | "Virtual Key Created" | "New User Invitation" | "Virtual Key Rotated" | "Soft Budget Crossed" | "Max Budget Alert";
|
||||
/** EmailEventSettings */
|
||||
EmailEventSettings: {
|
||||
/** Enabled */
|
||||
|
|
@ -40207,6 +40213,14 @@ export interface components {
|
|||
*/
|
||||
team_public_model_name?: string | null;
|
||||
};
|
||||
/** ModelDeprecationMonitoringUpdate */
|
||||
ModelDeprecationMonitoringUpdate: {
|
||||
/**
|
||||
* Model Deprecations Enabled
|
||||
* @description Enable or disable model deprecation alerts.
|
||||
*/
|
||||
model_deprecations_enabled: boolean;
|
||||
};
|
||||
/** ModelDeprecationResponse */
|
||||
ModelDeprecationResponse: {
|
||||
/**
|
||||
|
|
@ -55078,7 +55092,7 @@ export interface operations {
|
|||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["AlertMonitoringUpdate"];
|
||||
"application/json": components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Final
|
||||
|
|
@ -22,6 +23,7 @@ from litellm.models.alert import (
|
|||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.repositories.alert_repository import AlertRepository
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
from tests.integration._support.client import Gateway
|
||||
from tests.integration._support.database import read_rows, write_rows
|
||||
|
||||
|
|
@ -41,6 +43,7 @@ def saved_page(content: bytes) -> AlertPage:
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class Alerts:
|
||||
repository: AlertRepository
|
||||
config: ConfigRepository
|
||||
resource_id: str
|
||||
|
||||
def detection(self, kind: AlertType = "spend", day: int = 0) -> AlertDetection:
|
||||
|
|
@ -75,10 +78,12 @@ async def alerts() -> AsyncIterator[Alerts]:
|
|||
client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache()))
|
||||
await client.connect()
|
||||
try:
|
||||
yield Alerts(AlertRepository(client), resource_id)
|
||||
yield Alerts(AlertRepository(client), ConfigRepository(client, use_writer=True), resource_id)
|
||||
finally:
|
||||
try:
|
||||
write_rows('DELETE FROM "LiteLLM_Alert" WHERE resource_id=%s', (resource_id,))
|
||||
receipt: Final = "performance_alert_digest:" + hashlib.sha256(f"test:{resource_id}".encode()).hexdigest()
|
||||
write_rows('DELETE FROM "LiteLLM_Config" WHERE param_name=%s', (receipt,))
|
||||
finally:
|
||||
await client.disconnect()
|
||||
|
||||
|
|
@ -372,3 +377,118 @@ async def test_deleted_resource_preserves_history_and_history_timeout_preserves_
|
|||
finally:
|
||||
write_rows('DELETE FROM "LiteLLM_TeamTable" WHERE team_id=%s', (team,))
|
||||
write_rows('DELETE FROM "LiteLLM_DailyTeamSpend" WHERE team_id=%s', (team,))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daily_performance_summary_deduplicates_workers_and_keeps_failed_deliveries(alerts: Alerts) -> None:
|
||||
from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier, slack_text
|
||||
from litellm.models.alert import AlertSummary
|
||||
|
||||
destination: Final = f"test:{alerts.resource_id}"
|
||||
notifier: Final = PerformanceAlertNotifier(alerts.config, alerts.repository)
|
||||
first: Final = await alerts.repository.record(alerts.detection())
|
||||
now: Final = first.created_at + timedelta(minutes=1)
|
||||
delivered: Final[list[str]] = [] # mutable-ok: captures the external delivery boundary
|
||||
|
||||
async def send(rows: Sequence[AlertSummary], receipt: str) -> bool:
|
||||
delivered.append(slack_text(rows, "https://gateway.example"))
|
||||
return True
|
||||
|
||||
async def unavailable(rows: Sequence[AlertSummary], receipt: str) -> bool:
|
||||
return False
|
||||
|
||||
await notifier.deliver(destination, send, now)
|
||||
assert len(delivered) == 1
|
||||
for day in range(1, 8):
|
||||
await alerts.repository.record(alerts.detection(day=day))
|
||||
restarted: Final = PerformanceAlertNotifier(alerts.config, alerts.repository)
|
||||
await restarted.deliver(destination, send, now + timedelta(hours=23))
|
||||
assert len(delivered) == 1
|
||||
tomorrow: Final = now + timedelta(days=1)
|
||||
await restarted.deliver(destination, unavailable, tomorrow)
|
||||
assert len(delivered) == 1
|
||||
await restarted.deliver(destination, send, tomorrow + timedelta(minutes=3))
|
||||
assert len(delivered) == 1
|
||||
retry: Final = tomorrow + timedelta(days=1, minutes=3)
|
||||
await asyncio.gather(
|
||||
notifier.deliver(destination, send, retry),
|
||||
restarted.deliver(destination, send, retry),
|
||||
)
|
||||
assert len(delivered) == 2
|
||||
assert delivered[-1].count("?alert=") == 5
|
||||
assert f"?alert={first.id}|" not in delivered[-1]
|
||||
await restarted.deliver(destination, send, retry + timedelta(days=1, minutes=1))
|
||||
assert len(delivered) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_summary_does_not_replay_old_alerts(alerts: Alerts) -> None:
|
||||
from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier
|
||||
from litellm.models.alert import AlertSummary
|
||||
|
||||
saved: Final = await alerts.repository.record(alerts.detection())
|
||||
delivered: Final[list[tuple[AlertSummary, ...]]] = [] # mutable-ok: captures the external delivery boundary
|
||||
|
||||
async def send(rows: Sequence[AlertSummary], receipt: str) -> bool:
|
||||
delivered.append(tuple(rows))
|
||||
return True
|
||||
|
||||
notifier: Final = PerformanceAlertNotifier(alerts.config, alerts.repository)
|
||||
now: Final = saved.created_at + timedelta(days=2)
|
||||
await notifier.deliver(f"test:{alerts.resource_id}", send, now)
|
||||
assert delivered == []
|
||||
recent: Final = await alerts.repository.record(alerts.detection(day=1))
|
||||
await notifier.deliver(f"test:{alerts.resource_id}", send, now + timedelta(minutes=5))
|
||||
assert tuple(row.id for row in delivered[0]) == (recent.id,)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("crash", (False, True))
|
||||
async def test_failed_first_summary_does_not_replay_old_alerts_on_retry(alerts: Alerts, crash: bool) -> None:
|
||||
from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier
|
||||
from litellm.models.alert import AlertSummary
|
||||
|
||||
old: Final = await alerts.repository.record(alerts.detection())
|
||||
recent: Final = await alerts.repository.record(alerts.detection(day=1))
|
||||
now: Final = recent.created_at + timedelta(days=2)
|
||||
write_rows('UPDATE "LiteLLM_Alert" SET created_at=%s WHERE id=%s', (now, int(recent.id)))
|
||||
write_rows('UPDATE "LiteLLM_Alert" SET created_at=%s WHERE id=%s', (now - timedelta(days=2), int(old.id)))
|
||||
delivered: Final[list[tuple[str, ...]]] = [] # mutable-ok: captures the external delivery boundary
|
||||
|
||||
async def unavailable(rows: Sequence[AlertSummary], receipt: str) -> bool:
|
||||
if crash:
|
||||
raise RuntimeError("Connection lost after sending")
|
||||
return False
|
||||
|
||||
async def send(rows: Sequence[AlertSummary], receipt: str) -> bool:
|
||||
delivered.append(tuple(row.id for row in rows))
|
||||
return True
|
||||
|
||||
notifier: Final = PerformanceAlertNotifier(alerts.config, alerts.repository)
|
||||
if crash:
|
||||
with pytest.raises(RuntimeError, match="Connection lost after sending"):
|
||||
await notifier.deliver(f"test:{alerts.resource_id}", unavailable, now)
|
||||
else:
|
||||
await notifier.deliver(f"test:{alerts.resource_id}", unavailable, now)
|
||||
await notifier.deliver(f"test:{alerts.resource_id}", send, now + timedelta(minutes=3))
|
||||
assert delivered == []
|
||||
await notifier.deliver(f"test:{alerts.resource_id}", send, now + timedelta(days=1, minutes=3))
|
||||
assert delivered == [(recent.id,)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("enabled", (False, True))
|
||||
async def test_model_deprecation_preference_persists_without_changing_performance(
|
||||
alerts: Alerts, gateway: Gateway, enabled: bool
|
||||
) -> None:
|
||||
previous: Final = gateway.request("GET", "/alerts/monitoring")
|
||||
assert previous.status_code == 200, previous.text
|
||||
before: Final = previous.json()
|
||||
try:
|
||||
updated: Final = gateway.request("PATCH", "/alerts/monitoring", {"model_deprecations_enabled": enabled})
|
||||
assert updated.status_code == 200, updated.text
|
||||
assert updated.json() == {**before, "model_deprecations_enabled": enabled}
|
||||
refreshed: Final = gateway.request("GET", "/alerts/monitoring")
|
||||
assert refreshed.json() == updated.json()
|
||||
finally:
|
||||
await alerts.config.set_model_deprecations_enabled(before["model_deprecations_enabled"])
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
|
||||
import asyncio
|
||||
from itertools import chain, repeat
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm.constants import SLACK_MODEL_DEPRECATION_LOCK_ID
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
|
|
@ -53,6 +53,21 @@ async def test_should_skip_when_no_alerting_configured():
|
|||
assert sent is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gateway_opt_out_stops_deprecation_checks() -> None:
|
||||
async def disabled() -> bool:
|
||||
return False
|
||||
|
||||
alerting: Final = SlackAlerting(
|
||||
alerting=["slack"],
|
||||
alert_types=[AlertType.model_deprecation_warnings],
|
||||
model_deprecations_enabled=disabled,
|
||||
)
|
||||
router: Final = MagicMock()
|
||||
assert not await alerting.send_model_deprecation_alert(llm_router=router)
|
||||
router.registry.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_skip_when_no_deprecations_found(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "model_cost", {})
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import asyncio
|
|||
from dataclasses import replace
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -46,6 +47,9 @@ class MemoryConfig:
|
|||
def __init__(self):
|
||||
self.rows = {"general_settings": {"email_settings": {"Model Deprecations": True}}}
|
||||
|
||||
async def model_deprecations_enabled(self):
|
||||
return self.rows.get("model_deprecation_alerts_enabled", True)
|
||||
|
||||
async def get_param(self, key):
|
||||
return ConfigParam(key, self.rows[key]) if key in self.rows else None
|
||||
|
||||
|
|
@ -63,6 +67,8 @@ class MemoryConfig:
|
|||
@pytest.fixture()
|
||||
def sender(monkeypatch):
|
||||
monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example")
|
||||
monkeypatch.setenv("SMTP_HOST", "smtp.example.com")
|
||||
monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com")
|
||||
instance = module.ModelDeprecationEmailSender(MagicMock())
|
||||
instance.config = MemoryConfig()
|
||||
instance._impacts = AsyncMock(
|
||||
|
|
@ -296,3 +302,32 @@ async def test_should_keep_the_email_loop_running_after_a_delivery_error(monkeyp
|
|||
await module.run_scheduled_model_deprecation_emails()
|
||||
assert runner.run.await_count == 2
|
||||
assert all(call.args == (router,) for call in runner.run.await_args_list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gateway_opt_out_stops_owner_usage_scans_and_email(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("SMTP_HOST", "smtp.example.com")
|
||||
monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com")
|
||||
client: Final = MagicMock()
|
||||
client.writer_db.litellm_config.find_unique = AsyncMock(return_value=MagicMock(param_value=False))
|
||||
client.db.litellm_dailyuserspend.group_by = AsyncMock()
|
||||
instance: Final = module.ModelDeprecationEmailSender(client)
|
||||
assert await instance.run(MagicMock(), now=NOW) == 0
|
||||
client.db.litellm_dailyuserspend.group_by.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("missing", ("SMTP_HOST", "SMTP_SENDER_EMAIL"))
|
||||
async def test_missing_email_connection_does_not_scan_owner_usage(
|
||||
monkeypatch: pytest.MonkeyPatch, missing: str
|
||||
) -> None:
|
||||
monkeypatch.setenv("SMTP_HOST", "smtp.example.com")
|
||||
monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com")
|
||||
monkeypatch.delenv(missing)
|
||||
client: Final = MagicMock()
|
||||
client.writer_db.litellm_config.find_unique = AsyncMock()
|
||||
client.db.litellm_dailyuserspend.group_by = AsyncMock()
|
||||
instance: Final = module.ModelDeprecationEmailSender(client)
|
||||
assert await instance.run(MagicMock(), now=NOW) == 0
|
||||
client.writer_db.litellm_config.find_unique.assert_not_awaited()
|
||||
client.db.litellm_dailyuserspend.group_by.assert_not_awaited()
|
||||
|
|
|
|||
213
tests/unit/integrations/test_performance_alert_notifications.py
Normal file
213
tests/unit/integrations/test_performance_alert_notifications.py
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
import smtplib
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier, email_body, slack_text
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.models.alert import AlertSummary, MetricEvidence
|
||||
from litellm.repositories.alert_repository import AlertRepository
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
from litellm.types.integrations.slack_alerting import AlertType
|
||||
|
||||
BASE_URL: Final = "https://gateway.example"
|
||||
WEBHOOK: Final = "https://hooks.slack.example/alerts"
|
||||
_TEXT: Final = TypeAdapter(dict[str, str])
|
||||
|
||||
|
||||
def summary(index: int) -> AlertSummary:
|
||||
return AlertSummary(
|
||||
id=str(index),
|
||||
type="spend",
|
||||
resource_type="team",
|
||||
resource_name=f"Research {index} <team>",
|
||||
created_at=datetime(2026, 1, 9, tzinfo=timezone.utc),
|
||||
evidence=MetricEvidence(value=1200, baseline=200, unit="usd", sample_size=2000),
|
||||
)
|
||||
|
||||
|
||||
def test_daily_messages_include_five_findings_and_escape_resource_names() -> None:
|
||||
alerts: Final = tuple(summary(index) for index in range(1, 7))
|
||||
body, text = email_body(alerts, BASE_URL)
|
||||
slack: Final = slack_text(alerts, BASE_URL)
|
||||
assert body.count("<li>") == 5
|
||||
assert slack.count("?alert=") == 5
|
||||
assert text.count("?alert=") == 5
|
||||
assert "$1,200.00 vs $200.00" in text
|
||||
assert "Research 6" not in body + text + slack
|
||||
assert "Research 1 <team>" in body and "Research 1 <team>" in slack
|
||||
assert f'<a href="{BASE_URL}/ui/alerts">See all alerts</a>' in body
|
||||
assert f"<{BASE_URL}/ui/alerts|See all alerts>" in slack
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kind", "metric", "expected"),
|
||||
(
|
||||
("reliability", MetricEvidence(value=0.25, baseline=0.02, unit="ratio", sample_size=2000), "25.0% vs 2.0%"),
|
||||
(
|
||||
"latency",
|
||||
MetricEvidence(value=4000, baseline=1500, unit="milliseconds", sample_size=2000),
|
||||
"4,000 ms vs 1,500 ms",
|
||||
),
|
||||
(
|
||||
"traffic",
|
||||
MetricEvidence(value=12000, baseline=2000, unit="requests", sample_size=2000),
|
||||
"12,000 requests vs 2,000 requests",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_daily_messages_preserve_metric_units(
|
||||
kind: Literal["reliability", "latency", "traffic"], metric: MetricEvidence, expected: str
|
||||
) -> None:
|
||||
alert: Final = summary(1).model_copy(update={"type": kind, "evidence": metric})
|
||||
body, text = email_body((alert,), BASE_URL)
|
||||
assert expected in body and expected in text
|
||||
assert expected in slack_text((alert,), BASE_URL)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connected_channels_do_not_subscribe_to_performance_summaries(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("PROXY_BASE_URL", BASE_URL)
|
||||
monkeypatch.setenv("SMTP_HOST", "smtp.example.com")
|
||||
monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com")
|
||||
client: Final = MagicMock()
|
||||
client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
|
||||
client.writer_db.tx = MagicMock()
|
||||
slack: Final = SlackAlerting(alerting=["slack"], default_webhook_url=WEBHOOK)
|
||||
notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client))
|
||||
await notifier.run(slack)
|
||||
assert slack.performance_alert_webhooks() == ()
|
||||
client.writer_db.tx.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_slack_summary_reuses_the_configured_connection_and_reports_delivery() -> None:
|
||||
posted: Final = MagicMock(return_value=httpx.Response(200, text="ok"))
|
||||
handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(posted))
|
||||
try:
|
||||
slack: Final = SlackAlerting(
|
||||
alerting=["slack"],
|
||||
alert_types=[AlertType.performance_alerts],
|
||||
default_webhook_url=WEBHOOK,
|
||||
async_http_handler=handler,
|
||||
)
|
||||
assert slack.performance_alert_webhooks() == (WEBHOOK,)
|
||||
message: Final = slack_text((summary(1),), BASE_URL)
|
||||
assert await slack.send_performance_alert_summary(WEBHOOK, message)
|
||||
request: Final = posted.call_args.args[0]
|
||||
assert str(request.url) == WEBHOOK
|
||||
assert _TEXT.validate_json(request.content) == {"text": message}
|
||||
finally:
|
||||
await handler.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_opted_in_summary_reaches_slack_from_the_notification_job(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("PROXY_BASE_URL", BASE_URL)
|
||||
monkeypatch.delenv("SMTP_HOST", raising=False)
|
||||
alert: Final = summary(1)
|
||||
client: Final = MagicMock()
|
||||
client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
|
||||
client.writer_db.query_raw = AsyncMock(return_value=[{"param_name": "receipt"}])
|
||||
client.writer_db.tx.return_value.__aenter__.return_value.query_raw = AsyncMock(
|
||||
side_effect=((), ({"id": 1},), (), (alert.model_dump(mode="json"),))
|
||||
)
|
||||
posted: Final = MagicMock(return_value=httpx.Response(200, text="ok"))
|
||||
handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(posted))
|
||||
try:
|
||||
slack: Final = SlackAlerting(
|
||||
alerting=["slack"],
|
||||
alert_types=[AlertType.performance_alerts],
|
||||
default_webhook_url=WEBHOOK,
|
||||
async_http_handler=handler,
|
||||
)
|
||||
notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client))
|
||||
await notifier.run(slack, now=alert.created_at)
|
||||
posted.assert_called_once()
|
||||
assert _TEXT.validate_json(posted.call_args.args[0].content) == {"text": slack_text((alert,), BASE_URL)}
|
||||
finally:
|
||||
await handler.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", (httpx.ReadTimeout("Webhook timed out"), httpx.Response(503)))
|
||||
async def test_unavailable_webhook_does_not_block_other_destinations(
|
||||
monkeypatch: pytest.MonkeyPatch, failure: Exception | httpx.Response
|
||||
) -> None:
|
||||
monkeypatch.setenv("PROXY_BASE_URL", BASE_URL)
|
||||
monkeypatch.delenv("SMTP_HOST", raising=False)
|
||||
alert: Final = summary(1)
|
||||
client: Final = MagicMock()
|
||||
client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
|
||||
client.writer_db.query_raw = AsyncMock(return_value=[{"param_name": "receipt"}])
|
||||
client.writer_db.tx.return_value.__aenter__.return_value.query_raw = AsyncMock(
|
||||
side_effect=((), ({"id": 1},), (), (alert.model_dump(mode="json"),)) * 2
|
||||
)
|
||||
posted: Final = MagicMock(side_effect=(failure, httpx.Response(200)))
|
||||
handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(posted))
|
||||
try:
|
||||
slack: Final = SlackAlerting(
|
||||
alerting=["slack"],
|
||||
alert_types=[AlertType.performance_alerts],
|
||||
alert_to_webhook_url={AlertType.performance_alerts: [WEBHOOK, WEBHOOK + "/second", WEBHOOK]},
|
||||
async_http_handler=handler,
|
||||
)
|
||||
notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client))
|
||||
await notifier.run(slack, now=alert.created_at)
|
||||
assert tuple(str(call.args[0].url) for call in posted.call_args_list) == (WEBHOOK, WEBHOOK + "/second")
|
||||
assert _TEXT.validate_json(posted.call_args.args[0].content) == {"text": slack_text((alert,), BASE_URL)}
|
||||
finally:
|
||||
await handler.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_opted_in_email_reaches_recipients_across_pages(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("PROXY_BASE_URL", BASE_URL)
|
||||
monkeypatch.setenv("SMTP_HOST", "smtp.example.com")
|
||||
monkeypatch.setenv("SMTP_PORT", "25")
|
||||
monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com")
|
||||
monkeypatch.setenv("SMTP_TLS", "False")
|
||||
monkeypatch.setenv("SMTP_USE_SSL", "False")
|
||||
monkeypatch.delenv("SMTP_USERNAME", raising=False)
|
||||
monkeypatch.delenv("SMTP_PASSWORD", raising=False)
|
||||
smtp: Final = MagicMock()
|
||||
monkeypatch.setattr(smtplib, "SMTP", MagicMock(return_value=smtp))
|
||||
alert: Final = summary(1)
|
||||
client: Final = MagicMock()
|
||||
client.writer_db.litellm_config.find_unique = AsyncMock(
|
||||
side_effect=(MagicMock(param_value={"email_settings": {"Performance Alerts": True}}), None, None)
|
||||
)
|
||||
client.writer_db.query_raw = AsyncMock(return_value=[{"param_name": "receipt"}])
|
||||
client.writer_db.tx.return_value.__aenter__.return_value.query_raw = AsyncMock(
|
||||
side_effect=(
|
||||
(),
|
||||
({"user_id": "a", "email": "first@example.com"},),
|
||||
(),
|
||||
({"id": 1},),
|
||||
(),
|
||||
(alert.model_dump(mode="json"),),
|
||||
(),
|
||||
({"user_id": "b", "email": "second@example.com"},),
|
||||
(),
|
||||
({"id": 1},),
|
||||
(),
|
||||
(alert.model_dump(mode="json"),),
|
||||
(),
|
||||
(),
|
||||
)
|
||||
)
|
||||
notifier: Final = PerformanceAlertNotifier(ConfigRepository(client, use_writer=True), AlertRepository(client))
|
||||
await notifier.run(None, now=alert.created_at)
|
||||
sends: Final = smtp.__enter__.return_value.send_message.call_args_list
|
||||
assert tuple(call.kwargs["to_addrs"] for call in sends) == ("first@example.com", "second@example.com")
|
||||
for call in sends:
|
||||
message: Final = call.kwargs["msg"]
|
||||
assert message["Subject"] == "New performance alerts"
|
||||
assert f"{BASE_URL}/ui/alerts?alert=1" in message.get_payload(0).get_payload(decode=True).decode()
|
||||
assert message["Message-ID"]
|
||||
assert sends[0].kwargs["msg"]["Message-ID"] != sends[1].kwargs["msg"]["Message-ID"]
|
||||
|
|
@ -55,6 +55,7 @@ def app(table: MagicMock, history_query: AsyncMock) -> FastAPI:
|
|||
app.include_router(router)
|
||||
prisma: Final = MagicMock()
|
||||
prisma.writer_db.litellm_alert = table
|
||||
prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
|
||||
prisma.writer_db.tx.return_value.__aenter__.return_value.query_raw = history_query
|
||||
app.dependency_overrides[get_database] = lambda: Database(prisma, native_database())
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
|
|
@ -123,8 +124,38 @@ def test_monitoring_only_accepts_a_boolean_opt_out(
|
|||
history_query.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enabled", (False, True))
|
||||
def test_deprecation_preference_updates_without_changing_performance(app: FastAPI, enabled: bool) -> None:
|
||||
prisma: Final = MagicMock()
|
||||
prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=MagicMock(param_value=enabled))
|
||||
prisma.writer_db.litellm_config.upsert = AsyncMock()
|
||||
transaction: Final = prisma.writer_db.tx.return_value.__aenter__.return_value
|
||||
transaction.query_raw = AsyncMock(side_effect=((), ({"enabled": False, "alerts_used": 3},)))
|
||||
transaction.execute_raw = AsyncMock()
|
||||
app.dependency_overrides[get_alert_monitoring_repository] = lambda: PerformanceAlertRepository(prisma)
|
||||
with TestClient(app) as client:
|
||||
response: Final = client.patch("/alerts/monitoring", json={"model_deprecations_enabled": enabled})
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {
|
||||
"enabled": False,
|
||||
"model_deprecations_enabled": enabled,
|
||||
"alerts_used": 3,
|
||||
"alert_limit": 5,
|
||||
"disabled_by_config": False,
|
||||
}
|
||||
prisma.writer_db.litellm_config.upsert.assert_awaited_once_with(
|
||||
where={"param_name": "model_deprecation_alerts_enabled"},
|
||||
data={
|
||||
"create": {"param_name": "model_deprecation_alerts_enabled", "param_value": str(enabled).lower()},
|
||||
"update": {"param_value": str(enabled).lower()},
|
||||
},
|
||||
)
|
||||
transaction.execute_raw.assert_not_awaited()
|
||||
|
||||
|
||||
def test_config_opt_out_cannot_be_enabled_from_the_api(app: FastAPI) -> None:
|
||||
prisma: Final = MagicMock()
|
||||
prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
|
||||
transaction: Final = prisma.writer_db.tx.return_value.__aenter__.return_value
|
||||
transaction.query_raw = AsyncMock(return_value=[{"enabled": True, "alerts_used": 0}])
|
||||
transaction.execute_raw = AsyncMock()
|
||||
|
|
@ -134,7 +165,13 @@ def test_config_opt_out_cannot_be_enabled_from_the_api(app: FastAPI) -> None:
|
|||
with TestClient(app) as client:
|
||||
status: Final = client.get("/alerts/monitoring")
|
||||
response: Final = client.patch("/alerts/monitoring", json={"enabled": True})
|
||||
assert status.json() == {"enabled": False, "alerts_used": 0, "alert_limit": 5, "disabled_by_config": True}
|
||||
assert status.json() == {
|
||||
"enabled": False,
|
||||
"alerts_used": 0,
|
||||
"alert_limit": 5,
|
||||
"disabled_by_config": True,
|
||||
"model_deprecations_enabled": True,
|
||||
}
|
||||
assert response.status_code == 409, response.text
|
||||
transaction.execute_raw.assert_not_awaited()
|
||||
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ def _repository(
|
|||
transaction: Final = MagicMock(query_raw=query, execute_raw=execute)
|
||||
context: Final = MagicMock(__aenter__=AsyncMock(return_value=transaction), __aexit__=AsyncMock(return_value=False))
|
||||
client: Final = MagicMock(writer_db=MagicMock(tx=MagicMock(return_value=context)))
|
||||
client.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
|
||||
return PerformanceAlertRepository(client, enterprise=enterprise), query, execute
|
||||
|
||||
|
||||
|
|
@ -191,7 +192,12 @@ async def test_first_read_initializes_the_allowance_once(enterprise: bool) -> No
|
|||
@pytest.mark.parametrize("enabled", (False, True))
|
||||
async def test_admin_toggle_persists_the_requested_value(enabled: bool) -> None:
|
||||
repository, query, execute = _repository(())
|
||||
query.side_effect = ((), ({"enabled": not enabled, "alerts_used": 2},), ({"enabled": enabled, "alerts_used": 2},))
|
||||
query.side_effect = (
|
||||
(),
|
||||
({"enabled": not enabled, "alerts_used": 2},),
|
||||
(),
|
||||
({"enabled": enabled, "alerts_used": 2},),
|
||||
)
|
||||
result: Final = await repository.set_enabled(enabled)
|
||||
assert result.enabled is enabled
|
||||
assert result.active is enabled
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue