mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-07 07:58:09 +00:00
feat: email affected owners before model deprecations (#50172)
* feat: opt-in model deprecation emails for affected owners * fix: annotate deprecation sender initialization * fix: clarify deprecation emails and use consistent settings toggles * fix: exclude former team admins from deprecation emails * feat: clarify provider deprecations and model migration steps
This commit is contained in:
parent
3f4dc93b40
commit
da2ecbc5bb
22 changed files with 1006 additions and 26 deletions
|
|
@ -1,9 +1,9 @@
|
|||
import enum
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from litellm.proxy._types import WebhookEvent
|
||||
from litellm.types.integrations.model_deprecation_email import MODEL_DEPRECATION_EMAIL_EVENT
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
||||
|
|
@ -26,7 +26,7 @@ class SendKeyCreatedEmailEvent(WebhookEvent):
|
|||
|
||||
class SendKeyRotatedEmailEvent(WebhookEvent):
|
||||
virtual_key: str
|
||||
key_alias: Optional[str] = None
|
||||
key_alias: str | None = None
|
||||
"""
|
||||
The virtual key that was rotated
|
||||
this will be sk-123xxx, since we will be emailing this to the user to start using the new key
|
||||
|
|
@ -34,23 +34,33 @@ class SendKeyRotatedEmailEvent(WebhookEvent):
|
|||
|
||||
|
||||
class EmailEvent(str, enum.Enum):
|
||||
model_deprecations = MODEL_DEPRECATION_EMAIL_EVENT
|
||||
virtual_key_created = "Virtual Key Created"
|
||||
new_user_invitation = "New User Invitation"
|
||||
virtual_key_rotated = "Virtual Key Rotated"
|
||||
soft_budget_crossed = "Soft Budget Crossed"
|
||||
max_budget_alert = "Max Budget Alert"
|
||||
|
||||
|
||||
class EmailEventSettings(LiteLLMBaseModel):
|
||||
event: EmailEvent
|
||||
enabled: bool
|
||||
|
||||
|
||||
class EmailEventSettingsUpdateRequest(LiteLLMBaseModel):
|
||||
settings: List[EmailEventSettings]
|
||||
settings: list[EmailEventSettings]
|
||||
|
||||
|
||||
class EmailEventSettingsResponse(LiteLLMBaseModel):
|
||||
settings: List[EmailEventSettings]
|
||||
settings: list[EmailEventSettings]
|
||||
|
||||
|
||||
class DefaultEmailSettings(LiteLLMBaseModel):
|
||||
"""Default settings for email events"""
|
||||
settings: Dict[EmailEvent, bool] = Field(
|
||||
|
||||
settings: dict[EmailEvent, bool] = Field(
|
||||
default_factory=lambda: {
|
||||
EmailEvent.model_deprecations: False, # Explicit opt-in for owner retirement reminders.
|
||||
EmailEvent.virtual_key_created: True, # On by default
|
||||
EmailEvent.new_user_invitation: True, # On by default
|
||||
EmailEvent.virtual_key_rotated: True, # On by default
|
||||
|
|
@ -58,10 +68,12 @@ class DefaultEmailSettings(LiteLLMBaseModel):
|
|||
EmailEvent.max_budget_alert: True, # On by default
|
||||
}
|
||||
)
|
||||
def to_dict(self) -> Dict[str, bool]:
|
||||
|
||||
def to_dict(self) -> dict[str, bool]:
|
||||
"""Convert to dictionary with string keys for storage"""
|
||||
return {event.value: enabled for event, enabled in self.settings.items()}
|
||||
|
||||
@classmethod
|
||||
def get_defaults(cls) -> Dict[str, bool]:
|
||||
def get_defaults(cls) -> dict[str, bool]:
|
||||
"""Get the default settings as a dictionary with string keys"""
|
||||
return cls().to_dict()
|
||||
return cls().to_dict()
|
||||
|
|
|
|||
181
litellm/integrations/email_templates/model_deprecation_email.py
Normal file
181
litellm/integrations/email_templates/model_deprecation_email.py
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
"""Model deprecation emails with provider identity, a migration comparison and next steps."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from html import escape
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
|
||||
from litellm.types.integrations.model_deprecation_email import (
|
||||
DeprecationEmailMilestone,
|
||||
ModelDeprecationEmail,
|
||||
RenderedDeprecationEmail,
|
||||
)
|
||||
from litellm.types.proxy.model_deprecation import SuccessorModelSource
|
||||
|
||||
_LOGO_BASE_URL: Final = (
|
||||
"https://raw.githubusercontent.com/BerriAI/litellm/main/ui/litellm-dashboard/public/assets/logos"
|
||||
)
|
||||
_EMAIL_LOGO_URL: Final = f"{_LOGO_BASE_URL}/litellm_logo.png"
|
||||
# Reuse dashboard brand assets; PNGs also render in email clients that block SVG.
|
||||
_PROVIDER_BRANDS: Final[Mapping[str, tuple[str, str]]] = MappingProxyType(
|
||||
{
|
||||
"gemini": ("Google", "google_pse.png"),
|
||||
"vertex_ai": ("Google", "google_pse.png"),
|
||||
"openai": ("OpenAI", "openai.png"),
|
||||
"anthropic": ("Anthropic", "anthropic.png"),
|
||||
"azure": ("Microsoft Azure", "microsoft_azure.png"),
|
||||
"bedrock": ("AWS Bedrock", "aws.png"),
|
||||
"bedrock_converse": ("AWS Bedrock", "aws.png"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _recommendation_source(source: SuccessorModelSource | None) -> str:
|
||||
if source is SuccessorModelSource.DEPLOYMENT_CONFIG:
|
||||
return "Gateway configuration"
|
||||
if source is SuccessorModelSource.MODEL_COST_MAP:
|
||||
return "LiteLLM model cost map"
|
||||
return "Not provided"
|
||||
|
||||
|
||||
def _http_url(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
parsed: Final = urlsplit(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return value if parsed.scheme in ("http", "https") and parsed.netloc and not parsed.username else None
|
||||
|
||||
|
||||
def _provider(model: str, fallback: str | None = None) -> str:
|
||||
prefix, separator, _ = model.partition("/")
|
||||
return prefix if separator else fallback or ""
|
||||
|
||||
|
||||
def _provider_name(provider: str) -> str:
|
||||
brand: Final = _PROVIDER_BRANDS.get(provider)
|
||||
if brand:
|
||||
return brand[0]
|
||||
return provider.replace("_", " ") or "The provider"
|
||||
|
||||
|
||||
def _model_html(model: str, provider: str) -> str:
|
||||
brand: Final = _PROVIDER_BRANDS.get(provider)
|
||||
logo: Final = (
|
||||
f'<td width="38" valign="middle"><img src="{_LOGO_BASE_URL}/{brand[1]}" width="24" height="24" alt="{escape(brand[0], quote=True)}" style="display:block;object-fit:contain;border:0;"></td>'
|
||||
if brand
|
||||
else ""
|
||||
)
|
||||
return (
|
||||
f'<table role="presentation" width="100%" cellspacing="0" cellpadding="0"><tr>{logo}'
|
||||
f'<td><div style="font-size:11px;line-height:18px;color:#526079;">{escape(_provider_name(provider))}</div>'
|
||||
f"<div style=\"font:14px 'Courier New',monospace;line-height:22px;word-break:break-word;overflow-wrap:anywhere;color:#17213b;\">{escape(model)}</div></td></tr></table>"
|
||||
)
|
||||
|
||||
|
||||
def render_model_deprecation_email(email: ModelDeprecationEmail, *, base_url: str) -> RenderedDeprecationEmail:
|
||||
gateway: Final = _http_url(base_url)
|
||||
if gateway is None:
|
||||
raise ValueError("Model deprecation emails require a public HTTP(S) PROXY_BASE_URL")
|
||||
urgent: Final = email.milestone is DeprecationEmailMilestone.ONE_DAY
|
||||
days: Final = int(email.milestone)
|
||||
timing: Final = "tomorrow" if urgent else "in 30 days"
|
||||
provider: Final = _provider(email.usage_model, email.model.litellm_provider)
|
||||
provider_name: Final = _provider_name(provider)
|
||||
subject: Final = f"[LiteLLM] Model being deprecated by {provider_name} {timing}: {email.usage_model}"
|
||||
deadline: Final = email.model.deprecation_date.strftime("%d %b %Y")
|
||||
announcement: Final = f"{provider_name} is deprecating this model on {deadline} (UTC)."
|
||||
preheader: Final = f"{announcement} Review the model change and migrate your affected keys."
|
||||
query: Final = urlencode(
|
||||
{
|
||||
"tab": "models",
|
||||
"from": email.start_date.isoformat(),
|
||||
"to": email.end_date.isoformat(),
|
||||
}
|
||||
)
|
||||
review_url: Final = f"{gateway.rstrip('/')}/ui/usage?{query}"
|
||||
replacement: Final = email.model.successor_model
|
||||
after: Final = (
|
||||
_model_html(replacement, _provider(replacement, provider))
|
||||
if replacement
|
||||
else '<p style="margin:0;font-size:14px;line-height:22px;color:#17213b;">No recommended model listed. Choose a supported replacement.</p>'
|
||||
)
|
||||
source: Final = _recommendation_source(email.model.successor_model_source)
|
||||
notice: Final = _http_url(email.model.deprecation_source)
|
||||
source_link: Final = (
|
||||
f'<a href="{escape(notice, quote=True)}" style="color:#263cdb;text-decoration:underline;">Read the provider notice ↗</a>'
|
||||
if notice
|
||||
else ""
|
||||
)
|
||||
team_names: Final = ", ".join(email.team_names[:3]) or "Personal keys"
|
||||
more_teams: Final = f" + {len(email.team_names) - 3} more" if len(email.team_names) > 3 else ""
|
||||
day_label: Final = "DAY" if urgent else "DAYS"
|
||||
label: Final = "FINAL REMINDER" if urgent else "DEPRECATION NOTICE"
|
||||
test_step: Final = (
|
||||
"Test the recommended model with your workload."
|
||||
if replacement
|
||||
else "Choose a supported replacement and test it with your workload."
|
||||
)
|
||||
switch_step: Final = (
|
||||
f"Switch your app or gateway routing before {deadline}. Then check that requests to the old model reach zero."
|
||||
)
|
||||
html: Final = f"""<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>{escape(subject)}</title>
|
||||
<style>@import url('https://fonts.googleapis.com/css2?family=DM+Sans:wght@400;500;600;700&display=swap');
|
||||
@media only screen and (max-width:620px){{.pad{{padding:24px!important}}.headline{{font-size:30px!important;line-height:36px!important}}.brand{{padding:20px 24px!important}}.brand img{{width:96px!important}}.comparison-half{{display:block!important;width:auto!important;border-right:0!important}}.comparison-after{{border-top:1px solid #e2e6ee!important}}}}
|
||||
</style></head>
|
||||
<body style="margin:0;padding:0;background:#f7f8fa;color:#17213b;font-family:'DM Sans',Arial,sans-serif;">
|
||||
<div style="display:none;max-height:0;overflow:hidden;mso-hide:all;">{escape(preheader)}</div>
|
||||
<table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0"><tr><td style="padding:0;">
|
||||
<table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0" style="width:100%;background:#ffffff;">
|
||||
<tr><td class="brand" style="padding:24px 36px;border-bottom:1px solid #e2e6ee;">
|
||||
<table role="presentation" width="100%" cellspacing="0" cellpadding="0"><tr><td><img src="{_EMAIL_LOGO_URL}" width="112" alt="LiteLLM" style="display:block;border:0;height:auto;"></td>
|
||||
<td align="right" style="font:11px 'Courier New',monospace;letter-spacing:1.5px;color:#526079;">/ MODEL DEPRECATIONS</td></tr></table></td></tr>
|
||||
<tr><td class="pad" style="padding:32px 36px;background:#263cdb;color:#ffffff;">
|
||||
<table role="presentation" width="100%" cellspacing="0" cellpadding="0"><tr><td style="font:11px 'Courier New',monospace;letter-spacing:1.8px;color:#ffffff;">{label}</td>
|
||||
<td align="right" style="font:11px 'Courier New',monospace;color:#ffffff;">{days:02d} {day_label} UNTIL DEPRECATION</td></tr></table>
|
||||
<h1 class="headline" style="margin:22px 0 16px;font-size:36px;line-height:42px;font-weight:600;letter-spacing:-1px;color:#ffffff;">Model being deprecated</h1>
|
||||
<p style="margin:0;font-size:14px;line-height:22px;color:#ffffff;">{escape(announcement)}</p>
|
||||
</td></tr>
|
||||
<tr><td class="pad" style="padding:28px 36px;">
|
||||
<table role="presentation" width="100%" cellspacing="0" cellpadding="0" style="border:1px solid #e2e6ee;border-collapse:collapse;">
|
||||
<tr><td class="comparison-half" width="50%" valign="top" style="padding:16px 18px;border-right:1px solid #e2e6ee;">
|
||||
<div style="margin-bottom:8px;font:10px 'Courier New',monospace;letter-spacing:1px;color:#526079;">BEFORE · MODEL TO REPLACE</div>
|
||||
{_model_html(email.usage_model, provider)}
|
||||
</td>
|
||||
<td class="comparison-half comparison-after" width="50%" valign="top" style="padding:16px 18px;background:#f5f7ff;">
|
||||
<div style="margin-bottom:8px;font:10px 'Courier New',monospace;letter-spacing:1px;color:#263cdb;">AFTER · RECOMMENDED MODEL</div>
|
||||
{after}
|
||||
</td></tr></table>
|
||||
<p style="margin:10px 0 24px;font-size:12px;line-height:20px;color:#526079;">Recommendation source: {escape(source)}<br>{source_link}</p>
|
||||
<h2 style="margin:0 0 12px;font-size:17px;font-weight:600;color:#17213b;">Your affected usage</h2>
|
||||
<table role="presentation" width="100%" cellspacing="0" cellpadding="0" style="border-top:1px solid #e2e6ee;border-bottom:1px solid #e2e6ee;">
|
||||
<tr><td width="50%" style="padding:14px 0;"><div style="font:24px 'Courier New',monospace;color:#17213b;">{email.key_count:,}</div><div style="margin-top:5px;font-size:12px;color:#526079;">Affected keys</div></td>
|
||||
<td width="50%" style="padding:14px 0 14px 20px;border-left:1px solid #e2e6ee;"><div style="font:24px 'Courier New',monospace;color:#17213b;">{email.requests:,}</div><div style="margin-top:5px;font-size:12px;color:#526079;">Requests · last 30 days</div></td></tr></table>
|
||||
<p style="margin:10px 0 24px;font-size:12px;line-height:20px;color:#526079;">{escape(team_names + more_teams)}<br>Your keys and teams you administer.</p>
|
||||
<h2 style="margin:0 0 10px;font-size:17px;font-weight:600;color:#17213b;">What to do</h2>
|
||||
<ol style="margin:0 0 24px;padding-left:20px;font-size:13px;line-height:22px;color:#526079;">
|
||||
<li style="padding:0 0 8px 4px;">Review the affected keys in your usage dashboard.</li>
|
||||
<li style="padding:0 0 8px 4px;">{escape(test_step)}</li>
|
||||
<li style="padding-left:4px;">{escape(switch_step)}</li></ol>
|
||||
<table role="presentation" cellspacing="0" cellpadding="0"><tr><td bgcolor="#263cdb" style="border-radius:4px;mso-padding-alt:14px 20px;">
|
||||
<a href="{escape(review_url, quote=True)}" style="display:inline-block;padding:14px 20px;color:#ffffff;font-size:13px;font-weight:600;line-height:18px;text-decoration:none;">Review affected usage ↗</a></td></tr></table>
|
||||
</td></tr>
|
||||
<tr><td class="pad" style="padding:20px 36px;background:#fafbfc;border-top:1px solid #e2e6ee;font-size:11px;line-height:18px;color:#526079;">
|
||||
Your gateway admin enabled these emails. Reminder schedule: 30 days and 1 day before deprecation.<br><br>LiteLLM · AI Gateway</td></tr>
|
||||
</table></td></tr></table></body></html>"""
|
||||
text: Final = (
|
||||
f"{subject}\n\n{announcement}\n\n"
|
||||
f"BEFORE - MODEL TO REPLACE\n{email.usage_model}\n\n"
|
||||
f"AFTER - RECOMMENDED MODEL\n{replacement or 'No recommended model listed. Choose a supported replacement.'}\n"
|
||||
f"Recommendation source: {source}\nProvider notice: {notice or 'Not provided'}\n\n"
|
||||
f"YOUR AFFECTED USAGE\n{email.key_count} keys | {email.requests} requests in the last 30 days\n"
|
||||
f"{team_names}{more_teams}\nYour keys and teams you administer.\n\n"
|
||||
f"WHAT TO DO\n1. Review the affected keys in your usage dashboard.\n2. {test_step}\n3. {switch_step}\n\n"
|
||||
f"Review affected usage: {review_url}\n\n"
|
||||
"Your gateway admin enabled these emails. Reminder schedule: 30 days and 1 day before deprecation."
|
||||
)
|
||||
return RenderedDeprecationEmail(subject=subject, html=html, text=text)
|
||||
267
litellm/integrations/model_deprecation_email.py
Normal file
267
litellm/integrations/model_deprecation_email.py
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
"""Two owner-scoped retirement reminders, with durable per-recipient delivery receipts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from enum import Enum
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import AliasPath, BaseModel, ConfigDict, Field, TypeAdapter
|
||||
|
||||
from litellm.integrations.email_templates.model_deprecation_email import (
|
||||
render_model_deprecation_email,
|
||||
)
|
||||
from litellm.models.team import LiteLLM_TeamTable
|
||||
from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
from litellm.repositories.table_repositories import DailyUserSpendRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.verification_token_repository import (
|
||||
VerificationTokenRepository,
|
||||
)
|
||||
from litellm.types.integrations.model_deprecation_email import (
|
||||
MODEL_DEPRECATION_EMAIL_EVENT,
|
||||
DeprecationEmailMilestone,
|
||||
ModelDeprecationEmail,
|
||||
)
|
||||
from litellm.types.proxy.model_deprecation import (
|
||||
DEPRECATION_IDLE_POLL_SECONDS,
|
||||
ModelDeprecationInfo,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm import Router
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
_RECEIPT_PREFIX: Final = "model_deprecation_email:"
|
||||
_PAGE_SIZE: Final = 100
|
||||
|
||||
|
||||
class _OptIn(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
|
||||
enabled: bool = Field(
|
||||
False,
|
||||
strict=True,
|
||||
validation_alias=AliasPath("email_settings", MODEL_DEPRECATION_EMAIL_EVENT),
|
||||
)
|
||||
|
||||
|
||||
class _RequestCount(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
|
||||
api_requests: int
|
||||
|
||||
|
||||
class _UsedKey(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
|
||||
api_key: str
|
||||
metrics: _RequestCount = Field(alias="_sum")
|
||||
|
||||
|
||||
_USED_KEYS: Final = TypeAdapter(tuple[_UsedKey, ...])
|
||||
|
||||
|
||||
class _DeliveryStatus(str, Enum):
|
||||
SENDING = "sending"
|
||||
SENT = "sent"
|
||||
|
||||
|
||||
class _Receipt(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
|
||||
revision: int = 0
|
||||
status: _DeliveryStatus = _DeliveryStatus.SENDING
|
||||
lease_until: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Impact:
|
||||
key_count: int = 0
|
||||
requests: int = 0
|
||||
teams: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
def _admin_ids(team: LiteLLM_TeamTable | None) -> frozenset[str]:
|
||||
if team is None:
|
||||
return frozenset()
|
||||
return frozenset(
|
||||
member.user_id for member in (team.members_with_roles or ()) if member.role == "admin" and member.user_id
|
||||
)
|
||||
|
||||
|
||||
def _recipient_emails(
|
||||
user_id: str | None, team: LiteLLM_TeamTable | None, addresses: Mapping[str, str]
|
||||
) -> frozenset[str]:
|
||||
owner_ids: Final[frozenset[str]] = frozenset((user_id,)) if user_id else frozenset()
|
||||
resolved: Final = frozenset(addresses[owner] for owner in owner_ids | _admin_ids(team) if owner in addresses)
|
||||
if team is None:
|
||||
return resolved
|
||||
return resolved | frozenset(
|
||||
member.user_email.strip()
|
||||
for member in (team.members_with_roles or ())
|
||||
if member.role == "admin"
|
||||
and member.user_email
|
||||
and member.user_email.strip()
|
||||
and member.user_id not in addresses
|
||||
)
|
||||
|
||||
|
||||
def _usage_model(model: ModelDeprecationInfo) -> str:
|
||||
name: Final = model.litellm_model or model.model_name
|
||||
provider: Final = model.litellm_provider
|
||||
if provider and not name.startswith(f"{provider}/"):
|
||||
return f"{provider}/{name}"
|
||||
return name
|
||||
|
||||
|
||||
def _receipt_key(email: ModelDeprecationEmail) -> str:
|
||||
identity: Final = json.dumps(
|
||||
[
|
||||
email.usage_model,
|
||||
email.model.deprecation_date.isoformat(),
|
||||
int(email.milestone),
|
||||
email.recipient_email.casefold(),
|
||||
]
|
||||
)
|
||||
return _RECEIPT_PREFIX + hashlib.sha256(identity.encode()).hexdigest()
|
||||
|
||||
|
||||
class ModelDeprecationEmailSender:
|
||||
def __init__(self, prisma_client: PrismaClient) -> None:
|
||||
self.prisma: Final = prisma_client
|
||||
self.config: Final = ConfigRepository(prisma_client, use_writer=True)
|
||||
|
||||
async def _impacts(self, model: str, start: date, end: date) -> Mapping[str, _Impact]:
|
||||
impacts: Final[dict[str, _Impact]] = {} # mutable-ok: combines per-recipient counts across bounded usage pages
|
||||
offset: int = 0 # rebind-ok: each page advances the stable key grouping cursor
|
||||
while True:
|
||||
raw = await DailyUserSpendRepository(self.prisma).table.group_by(
|
||||
by=["api_key"],
|
||||
where={
|
||||
"model": model,
|
||||
"date": {"gte": start.isoformat(), "lte": end.isoformat()},
|
||||
"api_requests": {"gt": 0},
|
||||
},
|
||||
sum={"api_requests": True},
|
||||
order={"api_key": "asc"},
|
||||
skip=offset,
|
||||
take=_PAGE_SIZE,
|
||||
)
|
||||
usage = _USED_KEYS.validate_python(raw)
|
||||
if not usage:
|
||||
return MappingProxyType(impacts)
|
||||
keys = await VerificationTokenRepository(self.prisma).find_many_in("token", (row.api_key for row in usage))
|
||||
teams = await TeamRepository(self.prisma).find_many_in(
|
||||
"team_id", frozenset(key.team_id for key in keys if key.team_id)
|
||||
)
|
||||
team_by_id = {team.team_id: team for team in teams}
|
||||
ids = frozenset(key.user_id for key in keys if key.user_id) | frozenset(
|
||||
chain.from_iterable(_admin_ids(team) for team in teams)
|
||||
)
|
||||
users = await UserRepository(self.prisma).find_many_in("user_id", ids)
|
||||
addresses = {
|
||||
user.user_id: user.user_email.strip() for user in users if user.user_email and user.user_email.strip()
|
||||
}
|
||||
counts = {row.api_key: row.metrics.api_requests for row in usage}
|
||||
for key in keys:
|
||||
team = team_by_id.get(key.team_id) if key.team_id else None
|
||||
if key.token is None:
|
||||
continue
|
||||
recipients = _recipient_emails(key.user_id, team, addresses)
|
||||
for address in recipients:
|
||||
previous = impacts.get(address, _Impact())
|
||||
names: frozenset[str] = frozenset((team.team_alias or team.team_id,)) if team else frozenset()
|
||||
impacts[address] = _Impact(
|
||||
key_count=previous.key_count + 1,
|
||||
requests=previous.requests + counts[key.token],
|
||||
teams=previous.teams | names,
|
||||
)
|
||||
if len(usage) < _PAGE_SIZE:
|
||||
return MappingProxyType(impacts)
|
||||
offset += _PAGE_SIZE
|
||||
|
||||
async def _deliver(self, email: ModelDeprecationEmail, now: datetime, base_url: str) -> bool:
|
||||
from litellm.proxy.utils import send_email
|
||||
|
||||
key: Final = _receipt_key(email)
|
||||
record: Final = await self.config.get_param(key)
|
||||
previous: Final = _Receipt.model_validate(record.param_value) if record else None
|
||||
if previous and (previous.status is _DeliveryStatus.SENT or previous.lease_until > now):
|
||||
return False
|
||||
revision: Final = previous.revision if previous else 0
|
||||
claim: Final = _Receipt(revision=revision + 1, lease_until=now + timedelta(minutes=5))
|
||||
if not await self.config.set_param_if_revision(key, claim.model_dump(mode="json"), revision):
|
||||
return False
|
||||
content: Final = render_model_deprecation_email(email, base_url=base_url)
|
||||
if not await send_email(
|
||||
receiver_email=email.recipient_email,
|
||||
subject=content.subject,
|
||||
html=content.html,
|
||||
text=content.text,
|
||||
message_id=f"<litellm-deprecation-{key.removeprefix(_RECEIPT_PREFIX)}@alerts.litellm.ai>",
|
||||
):
|
||||
return False
|
||||
sent: Final = claim.model_copy(update={"revision": claim.revision + 1, "status": _DeliveryStatus.SENT})
|
||||
if not await self.config.set_param_if_revision(key, sent.model_dump(mode="json"), claim.revision):
|
||||
raise RuntimeError("Could not record the delivered model deprecation reminder")
|
||||
return True
|
||||
|
||||
async def run(self, llm_router: Router, *, now: datetime | None = None) -> int:
|
||||
current: Final = now or datetime.now(timezone.utc)
|
||||
settings: Final = await self.config.get_param("general_settings")
|
||||
if settings is None or not _OptIn.model_validate(settings.param_value).enabled:
|
||||
return 0
|
||||
base_url: Final = os.getenv("PROXY_BASE_URL", "")
|
||||
if not base_url:
|
||||
return 0
|
||||
await self.config.delete_params_older_than(_RECEIPT_PREFIX, current - timedelta(days=60))
|
||||
snapshot: Final = collect_model_deprecations(llm_router=llm_router, today=current.date())
|
||||
start: Final = current.date() - timedelta(days=29)
|
||||
seen: Final[set[tuple[str, date]]] = set() # mutable-ok: deduplicates aliases in this pass
|
||||
sent: int = 0 # rebind-ok: counts successfully delivered reminders in this pass
|
||||
for model in snapshot.imminent:
|
||||
if model.days_until_deprecation not in tuple(DeprecationEmailMilestone):
|
||||
continue
|
||||
name = _usage_model(model)
|
||||
identity = (name, model.deprecation_date)
|
||||
if identity in seen:
|
||||
continue
|
||||
seen.add(identity)
|
||||
impacts = await self._impacts(name, start, current.date())
|
||||
for address, impact in sorted(impacts.items()):
|
||||
email = ModelDeprecationEmail(
|
||||
model=model,
|
||||
usage_model=name,
|
||||
recipient_email=address,
|
||||
milestone=DeprecationEmailMilestone(model.days_until_deprecation),
|
||||
key_count=impact.key_count,
|
||||
requests=impact.requests,
|
||||
team_names=tuple(sorted(impact.teams)),
|
||||
start_date=start,
|
||||
end_date=current.date(),
|
||||
)
|
||||
if await self._deliver(email, now or datetime.now(timezone.utc), base_url):
|
||||
sent += 1
|
||||
return sent
|
||||
|
||||
|
||||
async def run_scheduled_model_deprecation_emails() -> None:
|
||||
"""Poll the email opt-in independently; a large owner mailing never delays Slack checks."""
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
while True:
|
||||
try:
|
||||
from litellm.proxy.proxy_server import llm_router, prisma_client
|
||||
|
||||
if llm_router is not None and prisma_client is not None:
|
||||
await ModelDeprecationEmailSender(prisma_client).run(llm_router)
|
||||
except Exception: # noqa: BLE001 # transient SMTP/DB failures must not terminate the reminder loop
|
||||
verbose_proxy_logger.exception("Error sending model deprecation email reminders")
|
||||
await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS)
|
||||
|
|
@ -11,6 +11,7 @@ from typing import Final
|
|||
from litellm.proxy.config_resolvers._descriptors import FieldDescriptor
|
||||
|
||||
EMAIL_DESCRIPTORS: Final[tuple[FieldDescriptor, ...]] = (
|
||||
FieldDescriptor("PROXY_BASE_URL", "PROXY_BASE_URL", "PROXY_BASE_URL"),
|
||||
FieldDescriptor("SMTP_HOST", "SMTP_HOST", "SMTP_HOST"),
|
||||
FieldDescriptor("SMTP_PORT", "SMTP_PORT", "SMTP_PORT", default="587"),
|
||||
FieldDescriptor("SMTP_TLS", "SMTP_TLS", "SMTP_TLS", default="True"),
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from dataclasses import dataclass, field
|
|||
from datetime import date, datetime, timedelta, timezone
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from email.utils import formatdate
|
||||
from functools import partial, reduce
|
||||
from itertools import takewhile
|
||||
from operator import attrgetter
|
||||
|
|
@ -1952,6 +1953,7 @@ class ProxyLogging:
|
|||
self.daily_report_started: bool = False
|
||||
self.hanging_requests_check_started: bool = False
|
||||
self.deprecation_check_started: bool = False
|
||||
self.deprecation_email_check_started: bool = False
|
||||
|
||||
def startup_event(
|
||||
self,
|
||||
|
|
@ -1995,6 +1997,20 @@ class ProxyLogging:
|
|||
self.hanging_requests_check_started = True
|
||||
|
||||
self._ensure_deprecation_check_scheduled()
|
||||
self._ensure_deprecation_email_check_scheduled()
|
||||
|
||||
def _ensure_deprecation_email_check_scheduled(self) -> None:
|
||||
"""The email event opt-in controls reminders independently of Slack integration settings."""
|
||||
if self.deprecation_email_check_started:
|
||||
return
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return
|
||||
from litellm.integrations.model_deprecation_email import run_scheduled_model_deprecation_emails
|
||||
|
||||
asyncio.create_task(run_scheduled_model_deprecation_emails())
|
||||
self.deprecation_email_check_started = True
|
||||
|
||||
def _ensure_deprecation_check_scheduled(self) -> None:
|
||||
"""Alerting can be configured at startup or by a later config reload, so schedule from either path"""
|
||||
|
|
@ -2041,6 +2057,7 @@ class ProxyLogging:
|
|||
|
||||
if updated_slack_alerting is True:
|
||||
self._ensure_deprecation_check_scheduled()
|
||||
self._ensure_deprecation_email_check_scheduled()
|
||||
self.slack_alerting_instance.update_values(
|
||||
alerting=self.alerting,
|
||||
alerting_threshold=self.alerting_threshold,
|
||||
|
|
@ -8263,7 +8280,9 @@ async def send_email(
|
|||
receiver_email: str | None = None,
|
||||
subject: str | None = None,
|
||||
html: str | None = None,
|
||||
):
|
||||
text: str | None = None,
|
||||
message_id: str | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
smtp_host,
|
||||
smtp_port,
|
||||
|
|
@ -8289,16 +8308,21 @@ async def send_email(
|
|||
raise ValueError(f"No HTML body provided for SMTP email. {html}")
|
||||
|
||||
## EMAIL SETUP ##
|
||||
email_message: Final = MIMEMultipart()
|
||||
email_message: Final = MIMEMultipart("alternative" if text is not None else "mixed")
|
||||
email_message["From"] = sender_email
|
||||
email_message["To"] = receiver_email
|
||||
email_message["Subject"] = subject
|
||||
if message_id is not None:
|
||||
email_message["Message-ID"] = message_id
|
||||
email_message["Date"] = formatdate(usegmt=True)
|
||||
verbose_proxy_logger.debug("sending email from %s to %s", sender_email, receiver_email)
|
||||
|
||||
if smtp_host is None:
|
||||
raise ValueError("Trying to use SMTP, but SMTP_HOST is not set")
|
||||
|
||||
# Attach the body to the email
|
||||
if text is not None:
|
||||
email_message.attach(MIMEText(text, "plain"))
|
||||
email_message.attach(MIMEText(html, "html"))
|
||||
|
||||
try:
|
||||
|
|
@ -8317,6 +8341,8 @@ async def send_email(
|
|||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("An error occurred while sending the email:" + str(e))
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def hash_token(token: str):
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Final, Protocol, cast
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -35,6 +36,8 @@ class _ConfigTable(Protocol):
|
|||
|
||||
async def delete(self, *, where: Mapping[str, str]) -> _ConfigRow | None: ...
|
||||
|
||||
async def delete_many(self, *, where: Mapping[str, object]) -> int: ...
|
||||
|
||||
|
||||
class _ConfigDatabase(Protocol):
|
||||
async def query_raw(self, query: str, *args: object) -> object: ...
|
||||
|
|
@ -130,3 +133,9 @@ class ConfigRepository:
|
|||
param_value = _decoded_json(param_value)
|
||||
result[record.param_name] = param_value
|
||||
return result
|
||||
|
||||
async def delete_params_older_than(self, prefix: str, before: datetime) -> int:
|
||||
"""Expire operational config records using the timestamp written by revision claims."""
|
||||
return await self._config_table.delete_many(
|
||||
where={"param_name": {"startswith": prefix}, "last_run_at": {"lt": before}}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -387,3 +387,7 @@ class BackgroundInteractionSettlementRepository(
|
|||
|
||||
class RetiredAgentRepository(PrismaTableRepository["prisma_models.LiteLLM_RetiredAgent"]):
|
||||
table_name = "litellm_retiredagent"
|
||||
|
||||
|
||||
class DailyUserSpendRepository(PrismaTableRepository["prisma_models.LiteLLM_DailyUserSpend"]):
|
||||
table_name = "litellm_dailyuserspend"
|
||||
|
|
|
|||
35
litellm/types/integrations/model_deprecation_email.py
Normal file
35
litellm/types/integrations/model_deprecation_email.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
"""Model retirement email milestones and recipient-scoped content."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from enum import IntEnum
|
||||
from typing import Final
|
||||
|
||||
from litellm.types.proxy.model_deprecation import ModelDeprecationInfo
|
||||
|
||||
MODEL_DEPRECATION_EMAIL_EVENT: Final = "Model Deprecations"
|
||||
|
||||
|
||||
class DeprecationEmailMilestone(IntEnum):
|
||||
THIRTY_DAYS = 30
|
||||
ONE_DAY = 1
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelDeprecationEmail:
|
||||
model: ModelDeprecationInfo
|
||||
usage_model: str
|
||||
recipient_email: str
|
||||
milestone: DeprecationEmailMilestone
|
||||
key_count: int
|
||||
requests: int
|
||||
team_names: tuple[str, ...]
|
||||
start_date: date
|
||||
end_date: date
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RenderedDeprecationEmail:
|
||||
subject: str
|
||||
html: str
|
||||
text: str
|
||||
0
tests/unit/integrations/email_alerting/__init__.py
Normal file
0
tests/unit/integrations/email_alerting/__init__.py
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
"""Retirement reminder schedule, durable dedupe, ownership scope and safe rendering."""
|
||||
|
||||
import asyncio
|
||||
from dataclasses import replace
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from types import MappingProxyType
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.integrations import model_deprecation_email as module
|
||||
from litellm.integrations.email_templates.model_deprecation_email import (
|
||||
render_model_deprecation_email,
|
||||
)
|
||||
from litellm.models.team import LiteLLM_TeamTable
|
||||
from litellm.models.user import LiteLLM_UserTable
|
||||
from litellm.proxy._types import LiteLLM_VerificationToken
|
||||
from litellm.repositories.config_repository import ConfigParam
|
||||
from litellm.types.integrations.model_deprecation_email import (
|
||||
DeprecationEmailMilestone,
|
||||
ModelDeprecationEmail,
|
||||
)
|
||||
from litellm.types.proxy.model_deprecation import (
|
||||
ModelDeprecationInfo,
|
||||
ModelDeprecationResponse,
|
||||
SuccessorModelSource,
|
||||
)
|
||||
|
||||
NOW = datetime(2026, 11, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def info(days=30, model="gemini/gemini-2.5-pro"):
|
||||
return ModelDeprecationInfo(
|
||||
model_name="research",
|
||||
litellm_model=model,
|
||||
deprecation_date=NOW.date() + timedelta(days=days),
|
||||
days_until_deprecation=days,
|
||||
status="imminent",
|
||||
successor_model="gemini/gemini-3.8-flash",
|
||||
successor_model_source=SuccessorModelSource.MODEL_COST_MAP,
|
||||
deprecation_source="https://ai.google.dev/gemini-api/docs/deprecations",
|
||||
)
|
||||
|
||||
|
||||
class MemoryConfig:
|
||||
def __init__(self):
|
||||
self.rows = {"general_settings": {"email_settings": {"Model Deprecations": True}}}
|
||||
|
||||
async def get_param(self, key):
|
||||
return ConfigParam(key, self.rows[key]) if key in self.rows else None
|
||||
|
||||
async def set_param_if_revision(self, key, value, revision):
|
||||
current = self.rows.get(key, {}).get("revision", 0)
|
||||
if current != revision:
|
||||
return False
|
||||
self.rows[key] = value
|
||||
return True
|
||||
|
||||
async def delete_params_older_than(self, prefix, before):
|
||||
return 0
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def sender(monkeypatch):
|
||||
monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example")
|
||||
instance = module.ModelDeprecationEmailSender(MagicMock())
|
||||
instance.config = MemoryConfig()
|
||||
instance._impacts = AsyncMock(
|
||||
return_value=MappingProxyType(
|
||||
{"owner@example.com": module._Impact(key_count=3, requests=12, teams=frozenset({"Research"}))}
|
||||
)
|
||||
)
|
||||
send = AsyncMock(return_value=True)
|
||||
monkeypatch.setattr("litellm.proxy.utils.send_email", send)
|
||||
return instance, send
|
||||
|
||||
|
||||
def schedule(monkeypatch, models):
|
||||
snapshot = ModelDeprecationResponse(
|
||||
deprecated=[], imminent=models, upcoming=[], warn_within_days=30, checked_at=NOW
|
||||
)
|
||||
monkeypatch.setattr(module, "collect_model_deprecations", lambda **kwargs: snapshot)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_send_only_the_two_milestones_and_keep_receipts_across_restart(sender, monkeypatch):
|
||||
instance, send = sender
|
||||
schedule(monkeypatch, [info()])
|
||||
assert await instance.run(MagicMock(), now=NOW) == 1
|
||||
assert await instance.run(MagicMock(), now=NOW) == 0
|
||||
restarted = module.ModelDeprecationEmailSender(MagicMock())
|
||||
restarted.config = instance.config
|
||||
restarted._impacts = instance._impacts
|
||||
assert await restarted.run(MagicMock(), now=NOW) == 0
|
||||
final_day = NOW + timedelta(days=29)
|
||||
schedule(monkeypatch, [info().model_copy(update={"days_until_deprecation": 1})])
|
||||
assert await restarted.run(MagicMock(), now=final_day) == 1
|
||||
assert await restarted.run(MagicMock(), now=final_day) == 0
|
||||
assert send.await_count == 2
|
||||
assert "in 30 days" in send.await_args_list[0].kwargs["subject"]
|
||||
assert "tomorrow" in send.await_args_list[1].kwargs["subject"]
|
||||
assert send.await_args_list[0].kwargs["message_id"] != send.await_args_list[1].kwargs["message_id"]
|
||||
assert "<html" in send.await_args_list[0].kwargs["html"]
|
||||
assert "3 keys | 12 requests in the last 30 days" in send.await_args_list[0].kwargs["text"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("days", [31, 29, 2, 0, -1])
|
||||
async def test_should_not_send_on_other_days(sender, monkeypatch, days):
|
||||
instance, send = sender
|
||||
schedule(monkeypatch, [info(days)])
|
||||
assert await instance.run(MagicMock(), now=NOW) == 0
|
||||
send.assert_not_awaited()
|
||||
instance._impacts.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"settings",
|
||||
[{}, {"email_settings": {}}, {"email_settings": {"Model Deprecations": False}}],
|
||||
)
|
||||
async def test_should_require_explicit_opt_in(sender, settings):
|
||||
instance, send = sender
|
||||
instance.config.rows["general_settings"] = settings
|
||||
assert await instance.run(MagicMock(), now=NOW) == 0
|
||||
send.assert_not_awaited()
|
||||
instance._impacts.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_claim_delivery_once_across_concurrent_workers(sender, monkeypatch):
|
||||
instance, send = sender
|
||||
schedule(monkeypatch, [info()])
|
||||
second = module.ModelDeprecationEmailSender(MagicMock())
|
||||
second.config = instance.config
|
||||
second._impacts = instance._impacts
|
||||
assert sum(await asyncio.gather(instance.run(MagicMock(), now=NOW), second.run(MagicMock(), now=NOW))) == 1
|
||||
send.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_retry_failed_delivery_after_the_lease_expires(sender, monkeypatch):
|
||||
instance, send = sender
|
||||
schedule(monkeypatch, [info()])
|
||||
send.side_effect = [False, True]
|
||||
assert await instance.run(MagicMock(), now=NOW) == 0
|
||||
assert await instance.run(MagicMock(), now=NOW + timedelta(minutes=1)) == 0
|
||||
assert await instance.run(MagicMock(), now=NOW + timedelta(minutes=6)) == 1
|
||||
assert await instance.run(MagicMock(), now=NOW + timedelta(minutes=7)) == 0
|
||||
assert send.await_count == 2
|
||||
assert send.await_args_list[0].kwargs["message_id"] == send.await_args_list[1].kwargs["message_id"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_dedupe_model_aliases_and_skip_models_with_no_observed_owners(sender, monkeypatch):
|
||||
instance, send = sender
|
||||
schedule(monkeypatch, [info(), info().model_copy(update={"model_name": "second-alias"})])
|
||||
assert await instance.run(MagicMock(), now=NOW) == 1
|
||||
send.assert_awaited_once()
|
||||
instance.config.rows = {"general_settings": {"email_settings": {"Model Deprecations": True}}}
|
||||
instance._impacts.return_value = {}
|
||||
assert await instance.run(MagicMock(), now=NOW) == 0
|
||||
assert send.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_scope_usage_to_current_owners_and_admins_excluding_former_admins_across_pages(
|
||||
monkeypatch,
|
||||
):
|
||||
instance = module.ModelDeprecationEmailSender(MagicMock())
|
||||
monkeypatch.setattr(module, "_PAGE_SIZE", 2)
|
||||
grouped = AsyncMock(
|
||||
side_effect=[
|
||||
[
|
||||
{"api_key": "one", "_sum": {"api_requests": 10}},
|
||||
{"api_key": "two", "_sum": {"api_requests": 20}},
|
||||
],
|
||||
[{"api_key": "three", "_sum": {"api_requests": 30}}],
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module.DailyUserSpendRepository,
|
||||
"table",
|
||||
property(lambda _: MagicMock(group_by=grouped)),
|
||||
)
|
||||
keys = [
|
||||
LiteLLM_VerificationToken(token="one", user_id="alice", team_id="research"),
|
||||
LiteLLM_VerificationToken(token="two", user_id="alice"),
|
||||
LiteLLM_VerificationToken(token="three", user_id="bob", team_id="research"),
|
||||
]
|
||||
|
||||
async def key_rows(_, field, values):
|
||||
ids = set(values)
|
||||
return [key for key in keys if key.token in ids]
|
||||
|
||||
monkeypatch.setattr(module.VerificationTokenRepository, "find_many_in", key_rows)
|
||||
team = LiteLLM_TeamTable(
|
||||
team_id="research",
|
||||
team_alias="Research",
|
||||
admins=["bob", "mallory", "removed-admin"],
|
||||
members_with_roles=[
|
||||
{"user_id": "bob", "user_email": "BOB@example.com", "role": "admin"},
|
||||
{"user_id": "mallory", "user_email": "mallory@example.com", "role": "user"},
|
||||
],
|
||||
)
|
||||
|
||||
async def team_rows(_, field, values):
|
||||
return [team] if "research" in values else []
|
||||
|
||||
monkeypatch.setattr(module.TeamRepository, "find_many_in", team_rows)
|
||||
|
||||
async def user_rows(_, field, values):
|
||||
return [LiteLLM_UserTable(user_id=user_id, user_email=f"{user_id}@example.com") for user_id in values]
|
||||
|
||||
monkeypatch.setattr(module.UserRepository, "find_many_in", user_rows)
|
||||
impacts = await instance._impacts("gemini/gemini-2.5-pro", date(2026, 10, 3), NOW.date())
|
||||
assert set(impacts) == {"alice@example.com", "bob@example.com"}
|
||||
assert impacts["alice@example.com"].requests == 30
|
||||
assert impacts["alice@example.com"].key_count == 2
|
||||
assert impacts["bob@example.com"].requests == 40
|
||||
assert impacts["bob@example.com"].key_count == 2
|
||||
assert impacts["bob@example.com"].teams == frozenset({"Research"})
|
||||
assert grouped.await_args_list[0].kwargs["where"] == {
|
||||
"model": "gemini/gemini-2.5-pro",
|
||||
"date": {"gte": "2026-10-03", "lte": "2026-11-01"},
|
||||
"api_requests": {"gt": 0},
|
||||
}
|
||||
|
||||
|
||||
def test_should_render_the_current_ui_logo_safe_model_text_and_actionable_links():
|
||||
model = info().model_copy(
|
||||
update={
|
||||
"successor_model": '<script>alert("x")</script>',
|
||||
"deprecation_source": "javascript:alert(1)",
|
||||
}
|
||||
)
|
||||
email = ModelDeprecationEmail(
|
||||
model=model,
|
||||
usage_model=model.litellm_model,
|
||||
recipient_email="owner@example.com",
|
||||
milestone=DeprecationEmailMilestone.THIRTY_DAYS,
|
||||
key_count=3,
|
||||
requests=12,
|
||||
team_names=("Research",),
|
||||
start_date=date(2026, 10, 3),
|
||||
end_date=NOW.date(),
|
||||
)
|
||||
rendered = render_model_deprecation_email(email, base_url="https://gateway.example")
|
||||
assert "/public/assets/logos/litellm_logo.png" in rendered.html
|
||||
assert "litellm-listing.s3.amazonaws.com" not in rendered.html
|
||||
assert "<script>" not in rendered.html
|
||||
assert "javascript:" not in rendered.html
|
||||
assert "<script>" in rendered.html
|
||||
assert "/ui/usage?tab=models&from=2026-10-03&to=2026-11-01" in rendered.html
|
||||
assert "Recommendation source: LiteLLM model cost map" in rendered.text
|
||||
assert "Google is deprecating this model on 01 Dec 2026 (UTC)." in rendered.html
|
||||
assert "Google is deprecating this model on 01 Dec 2026 (UTC)." in rendered.text
|
||||
assert "google_pse.png" in rendered.html
|
||||
assert 'alt="Google"' in rendered.html
|
||||
assert "BEFORE · MODEL TO REPLACE" in rendered.html
|
||||
assert "AFTER · RECOMMENDED MODEL" in rendered.html
|
||||
assert "What to do" in rendered.html
|
||||
assert "check that requests to the old model reach zero" in rendered.text
|
||||
missing = render_model_deprecation_email(
|
||||
replace(email, model=model.model_copy(update={"successor_model": None})),
|
||||
base_url="https://gateway.example",
|
||||
)
|
||||
assert "No recommended model listed. Choose a supported replacement." in missing.html
|
||||
assert "Choose a supported replacement and test it with your workload." in missing.text
|
||||
tomorrow = render_model_deprecation_email(
|
||||
replace(email, milestone=DeprecationEmailMilestone.ONE_DAY),
|
||||
base_url="https://gateway.example",
|
||||
)
|
||||
assert "01 DAY UNTIL DEPRECATION" in tomorrow.html
|
||||
assert "Model being deprecated" in rendered.html
|
||||
assert "Model being deprecated" in rendered.subject
|
||||
assert "Model being deprecated" in tomorrow.html
|
||||
assert "Model being deprecated" in tomorrow.subject
|
||||
assert "FINAL REMINDER" in tomorrow.html
|
||||
assert "tomorrow" in tomorrow.subject
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_keep_the_email_loop_running_after_a_delivery_error(monkeypatch):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
router = MagicMock()
|
||||
database = MagicMock()
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", database)
|
||||
runner = MagicMock()
|
||||
runner.run = AsyncMock(side_effect=[RuntimeError("temporary SMTP failure"), 0])
|
||||
monkeypatch.setattr(module, "ModelDeprecationEmailSender", lambda _: runner)
|
||||
monkeypatch.setattr(module.asyncio, "sleep", AsyncMock(side_effect=[None, asyncio.CancelledError()]))
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await module.run_scheduled_model_deprecation_emails()
|
||||
assert runner.run.await_count == 2
|
||||
assert all(call.args == (router,) for call in runner.run.await_args_list)
|
||||
|
|
@ -8,6 +8,7 @@ from __future__ import annotations
|
|||
|
||||
import threading
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -27,11 +28,12 @@ def _smtp_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_email_dispatches_via_smtp(in_memory_smtp: Any) -> None:
|
||||
await send_email(
|
||||
delivered = await send_email(
|
||||
receiver_email="to@invalid",
|
||||
subject="Hello",
|
||||
html="<p>body</p>",
|
||||
)
|
||||
assert delivered is True
|
||||
assert len(in_memory_smtp.sent) == 1
|
||||
sent = in_memory_smtp.sent[0]
|
||||
pinned = {
|
||||
|
|
@ -153,5 +155,24 @@ async def test_send_email_smtp_failure_is_swallowed(
|
|||
does not raise so a failing email never blocks the proxy.
|
||||
"""
|
||||
in_memory_smtp.raise_on_send = RuntimeError("smtp boom")
|
||||
await send_email(receiver_email="to@invalid", subject="Hi", html="<p>x</p>")
|
||||
delivered = await send_email(receiver_email="to@invalid", subject="Hi", html="<p>x</p>")
|
||||
assert delivered is False
|
||||
assert in_memory_smtp.sent == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_send_plain_text_and_html_as_multipart_alternative(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
transport = Mock()
|
||||
monkeypatch.setattr("litellm.proxy.utils._send_smtp_message", transport)
|
||||
assert (
|
||||
await send_email(
|
||||
receiver_email="to@invalid", subject="Retirement", html="<p>Model retirement</p>", text="Model retirement"
|
||||
)
|
||||
is True
|
||||
)
|
||||
message = transport.call_args.kwargs["email_message"]
|
||||
assert message.get_content_type() == "multipart/alternative"
|
||||
parts = tuple(part for part in message.walk() if not part.is_multipart())
|
||||
assert tuple(part.get_content_type() for part in parts) == ("text/plain", "text/html")
|
||||
assert parts[0].get_payload(decode=True).decode() == "Model retirement"
|
||||
assert parts[1].get_payload(decode=True).decode() == "<p>Model retirement</p>"
|
||||
|
|
|
|||
BIN
ui/litellm-dashboard/public/assets/logos/anthropic.png
Normal file
BIN
ui/litellm-dashboard/public/assets/logos/anthropic.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 3.3 KiB |
BIN
ui/litellm-dashboard/public/assets/logos/aws.png
Normal file
BIN
ui/litellm-dashboard/public/assets/logos/aws.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 2.3 KiB |
BIN
ui/litellm-dashboard/public/assets/logos/microsoft_azure.png
Normal file
BIN
ui/litellm-dashboard/public/assets/logos/microsoft_azure.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
BIN
ui/litellm-dashboard/public/assets/logos/openai.png
Normal file
BIN
ui/litellm-dashboard/public/assets/logos/openai.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 5.5 KiB |
|
|
@ -0,0 +1,26 @@
|
|||
import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import DeprecationEmailSetup from "./DeprecationEmailSetup";
|
||||
|
||||
describe("deprecation email setup", () => {
|
||||
it("should render an ambient setup action", () => {
|
||||
render(<DeprecationEmailSetup />);
|
||||
expect(screen.getByRole("button", { name: "Set up email reminders" })).toBeInTheDocument();
|
||||
expect(screen.queryByRole("dialog")).not.toBeInTheDocument();
|
||||
});
|
||||
it("should explain the three setup steps and link to Email Settings", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<DeprecationEmailSetup />);
|
||||
await user.click(screen.getByRole("button", { name: "Set up email reminders" }));
|
||||
expect(await screen.findByRole("dialog", { name: "Set up model deprecation emails" })).toBeInTheDocument();
|
||||
expect(screen.getByText("Connect email delivery")).toBeInTheDocument();
|
||||
expect(screen.getByText("Enable model deprecation emails")).toBeInTheDocument();
|
||||
expect(screen.getByText("Owners get two reminders")).toBeInTheDocument();
|
||||
expect(screen.getByText(/This is off by default/)).toBeInTheDocument();
|
||||
expect(screen.getByRole("link", { name: "Open Email Settings" })).toHaveAttribute(
|
||||
"href",
|
||||
"/ui/logging-and-alerts?tab=email-alerts",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
"use client";
|
||||
|
||||
import { Mail } from "lucide-react";
|
||||
import { Button, buttonVariants } from "@/components/ui/button";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
DialogTrigger,
|
||||
} from "@/components/ui/dialog";
|
||||
import { uiHref } from "@/utils/uiHref";
|
||||
|
||||
const emailSettingsHref = `${uiHref("logging-and-alerts")}?tab=email-alerts`;
|
||||
const setupSteps = [
|
||||
{
|
||||
title: "Connect email delivery",
|
||||
description:
|
||||
"Add your SMTP settings and public gateway URL in Email Settings. Send a test email to verify delivery.",
|
||||
},
|
||||
{
|
||||
title: "Enable model deprecation emails",
|
||||
description: "In Email events, turn on Model Deprecations and save. This is off by default.",
|
||||
},
|
||||
{
|
||||
title: "Owners get two reminders",
|
||||
description:
|
||||
"Affected key owners and team admins receive a reminder 30 days before retirement, and one final reminder 1 day before. Recipients are based on actual usage in the last 30 days. Dates are checked in UTC.",
|
||||
},
|
||||
];
|
||||
|
||||
export default function DeprecationEmailSetup() {
|
||||
return (
|
||||
<Dialog>
|
||||
<div className="flex flex-wrap items-center justify-between gap-2 border-b pb-3 text-xs text-muted-foreground">
|
||||
<span className="flex items-center gap-2">
|
||||
<Mail className="size-3.5 shrink-0" aria-hidden />
|
||||
Give affected owners time to migrate. Email reminders at 30 days and 1 day.
|
||||
</span>
|
||||
<DialogTrigger render={<Button variant="link" size="sm" />}>Set up email reminders</DialogTrigger>
|
||||
</div>
|
||||
<DialogContent>
|
||||
<DialogHeader>
|
||||
<DialogTitle>Set up model deprecation emails</DialogTitle>
|
||||
<DialogDescription>
|
||||
Help key owners and team admins act before a provider retires a model. Only their affected usage is
|
||||
included.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<ol className="my-3 space-y-5">
|
||||
{setupSteps.map((step, index) => (
|
||||
<li key={step.title} className="flex gap-3">
|
||||
<span
|
||||
className="flex size-7 shrink-0 items-center justify-center rounded-full border text-xs font-medium"
|
||||
aria-hidden
|
||||
>
|
||||
{index + 1}
|
||||
</span>
|
||||
<div>
|
||||
<h3 className="text-sm font-medium">{step.title}</h3>
|
||||
<p className="mt-1 text-sm leading-relaxed text-muted-foreground">{step.description}</p>
|
||||
</div>
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
<DialogFooter>
|
||||
<a className={buttonVariants()} href={emailSettingsHref}>
|
||||
Open Email Settings
|
||||
</a>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@ import { Button, buttonVariants } from "@/components/ui/button";
|
|||
import { useDeprecationAlerts } from "@/components/ModelDeprecationAlerts/useDeprecationAlerts";
|
||||
import { hasCapability } from "@/utils/capabilities";
|
||||
import { migrationUrlParsers, migrationHref } from "./migrationRoute";
|
||||
import DeprecationEmailSetup from "./DeprecationEmailSetup";
|
||||
import AlertsInbox from "./AlertsInbox";
|
||||
import MigrationDetail from "./MigrationDetail";
|
||||
|
||||
|
|
@ -45,6 +46,7 @@ export default function ModelDeprecationsPanel() {
|
|||
</a>
|
||||
</header>
|
||||
)}
|
||||
<DeprecationEmailSetup />
|
||||
{alerts.isError ? (
|
||||
<div role="alert">
|
||||
<p>Could not load alerts. No alert status has been changed.</p>
|
||||
|
|
|
|||
|
|
@ -39,15 +39,15 @@ describe("EmailEventSettings", () => {
|
|||
expect(screen.getByText("Select which events should trigger email notifications.")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders one checkbox per event, reflecting the persisted enabled state", async () => {
|
||||
it("should render one toggle per event, reflecting the persisted enabled state", async () => {
|
||||
renderWithProviders(<EmailEventSettings accessToken="sk-test" />);
|
||||
|
||||
await screen.findByText("Virtual Key Created");
|
||||
const checkboxes = screen.getAllByRole("checkbox");
|
||||
const toggles = screen.getAllByRole("switch");
|
||||
|
||||
expect(checkboxes).toHaveLength(2);
|
||||
expect(checkboxes[0]).toBeChecked();
|
||||
expect(checkboxes[1]).not.toBeChecked();
|
||||
expect(toggles).toHaveLength(2);
|
||||
expect(toggles[0]).toBeChecked();
|
||||
expect(toggles[1]).not.toBeChecked();
|
||||
});
|
||||
|
||||
it("renders a per-event description", async () => {
|
||||
|
|
@ -66,7 +66,7 @@ describe("EmailEventSettings", () => {
|
|||
renderWithProviders(<EmailEventSettings accessToken="sk-test" />);
|
||||
|
||||
await screen.findByText("Virtual Key Created");
|
||||
await user.click(screen.getAllByRole("checkbox")[1]);
|
||||
await user.click(screen.getAllByRole("switch")[1]);
|
||||
await user.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
|
||||
await waitFor(() => {
|
||||
|
|
@ -84,7 +84,7 @@ describe("EmailEventSettings", () => {
|
|||
renderWithProviders(<EmailEventSettings accessToken="sk-test" />);
|
||||
|
||||
await screen.findByText("Virtual Key Created");
|
||||
await user.click(screen.getAllByRole("checkbox")[0]);
|
||||
await user.click(screen.getAllByRole("switch")[0]);
|
||||
await user.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
|
||||
await waitFor(() => {
|
||||
|
|
@ -118,7 +118,7 @@ describe("EmailEventSettings", () => {
|
|||
getEmailEventSettings.mockReturnValue(new Promise(() => {}));
|
||||
renderWithProviders(<EmailEventSettings accessToken="sk-test" />);
|
||||
|
||||
expect(screen.queryByRole("checkbox")).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("switch")).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled();
|
||||
});
|
||||
|
||||
|
|
@ -129,4 +129,20 @@ describe("EmailEventSettings", () => {
|
|||
expect(getEmailEventSettings).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
it("should show model deprecation reminders off and save explicit opt-in", async () => {
|
||||
const user = userEvent.setup();
|
||||
getEmailEventSettings.mockResolvedValue({ settings: [{ event: "Model Deprecations", enabled: false }] });
|
||||
renderWithProviders(<EmailEventSettings accessToken="test-access" />);
|
||||
const toggle = await screen.findByRole("switch", { name: "Model Deprecations" });
|
||||
expect(toggle).not.toBeChecked();
|
||||
expect(screen.queryByRole("checkbox")).not.toBeInTheDocument();
|
||||
expect(screen.getByText(/30 days and 1 day before model deprecation/)).toBeInTheDocument();
|
||||
await user.click(toggle);
|
||||
await user.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
await waitFor(() =>
|
||||
expect(updateEmailEventSettings).toHaveBeenCalledWith("test-access", {
|
||||
settings: [{ event: "Model Deprecations", enabled: true }],
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import React, { useState, useEffect } from "react";
|
|||
import { useTranslation } from "react-i18next";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { Separator } from "@/components/ui/separator";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { toast } from "@/lib/toast";
|
||||
|
|
@ -39,7 +39,7 @@ const EmailEventSettings: React.FC<EmailEventSettingsProps> = ({ accessToken })
|
|||
}
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (event: EmailEvent, checked: boolean) => {
|
||||
const handleEnabledChange = (event: EmailEvent, checked: boolean) => {
|
||||
const updatedSettings = eventSettings.map((setting) =>
|
||||
setting.event === event ? { ...setting, enabled: checked } : setting,
|
||||
);
|
||||
|
|
@ -74,6 +74,9 @@ const EmailEventSettings: React.FC<EmailEventSettingsProps> = ({ accessToken })
|
|||
|
||||
// Helper function to get a description for each event type
|
||||
const getEventDescription = (event: EmailEvent): string => {
|
||||
if (event === "Model Deprecations") {
|
||||
return "Email affected key owners and team admins 30 days and 1 day before model deprecation. Uses requests from the last 30 days. Off by default.";
|
||||
}
|
||||
// Convert event name to a sentence with more context
|
||||
if (event.includes("Virtual Key Created")) {
|
||||
return t("loggingAndAlerts.emailEvents.descriptions.virtualKeyCreated");
|
||||
|
|
@ -108,9 +111,10 @@ const EmailEventSettings: React.FC<EmailEventSettingsProps> = ({ accessToken })
|
|||
<div className="space-y-4">
|
||||
{eventSettings.map((setting) => (
|
||||
<div key={setting.event} className="flex items-start">
|
||||
<Checkbox
|
||||
<Switch
|
||||
checked={setting.enabled}
|
||||
onCheckedChange={(checked) => handleCheckboxChange(setting.event, checked === true)}
|
||||
onCheckedChange={(checked) => handleEnabledChange(setting.event, checked)}
|
||||
aria-label={setting.event}
|
||||
className="mt-1"
|
||||
/>
|
||||
<div className="ml-3">
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ const REQUIRED_MARKER = (
|
|||
);
|
||||
|
||||
const FIELD_HELP: Record<string, React.ReactNode> = {
|
||||
PROXY_BASE_URL: "Public HTTP(S) gateway URL. Required for model deprecation emails so owners can review their usage.",
|
||||
SMTP_HOST: (
|
||||
<>
|
||||
<Trans i18nKey="loggingAndAlerts.emailSettings.help.smtpHost" />
|
||||
|
|
@ -154,11 +155,12 @@ const EmailSettings: React.FC<EmailSettingsProps> = ({ accessToken, premiumUser,
|
|||
✨ {key}
|
||||
</a>
|
||||
) : (
|
||||
<p className="text-sm">{key}</p>
|
||||
<p className="text-sm">{key === "PROXY_BASE_URL" ? "Gateway URL" : key}</p>
|
||||
)}
|
||||
<InputGroup className="max-w-100">
|
||||
<InputGroupInput
|
||||
name={key}
|
||||
aria-label={key === "PROXY_BASE_URL" ? "Gateway URL" : key}
|
||||
defaultValue={value as string}
|
||||
type={isSensitive && !isVisible ? "password" : "text"}
|
||||
disabled={isLocked}
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -32840,7 +32840,7 @@ export interface components {
|
|||
* EmailEvent
|
||||
* @enum {string}
|
||||
*/
|
||||
EmailEvent: "Virtual Key Created" | "New User Invitation" | "Virtual Key Rotated" | "Soft Budget Crossed" | "Max Budget Alert";
|
||||
EmailEvent: "Model Deprecations" | "Virtual Key Created" | "New User Invitation" | "Virtual Key Rotated" | "Soft Budget Crossed" | "Max Budget Alert";
|
||||
/** EmailEventSettings */
|
||||
EmailEventSettings: {
|
||||
/** Enabled */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue