diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105110000_usage_alert_flush/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105110000_usage_alert_flush/migration.sql new file mode 100644 index 00000000000..217fa4f1510 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105110000_usage_alert_flush/migration.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertFlush" ( + source TEXT NOT NULL, + hour TIMESTAMP(3) NOT NULL, + observed_at TIMESTAMP(3) NOT NULL, + PRIMARY KEY (source, hour) +); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 8a67d1b123a..3a08c094b3d 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule { @@id([delivery_id, rule_id, source_id]) } + +// Hourly samples of successful background daily-aggregate flushes, retained for 35 days. +model LiteLLM_UsageAlertFlush { + source String + hour DateTime + observed_at DateTime + + @@id([source, hour]) +} diff --git a/litellm/integrations/SlackAlerting/usage_alerts.py b/litellm/integrations/SlackAlerting/usage_alerts.py new file mode 100644 index 00000000000..9b09d8fb05c --- /dev/null +++ b/litellm/integrations/SlackAlerting/usage_alerts.py @@ -0,0 +1,88 @@ +"""Deliver persisted usage incidents through the existing configured transports.""" + +import json +import os +from html import escape +from typing import TYPE_CHECKING, Final +from urllib.parse import urlsplit + +from pydantic import TypeAdapter + +from litellm.types.integrations.slack_alerting import AlertType +from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertChannel, UsageAlertRule + +if TYPE_CHECKING: + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + +_URLS: Final = TypeAdapter(tuple[str, ...]) + + +def configured_destinations(slack: "SlackAlerting") -> tuple[str, ...]: + return _URLS.validate_python(slack.alerting or ()) # pyright: ignore[reportUnknownMemberType] # validate legacy destination config + + +def activity_url(alert: UsageAlert, base_url: str | None) -> str: + if not base_url: + return alert.activity_path + parsed: Final = urlsplit(base_url) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + return alert.activity_path + return base_url.rstrip("/") + "/ui" + alert.activity_path + + +def incident_message(alert: UsageAlert, url: str) -> str: + scope: Final = "Gateway" if alert.scope_type == "deployment" else f"{alert.scope_type}: {alert.scope_id}" + unit: Final = " ms" if alert.metric == "latency" else " USD" if alert.metric == "spend" else "" + return ( + f"{alert.rule_name}\n{scope}\n{alert.day.isoformat()} UTC: " + f"{alert.observed_value:,.2f}{unit}; baseline {alert.baseline_value:,.2f}{unit}; " + f"threshold {alert.threshold:,.2f}{unit}.\nView activity: {url}" + ) + + +class UsageAlertDelivery: + def __init__(self, slack: "SlackAlerting") -> None: + self._slack = slack + + async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool: + configured: Final = configured_destinations(self._slack) + if channel not in configured: + return False + url: Final = activity_url(alert, os.getenv("PROXY_BASE_URL")) + message: Final = incident_message(alert, url) + if channel == "email": + from litellm.proxy.utils import send_email + + for recipient in rule.email_recipients: + await send_email( + receiver_email=recipient, + subject=f"LiteLLM usage alert: {alert.rule_name}", + html=( + f"

{escape(message).replace(chr(10), '
')}

" + f'

View activity

' + ), + raise_on_error=True, + ) + return bool(rule.email_recipients) + overrides: Final = self._slack.alert_to_webhook_url or {} + configured_url: Final = ( + overrides.get(AlertType.usage_alerts) + or self._slack.default_webhook_url + or os.getenv("SLACK_WEBHOOK_URL") + or os.getenv("ALERTING_WEBHOOK_URL") + ) + if not configured_url: + return False + urls: Final = (configured_url,) if isinstance(configured_url, str) else _URLS.validate_python(configured_url) + for webhook in urls: + await self._post(webhook, message) + return True + + async def _post(self, webhook: str, message: str) -> None: + response: Final = await self._slack.async_http_handler.post( # pyright: ignore[reportUnknownMemberType] # legacy HTTP signature; supplied arguments are concrete + url=webhook, + headers={"Content-type": "application/json"}, + data=json.dumps({"text": message}), + timeout=10, + ) + response.raise_for_status() diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 80f09df6c86..53b95e6e889 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -665,6 +665,9 @@ class DBSpendUpdateWriter: self, redis_cache: RedisCache | None = None, ): + from litellm.repositories.usage_alert_health import UsageAlertFlushRecorder + + self.usage_alert_flush_recorder = UsageAlertFlushRecorder() self.redis_cache = redis_cache self.redis_update_buffer = RedisUpdateBuffer(redis_cache=self.redis_cache) self.pod_lock_manager = PodLockManager() @@ -1879,6 +1882,7 @@ class DBSpendUpdateWriter: daily_spend_transactions=daily_spend_update_transactions, ) uncommitted.pop("daily_spend_update_transactions", None) + await self.usage_alert_flush_recorder.record(prisma_client, "user") if daily_team_spend_update_transactions is not None: await DBSpendUpdateWriter.update_daily_team_spend( @@ -1888,6 +1892,7 @@ class DBSpendUpdateWriter: daily_spend_transactions=daily_team_spend_update_transactions, ) uncommitted.pop("daily_team_spend_update_transactions", None) + await self.usage_alert_flush_recorder.record(prisma_client, "team") if daily_org_spend_update_transactions is not None: await DBSpendUpdateWriter.update_daily_org_spend( @@ -1970,6 +1975,8 @@ class DBSpendUpdateWriter: ) try: await asyncio.shield(commit_task) + if entity_type in ("user", "team"): + await self.usage_alert_flush_recorder.record(prisma_client, entity_type) except asyncio.CancelledError: if commit_started.is_set(): queue.track_interrupted_commit( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4d5f8439f37..144819876bd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11678,6 +11678,21 @@ class ProxyStartupEvent: misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) + from litellm.proxy.spend_tracking.usage_alert_job import USAGE_ALERT_JOB_ID, UsageAlertJob + + usage_alert_job: Final = UsageAlertJob.from_prisma(prisma_client, proxy_logging_obj.slack_alerting_instance) + scheduler.add_job( + usage_alert_job.run, + "interval", + seconds=3600, + next_run_time=datetime.now() + timedelta(seconds=batch_writing_interval + 10 + random.randint(0, 20)), + id=USAGE_ALERT_JOB_ID, + replace_existing=True, + max_instances=1, + coalesce=True, + misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, + ) + ### TOOL DISCOVERY (own lock, same interval, never rides the spend commit) ### from litellm.proxy.utils import commit_tool_discoveries diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 8a67d1b123a..3a08c094b3d 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule { @@id([delivery_id, rule_id, source_id]) } + +// Hourly samples of successful background daily-aggregate flushes, retained for 35 days. +model LiteLLM_UsageAlertFlush { + source String + hour DateTime + observed_at DateTime + + @@id([source, hour]) +} diff --git a/litellm/proxy/spend_tracking/daily_usage_alerts.py b/litellm/proxy/spend_tracking/daily_usage_alerts.py new file mode 100644 index 00000000000..a79adaaedfa --- /dev/null +++ b/litellm/proxy/spend_tracking/daily_usage_alerts.py @@ -0,0 +1,86 @@ +"""Daily comparisons over existing aggregated counters, outside the request path.""" + +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date, timedelta +from typing import Final, Literal, Protocol + +from litellm.repositories.usage_alert_metrics import DailyAlertMetrics + + +class ComparisonRule(Protocol): + @property + def metric(self) -> Literal["spend", "requests", "latency", "errors"]: ... + + @property + def direction(self) -> Literal["increase", "decrease"]: ... + + @property + def multiplier(self) -> float: ... + + @property + def baseline_days(self) -> int: ... + + @property + def min_history_days(self) -> int: ... + + @property + def min_requests(self) -> int: ... + + @property + def min_value(self) -> float: ... + + +@dataclass(frozen=True, slots=True) +class DailyComparison: + baseline_sample_days: int + observed: float + baseline: float + threshold: float + breached: bool + + +def _value(metric: str, rows: Sequence[DailyAlertMetrics], days: int) -> float | None: + if metric == "latency": + timed: Final = sum(row.timed_requests for row in rows) + return sum(row.response_time_ms for row in rows) / timed if timed else None + if metric == "spend": + return sum(row.spend for row in rows) / days + if metric == "errors": + return sum(row.errors for row in rows) / days + return sum(row.requests for row in rows) / days + + +def compare_daily_usage( + rule: ComparisonRule, rows: Sequence[DailyAlertMetrics], target: date +) -> DailyComparison | None: + """None means insufficient evidence; it must not resolve an existing incident.""" + start: Final = target - timedelta(days=rule.baseline_days) + history: Final = tuple(row for row in rows if start <= row.day < target) + current: Final = tuple(row for row in rows if row.day == target) + sample_days: Final = len({row.day for row in history}) + if sample_days < rule.min_history_days: + return None + baseline_requests: Final = sum(row.requests for row in history) / sample_days + current_requests: Final = sum(row.requests for row in current) + if baseline_requests < rule.min_requests: + return None + if rule.direction == "increase" and current_requests < rule.min_requests: + return None + if rule.metric == "latency": + baseline_timed: Final = sum(row.timed_requests for row in history) / sample_days + if baseline_timed < rule.min_requests or sum(row.timed_requests for row in current) < rule.min_requests: + return None + baseline: Final = _value(rule.metric, history, sample_days) + observed: Final = _value(rule.metric, current, 1) + if baseline is None or baseline < 0 or observed is None: + return None + if baseline == 0: + if rule.direction == "decrease": + return None + return DailyComparison(sample_days, observed, 0, rule.min_value, observed > rule.min_value) + threshold: Final = baseline * rule.multiplier if rule.direction == "increase" else baseline / rule.multiplier + crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold + return DailyComparison( + sample_days, observed, baseline, threshold, crossed and abs(observed - baseline) >= rule.min_value + ) diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py new file mode 100644 index 00000000000..18bd2059adb --- /dev/null +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -0,0 +1,286 @@ +"""Once-daily evaluation, polled hourly by one database-elected worker.""" + +import asyncio +from collections.abc import Sequence +from datetime import date, datetime, time, timedelta, timezone +from itertools import groupby, product +from typing import ( + TYPE_CHECKING, + Final, + Protocol, + cast, # noqa: TID251 # generated Prisma delegate supplies the repository protocol +) +from uuid import NAMESPACE_URL, uuid4, uuid5 + +from litellm._logging import verbose_proxy_logger +from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, configured_destinations +from litellm.proxy.spend_tracking.daily_usage_alerts import compare_daily_usage +from litellm.repositories.usage_alert_health import UsageAlertHealthRepository +from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, MetricsDatabase, UsageAlertMetricsRepository +from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.types.integrations.slack_alerting import AlertType +from litellm.types.proxy.usage_alerts import ( + UsageAlert, + UsageAlertChannel, + UsageAlertMetric, + UsageAlertNotification, + UsageAlertObservation, + UsageAlertRule, + UsageAlertRuleInput, + UsageAlertScope, +) + +if TYPE_CHECKING: + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + from litellm.proxy.utils import PrismaClient + +USAGE_ALERT_JOB_ID: Final = "daily_usage_alerts" +_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("deployment", "key", "user", "team", "model") +_METRICS: Final[tuple[UsageAlertMetric, ...]] = ("spend", "requests", "latency", "errors") +_MIN_VALUES: Final = {"spend": 10, "requests": 100, "latency": 100, "errors": 10} + + +def default_usage_rules( + channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False +) -> tuple[UsageAlertRuleInput, ...]: + return tuple( + UsageAlertRuleInput( + name=f"Daily {scope} {metric} {direction}", + scope_type=scope, + metric=metric, + direction=direction, + min_value=_MIN_VALUES[metric], + notification_channels=( + () + if legacy_user_spend_alerts and (scope, metric, direction) == ("user", "spend", "increase") + else channels + ), + ) + for scope, metric, direction in product(_SCOPES, _METRICS, ("increase", "decrease")) + ) + + +def completed_day(now: datetime) -> date: + """A UTC day gets an hour for normal buffered writes before it is eligible.""" + return (now.astimezone(timezone.utc) - timedelta(hours=1)).date() - timedelta(days=1) + + +def _has_activity_page(scope: UsageAlertScope, entity: str) -> bool: + if not entity.strip(): + return False + if scope == "key": + return entity != "unattributed" and not entity.startswith("sk-") + if scope == "user": + return entity != "default_user_id" + if scope == "team": + return entity != "litellm-dashboard" + return True + + +def _observations( + rule: UsageAlertRule, + rows: Sequence[DailyAlertMetrics], + target: date, + suppressed: frozenset[str | None] = frozenset(), +) -> tuple[UsageAlertObservation, ...]: + def observation(entity: str, entity_rows: Sequence[DailyAlertMetrics]) -> UsageAlertObservation | None: + if None in suppressed or entity in suppressed: + return None + if not _has_activity_page(rule.scope_type, entity): + return None + if rule.scope_id is not None and rule.scope_id != entity: + return None + comparison: Final = compare_daily_usage(rule, entity_rows, target) + if comparison is None: + return None + identity: Final = None if rule.scope_type == "deployment" else entity + return UsageAlertObservation( + entity_id=identity, + baseline_sample_days=comparison.baseline_sample_days, + day=target, + observed=comparison.observed, + baseline=comparison.baseline, + threshold=comparison.threshold, + breached=comparison.breached, + ) + + candidates: Final = tuple( + observation(entity, tuple(group)) for entity, group in groupby(rows, lambda row: row.entity_id) + ) + return tuple(item for item in candidates if item is not None) + + +def _default_rule(rule: UsageAlertRule) -> bool: + return rule.id == str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}")) + + +def _suppressed_entities(rule: UsageAlertRule, enabled: tuple[UsageAlertRule, ...]) -> frozenset[str | None]: + if not _default_rule(rule): + return frozenset() + matching: Final = tuple( + other + for other in enabled + if (other.scope_type, other.metric, other.direction) == (rule.scope_type, rule.metric, rule.direction) + ) + return frozenset(other.scope_id for other in matching if not _default_rule(other)) + + +class IncidentDelivery(Protocol): + async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool: ... + + +class UsageAlertJob: + def __init__( + self, + store: UsageAlertsRepository, + metrics: UsageAlertMetricsRepository, + health: UsageAlertHealthRepository, + delivery: IncidentDelivery, + defaults: tuple[UsageAlertRuleInput, ...] | None = None, + ) -> None: + self._store = store + self._metrics = metrics + self._health = health + self._delivery = delivery + self._defaults = default_usage_rules() if defaults is None else defaults + self._defaults_registered = False + + @classmethod + def from_prisma(cls, prisma: "PrismaClient", slack: "SlackAlerting") -> "UsageAlertJob": + database: Final = cast(MetricsDatabase, prisma.writer_db) # cast-ok: generated Prisma delegate + return cls( + UsageAlertsRepository.from_prisma(prisma), + UsageAlertMetricsRepository(database), + UsageAlertHealthRepository(database), + UsageAlertDelivery(slack), + default_usage_rules( + ("slack",) + if "slack" in configured_destinations(slack) and AlertType.usage_alerts in slack.alert_types + else (), + legacy_user_spend_alerts=AlertType.user_spend_anomalies in slack.alert_types, + ), + ) + + async def run(self, now: datetime | None = None) -> None: + owner: Final = str(uuid4()) + if not await self._store.claim_job(owner): + return + try: + await asyncio.wait_for(self._run(now or datetime.now(timezone.utc)), timeout=840) + except asyncio.TimeoutError: + verbose_proxy_logger.warning( + "Usage alert evaluation exceeded its 14-minute lease budget; retrying next hour" + ) + except Exception: # noqa: BLE001 # the scheduler retries; completed rule/day checkpoints survive a failed pass + verbose_proxy_logger.exception("Daily usage alert evaluation failed") + finally: + await self._store.release_job(owner) + + async def _rules(self) -> tuple[UsageAlertRule, ...]: + pages: Final[list[UsageAlertRule]] = [] # mutable-ok: collect configuration pages + offset = 0 # rebind-ok: configuration pagination cursor + while page := await self._store.list_enabled_rules(limit=100, offset=offset): + pages.extend(page) + if len(page) < 100: + return tuple(pages) + offset += len(page) + return tuple(pages) + + async def _run(self, now: datetime) -> None: + if not self._defaults_registered: + await self._store.ensure_default_rules(self._defaults) + self._defaults_registered = True + target: Final = completed_day(now) + rules: Final = await self._rules() + due: Final = tuple([rule for rule in rules if not await self._store.is_day_complete(rule, target)]) + for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in _SCOPES): + if scoped: + await self._scope(scoped, target, rules) + await self._notify() + await self._store.prune() + + async def _scope( + self, rules: tuple[UsageAlertRule, ...], target: date, enabled: tuple[UsageAlertRule, ...] + ) -> None: + scope: Final = rules[0].scope_type + start: Final = datetime.combine(target, time(), tzinfo=timezone.utc) + coverage: Final = await self._health.coverage("team" if scope == "team" else "user", start) + eligible: Final = tuple( + rule for rule in rules if coverage.after_grace and (rule.direction == "increase" or coverage.hours == 24) + ) + suppressed: Final = {rule.id: _suppressed_entities(rule, enabled) for rule in eligible} + for rule in rules: + if rule not in eligible: + await self._store.record_evaluation_status(rule, target, "insufficient_coverage") + if not eligible: + return + evaluated: Final[set[str]] = set() # mutable-ok: at most one entry per configured rule across streaming pages + async for rows in self._metrics.batches(scope, target, max(rule.baseline_days for rule in eligible)): + evaluated.update(await self._page(eligible, rows, target, enabled)) + for rule in eligible: + await self._store.finish_day(rule, target) + await self._store.record_evaluation_status( + rule, target, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history" + ) + + async def _page( + self, + rules: tuple[UsageAlertRule, ...], + rows: tuple[DailyAlertMetrics, ...], + target: date, + enabled: tuple[UsageAlertRule, ...], + ) -> tuple[str, ...]: + entities: Final = ( + (None,) if rules[0].scope_type == "deployment" else tuple(dict.fromkeys(row.entity_id for row in rows)) + ) + active: Final = await self._store.breached_entities(tuple(rule.id for rule in rules), entities) + return tuple( + [ + rule.id + for rule in rules + if await self._rule_page( + rule, rows, target, active.get(rule.id, frozenset()), _suppressed_entities(rule, enabled) + ) + ] + ) + + async def _rule_page( + self, + rule: UsageAlertRule, + rows: tuple[DailyAlertMetrics, ...], + target: date, + active: frozenset[str | None], + suppressed: frozenset[str | None], + ) -> bool: + observations: Final = _observations(rule, rows, target, suppressed) + changed: Final = tuple(item for item in observations if item.breached or item.entity_id in active) + if changed: + await self._store.record_observations(rule, changed) + return bool(observations) + + async def _deliver_channel(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> str | None: + try: + delivered: Final = await asyncio.wait_for(self._delivery.send(alert, rule, channel), timeout=30) + if not delivered: + return "transport_not_configured" + await self._store.mark_notified(alert.id, channel) + except Exception: # noqa: BLE001 # retry only failed channels; persisted acknowledgements survive restarts + verbose_proxy_logger.warning("Usage alert delivery failed for incident %s via %s", alert.id, channel) + return "delivery_failed" + return None + + async def _notify(self) -> None: + while page := await self._store.pending_notifications(limit=100): + for pending in page: + await self._notify_incident(pending) + + async def _notify_incident(self, pending: UsageAlertNotification) -> None: + failures: Final = tuple( + [ + await self._deliver_channel(pending.incident, pending.rule, channel) + for channel in pending.pending_channels + ] + ) + if any(failures): + error: Final = "delivery_failed" if "delivery_failed" in failures else "transport_not_configured" + await self._store.defer_notification(pending.incident.id, error) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a2928bba287..732eb0b4b35 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -8282,6 +8282,8 @@ async def send_email( html: str | None = None, text: str | None = None, message_id: str | None = None, + *, + raise_on_error: bool = False, ) -> bool: """ smtp_host, @@ -8340,6 +8342,8 @@ async def send_email( ) except Exception as e: + if raise_on_error: + raise verbose_proxy_logger.exception("An error occurred while sending the email:" + str(e)) return False return True diff --git a/litellm/repositories/usage_alert_health.py b/litellm/repositories/usage_alert_health.py new file mode 100644 index 00000000000..50049eb126a --- /dev/null +++ b/litellm/repositories/usage_alert_health.py @@ -0,0 +1,95 @@ +"""Sample successful aggregate flushes without adding request-time telemetry.""" + +from datetime import datetime, timedelta, timezone +from typing import ( + TYPE_CHECKING, + Final, + Literal, + TypeAlias, + cast, # noqa: TID251 # generated Prisma delegate supplies the repository protocol +) + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.repositories.usage_alert_metrics import MetricsDatabase + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +FlushSource: TypeAlias = Literal["user", "team"] + + +class FlushCoverage(BaseModel): + model_config = ConfigDict(frozen=True) + hours: int + after_grace: bool + + +_COVERAGE: Final = TypeAdapter(tuple[FlushCoverage, ...]) + + +class UsageAlertHealthRepository: + def __init__(self, database: MetricsDatabase) -> None: + self._database = database + + async def record(self, source: FlushSource, now: datetime) -> bool: + hour: Final = now.astimezone(timezone.utc).replace(minute=0, second=0, microsecond=0, tzinfo=None) + rows: Final = await self._database.query_raw( + """INSERT INTO "LiteLLM_UsageAlertFlush" (source,hour,observed_at) + SELECT $1,$2,$3 WHERE NOT EXISTS ( + SELECT 1 FROM "LiteLLM_SpendLogDelivery" WHERE state='pending' AND created_at<$2 LIMIT 1 + ) ON CONFLICT (source,hour) DO UPDATE SET observed_at=EXCLUDED.observed_at RETURNING source""", + source, + hour, + now.astimezone(timezone.utc).replace(tzinfo=None), + ) + await self._database.query_raw( + 'DELETE FROM "LiteLLM_UsageAlertFlush" WHERE hour<$1 RETURNING source', hour - timedelta(days=35) + ) + return bool(rows) + + async def coverage(self, source: FlushSource, start: datetime) -> FlushCoverage: + utc_start: Final = start.astimezone(timezone.utc).replace(tzinfo=None) + end: Final = utc_start + timedelta(days=1) + rows: Final = _COVERAGE.validate_python( + await self._database.query_raw( + """SELECT COUNT(*) FILTER(WHERE hour >= $2 AND hour < $3)::integer AS hours, + COALESCE(BOOL_OR(observed_at >= $3 + INTERVAL '1 hour'),FALSE) AS after_grace + FROM "LiteLLM_UsageAlertFlush" WHERE source=$1 AND hour >= $2""", + source, + utc_start, + end, + ) + ) + return rows[0] if rows else FlushCoverage(hours=0, after_grace=False) + + +class UsageAlertFlushRecorder: + """At most one heartbeat attempt per source per worker per hour after success.""" + + def __init__(self) -> None: + self._user_hour: datetime | None = None + self._team_hour: datetime | None = None + + async def record(self, prisma_client: "PrismaClient", source: FlushSource) -> None: + from litellm._logging import verbose_proxy_logger + from litellm.proxy.utils import ProxyUpdateSpend + + if ProxyUpdateSpend.disable_spend_updates(): + return + now: Final = datetime.now(timezone.utc) + hour: Final = now.replace(minute=0, second=0, microsecond=0) + prior: Final = self._user_hour if source == "user" else self._team_hour + if prior == hour: + return + database: Final = cast(MetricsDatabase, prisma_client.writer_db) # cast-ok: generated Prisma delegate + try: + if not await UsageAlertHealthRepository(database).record(source, now): + return + except Exception: # noqa: BLE001 # observability must not fail or requeue a committed spend flush + verbose_proxy_logger.warning("Could not record usage-alert aggregate freshness") + return + if source == "user": + self._user_hour = hour + else: + self._team_hour = hour diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py new file mode 100644 index 00000000000..1623a698d2c --- /dev/null +++ b/litellm/repositories/usage_alert_metrics.py @@ -0,0 +1,92 @@ +"""Bounded reads of the daily rollups used by the Usage dashboard.""" + +from collections.abc import AsyncIterator, Mapping, Sequence +from datetime import date, timedelta +from typing import Final, Literal, Protocol, TypeAlias + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.repositories.daily_activity_buckets import api_key_bucket, model_bucket + +Scope: TypeAlias = Literal["deployment", "key", "user", "team", "model"] + + +class DailyAlertMetrics(BaseModel): + model_config = ConfigDict(frozen=True) + + entity_id: str + day: date + spend: float + requests: int + errors: int + response_time_ms: float + timed_requests: int + + +class MetricsDatabase(Protocol): + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... + + +_ROWS: Final = TypeAdapter(tuple[DailyAlertMetrics, ...]) +_BATCH_SIZE: Final = 500 + + +def _source(scope: Scope) -> tuple[str, str]: + if scope == "team": + return '"LiteLLM_DailyTeamSpend"', "team_id" + if scope == "user": + return '"LiteLLM_DailyUserSpend"', "user_id" + if scope == "key": + return '"LiteLLM_DailyUserSpend"', "api_key" + if scope == "model": + return '"LiteLLM_DailyUserSpend"', "COALESCE(NULLIF(model, ''), 'unknown-model')" + return '"LiteLLM_DailyUserSpend"', "'deployment'::text" + + +def _query(scope: Scope) -> str: + table, entity = _source(scope) + # Identifiers come only from the closed scope mapping; request values are parameters. + return f""" +WITH entities AS MATERIALIZED ( + SELECT DISTINCT {entity} AS entity_id FROM {table} + WHERE date >= $1 AND date <= $2 AND {entity} >= COALESCE($3::text, '') AND ($3::text IS NULL OR {entity} <> $3) + ORDER BY entity_id LIMIT $4 +) +SELECT entities.entity_id, activity.date AS day, + SUM(activity.spend) AS spend, + SUM(activity.api_requests)::bigint AS requests, + SUM(activity.failed_requests)::bigint AS errors, + SUM(activity.total_response_time_ms) AS response_time_ms, + SUM(activity.timed_requests)::bigint AS timed_requests +FROM {table} activity JOIN entities ON {entity} = entities.entity_id +WHERE activity.date >= $1 AND activity.date <= $2 +GROUP BY entities.entity_id, activity.date +ORDER BY entities.entity_id, activity.date +""" + + +def _reporting_id(scope: Scope, row: DailyAlertMetrics) -> DailyAlertMetrics: + if scope == "key": + return row.model_copy(update={"entity_id": api_key_bucket(row.entity_id)}) + if scope == "model": + return row.model_copy(update={"entity_id": model_bucket(row.entity_id)}) + return row + + +class UsageAlertMetricsRepository: + def __init__(self, database: MetricsDatabase) -> None: + self._database = database + + async def batches( + self, scope: Scope, target: date, baseline_days: int + ) -> AsyncIterator[tuple[DailyAlertMetrics, ...]]: + start: Final = (target - timedelta(days=baseline_days)).isoformat() + query: Final = _query(scope) + cursor: str | None = None # rebind-ok: keyset pagination retains only the current page + while rows := _ROWS.validate_python( + await self._database.query_raw(query, start, target.isoformat(), cursor, _BATCH_SIZE) + ): + yield tuple(_reporting_id(scope, row) for row in rows) + cursor = rows[-1].entity_id + if scope == "deployment": + return diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 3ecfe60d679..dcab3eb7cac 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -3,6 +3,7 @@ import json from collections.abc import Mapping from datetime import date, datetime, timedelta, timezone +from itertools import groupby from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -13,7 +14,7 @@ from typing import ( from urllib.parse import urlencode from uuid import NAMESPACE_URL, uuid4, uuid5 -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter from litellm.repositories.daily_activity_buckets import api_key_from_bucket from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS @@ -45,6 +46,12 @@ if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient +class _BreachedEntity(BaseModel): + model_config = ConfigDict(frozen=True) + rule_id: str + scope_id: str | None + + class AlertDatabase(Protocol): async def query_raw(self, query: str, *args: object) -> object: ... @@ -672,7 +679,9 @@ class UsageAlertsRepository: rows: Final = TypeAdapter(tuple[_Pending, ...]).validate_python( await self.database.query_raw( """SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id - WHERE i.status='open' AND r.enabled AND NOT r.deleted AND EXISTS ( + WHERE i.status='open' AND r.enabled AND NOT r.deleted + AND COALESCE((i.record->>'next_notification_attempt')::timestamptz,'-infinity'::timestamptz)<=CURRENT_TIMESTAMP + AND EXISTS ( SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels))) ORDER BY i.updated_at,i.id LIMIT $1""", limit, @@ -693,7 +702,16 @@ class UsageAlertsRepository: async def mark_notified(self, incident_id: str, channel: UsageAlertChannel) -> None: await self.database.query_raw( - 'UPDATE "LiteLLM_UsageAlertIncident" SET notified_channels=array_append(notified_channels,$2::text) WHERE id=$1 AND NOT ($2::text=ANY(notified_channels)) RETURNING id', + """UPDATE "LiteLLM_UsageAlertIncident" i SET + notified_channels=array_append(i.notified_channels,$2::text), + record=i.record || jsonb_build_object('notified_channels',array_append(i.notified_channels,$2::text), 'last_notification_attempt',CURRENT_TIMESTAMP) || + CASE WHEN NOT EXISTS ( + SELECT 1 FROM "LiteLLM_UsageAlertRule" r, + jsonb_array_elements_text(r.record->'notification_channels') channel + WHERE r.id=i.rule_id AND NOT (channel=ANY(array_append(i.notified_channels,$2::text))) + ) THEN jsonb_build_object('notification_error',NULL,'next_notification_attempt',NULL) + ELSE '{}'::jsonb END + WHERE i.id=$1 AND NOT ($2::text=ANY(i.notified_channels)) RETURNING i.id""", incident_id, channel, ) @@ -718,3 +736,35 @@ class UsageAlertsRepository: ) ) return rows[0]["role"] if rows else None + + async def breached_entities( + self, rule_ids: tuple[str, ...], entity_ids: tuple[str | None, ...] + ) -> Mapping[str, frozenset[str | None]]: + rows: Final = TypeAdapter(tuple[_BreachedEntity, ...]).validate_python( + await self.database.query_raw( + 'SELECT DISTINCT rule_id,scope_id FROM "LiteLLM_UsageAlertIncident" ' + "WHERE rule_id=ANY($1::text[]) AND scope_key=ANY($2::text[]) AND breached ORDER BY rule_id", + rule_ids, + tuple(entity or "" for entity in entity_ids), + ) + ) + return MappingProxyType( + { + rule_id: frozenset(row.scope_id for row in group) + for rule_id, group in groupby(rows, lambda row: row.rule_id) + } + ) + + async def defer_notification(self, incident_id: str, error: str) -> None: + await self.database.query_raw( + """UPDATE "LiteLLM_UsageAlertIncident" SET updated_at=CURRENT_TIMESTAMP, + record=record || jsonb_build_object( + 'notification_error',$2::text, + 'notification_attempts',COALESCE((record->>'notification_attempts')::int,0)+1, + 'last_notification_attempt',CURRENT_TIMESTAMP, + 'next_notification_attempt',CURRENT_TIMESTAMP + make_interval(secs => + LEAST(86400,3600*power(2,LEAST(5,COALESCE((record->>'notification_attempts')::int,0)))::int)) + ) WHERE id=$1 RETURNING id""", + incident_id, + error, + ) diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py index 89c2ad841e8..19e6437dadf 100644 --- a/litellm/types/integrations/slack_alerting.py +++ b/litellm/types/integrations/slack_alerting.py @@ -177,6 +177,7 @@ class AlertType(str, Enum): failed_tracking_spend = "failed_tracking_spend" user_spend_thresholds = "user_spend_thresholds" user_spend_anomalies = "user_spend_anomalies" + usage_alerts = "usage_alerts" # Database alerts db_exceptions = "db_exceptions" diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index 64336441fbe..cc967e03e66 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -85,6 +85,11 @@ class UsageAlert(BaseModel): first_seen: datetime last_seen: datetime activity_path: str + notified_channels: tuple[UsageAlertChannel, ...] = () + notification_attempts: int = 0 + last_notification_attempt: datetime | None = None + next_notification_attempt: datetime | None = None + notification_error: Literal["delivery_failed", "transport_not_configured"] | None = None class UsageAlertUpdate(BaseModel): diff --git a/schema.prisma b/schema.prisma index 8a67d1b123a..3a08c094b3d 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule { @@id([delivery_id, rule_id, source_id]) } + +// Hourly samples of successful background daily-aggregate flushes, retained for 35 days. +model LiteLLM_UsageAlertFlush { + source String + hour DateTime + observed_at DateTime + + @@id([source, hour]) +} diff --git a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py new file mode 100644 index 00000000000..a0f6088a807 --- /dev/null +++ b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py @@ -0,0 +1,78 @@ +from datetime import date, datetime, timezone +from typing import Final + +import httpx +import pytest + +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, activity_url +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule + +_NOW: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) +_RULE: Final = UsageAlertRule( + name="Request spike", + id="rule", + scope_type="key", + scope_id="reporting-key", + metric="requests", + direction="increase", + created_at=_NOW, + updated_at=_NOW, + created_by=None, + notification_channels=("slack",), +) +_ALERT: Final = UsageAlert( + id="incident", + rule_id="rule", + rule_name="Request spike", + scope_type="key", + scope_id="reporting-key", + metric="requests", + direction="increase", + status="open", + day=date(2026, 10, 20), + observed_value=1000, + baseline_value=200, + threshold=600, + first_seen=_NOW, + last_seen=_NOW, + baseline_start=date(2026, 10, 13), + baseline_end=date(2026, 10, 19), + baseline_sample_days=7, + activity_path="/api-keys?key=reporting-key&detail_tab=activity", +) + + +def test_activity_link_retains_dashboard_mount_and_entity_query() -> None: + assert activity_url(_ALERT, "https://gateway.example.com/") == ( + "https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" + ) + assert activity_url(_ALERT, None) == _ALERT.activity_path + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [200, 503]) +async def test_slack_ack_requires_successful_http_response(status: int, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + requests: Final[list[httpx.Request]] = [] + + def transport(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(status) + + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(transport)) + slack: Final = SlackAlerting( + alerting=["slack"], default_webhook_url="https://hooks.example.com/test", async_http_handler=handler + ) + delivery: Final = UsageAlertDelivery(slack) + try: + if status == 200: + assert await delivery.send(_ALERT, _RULE, "slack") is True + else: + with pytest.raises(httpx.HTTPStatusError): + await delivery.send(_ALERT, _RULE, "slack") + assert len(requests) == 1 + assert b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content + finally: + await handler.close() diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py new file mode 100644 index 00000000000..4aa657de48b --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -0,0 +1,155 @@ +from datetime import date, datetime, timedelta, timezone +from typing import Final + +import pytest + +from litellm.proxy.spend_tracking.daily_usage_alerts import DailyComparison, compare_daily_usage +from litellm.proxy.spend_tracking.usage_alert_job import completed_day +from litellm.repositories.usage_alert_metrics import DailyAlertMetrics +from litellm.types.proxy.usage_alerts import UsageAlertRuleInput + +_DAY: Final = date(2026, 10, 20) + + +def _row(day: date, requests: int = 200, spend: float = 20, ms: float = 100_000, timed: int = 200) -> DailyAlertMetrics: + return DailyAlertMetrics( + entity_id="key-a", day=day, spend=spend, requests=requests, errors=10, response_time_ms=ms, timed_requests=timed + ) + + +@pytest.mark.parametrize( + "metric,observed,baseline", [("spend", 100, 20), ("requests", 1000, 200), ("latency", 2000, 500)] +) +def test_spikes_use_recorded_reference_days_and_weighted_latency(metric: str, observed: float, baseline: float) -> None: + rule: Final = UsageAlertRuleInput.model_validate( + {"name": "spike", "scope_type": "key", "metric": metric, "direction": "increase"} + ) + history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 3, 7)) + current: Final = _row(_DAY, requests=1000, spend=100, ms=400_000) + assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison( + 3, observed, baseline, baseline * 3, True + ) + + +def test_missing_target_is_zero_for_counts_but_never_zero_latency() -> None: + history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) + rule: Final = UsageAlertRuleInput(name="drop", scope_type="key", metric="requests", direction="decrease") + assert compare_daily_usage(rule, history, _DAY) == DailyComparison(3, 0, 200, 200 / 3, True) + assert compare_daily_usage(rule.model_copy(update={"metric": "latency"}), history, _DAY) is None + + +def test_low_volume_is_insufficient_and_recorded_zero_baseline_uses_absolute_floor() -> None: + rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase") + history: Final = tuple(_row(_DAY - timedelta(days=days), requests=10, spend=0) for days in (1, 2, 3)) + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) is None + enough_requests: Final = tuple(row.model_copy(update={"requests": 200}) for row in history) + assert compare_daily_usage(rule, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison( + 3, 100, 0, 0, True + ) + floor: Final = rule.model_copy(update={"min_value": 100}) + assert compare_daily_usage(floor, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison( + 3, 100, 0, 100, False + ) + + +def test_absolute_floor_suppresses_small_changes() -> None: + history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) + rule: Final = UsageAlertRuleInput( + name="spike", scope_type="key", metric="spend", direction="increase", min_value=100 + ) + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 60, False) + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, False) + + +def test_day_is_eligible_only_after_one_hour_utc_grace() -> None: + assert completed_day(datetime(2026, 10, 21, 0, 59, tzinfo=timezone.utc)) == date(2026, 10, 19) + assert completed_day(datetime(2026, 10, 21, 1, 0, tzinfo=timezone.utc)) == _DAY + + +def test_latency_weights_request_samples_instead_of_averaging_daily_averages() -> None: + rule: Final = UsageAlertRuleInput( + name="latency", scope_type="key", metric="latency", direction="increase", min_history_days=2 + ) + rows: Final = ( + _row(_DAY - timedelta(days=1), requests=100, ms=100_000, timed=100), + _row(_DAY - timedelta(days=2), requests=400, ms=1_600_000, timed=400), + _row(_DAY, requests=100, ms=1_100_000, timed=100), + ) + assert compare_daily_usage(rule, rows, _DAY) == DailyComparison(2, 11000, 3400, 10200, True) + + +def test_errors_are_failed_request_count_and_require_absolute_change_floor() -> None: + rule: Final = UsageAlertRuleInput( + name="errors", scope_type="key", metric="errors", direction="increase", min_value=10 + ) + history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) + current: Final = _row(_DAY).model_copy(update={"errors": 40}) + assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(3, 40, 10, 30, True) + + +def test_a_value_at_the_configured_multiplier_alerts() -> None: + history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) + rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase") + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, True) + + +def test_default_notifications_require_opt_in_and_do_not_duplicate_legacy_user_spend() -> None: + from litellm.proxy.spend_tracking.usage_alert_job import default_usage_rules + + defaults: Final = default_usage_rules() + assert len(defaults) == 40 + assert all(not rule.notification_channels for rule in defaults) + opted_in: Final = default_usage_rules(("slack",), legacy_user_spend_alerts=True) + quiet: Final = tuple(rule for rule in opted_in if not rule.notification_channels) + assert tuple((rule.scope_type, rule.metric, rule.direction) for rule in quiet) == (("user", "spend", "increase"),) + + +@pytest.mark.parametrize( + "scope,entity", + [("key", "unattributed"), ("key", "sk-example"), ("user", "default_user_id"), ("team", "litellm-dashboard")], +) +def test_sentinel_entities_cannot_create_alerts_with_unusable_activity_links(scope: str, entity: str) -> None: + from litellm.proxy.spend_tracking.usage_alert_job import _observations + from litellm.types.proxy.usage_alerts import UsageAlertRule + + rule: Final = UsageAlertRule.model_validate( + { + "id": "rule", + "name": "spike", + "scope_type": scope, + "metric": "spend", + "direction": "increase", + "created_at": "2026-10-01T00:00:00Z", + "updated_at": "2026-10-01T00:00:00Z", + "created_by": None, + } + ) + history: Final = tuple( + _row(_DAY - timedelta(days=days)).model_copy(update={"entity_id": entity}) for days in (1, 2, 3) + ) + current: Final = _row(_DAY, spend=100).model_copy(update={"entity_id": entity}) + assert _observations(rule, (*history, current), _DAY) == () + + +def test_custom_entity_rule_replaces_only_the_matching_default() -> None: + from uuid import NAMESPACE_URL, uuid5 + + from litellm.proxy.spend_tracking.usage_alert_job import _suppressed_entities + from litellm.types.proxy.usage_alerts import UsageAlertRule + + default: Final = UsageAlertRule( + id=str(uuid5(NAMESPACE_URL, "litellm:usage-alert:key:spend:increase")), + name="default", + scope_type="key", + metric="spend", + direction="increase", + created_at=datetime(2026, 10, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 10, 1, tzinfo=timezone.utc), + created_by=None, + ) + custom: Final = default.model_copy(update={"id": "custom", "scope_id": "key-a"}) + other_metric: Final = custom.model_copy(update={"id": "latency", "metric": "latency", "scope_id": "key-b"}) + assert _suppressed_entities(default, (default, custom, other_metric)) == frozenset({"key-a"}) + assert _suppressed_entities(custom, (default, custom, other_metric)) == frozenset() + wildcard: Final = custom.model_copy(update={"scope_id": None}) + assert _suppressed_entities(default, (default, wildcard)) == frozenset({None}) diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py b/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py index d4965c64559..c4f36883270 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py @@ -7,13 +7,16 @@ Symbols pinned here: from __future__ import annotations import threading -from typing import Any +from typing import Any, TYPE_CHECKING from unittest.mock import Mock import pytest from litellm.proxy.utils import send_email +if TYPE_CHECKING: + from tests.unit.proxy.utils.prisma_and_spend.conftest import InMemorySMTP + @pytest.fixture(autouse=True) def _smtp_env(monkeypatch: pytest.MonkeyPatch) -> None: @@ -176,3 +179,12 @@ async def test_should_send_plain_text_and_html_as_multipart_alternative(monkeypa assert tuple(part.get_content_type() for part in parts) == ("text/plain", "text/html") assert parts[0].get_payload(decode=True).decode() == "Model retirement" assert parts[1].get_payload(decode=True).decode() == "

Model retirement

" + + +@pytest.mark.asyncio +async def test_durable_alert_delivery_can_observe_smtp_failure(in_memory_smtp: InMemorySMTP) -> None: + in_memory_smtp.raise_on_send = RuntimeError("smtp unavailable") + with pytest.raises(RuntimeError, match="smtp unavailable"): + await send_email(receiver_email="alerts@example.com", subject="Daily usage", html="

Alert

", + raise_on_error=True) + assert in_memory_smtp.sent == []