diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105110000_usage_alert_flush/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105110000_usage_alert_flush/migration.sql
new file mode 100644
index 00000000000..217fa4f1510
--- /dev/null
+++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105110000_usage_alert_flush/migration.sql
@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertFlush" (
+ source TEXT NOT NULL,
+ hour TIMESTAMP(3) NOT NULL,
+ observed_at TIMESTAMP(3) NOT NULL,
+ PRIMARY KEY (source, hour)
+);
diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
index 8a67d1b123a..3a08c094b3d 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
@@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule {
@@id([delivery_id, rule_id, source_id])
}
+
+// Hourly samples of successful background daily-aggregate flushes, retained for 35 days.
+model LiteLLM_UsageAlertFlush {
+ source String
+ hour DateTime
+ observed_at DateTime
+
+ @@id([source, hour])
+}
diff --git a/litellm/integrations/SlackAlerting/usage_alerts.py b/litellm/integrations/SlackAlerting/usage_alerts.py
new file mode 100644
index 00000000000..9b09d8fb05c
--- /dev/null
+++ b/litellm/integrations/SlackAlerting/usage_alerts.py
@@ -0,0 +1,88 @@
+"""Deliver persisted usage incidents through the existing configured transports."""
+
+import json
+import os
+from html import escape
+from typing import TYPE_CHECKING, Final
+from urllib.parse import urlsplit
+
+from pydantic import TypeAdapter
+
+from litellm.types.integrations.slack_alerting import AlertType
+from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertChannel, UsageAlertRule
+
+if TYPE_CHECKING:
+ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
+
+_URLS: Final = TypeAdapter(tuple[str, ...])
+
+
+def configured_destinations(slack: "SlackAlerting") -> tuple[str, ...]:
+ return _URLS.validate_python(slack.alerting or ()) # pyright: ignore[reportUnknownMemberType] # validate legacy destination config
+
+
+def activity_url(alert: UsageAlert, base_url: str | None) -> str:
+ if not base_url:
+ return alert.activity_path
+ parsed: Final = urlsplit(base_url)
+ if parsed.scheme not in ("http", "https") or not parsed.netloc:
+ return alert.activity_path
+ return base_url.rstrip("/") + "/ui" + alert.activity_path
+
+
+def incident_message(alert: UsageAlert, url: str) -> str:
+ scope: Final = "Gateway" if alert.scope_type == "deployment" else f"{alert.scope_type}: {alert.scope_id}"
+ unit: Final = " ms" if alert.metric == "latency" else " USD" if alert.metric == "spend" else ""
+ return (
+ f"{alert.rule_name}\n{scope}\n{alert.day.isoformat()} UTC: "
+ f"{alert.observed_value:,.2f}{unit}; baseline {alert.baseline_value:,.2f}{unit}; "
+ f"threshold {alert.threshold:,.2f}{unit}.\nView activity: {url}"
+ )
+
+
+class UsageAlertDelivery:
+ def __init__(self, slack: "SlackAlerting") -> None:
+ self._slack = slack
+
+ async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool:
+ configured: Final = configured_destinations(self._slack)
+ if channel not in configured:
+ return False
+ url: Final = activity_url(alert, os.getenv("PROXY_BASE_URL"))
+ message: Final = incident_message(alert, url)
+ if channel == "email":
+ from litellm.proxy.utils import send_email
+
+ for recipient in rule.email_recipients:
+ await send_email(
+ receiver_email=recipient,
+ subject=f"LiteLLM usage alert: {alert.rule_name}",
+ html=(
+ f"
{escape(message).replace(chr(10), '
')}
"
+ f'View activity
'
+ ),
+ raise_on_error=True,
+ )
+ return bool(rule.email_recipients)
+ overrides: Final = self._slack.alert_to_webhook_url or {}
+ configured_url: Final = (
+ overrides.get(AlertType.usage_alerts)
+ or self._slack.default_webhook_url
+ or os.getenv("SLACK_WEBHOOK_URL")
+ or os.getenv("ALERTING_WEBHOOK_URL")
+ )
+ if not configured_url:
+ return False
+ urls: Final = (configured_url,) if isinstance(configured_url, str) else _URLS.validate_python(configured_url)
+ for webhook in urls:
+ await self._post(webhook, message)
+ return True
+
+ async def _post(self, webhook: str, message: str) -> None:
+ response: Final = await self._slack.async_http_handler.post( # pyright: ignore[reportUnknownMemberType] # legacy HTTP signature; supplied arguments are concrete
+ url=webhook,
+ headers={"Content-type": "application/json"},
+ data=json.dumps({"text": message}),
+ timeout=10,
+ )
+ response.raise_for_status()
diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py
index 80f09df6c86..53b95e6e889 100644
--- a/litellm/proxy/db/db_spend_update_writer.py
+++ b/litellm/proxy/db/db_spend_update_writer.py
@@ -665,6 +665,9 @@ class DBSpendUpdateWriter:
self,
redis_cache: RedisCache | None = None,
):
+ from litellm.repositories.usage_alert_health import UsageAlertFlushRecorder
+
+ self.usage_alert_flush_recorder = UsageAlertFlushRecorder()
self.redis_cache = redis_cache
self.redis_update_buffer = RedisUpdateBuffer(redis_cache=self.redis_cache)
self.pod_lock_manager = PodLockManager()
@@ -1879,6 +1882,7 @@ class DBSpendUpdateWriter:
daily_spend_transactions=daily_spend_update_transactions,
)
uncommitted.pop("daily_spend_update_transactions", None)
+ await self.usage_alert_flush_recorder.record(prisma_client, "user")
if daily_team_spend_update_transactions is not None:
await DBSpendUpdateWriter.update_daily_team_spend(
@@ -1888,6 +1892,7 @@ class DBSpendUpdateWriter:
daily_spend_transactions=daily_team_spend_update_transactions,
)
uncommitted.pop("daily_team_spend_update_transactions", None)
+ await self.usage_alert_flush_recorder.record(prisma_client, "team")
if daily_org_spend_update_transactions is not None:
await DBSpendUpdateWriter.update_daily_org_spend(
@@ -1970,6 +1975,8 @@ class DBSpendUpdateWriter:
)
try:
await asyncio.shield(commit_task)
+ if entity_type in ("user", "team"):
+ await self.usage_alert_flush_recorder.record(prisma_client, entity_type)
except asyncio.CancelledError:
if commit_started.is_set():
queue.track_interrupted_commit(
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 4d5f8439f37..144819876bd 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -11678,6 +11678,21 @@ class ProxyStartupEvent:
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
)
+ from litellm.proxy.spend_tracking.usage_alert_job import USAGE_ALERT_JOB_ID, UsageAlertJob
+
+ usage_alert_job: Final = UsageAlertJob.from_prisma(prisma_client, proxy_logging_obj.slack_alerting_instance)
+ scheduler.add_job(
+ usage_alert_job.run,
+ "interval",
+ seconds=3600,
+ next_run_time=datetime.now() + timedelta(seconds=batch_writing_interval + 10 + random.randint(0, 20)),
+ id=USAGE_ALERT_JOB_ID,
+ replace_existing=True,
+ max_instances=1,
+ coalesce=True,
+ misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
+ )
+
### TOOL DISCOVERY (own lock, same interval, never rides the spend commit) ###
from litellm.proxy.utils import commit_tool_discoveries
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index 8a67d1b123a..3a08c094b3d 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule {
@@id([delivery_id, rule_id, source_id])
}
+
+// Hourly samples of successful background daily-aggregate flushes, retained for 35 days.
+model LiteLLM_UsageAlertFlush {
+ source String
+ hour DateTime
+ observed_at DateTime
+
+ @@id([source, hour])
+}
diff --git a/litellm/proxy/spend_tracking/daily_usage_alerts.py b/litellm/proxy/spend_tracking/daily_usage_alerts.py
new file mode 100644
index 00000000000..a79adaaedfa
--- /dev/null
+++ b/litellm/proxy/spend_tracking/daily_usage_alerts.py
@@ -0,0 +1,86 @@
+"""Daily comparisons over existing aggregated counters, outside the request path."""
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import date, timedelta
+from typing import Final, Literal, Protocol
+
+from litellm.repositories.usage_alert_metrics import DailyAlertMetrics
+
+
+class ComparisonRule(Protocol):
+ @property
+ def metric(self) -> Literal["spend", "requests", "latency", "errors"]: ...
+
+ @property
+ def direction(self) -> Literal["increase", "decrease"]: ...
+
+ @property
+ def multiplier(self) -> float: ...
+
+ @property
+ def baseline_days(self) -> int: ...
+
+ @property
+ def min_history_days(self) -> int: ...
+
+ @property
+ def min_requests(self) -> int: ...
+
+ @property
+ def min_value(self) -> float: ...
+
+
+@dataclass(frozen=True, slots=True)
+class DailyComparison:
+ baseline_sample_days: int
+ observed: float
+ baseline: float
+ threshold: float
+ breached: bool
+
+
+def _value(metric: str, rows: Sequence[DailyAlertMetrics], days: int) -> float | None:
+ if metric == "latency":
+ timed: Final = sum(row.timed_requests for row in rows)
+ return sum(row.response_time_ms for row in rows) / timed if timed else None
+ if metric == "spend":
+ return sum(row.spend for row in rows) / days
+ if metric == "errors":
+ return sum(row.errors for row in rows) / days
+ return sum(row.requests for row in rows) / days
+
+
+def compare_daily_usage(
+ rule: ComparisonRule, rows: Sequence[DailyAlertMetrics], target: date
+) -> DailyComparison | None:
+ """None means insufficient evidence; it must not resolve an existing incident."""
+ start: Final = target - timedelta(days=rule.baseline_days)
+ history: Final = tuple(row for row in rows if start <= row.day < target)
+ current: Final = tuple(row for row in rows if row.day == target)
+ sample_days: Final = len({row.day for row in history})
+ if sample_days < rule.min_history_days:
+ return None
+ baseline_requests: Final = sum(row.requests for row in history) / sample_days
+ current_requests: Final = sum(row.requests for row in current)
+ if baseline_requests < rule.min_requests:
+ return None
+ if rule.direction == "increase" and current_requests < rule.min_requests:
+ return None
+ if rule.metric == "latency":
+ baseline_timed: Final = sum(row.timed_requests for row in history) / sample_days
+ if baseline_timed < rule.min_requests or sum(row.timed_requests for row in current) < rule.min_requests:
+ return None
+ baseline: Final = _value(rule.metric, history, sample_days)
+ observed: Final = _value(rule.metric, current, 1)
+ if baseline is None or baseline < 0 or observed is None:
+ return None
+ if baseline == 0:
+ if rule.direction == "decrease":
+ return None
+ return DailyComparison(sample_days, observed, 0, rule.min_value, observed > rule.min_value)
+ threshold: Final = baseline * rule.multiplier if rule.direction == "increase" else baseline / rule.multiplier
+ crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold
+ return DailyComparison(
+ sample_days, observed, baseline, threshold, crossed and abs(observed - baseline) >= rule.min_value
+ )
diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py
new file mode 100644
index 00000000000..18bd2059adb
--- /dev/null
+++ b/litellm/proxy/spend_tracking/usage_alert_job.py
@@ -0,0 +1,286 @@
+"""Once-daily evaluation, polled hourly by one database-elected worker."""
+
+import asyncio
+from collections.abc import Sequence
+from datetime import date, datetime, time, timedelta, timezone
+from itertools import groupby, product
+from typing import (
+ TYPE_CHECKING,
+ Final,
+ Protocol,
+ cast, # noqa: TID251 # generated Prisma delegate supplies the repository protocol
+)
+from uuid import NAMESPACE_URL, uuid4, uuid5
+
+from litellm._logging import verbose_proxy_logger
+from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, configured_destinations
+from litellm.proxy.spend_tracking.daily_usage_alerts import compare_daily_usage
+from litellm.repositories.usage_alert_health import UsageAlertHealthRepository
+from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, MetricsDatabase, UsageAlertMetricsRepository
+from litellm.repositories.usage_alerts import UsageAlertsRepository
+from litellm.types.integrations.slack_alerting import AlertType
+from litellm.types.proxy.usage_alerts import (
+ UsageAlert,
+ UsageAlertChannel,
+ UsageAlertMetric,
+ UsageAlertNotification,
+ UsageAlertObservation,
+ UsageAlertRule,
+ UsageAlertRuleInput,
+ UsageAlertScope,
+)
+
+if TYPE_CHECKING:
+ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
+ from litellm.proxy.utils import PrismaClient
+
+USAGE_ALERT_JOB_ID: Final = "daily_usage_alerts"
+_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("deployment", "key", "user", "team", "model")
+_METRICS: Final[tuple[UsageAlertMetric, ...]] = ("spend", "requests", "latency", "errors")
+_MIN_VALUES: Final = {"spend": 10, "requests": 100, "latency": 100, "errors": 10}
+
+
+def default_usage_rules(
+ channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False
+) -> tuple[UsageAlertRuleInput, ...]:
+ return tuple(
+ UsageAlertRuleInput(
+ name=f"Daily {scope} {metric} {direction}",
+ scope_type=scope,
+ metric=metric,
+ direction=direction,
+ min_value=_MIN_VALUES[metric],
+ notification_channels=(
+ ()
+ if legacy_user_spend_alerts and (scope, metric, direction) == ("user", "spend", "increase")
+ else channels
+ ),
+ )
+ for scope, metric, direction in product(_SCOPES, _METRICS, ("increase", "decrease"))
+ )
+
+
+def completed_day(now: datetime) -> date:
+ """A UTC day gets an hour for normal buffered writes before it is eligible."""
+ return (now.astimezone(timezone.utc) - timedelta(hours=1)).date() - timedelta(days=1)
+
+
+def _has_activity_page(scope: UsageAlertScope, entity: str) -> bool:
+ if not entity.strip():
+ return False
+ if scope == "key":
+ return entity != "unattributed" and not entity.startswith("sk-")
+ if scope == "user":
+ return entity != "default_user_id"
+ if scope == "team":
+ return entity != "litellm-dashboard"
+ return True
+
+
+def _observations(
+ rule: UsageAlertRule,
+ rows: Sequence[DailyAlertMetrics],
+ target: date,
+ suppressed: frozenset[str | None] = frozenset(),
+) -> tuple[UsageAlertObservation, ...]:
+ def observation(entity: str, entity_rows: Sequence[DailyAlertMetrics]) -> UsageAlertObservation | None:
+ if None in suppressed or entity in suppressed:
+ return None
+ if not _has_activity_page(rule.scope_type, entity):
+ return None
+ if rule.scope_id is not None and rule.scope_id != entity:
+ return None
+ comparison: Final = compare_daily_usage(rule, entity_rows, target)
+ if comparison is None:
+ return None
+ identity: Final = None if rule.scope_type == "deployment" else entity
+ return UsageAlertObservation(
+ entity_id=identity,
+ baseline_sample_days=comparison.baseline_sample_days,
+ day=target,
+ observed=comparison.observed,
+ baseline=comparison.baseline,
+ threshold=comparison.threshold,
+ breached=comparison.breached,
+ )
+
+ candidates: Final = tuple(
+ observation(entity, tuple(group)) for entity, group in groupby(rows, lambda row: row.entity_id)
+ )
+ return tuple(item for item in candidates if item is not None)
+
+
+def _default_rule(rule: UsageAlertRule) -> bool:
+ return rule.id == str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}"))
+
+
+def _suppressed_entities(rule: UsageAlertRule, enabled: tuple[UsageAlertRule, ...]) -> frozenset[str | None]:
+ if not _default_rule(rule):
+ return frozenset()
+ matching: Final = tuple(
+ other
+ for other in enabled
+ if (other.scope_type, other.metric, other.direction) == (rule.scope_type, rule.metric, rule.direction)
+ )
+ return frozenset(other.scope_id for other in matching if not _default_rule(other))
+
+
+class IncidentDelivery(Protocol):
+ async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool: ...
+
+
+class UsageAlertJob:
+ def __init__(
+ self,
+ store: UsageAlertsRepository,
+ metrics: UsageAlertMetricsRepository,
+ health: UsageAlertHealthRepository,
+ delivery: IncidentDelivery,
+ defaults: tuple[UsageAlertRuleInput, ...] | None = None,
+ ) -> None:
+ self._store = store
+ self._metrics = metrics
+ self._health = health
+ self._delivery = delivery
+ self._defaults = default_usage_rules() if defaults is None else defaults
+ self._defaults_registered = False
+
+ @classmethod
+ def from_prisma(cls, prisma: "PrismaClient", slack: "SlackAlerting") -> "UsageAlertJob":
+ database: Final = cast(MetricsDatabase, prisma.writer_db) # cast-ok: generated Prisma delegate
+ return cls(
+ UsageAlertsRepository.from_prisma(prisma),
+ UsageAlertMetricsRepository(database),
+ UsageAlertHealthRepository(database),
+ UsageAlertDelivery(slack),
+ default_usage_rules(
+ ("slack",)
+ if "slack" in configured_destinations(slack) and AlertType.usage_alerts in slack.alert_types
+ else (),
+ legacy_user_spend_alerts=AlertType.user_spend_anomalies in slack.alert_types,
+ ),
+ )
+
+ async def run(self, now: datetime | None = None) -> None:
+ owner: Final = str(uuid4())
+ if not await self._store.claim_job(owner):
+ return
+ try:
+ await asyncio.wait_for(self._run(now or datetime.now(timezone.utc)), timeout=840)
+ except asyncio.TimeoutError:
+ verbose_proxy_logger.warning(
+ "Usage alert evaluation exceeded its 14-minute lease budget; retrying next hour"
+ )
+ except Exception: # noqa: BLE001 # the scheduler retries; completed rule/day checkpoints survive a failed pass
+ verbose_proxy_logger.exception("Daily usage alert evaluation failed")
+ finally:
+ await self._store.release_job(owner)
+
+ async def _rules(self) -> tuple[UsageAlertRule, ...]:
+ pages: Final[list[UsageAlertRule]] = [] # mutable-ok: collect configuration pages
+ offset = 0 # rebind-ok: configuration pagination cursor
+ while page := await self._store.list_enabled_rules(limit=100, offset=offset):
+ pages.extend(page)
+ if len(page) < 100:
+ return tuple(pages)
+ offset += len(page)
+ return tuple(pages)
+
+ async def _run(self, now: datetime) -> None:
+ if not self._defaults_registered:
+ await self._store.ensure_default_rules(self._defaults)
+ self._defaults_registered = True
+ target: Final = completed_day(now)
+ rules: Final = await self._rules()
+ due: Final = tuple([rule for rule in rules if not await self._store.is_day_complete(rule, target)])
+ for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in _SCOPES):
+ if scoped:
+ await self._scope(scoped, target, rules)
+ await self._notify()
+ await self._store.prune()
+
+ async def _scope(
+ self, rules: tuple[UsageAlertRule, ...], target: date, enabled: tuple[UsageAlertRule, ...]
+ ) -> None:
+ scope: Final = rules[0].scope_type
+ start: Final = datetime.combine(target, time(), tzinfo=timezone.utc)
+ coverage: Final = await self._health.coverage("team" if scope == "team" else "user", start)
+ eligible: Final = tuple(
+ rule for rule in rules if coverage.after_grace and (rule.direction == "increase" or coverage.hours == 24)
+ )
+ suppressed: Final = {rule.id: _suppressed_entities(rule, enabled) for rule in eligible}
+ for rule in rules:
+ if rule not in eligible:
+ await self._store.record_evaluation_status(rule, target, "insufficient_coverage")
+ if not eligible:
+ return
+ evaluated: Final[set[str]] = set() # mutable-ok: at most one entry per configured rule across streaming pages
+ async for rows in self._metrics.batches(scope, target, max(rule.baseline_days for rule in eligible)):
+ evaluated.update(await self._page(eligible, rows, target, enabled))
+ for rule in eligible:
+ await self._store.finish_day(rule, target)
+ await self._store.record_evaluation_status(
+ rule, target, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history"
+ )
+
+ async def _page(
+ self,
+ rules: tuple[UsageAlertRule, ...],
+ rows: tuple[DailyAlertMetrics, ...],
+ target: date,
+ enabled: tuple[UsageAlertRule, ...],
+ ) -> tuple[str, ...]:
+ entities: Final = (
+ (None,) if rules[0].scope_type == "deployment" else tuple(dict.fromkeys(row.entity_id for row in rows))
+ )
+ active: Final = await self._store.breached_entities(tuple(rule.id for rule in rules), entities)
+ return tuple(
+ [
+ rule.id
+ for rule in rules
+ if await self._rule_page(
+ rule, rows, target, active.get(rule.id, frozenset()), _suppressed_entities(rule, enabled)
+ )
+ ]
+ )
+
+ async def _rule_page(
+ self,
+ rule: UsageAlertRule,
+ rows: tuple[DailyAlertMetrics, ...],
+ target: date,
+ active: frozenset[str | None],
+ suppressed: frozenset[str | None],
+ ) -> bool:
+ observations: Final = _observations(rule, rows, target, suppressed)
+ changed: Final = tuple(item for item in observations if item.breached or item.entity_id in active)
+ if changed:
+ await self._store.record_observations(rule, changed)
+ return bool(observations)
+
+ async def _deliver_channel(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> str | None:
+ try:
+ delivered: Final = await asyncio.wait_for(self._delivery.send(alert, rule, channel), timeout=30)
+ if not delivered:
+ return "transport_not_configured"
+ await self._store.mark_notified(alert.id, channel)
+ except Exception: # noqa: BLE001 # retry only failed channels; persisted acknowledgements survive restarts
+ verbose_proxy_logger.warning("Usage alert delivery failed for incident %s via %s", alert.id, channel)
+ return "delivery_failed"
+ return None
+
+ async def _notify(self) -> None:
+ while page := await self._store.pending_notifications(limit=100):
+ for pending in page:
+ await self._notify_incident(pending)
+
+ async def _notify_incident(self, pending: UsageAlertNotification) -> None:
+ failures: Final = tuple(
+ [
+ await self._deliver_channel(pending.incident, pending.rule, channel)
+ for channel in pending.pending_channels
+ ]
+ )
+ if any(failures):
+ error: Final = "delivery_failed" if "delivery_failed" in failures else "transport_not_configured"
+ await self._store.defer_notification(pending.incident.id, error)
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index a2928bba287..732eb0b4b35 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -8282,6 +8282,8 @@ async def send_email(
html: str | None = None,
text: str | None = None,
message_id: str | None = None,
+ *,
+ raise_on_error: bool = False,
) -> bool:
"""
smtp_host,
@@ -8340,6 +8342,8 @@ async def send_email(
)
except Exception as e:
+ if raise_on_error:
+ raise
verbose_proxy_logger.exception("An error occurred while sending the email:" + str(e))
return False
return True
diff --git a/litellm/repositories/usage_alert_health.py b/litellm/repositories/usage_alert_health.py
new file mode 100644
index 00000000000..50049eb126a
--- /dev/null
+++ b/litellm/repositories/usage_alert_health.py
@@ -0,0 +1,95 @@
+"""Sample successful aggregate flushes without adding request-time telemetry."""
+
+from datetime import datetime, timedelta, timezone
+from typing import (
+ TYPE_CHECKING,
+ Final,
+ Literal,
+ TypeAlias,
+ cast, # noqa: TID251 # generated Prisma delegate supplies the repository protocol
+)
+
+from pydantic import BaseModel, ConfigDict, TypeAdapter
+
+from litellm.repositories.usage_alert_metrics import MetricsDatabase
+
+if TYPE_CHECKING:
+ from litellm.proxy.utils import PrismaClient
+
+FlushSource: TypeAlias = Literal["user", "team"]
+
+
+class FlushCoverage(BaseModel):
+ model_config = ConfigDict(frozen=True)
+ hours: int
+ after_grace: bool
+
+
+_COVERAGE: Final = TypeAdapter(tuple[FlushCoverage, ...])
+
+
+class UsageAlertHealthRepository:
+ def __init__(self, database: MetricsDatabase) -> None:
+ self._database = database
+
+ async def record(self, source: FlushSource, now: datetime) -> bool:
+ hour: Final = now.astimezone(timezone.utc).replace(minute=0, second=0, microsecond=0, tzinfo=None)
+ rows: Final = await self._database.query_raw(
+ """INSERT INTO "LiteLLM_UsageAlertFlush" (source,hour,observed_at)
+ SELECT $1,$2,$3 WHERE NOT EXISTS (
+ SELECT 1 FROM "LiteLLM_SpendLogDelivery" WHERE state='pending' AND created_at<$2 LIMIT 1
+ ) ON CONFLICT (source,hour) DO UPDATE SET observed_at=EXCLUDED.observed_at RETURNING source""",
+ source,
+ hour,
+ now.astimezone(timezone.utc).replace(tzinfo=None),
+ )
+ await self._database.query_raw(
+ 'DELETE FROM "LiteLLM_UsageAlertFlush" WHERE hour<$1 RETURNING source', hour - timedelta(days=35)
+ )
+ return bool(rows)
+
+ async def coverage(self, source: FlushSource, start: datetime) -> FlushCoverage:
+ utc_start: Final = start.astimezone(timezone.utc).replace(tzinfo=None)
+ end: Final = utc_start + timedelta(days=1)
+ rows: Final = _COVERAGE.validate_python(
+ await self._database.query_raw(
+ """SELECT COUNT(*) FILTER(WHERE hour >= $2 AND hour < $3)::integer AS hours,
+ COALESCE(BOOL_OR(observed_at >= $3 + INTERVAL '1 hour'),FALSE) AS after_grace
+ FROM "LiteLLM_UsageAlertFlush" WHERE source=$1 AND hour >= $2""",
+ source,
+ utc_start,
+ end,
+ )
+ )
+ return rows[0] if rows else FlushCoverage(hours=0, after_grace=False)
+
+
+class UsageAlertFlushRecorder:
+ """At most one heartbeat attempt per source per worker per hour after success."""
+
+ def __init__(self) -> None:
+ self._user_hour: datetime | None = None
+ self._team_hour: datetime | None = None
+
+ async def record(self, prisma_client: "PrismaClient", source: FlushSource) -> None:
+ from litellm._logging import verbose_proxy_logger
+ from litellm.proxy.utils import ProxyUpdateSpend
+
+ if ProxyUpdateSpend.disable_spend_updates():
+ return
+ now: Final = datetime.now(timezone.utc)
+ hour: Final = now.replace(minute=0, second=0, microsecond=0)
+ prior: Final = self._user_hour if source == "user" else self._team_hour
+ if prior == hour:
+ return
+ database: Final = cast(MetricsDatabase, prisma_client.writer_db) # cast-ok: generated Prisma delegate
+ try:
+ if not await UsageAlertHealthRepository(database).record(source, now):
+ return
+ except Exception: # noqa: BLE001 # observability must not fail or requeue a committed spend flush
+ verbose_proxy_logger.warning("Could not record usage-alert aggregate freshness")
+ return
+ if source == "user":
+ self._user_hour = hour
+ else:
+ self._team_hour = hour
diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py
new file mode 100644
index 00000000000..1623a698d2c
--- /dev/null
+++ b/litellm/repositories/usage_alert_metrics.py
@@ -0,0 +1,92 @@
+"""Bounded reads of the daily rollups used by the Usage dashboard."""
+
+from collections.abc import AsyncIterator, Mapping, Sequence
+from datetime import date, timedelta
+from typing import Final, Literal, Protocol, TypeAlias
+
+from pydantic import BaseModel, ConfigDict, TypeAdapter
+
+from litellm.repositories.daily_activity_buckets import api_key_bucket, model_bucket
+
+Scope: TypeAlias = Literal["deployment", "key", "user", "team", "model"]
+
+
+class DailyAlertMetrics(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ entity_id: str
+ day: date
+ spend: float
+ requests: int
+ errors: int
+ response_time_ms: float
+ timed_requests: int
+
+
+class MetricsDatabase(Protocol):
+ async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ...
+
+
+_ROWS: Final = TypeAdapter(tuple[DailyAlertMetrics, ...])
+_BATCH_SIZE: Final = 500
+
+
+def _source(scope: Scope) -> tuple[str, str]:
+ if scope == "team":
+ return '"LiteLLM_DailyTeamSpend"', "team_id"
+ if scope == "user":
+ return '"LiteLLM_DailyUserSpend"', "user_id"
+ if scope == "key":
+ return '"LiteLLM_DailyUserSpend"', "api_key"
+ if scope == "model":
+ return '"LiteLLM_DailyUserSpend"', "COALESCE(NULLIF(model, ''), 'unknown-model')"
+ return '"LiteLLM_DailyUserSpend"', "'deployment'::text"
+
+
+def _query(scope: Scope) -> str:
+ table, entity = _source(scope)
+ # Identifiers come only from the closed scope mapping; request values are parameters.
+ return f"""
+WITH entities AS MATERIALIZED (
+ SELECT DISTINCT {entity} AS entity_id FROM {table}
+ WHERE date >= $1 AND date <= $2 AND {entity} >= COALESCE($3::text, '') AND ($3::text IS NULL OR {entity} <> $3)
+ ORDER BY entity_id LIMIT $4
+)
+SELECT entities.entity_id, activity.date AS day,
+ SUM(activity.spend) AS spend,
+ SUM(activity.api_requests)::bigint AS requests,
+ SUM(activity.failed_requests)::bigint AS errors,
+ SUM(activity.total_response_time_ms) AS response_time_ms,
+ SUM(activity.timed_requests)::bigint AS timed_requests
+FROM {table} activity JOIN entities ON {entity} = entities.entity_id
+WHERE activity.date >= $1 AND activity.date <= $2
+GROUP BY entities.entity_id, activity.date
+ORDER BY entities.entity_id, activity.date
+"""
+
+
+def _reporting_id(scope: Scope, row: DailyAlertMetrics) -> DailyAlertMetrics:
+ if scope == "key":
+ return row.model_copy(update={"entity_id": api_key_bucket(row.entity_id)})
+ if scope == "model":
+ return row.model_copy(update={"entity_id": model_bucket(row.entity_id)})
+ return row
+
+
+class UsageAlertMetricsRepository:
+ def __init__(self, database: MetricsDatabase) -> None:
+ self._database = database
+
+ async def batches(
+ self, scope: Scope, target: date, baseline_days: int
+ ) -> AsyncIterator[tuple[DailyAlertMetrics, ...]]:
+ start: Final = (target - timedelta(days=baseline_days)).isoformat()
+ query: Final = _query(scope)
+ cursor: str | None = None # rebind-ok: keyset pagination retains only the current page
+ while rows := _ROWS.validate_python(
+ await self._database.query_raw(query, start, target.isoformat(), cursor, _BATCH_SIZE)
+ ):
+ yield tuple(_reporting_id(scope, row) for row in rows)
+ cursor = rows[-1].entity_id
+ if scope == "deployment":
+ return
diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py
index 3ecfe60d679..dcab3eb7cac 100644
--- a/litellm/repositories/usage_alerts.py
+++ b/litellm/repositories/usage_alerts.py
@@ -3,6 +3,7 @@
import json
from collections.abc import Mapping
from datetime import date, datetime, timedelta, timezone
+from itertools import groupby
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@@ -13,7 +14,7 @@ from typing import (
from urllib.parse import urlencode
from uuid import NAMESPACE_URL, uuid4, uuid5
-from pydantic import BaseModel, TypeAdapter
+from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS
@@ -45,6 +46,12 @@ if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
+class _BreachedEntity(BaseModel):
+ model_config = ConfigDict(frozen=True)
+ rule_id: str
+ scope_id: str | None
+
+
class AlertDatabase(Protocol):
async def query_raw(self, query: str, *args: object) -> object: ...
@@ -672,7 +679,9 @@ class UsageAlertsRepository:
rows: Final = TypeAdapter(tuple[_Pending, ...]).validate_python(
await self.database.query_raw(
"""SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id
- WHERE i.status='open' AND r.enabled AND NOT r.deleted AND EXISTS (
+ WHERE i.status='open' AND r.enabled AND NOT r.deleted
+ AND COALESCE((i.record->>'next_notification_attempt')::timestamptz,'-infinity'::timestamptz)<=CURRENT_TIMESTAMP
+ AND EXISTS (
SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels)))
ORDER BY i.updated_at,i.id LIMIT $1""",
limit,
@@ -693,7 +702,16 @@ class UsageAlertsRepository:
async def mark_notified(self, incident_id: str, channel: UsageAlertChannel) -> None:
await self.database.query_raw(
- 'UPDATE "LiteLLM_UsageAlertIncident" SET notified_channels=array_append(notified_channels,$2::text) WHERE id=$1 AND NOT ($2::text=ANY(notified_channels)) RETURNING id',
+ """UPDATE "LiteLLM_UsageAlertIncident" i SET
+ notified_channels=array_append(i.notified_channels,$2::text),
+ record=i.record || jsonb_build_object('notified_channels',array_append(i.notified_channels,$2::text), 'last_notification_attempt',CURRENT_TIMESTAMP) ||
+ CASE WHEN NOT EXISTS (
+ SELECT 1 FROM "LiteLLM_UsageAlertRule" r,
+ jsonb_array_elements_text(r.record->'notification_channels') channel
+ WHERE r.id=i.rule_id AND NOT (channel=ANY(array_append(i.notified_channels,$2::text)))
+ ) THEN jsonb_build_object('notification_error',NULL,'next_notification_attempt',NULL)
+ ELSE '{}'::jsonb END
+ WHERE i.id=$1 AND NOT ($2::text=ANY(i.notified_channels)) RETURNING i.id""",
incident_id,
channel,
)
@@ -718,3 +736,35 @@ class UsageAlertsRepository:
)
)
return rows[0]["role"] if rows else None
+
+ async def breached_entities(
+ self, rule_ids: tuple[str, ...], entity_ids: tuple[str | None, ...]
+ ) -> Mapping[str, frozenset[str | None]]:
+ rows: Final = TypeAdapter(tuple[_BreachedEntity, ...]).validate_python(
+ await self.database.query_raw(
+ 'SELECT DISTINCT rule_id,scope_id FROM "LiteLLM_UsageAlertIncident" '
+ "WHERE rule_id=ANY($1::text[]) AND scope_key=ANY($2::text[]) AND breached ORDER BY rule_id",
+ rule_ids,
+ tuple(entity or "" for entity in entity_ids),
+ )
+ )
+ return MappingProxyType(
+ {
+ rule_id: frozenset(row.scope_id for row in group)
+ for rule_id, group in groupby(rows, lambda row: row.rule_id)
+ }
+ )
+
+ async def defer_notification(self, incident_id: str, error: str) -> None:
+ await self.database.query_raw(
+ """UPDATE "LiteLLM_UsageAlertIncident" SET updated_at=CURRENT_TIMESTAMP,
+ record=record || jsonb_build_object(
+ 'notification_error',$2::text,
+ 'notification_attempts',COALESCE((record->>'notification_attempts')::int,0)+1,
+ 'last_notification_attempt',CURRENT_TIMESTAMP,
+ 'next_notification_attempt',CURRENT_TIMESTAMP + make_interval(secs =>
+ LEAST(86400,3600*power(2,LEAST(5,COALESCE((record->>'notification_attempts')::int,0)))::int))
+ ) WHERE id=$1 RETURNING id""",
+ incident_id,
+ error,
+ )
diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py
index 89c2ad841e8..19e6437dadf 100644
--- a/litellm/types/integrations/slack_alerting.py
+++ b/litellm/types/integrations/slack_alerting.py
@@ -177,6 +177,7 @@ class AlertType(str, Enum):
failed_tracking_spend = "failed_tracking_spend"
user_spend_thresholds = "user_spend_thresholds"
user_spend_anomalies = "user_spend_anomalies"
+ usage_alerts = "usage_alerts"
# Database alerts
db_exceptions = "db_exceptions"
diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py
index 64336441fbe..cc967e03e66 100644
--- a/litellm/types/proxy/usage_alerts.py
+++ b/litellm/types/proxy/usage_alerts.py
@@ -85,6 +85,11 @@ class UsageAlert(BaseModel):
first_seen: datetime
last_seen: datetime
activity_path: str
+ notified_channels: tuple[UsageAlertChannel, ...] = ()
+ notification_attempts: int = 0
+ last_notification_attempt: datetime | None = None
+ next_notification_attempt: datetime | None = None
+ notification_error: Literal["delivery_failed", "transport_not_configured"] | None = None
class UsageAlertUpdate(BaseModel):
diff --git a/schema.prisma b/schema.prisma
index 8a67d1b123a..3a08c094b3d 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule {
@@id([delivery_id, rule_id, source_id])
}
+
+// Hourly samples of successful background daily-aggregate flushes, retained for 35 days.
+model LiteLLM_UsageAlertFlush {
+ source String
+ hour DateTime
+ observed_at DateTime
+
+ @@id([source, hour])
+}
diff --git a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py
new file mode 100644
index 00000000000..a0f6088a807
--- /dev/null
+++ b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py
@@ -0,0 +1,78 @@
+from datetime import date, datetime, timezone
+from typing import Final
+
+import httpx
+import pytest
+
+from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
+from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, activity_url
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule
+
+_NOW: Final = datetime(2026, 10, 20, tzinfo=timezone.utc)
+_RULE: Final = UsageAlertRule(
+ name="Request spike",
+ id="rule",
+ scope_type="key",
+ scope_id="reporting-key",
+ metric="requests",
+ direction="increase",
+ created_at=_NOW,
+ updated_at=_NOW,
+ created_by=None,
+ notification_channels=("slack",),
+)
+_ALERT: Final = UsageAlert(
+ id="incident",
+ rule_id="rule",
+ rule_name="Request spike",
+ scope_type="key",
+ scope_id="reporting-key",
+ metric="requests",
+ direction="increase",
+ status="open",
+ day=date(2026, 10, 20),
+ observed_value=1000,
+ baseline_value=200,
+ threshold=600,
+ first_seen=_NOW,
+ last_seen=_NOW,
+ baseline_start=date(2026, 10, 13),
+ baseline_end=date(2026, 10, 19),
+ baseline_sample_days=7,
+ activity_path="/api-keys?key=reporting-key&detail_tab=activity",
+)
+
+
+def test_activity_link_retains_dashboard_mount_and_entity_query() -> None:
+ assert activity_url(_ALERT, "https://gateway.example.com/") == (
+ "https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity"
+ )
+ assert activity_url(_ALERT, None) == _ALERT.activity_path
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("status", [200, 503])
+async def test_slack_ack_requires_successful_http_response(status: int, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com")
+ requests: Final[list[httpx.Request]] = []
+
+ def transport(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ return httpx.Response(status)
+
+ handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(transport))
+ slack: Final = SlackAlerting(
+ alerting=["slack"], default_webhook_url="https://hooks.example.com/test", async_http_handler=handler
+ )
+ delivery: Final = UsageAlertDelivery(slack)
+ try:
+ if status == 200:
+ assert await delivery.send(_ALERT, _RULE, "slack") is True
+ else:
+ with pytest.raises(httpx.HTTPStatusError):
+ await delivery.send(_ALERT, _RULE, "slack")
+ assert len(requests) == 1
+ assert b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content
+ finally:
+ await handler.close()
diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py
new file mode 100644
index 00000000000..4aa657de48b
--- /dev/null
+++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py
@@ -0,0 +1,155 @@
+from datetime import date, datetime, timedelta, timezone
+from typing import Final
+
+import pytest
+
+from litellm.proxy.spend_tracking.daily_usage_alerts import DailyComparison, compare_daily_usage
+from litellm.proxy.spend_tracking.usage_alert_job import completed_day
+from litellm.repositories.usage_alert_metrics import DailyAlertMetrics
+from litellm.types.proxy.usage_alerts import UsageAlertRuleInput
+
+_DAY: Final = date(2026, 10, 20)
+
+
+def _row(day: date, requests: int = 200, spend: float = 20, ms: float = 100_000, timed: int = 200) -> DailyAlertMetrics:
+ return DailyAlertMetrics(
+ entity_id="key-a", day=day, spend=spend, requests=requests, errors=10, response_time_ms=ms, timed_requests=timed
+ )
+
+
+@pytest.mark.parametrize(
+ "metric,observed,baseline", [("spend", 100, 20), ("requests", 1000, 200), ("latency", 2000, 500)]
+)
+def test_spikes_use_recorded_reference_days_and_weighted_latency(metric: str, observed: float, baseline: float) -> None:
+ rule: Final = UsageAlertRuleInput.model_validate(
+ {"name": "spike", "scope_type": "key", "metric": metric, "direction": "increase"}
+ )
+ history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 3, 7))
+ current: Final = _row(_DAY, requests=1000, spend=100, ms=400_000)
+ assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(
+ 3, observed, baseline, baseline * 3, True
+ )
+
+
+def test_missing_target_is_zero_for_counts_but_never_zero_latency() -> None:
+ history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
+ rule: Final = UsageAlertRuleInput(name="drop", scope_type="key", metric="requests", direction="decrease")
+ assert compare_daily_usage(rule, history, _DAY) == DailyComparison(3, 0, 200, 200 / 3, True)
+ assert compare_daily_usage(rule.model_copy(update={"metric": "latency"}), history, _DAY) is None
+
+
+def test_low_volume_is_insufficient_and_recorded_zero_baseline_uses_absolute_floor() -> None:
+ rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase")
+ history: Final = tuple(_row(_DAY - timedelta(days=days), requests=10, spend=0) for days in (1, 2, 3))
+ assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) is None
+ enough_requests: Final = tuple(row.model_copy(update={"requests": 200}) for row in history)
+ assert compare_daily_usage(rule, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison(
+ 3, 100, 0, 0, True
+ )
+ floor: Final = rule.model_copy(update={"min_value": 100})
+ assert compare_daily_usage(floor, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison(
+ 3, 100, 0, 100, False
+ )
+
+
+def test_absolute_floor_suppresses_small_changes() -> None:
+ history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
+ rule: Final = UsageAlertRuleInput(
+ name="spike", scope_type="key", metric="spend", direction="increase", min_value=100
+ )
+ assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 60, False)
+ assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, False)
+
+
+def test_day_is_eligible_only_after_one_hour_utc_grace() -> None:
+ assert completed_day(datetime(2026, 10, 21, 0, 59, tzinfo=timezone.utc)) == date(2026, 10, 19)
+ assert completed_day(datetime(2026, 10, 21, 1, 0, tzinfo=timezone.utc)) == _DAY
+
+
+def test_latency_weights_request_samples_instead_of_averaging_daily_averages() -> None:
+ rule: Final = UsageAlertRuleInput(
+ name="latency", scope_type="key", metric="latency", direction="increase", min_history_days=2
+ )
+ rows: Final = (
+ _row(_DAY - timedelta(days=1), requests=100, ms=100_000, timed=100),
+ _row(_DAY - timedelta(days=2), requests=400, ms=1_600_000, timed=400),
+ _row(_DAY, requests=100, ms=1_100_000, timed=100),
+ )
+ assert compare_daily_usage(rule, rows, _DAY) == DailyComparison(2, 11000, 3400, 10200, True)
+
+
+def test_errors_are_failed_request_count_and_require_absolute_change_floor() -> None:
+ rule: Final = UsageAlertRuleInput(
+ name="errors", scope_type="key", metric="errors", direction="increase", min_value=10
+ )
+ history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
+ current: Final = _row(_DAY).model_copy(update={"errors": 40})
+ assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(3, 40, 10, 30, True)
+
+
+def test_a_value_at_the_configured_multiplier_alerts() -> None:
+ history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
+ rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase")
+ assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, True)
+
+
+def test_default_notifications_require_opt_in_and_do_not_duplicate_legacy_user_spend() -> None:
+ from litellm.proxy.spend_tracking.usage_alert_job import default_usage_rules
+
+ defaults: Final = default_usage_rules()
+ assert len(defaults) == 40
+ assert all(not rule.notification_channels for rule in defaults)
+ opted_in: Final = default_usage_rules(("slack",), legacy_user_spend_alerts=True)
+ quiet: Final = tuple(rule for rule in opted_in if not rule.notification_channels)
+ assert tuple((rule.scope_type, rule.metric, rule.direction) for rule in quiet) == (("user", "spend", "increase"),)
+
+
+@pytest.mark.parametrize(
+ "scope,entity",
+ [("key", "unattributed"), ("key", "sk-example"), ("user", "default_user_id"), ("team", "litellm-dashboard")],
+)
+def test_sentinel_entities_cannot_create_alerts_with_unusable_activity_links(scope: str, entity: str) -> None:
+ from litellm.proxy.spend_tracking.usage_alert_job import _observations
+ from litellm.types.proxy.usage_alerts import UsageAlertRule
+
+ rule: Final = UsageAlertRule.model_validate(
+ {
+ "id": "rule",
+ "name": "spike",
+ "scope_type": scope,
+ "metric": "spend",
+ "direction": "increase",
+ "created_at": "2026-10-01T00:00:00Z",
+ "updated_at": "2026-10-01T00:00:00Z",
+ "created_by": None,
+ }
+ )
+ history: Final = tuple(
+ _row(_DAY - timedelta(days=days)).model_copy(update={"entity_id": entity}) for days in (1, 2, 3)
+ )
+ current: Final = _row(_DAY, spend=100).model_copy(update={"entity_id": entity})
+ assert _observations(rule, (*history, current), _DAY) == ()
+
+
+def test_custom_entity_rule_replaces_only_the_matching_default() -> None:
+ from uuid import NAMESPACE_URL, uuid5
+
+ from litellm.proxy.spend_tracking.usage_alert_job import _suppressed_entities
+ from litellm.types.proxy.usage_alerts import UsageAlertRule
+
+ default: Final = UsageAlertRule(
+ id=str(uuid5(NAMESPACE_URL, "litellm:usage-alert:key:spend:increase")),
+ name="default",
+ scope_type="key",
+ metric="spend",
+ direction="increase",
+ created_at=datetime(2026, 10, 1, tzinfo=timezone.utc),
+ updated_at=datetime(2026, 10, 1, tzinfo=timezone.utc),
+ created_by=None,
+ )
+ custom: Final = default.model_copy(update={"id": "custom", "scope_id": "key-a"})
+ other_metric: Final = custom.model_copy(update={"id": "latency", "metric": "latency", "scope_id": "key-b"})
+ assert _suppressed_entities(default, (default, custom, other_metric)) == frozenset({"key-a"})
+ assert _suppressed_entities(custom, (default, custom, other_metric)) == frozenset()
+ wildcard: Final = custom.model_copy(update={"scope_id": None})
+ assert _suppressed_entities(default, (default, wildcard)) == frozenset({None})
diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py b/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py
index d4965c64559..c4f36883270 100644
--- a/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py
+++ b/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py
@@ -7,13 +7,16 @@ Symbols pinned here:
from __future__ import annotations
import threading
-from typing import Any
+from typing import Any, TYPE_CHECKING
from unittest.mock import Mock
import pytest
from litellm.proxy.utils import send_email
+if TYPE_CHECKING:
+ from tests.unit.proxy.utils.prisma_and_spend.conftest import InMemorySMTP
+
@pytest.fixture(autouse=True)
def _smtp_env(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -176,3 +179,12 @@ async def test_should_send_plain_text_and_html_as_multipart_alternative(monkeypa
assert tuple(part.get_content_type() for part in parts) == ("text/plain", "text/html")
assert parts[0].get_payload(decode=True).decode() == "Model retirement"
assert parts[1].get_payload(decode=True).decode() == "Model retirement
"
+
+
+@pytest.mark.asyncio
+async def test_durable_alert_delivery_can_observe_smtp_failure(in_memory_smtp: InMemorySMTP) -> None:
+ in_memory_smtp.raise_on_send = RuntimeError("smtp unavailable")
+ with pytest.raises(RuntimeError, match="smtp unavailable"):
+ await send_email(receiver_email="alerts@example.com", subject="Daily usage", html="Alert
",
+ raise_on_error=True)
+ assert in_memory_smtp.sent == []