feat: evaluate daily usage alerts from existing aggregates

This commit is contained in:
moe-berri 2026-11-05 11:55:27 +08:00 • committed by ishaan-j-berri
parent 466305d294
commit 2de895ffee
18 changed files with 1011 additions and 4 deletions

View file

@ -0,0 +1,6 @@
CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertFlush" (
source TEXT NOT NULL,
hour TIMESTAMP(3) NOT NULL,
observed_at TIMESTAMP(3) NOT NULL,
PRIMARY KEY (source, hour)
);

View file

@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule {
@@id([delivery_id, rule_id, source_id])
}
// Hourly samples of successful background daily-aggregate flushes, retained for 35 days.
model LiteLLM_UsageAlertFlush {
source String
hour DateTime
observed_at DateTime
@@id([source, hour])
}

View file

@ -0,0 +1,88 @@
"""Deliver persisted usage incidents through the existing configured transports."""
import json
import os
from html import escape
from typing import TYPE_CHECKING, Final
from urllib.parse import urlsplit
from pydantic import TypeAdapter
from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertChannel, UsageAlertRule
if TYPE_CHECKING:
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
_URLS: Final = TypeAdapter(tuple[str, ...])
def configured_destinations(slack: "SlackAlerting") -> tuple[str, ...]:
return _URLS.validate_python(slack.alerting or ()) # pyright: ignore[reportUnknownMemberType] # validate legacy destination config
def activity_url(alert: UsageAlert, base_url: str | None) -> str:
if not base_url:
return alert.activity_path
parsed: Final = urlsplit(base_url)
if parsed.scheme not in ("http", "https") or not parsed.netloc:
return alert.activity_path
return base_url.rstrip("/") + "/ui" + alert.activity_path
def incident_message(alert: UsageAlert, url: str) -> str:
scope: Final = "Gateway" if alert.scope_type == "deployment" else f"{alert.scope_type}: {alert.scope_id}"
unit: Final = " ms" if alert.metric == "latency" else " USD" if alert.metric == "spend" else ""
return (
f"{alert.rule_name}\n{scope}\n{alert.day.isoformat()} UTC: "
f"{alert.observed_value:,.2f}{unit}; baseline {alert.baseline_value:,.2f}{unit}; "
f"threshold {alert.threshold:,.2f}{unit}.\nView activity: {url}"
)
class UsageAlertDelivery:
def __init__(self, slack: "SlackAlerting") -> None:
self._slack = slack
async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool:
configured: Final = configured_destinations(self._slack)
if channel not in configured:
return False
url: Final = activity_url(alert, os.getenv("PROXY_BASE_URL"))
message: Final = incident_message(alert, url)
if channel == "email":
from litellm.proxy.utils import send_email
for recipient in rule.email_recipients:
await send_email(
receiver_email=recipient,
subject=f"LiteLLM usage alert: {alert.rule_name}",
html=(
f"<p>{escape(message).replace(chr(10), '<br>')}</p>"
f'<p><a href="{escape(url, quote=True)}">View activity</a></p>'
),
raise_on_error=True,
)
return bool(rule.email_recipients)
overrides: Final = self._slack.alert_to_webhook_url or {}
configured_url: Final = (
overrides.get(AlertType.usage_alerts)
or self._slack.default_webhook_url
or os.getenv("SLACK_WEBHOOK_URL")
or os.getenv("ALERTING_WEBHOOK_URL")
)
if not configured_url:
return False
urls: Final = (configured_url,) if isinstance(configured_url, str) else _URLS.validate_python(configured_url)
for webhook in urls:
await self._post(webhook, message)
return True
async def _post(self, webhook: str, message: str) -> None:
response: Final = await self._slack.async_http_handler.post( # pyright: ignore[reportUnknownMemberType] # legacy HTTP signature; supplied arguments are concrete
url=webhook,
headers={"Content-type": "application/json"},
data=json.dumps({"text": message}),
timeout=10,
)
response.raise_for_status()

View file

@ -665,6 +665,9 @@ class DBSpendUpdateWriter:
self,
redis_cache: RedisCache | None = None,
):
from litellm.repositories.usage_alert_health import UsageAlertFlushRecorder
self.usage_alert_flush_recorder = UsageAlertFlushRecorder()
self.redis_cache = redis_cache
self.redis_update_buffer = RedisUpdateBuffer(redis_cache=self.redis_cache)
self.pod_lock_manager = PodLockManager()
@ -1879,6 +1882,7 @@ class DBSpendUpdateWriter:
daily_spend_transactions=daily_spend_update_transactions,
)
uncommitted.pop("daily_spend_update_transactions", None)
await self.usage_alert_flush_recorder.record(prisma_client, "user")
if daily_team_spend_update_transactions is not None:
await DBSpendUpdateWriter.update_daily_team_spend(
@ -1888,6 +1892,7 @@ class DBSpendUpdateWriter:
daily_spend_transactions=daily_team_spend_update_transactions,
)
uncommitted.pop("daily_team_spend_update_transactions", None)
await self.usage_alert_flush_recorder.record(prisma_client, "team")
if daily_org_spend_update_transactions is not None:
await DBSpendUpdateWriter.update_daily_org_spend(
@ -1970,6 +1975,8 @@ class DBSpendUpdateWriter:
)
try:
await asyncio.shield(commit_task)
if entity_type in ("user", "team"):
await self.usage_alert_flush_recorder.record(prisma_client, entity_type)
except asyncio.CancelledError:
if commit_started.is_set():
queue.track_interrupted_commit(

View file

@ -11678,6 +11678,21 @@ class ProxyStartupEvent:
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
)
from litellm.proxy.spend_tracking.usage_alert_job import USAGE_ALERT_JOB_ID, UsageAlertJob
usage_alert_job: Final = UsageAlertJob.from_prisma(prisma_client, proxy_logging_obj.slack_alerting_instance)
scheduler.add_job(
usage_alert_job.run,
"interval",
seconds=3600,
next_run_time=datetime.now() + timedelta(seconds=batch_writing_interval + 10 + random.randint(0, 20)),
id=USAGE_ALERT_JOB_ID,
replace_existing=True,
max_instances=1,
coalesce=True,
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
)
### TOOL DISCOVERY (own lock, same interval, never rides the spend commit) ###
from litellm.proxy.utils import commit_tool_discoveries

View file

@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule {
@@id([delivery_id, rule_id, source_id])
}
// Hourly samples of successful background daily-aggregate flushes, retained for 35 days.
model LiteLLM_UsageAlertFlush {
source String
hour DateTime
observed_at DateTime
@@id([source, hour])
}

View file

@ -0,0 +1,86 @@
"""Daily comparisons over existing aggregated counters, outside the request path."""
from collections.abc import Sequence
from dataclasses import dataclass
from datetime import date, timedelta
from typing import Final, Literal, Protocol
from litellm.repositories.usage_alert_metrics import DailyAlertMetrics
class ComparisonRule(Protocol):
@property
def metric(self) -> Literal["spend", "requests", "latency", "errors"]: ...
@property
def direction(self) -> Literal["increase", "decrease"]: ...
@property
def multiplier(self) -> float: ...
@property
def baseline_days(self) -> int: ...
@property
def min_history_days(self) -> int: ...
@property
def min_requests(self) -> int: ...
@property
def min_value(self) -> float: ...
@dataclass(frozen=True, slots=True)
class DailyComparison:
baseline_sample_days: int
observed: float
baseline: float
threshold: float
breached: bool
def _value(metric: str, rows: Sequence[DailyAlertMetrics], days: int) -> float | None:
if metric == "latency":
timed: Final = sum(row.timed_requests for row in rows)
return sum(row.response_time_ms for row in rows) / timed if timed else None
if metric == "spend":
return sum(row.spend for row in rows) / days
if metric == "errors":
return sum(row.errors for row in rows) / days
return sum(row.requests for row in rows) / days
def compare_daily_usage(
rule: ComparisonRule, rows: Sequence[DailyAlertMetrics], target: date
) -> DailyComparison | None:
"""None means insufficient evidence; it must not resolve an existing incident."""
start: Final = target - timedelta(days=rule.baseline_days)
history: Final = tuple(row for row in rows if start <= row.day < target)
current: Final = tuple(row for row in rows if row.day == target)
sample_days: Final = len({row.day for row in history})
if sample_days < rule.min_history_days:
return None
baseline_requests: Final = sum(row.requests for row in history) / sample_days
current_requests: Final = sum(row.requests for row in current)
if baseline_requests < rule.min_requests:
return None
if rule.direction == "increase" and current_requests < rule.min_requests:
return None
if rule.metric == "latency":
baseline_timed: Final = sum(row.timed_requests for row in history) / sample_days
if baseline_timed < rule.min_requests or sum(row.timed_requests for row in current) < rule.min_requests:
return None
baseline: Final = _value(rule.metric, history, sample_days)
observed: Final = _value(rule.metric, current, 1)
if baseline is None or baseline < 0 or observed is None:
return None
if baseline == 0:
if rule.direction == "decrease":
return None
return DailyComparison(sample_days, observed, 0, rule.min_value, observed > rule.min_value)
threshold: Final = baseline * rule.multiplier if rule.direction == "increase" else baseline / rule.multiplier
crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold
return DailyComparison(
sample_days, observed, baseline, threshold, crossed and abs(observed - baseline) >= rule.min_value
)

View file

@ -0,0 +1,286 @@
"""Once-daily evaluation, polled hourly by one database-elected worker."""
import asyncio
from collections.abc import Sequence
from datetime import date, datetime, time, timedelta, timezone
from itertools import groupby, product
from typing import (
TYPE_CHECKING,
Final,
Protocol,
cast, # noqa: TID251 # generated Prisma delegate supplies the repository protocol
)
from uuid import NAMESPACE_URL, uuid4, uuid5
from litellm._logging import verbose_proxy_logger
from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, configured_destinations
from litellm.proxy.spend_tracking.daily_usage_alerts import compare_daily_usage
from litellm.repositories.usage_alert_health import UsageAlertHealthRepository
from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, MetricsDatabase, UsageAlertMetricsRepository
from litellm.repositories.usage_alerts import UsageAlertsRepository
from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.proxy.usage_alerts import (
UsageAlert,
UsageAlertChannel,
UsageAlertMetric,
UsageAlertNotification,
UsageAlertObservation,
UsageAlertRule,
UsageAlertRuleInput,
UsageAlertScope,
)
if TYPE_CHECKING:
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.proxy.utils import PrismaClient
USAGE_ALERT_JOB_ID: Final = "daily_usage_alerts"
_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("deployment", "key", "user", "team", "model")
_METRICS: Final[tuple[UsageAlertMetric, ...]] = ("spend", "requests", "latency", "errors")
_MIN_VALUES: Final = {"spend": 10, "requests": 100, "latency": 100, "errors": 10}
def default_usage_rules(
channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False
) -> tuple[UsageAlertRuleInput, ...]:
return tuple(
UsageAlertRuleInput(
name=f"Daily {scope} {metric} {direction}",
scope_type=scope,
metric=metric,
direction=direction,
min_value=_MIN_VALUES[metric],
notification_channels=(
()
if legacy_user_spend_alerts and (scope, metric, direction) == ("user", "spend", "increase")
else channels
),
)
for scope, metric, direction in product(_SCOPES, _METRICS, ("increase", "decrease"))
)
def completed_day(now: datetime) -> date:
"""A UTC day gets an hour for normal buffered writes before it is eligible."""
return (now.astimezone(timezone.utc) - timedelta(hours=1)).date() - timedelta(days=1)
def _has_activity_page(scope: UsageAlertScope, entity: str) -> bool:
if not entity.strip():
return False
if scope == "key":
return entity != "unattributed" and not entity.startswith("sk-")
if scope == "user":
return entity != "default_user_id"
if scope == "team":
return entity != "litellm-dashboard"
return True
def _observations(
rule: UsageAlertRule,
rows: Sequence[DailyAlertMetrics],
target: date,
suppressed: frozenset[str | None] = frozenset(),
) -> tuple[UsageAlertObservation, ...]:
def observation(entity: str, entity_rows: Sequence[DailyAlertMetrics]) -> UsageAlertObservation | None:
if None in suppressed or entity in suppressed:
return None
if not _has_activity_page(rule.scope_type, entity):
return None
if rule.scope_id is not None and rule.scope_id != entity:
return None
comparison: Final = compare_daily_usage(rule, entity_rows, target)
if comparison is None:
return None
identity: Final = None if rule.scope_type == "deployment" else entity
return UsageAlertObservation(
entity_id=identity,
baseline_sample_days=comparison.baseline_sample_days,
day=target,
observed=comparison.observed,
baseline=comparison.baseline,
threshold=comparison.threshold,
breached=comparison.breached,
)
candidates: Final = tuple(
observation(entity, tuple(group)) for entity, group in groupby(rows, lambda row: row.entity_id)
)
return tuple(item for item in candidates if item is not None)
def _default_rule(rule: UsageAlertRule) -> bool:
return rule.id == str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}"))
def _suppressed_entities(rule: UsageAlertRule, enabled: tuple[UsageAlertRule, ...]) -> frozenset[str | None]:
if not _default_rule(rule):
return frozenset()
matching: Final = tuple(
other
for other in enabled
if (other.scope_type, other.metric, other.direction) == (rule.scope_type, rule.metric, rule.direction)
)
return frozenset(other.scope_id for other in matching if not _default_rule(other))
class IncidentDelivery(Protocol):
async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool: ...
class UsageAlertJob:
def __init__(
self,
store: UsageAlertsRepository,
metrics: UsageAlertMetricsRepository,
health: UsageAlertHealthRepository,
delivery: IncidentDelivery,
defaults: tuple[UsageAlertRuleInput, ...] | None = None,
) -> None:
self._store = store
self._metrics = metrics
self._health = health
self._delivery = delivery
self._defaults = default_usage_rules() if defaults is None else defaults
self._defaults_registered = False
@classmethod
def from_prisma(cls, prisma: "PrismaClient", slack: "SlackAlerting") -> "UsageAlertJob":
database: Final = cast(MetricsDatabase, prisma.writer_db) # cast-ok: generated Prisma delegate
return cls(
UsageAlertsRepository.from_prisma(prisma),
UsageAlertMetricsRepository(database),
UsageAlertHealthRepository(database),
UsageAlertDelivery(slack),
default_usage_rules(
("slack",)
if "slack" in configured_destinations(slack) and AlertType.usage_alerts in slack.alert_types
else (),
legacy_user_spend_alerts=AlertType.user_spend_anomalies in slack.alert_types,
),
)
async def run(self, now: datetime | None = None) -> None:
owner: Final = str(uuid4())
if not await self._store.claim_job(owner):
return
try:
await asyncio.wait_for(self._run(now or datetime.now(timezone.utc)), timeout=840)
except asyncio.TimeoutError:
verbose_proxy_logger.warning(
"Usage alert evaluation exceeded its 14-minute lease budget; retrying next hour"
)
except Exception: # noqa: BLE001 # the scheduler retries; completed rule/day checkpoints survive a failed pass
verbose_proxy_logger.exception("Daily usage alert evaluation failed")
finally:
await self._store.release_job(owner)
async def _rules(self) -> tuple[UsageAlertRule, ...]:
pages: Final[list[UsageAlertRule]] = [] # mutable-ok: collect configuration pages
offset = 0 # rebind-ok: configuration pagination cursor
while page := await self._store.list_enabled_rules(limit=100, offset=offset):
pages.extend(page)
if len(page) < 100:
return tuple(pages)
offset += len(page)
return tuple(pages)
async def _run(self, now: datetime) -> None:
if not self._defaults_registered:
await self._store.ensure_default_rules(self._defaults)
self._defaults_registered = True
target: Final = completed_day(now)
rules: Final = await self._rules()
due: Final = tuple([rule for rule in rules if not await self._store.is_day_complete(rule, target)])
for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in _SCOPES):
if scoped:
await self._scope(scoped, target, rules)
await self._notify()
await self._store.prune()
async def _scope(
self, rules: tuple[UsageAlertRule, ...], target: date, enabled: tuple[UsageAlertRule, ...]
) -> None:
scope: Final = rules[0].scope_type
start: Final = datetime.combine(target, time(), tzinfo=timezone.utc)
coverage: Final = await self._health.coverage("team" if scope == "team" else "user", start)
eligible: Final = tuple(
rule for rule in rules if coverage.after_grace and (rule.direction == "increase" or coverage.hours == 24)
)
suppressed: Final = {rule.id: _suppressed_entities(rule, enabled) for rule in eligible}
for rule in rules:
if rule not in eligible:
await self._store.record_evaluation_status(rule, target, "insufficient_coverage")
if not eligible:
return
evaluated: Final[set[str]] = set() # mutable-ok: at most one entry per configured rule across streaming pages
async for rows in self._metrics.batches(scope, target, max(rule.baseline_days for rule in eligible)):
evaluated.update(await self._page(eligible, rows, target, enabled))
for rule in eligible:
await self._store.finish_day(rule, target)
await self._store.record_evaluation_status(
rule, target, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history"
)
async def _page(
self,
rules: tuple[UsageAlertRule, ...],
rows: tuple[DailyAlertMetrics, ...],
target: date,
enabled: tuple[UsageAlertRule, ...],
) -> tuple[str, ...]:
entities: Final = (
(None,) if rules[0].scope_type == "deployment" else tuple(dict.fromkeys(row.entity_id for row in rows))
)
active: Final = await self._store.breached_entities(tuple(rule.id for rule in rules), entities)
return tuple(
[
rule.id
for rule in rules
if await self._rule_page(
rule, rows, target, active.get(rule.id, frozenset()), _suppressed_entities(rule, enabled)
)
]
)
async def _rule_page(
self,
rule: UsageAlertRule,
rows: tuple[DailyAlertMetrics, ...],
target: date,
active: frozenset[str | None],
suppressed: frozenset[str | None],
) -> bool:
observations: Final = _observations(rule, rows, target, suppressed)
changed: Final = tuple(item for item in observations if item.breached or item.entity_id in active)
if changed:
await self._store.record_observations(rule, changed)
return bool(observations)
async def _deliver_channel(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> str | None:
try:
delivered: Final = await asyncio.wait_for(self._delivery.send(alert, rule, channel), timeout=30)
if not delivered:
return "transport_not_configured"
await self._store.mark_notified(alert.id, channel)
except Exception: # noqa: BLE001 # retry only failed channels; persisted acknowledgements survive restarts
verbose_proxy_logger.warning("Usage alert delivery failed for incident %s via %s", alert.id, channel)
return "delivery_failed"
return None
async def _notify(self) -> None:
while page := await self._store.pending_notifications(limit=100):
for pending in page:
await self._notify_incident(pending)
async def _notify_incident(self, pending: UsageAlertNotification) -> None:
failures: Final = tuple(
[
await self._deliver_channel(pending.incident, pending.rule, channel)
for channel in pending.pending_channels
]
)
if any(failures):
error: Final = "delivery_failed" if "delivery_failed" in failures else "transport_not_configured"
await self._store.defer_notification(pending.incident.id, error)

View file

@ -8282,6 +8282,8 @@ async def send_email(
html: str | None = None,
text: str | None = None,
message_id: str | None = None,
*,
raise_on_error: bool = False,
) -> bool:
"""
smtp_host,
@ -8340,6 +8342,8 @@ async def send_email(
)
except Exception as e:
if raise_on_error:
raise
verbose_proxy_logger.exception("An error occurred while sending the email:" + str(e))
return False
return True

View file

@ -0,0 +1,95 @@
"""Sample successful aggregate flushes without adding request-time telemetry."""
from datetime import datetime, timedelta, timezone
from typing import (
TYPE_CHECKING,
Final,
Literal,
TypeAlias,
cast, # noqa: TID251 # generated Prisma delegate supplies the repository protocol
)
from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm.repositories.usage_alert_metrics import MetricsDatabase
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
FlushSource: TypeAlias = Literal["user", "team"]
class FlushCoverage(BaseModel):
model_config = ConfigDict(frozen=True)
hours: int
after_grace: bool
_COVERAGE: Final = TypeAdapter(tuple[FlushCoverage, ...])
class UsageAlertHealthRepository:
def __init__(self, database: MetricsDatabase) -> None:
self._database = database
async def record(self, source: FlushSource, now: datetime) -> bool:
hour: Final = now.astimezone(timezone.utc).replace(minute=0, second=0, microsecond=0, tzinfo=None)
rows: Final = await self._database.query_raw(
"""INSERT INTO "LiteLLM_UsageAlertFlush" (source,hour,observed_at)
SELECT $1,$2,$3 WHERE NOT EXISTS (
SELECT 1 FROM "LiteLLM_SpendLogDelivery" WHERE state='pending' AND created_at<$2 LIMIT 1
) ON CONFLICT (source,hour) DO UPDATE SET observed_at=EXCLUDED.observed_at RETURNING source""",
source,
hour,
now.astimezone(timezone.utc).replace(tzinfo=None),
)
await self._database.query_raw(
'DELETE FROM "LiteLLM_UsageAlertFlush" WHERE hour<$1 RETURNING source', hour - timedelta(days=35)
)
return bool(rows)
async def coverage(self, source: FlushSource, start: datetime) -> FlushCoverage:
utc_start: Final = start.astimezone(timezone.utc).replace(tzinfo=None)
end: Final = utc_start + timedelta(days=1)
rows: Final = _COVERAGE.validate_python(
await self._database.query_raw(
"""SELECT COUNT(*) FILTER(WHERE hour >= $2 AND hour < $3)::integer AS hours,
COALESCE(BOOL_OR(observed_at >= $3 + INTERVAL '1 hour'),FALSE) AS after_grace
FROM "LiteLLM_UsageAlertFlush" WHERE source=$1 AND hour >= $2""",
source,
utc_start,
end,
)
)
return rows[0] if rows else FlushCoverage(hours=0, after_grace=False)
class UsageAlertFlushRecorder:
"""At most one heartbeat attempt per source per worker per hour after success."""
def __init__(self) -> None:
self._user_hour: datetime | None = None
self._team_hour: datetime | None = None
async def record(self, prisma_client: "PrismaClient", source: FlushSource) -> None:
from litellm._logging import verbose_proxy_logger
from litellm.proxy.utils import ProxyUpdateSpend
if ProxyUpdateSpend.disable_spend_updates():
return
now: Final = datetime.now(timezone.utc)
hour: Final = now.replace(minute=0, second=0, microsecond=0)
prior: Final = self._user_hour if source == "user" else self._team_hour
if prior == hour:
return
database: Final = cast(MetricsDatabase, prisma_client.writer_db) # cast-ok: generated Prisma delegate
try:
if not await UsageAlertHealthRepository(database).record(source, now):
return
except Exception: # noqa: BLE001 # observability must not fail or requeue a committed spend flush
verbose_proxy_logger.warning("Could not record usage-alert aggregate freshness")
return
if source == "user":
self._user_hour = hour
else:
self._team_hour = hour

View file

@ -0,0 +1,92 @@
"""Bounded reads of the daily rollups used by the Usage dashboard."""
from collections.abc import AsyncIterator, Mapping, Sequence
from datetime import date, timedelta
from typing import Final, Literal, Protocol, TypeAlias
from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm.repositories.daily_activity_buckets import api_key_bucket, model_bucket
Scope: TypeAlias = Literal["deployment", "key", "user", "team", "model"]
class DailyAlertMetrics(BaseModel):
model_config = ConfigDict(frozen=True)
entity_id: str
day: date
spend: float
requests: int
errors: int
response_time_ms: float
timed_requests: int
class MetricsDatabase(Protocol):
async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ...
_ROWS: Final = TypeAdapter(tuple[DailyAlertMetrics, ...])
_BATCH_SIZE: Final = 500
def _source(scope: Scope) -> tuple[str, str]:
if scope == "team":
return '"LiteLLM_DailyTeamSpend"', "team_id"
if scope == "user":
return '"LiteLLM_DailyUserSpend"', "user_id"
if scope == "key":
return '"LiteLLM_DailyUserSpend"', "api_key"
if scope == "model":
return '"LiteLLM_DailyUserSpend"', "COALESCE(NULLIF(model, ''), 'unknown-model')"
return '"LiteLLM_DailyUserSpend"', "'deployment'::text"
def _query(scope: Scope) -> str:
table, entity = _source(scope)
# Identifiers come only from the closed scope mapping; request values are parameters.
return f"""
WITH entities AS MATERIALIZED (
SELECT DISTINCT {entity} AS entity_id FROM {table}
WHERE date >= $1 AND date <= $2 AND {entity} >= COALESCE($3::text, '') AND ($3::text IS NULL OR {entity} <> $3)
ORDER BY entity_id LIMIT $4
)
SELECT entities.entity_id, activity.date AS day,
SUM(activity.spend) AS spend,
SUM(activity.api_requests)::bigint AS requests,
SUM(activity.failed_requests)::bigint AS errors,
SUM(activity.total_response_time_ms) AS response_time_ms,
SUM(activity.timed_requests)::bigint AS timed_requests
FROM {table} activity JOIN entities ON {entity} = entities.entity_id
WHERE activity.date >= $1 AND activity.date <= $2
GROUP BY entities.entity_id, activity.date
ORDER BY entities.entity_id, activity.date
"""
def _reporting_id(scope: Scope, row: DailyAlertMetrics) -> DailyAlertMetrics:
if scope == "key":
return row.model_copy(update={"entity_id": api_key_bucket(row.entity_id)})
if scope == "model":
return row.model_copy(update={"entity_id": model_bucket(row.entity_id)})
return row
class UsageAlertMetricsRepository:
def __init__(self, database: MetricsDatabase) -> None:
self._database = database
async def batches(
self, scope: Scope, target: date, baseline_days: int
) -> AsyncIterator[tuple[DailyAlertMetrics, ...]]:
start: Final = (target - timedelta(days=baseline_days)).isoformat()
query: Final = _query(scope)
cursor: str | None = None # rebind-ok: keyset pagination retains only the current page
while rows := _ROWS.validate_python(
await self._database.query_raw(query, start, target.isoformat(), cursor, _BATCH_SIZE)
):
yield tuple(_reporting_id(scope, row) for row in rows)
cursor = rows[-1].entity_id
if scope == "deployment":
return

View file

@ -3,6 +3,7 @@
import json
from collections.abc import Mapping
from datetime import date, datetime, timedelta, timezone
from itertools import groupby
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -13,7 +14,7 @@ from typing import (
from urllib.parse import urlencode
from uuid import NAMESPACE_URL, uuid4, uuid5
from pydantic import BaseModel, TypeAdapter
from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS
@ -45,6 +46,12 @@ if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
class _BreachedEntity(BaseModel):
model_config = ConfigDict(frozen=True)
rule_id: str
scope_id: str | None
class AlertDatabase(Protocol):
async def query_raw(self, query: str, *args: object) -> object: ...
@ -672,7 +679,9 @@ class UsageAlertsRepository:
rows: Final = TypeAdapter(tuple[_Pending, ...]).validate_python(
await self.database.query_raw(
"""SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id
WHERE i.status='open' AND r.enabled AND NOT r.deleted AND EXISTS (
WHERE i.status='open' AND r.enabled AND NOT r.deleted
AND COALESCE((i.record->>'next_notification_attempt')::timestamptz,'-infinity'::timestamptz)<=CURRENT_TIMESTAMP
AND EXISTS (
SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels)))
ORDER BY i.updated_at,i.id LIMIT $1""",
limit,
@ -693,7 +702,16 @@ class UsageAlertsRepository:
async def mark_notified(self, incident_id: str, channel: UsageAlertChannel) -> None:
await self.database.query_raw(
'UPDATE "LiteLLM_UsageAlertIncident" SET notified_channels=array_append(notified_channels,$2::text) WHERE id=$1 AND NOT ($2::text=ANY(notified_channels)) RETURNING id',
"""UPDATE "LiteLLM_UsageAlertIncident" i SET
notified_channels=array_append(i.notified_channels,$2::text),
record=i.record || jsonb_build_object('notified_channels',array_append(i.notified_channels,$2::text), 'last_notification_attempt',CURRENT_TIMESTAMP) ||
CASE WHEN NOT EXISTS (
SELECT 1 FROM "LiteLLM_UsageAlertRule" r,
jsonb_array_elements_text(r.record->'notification_channels') channel
WHERE r.id=i.rule_id AND NOT (channel=ANY(array_append(i.notified_channels,$2::text)))
) THEN jsonb_build_object('notification_error',NULL,'next_notification_attempt',NULL)
ELSE '{}'::jsonb END
WHERE i.id=$1 AND NOT ($2::text=ANY(i.notified_channels)) RETURNING i.id""",
incident_id,
channel,
)
@ -718,3 +736,35 @@ class UsageAlertsRepository:
)
)
return rows[0]["role"] if rows else None
async def breached_entities(
self, rule_ids: tuple[str, ...], entity_ids: tuple[str | None, ...]
) -> Mapping[str, frozenset[str | None]]:
rows: Final = TypeAdapter(tuple[_BreachedEntity, ...]).validate_python(
await self.database.query_raw(
'SELECT DISTINCT rule_id,scope_id FROM "LiteLLM_UsageAlertIncident" '
"WHERE rule_id=ANY($1::text[]) AND scope_key=ANY($2::text[]) AND breached ORDER BY rule_id",
rule_ids,
tuple(entity or "" for entity in entity_ids),
)
)
return MappingProxyType(
{
rule_id: frozenset(row.scope_id for row in group)
for rule_id, group in groupby(rows, lambda row: row.rule_id)
}
)
async def defer_notification(self, incident_id: str, error: str) -> None:
await self.database.query_raw(
"""UPDATE "LiteLLM_UsageAlertIncident" SET updated_at=CURRENT_TIMESTAMP,
record=record || jsonb_build_object(
'notification_error',$2::text,
'notification_attempts',COALESCE((record->>'notification_attempts')::int,0)+1,
'last_notification_attempt',CURRENT_TIMESTAMP,
'next_notification_attempt',CURRENT_TIMESTAMP + make_interval(secs =>
LEAST(86400,3600*power(2,LEAST(5,COALESCE((record->>'notification_attempts')::int,0)))::int))
) WHERE id=$1 RETURNING id""",
incident_id,
error,
)

View file

@ -177,6 +177,7 @@ class AlertType(str, Enum):
failed_tracking_spend = "failed_tracking_spend"
user_spend_thresholds = "user_spend_thresholds"
user_spend_anomalies = "user_spend_anomalies"
usage_alerts = "usage_alerts"
# Database alerts
db_exceptions = "db_exceptions"

View file

@ -85,6 +85,11 @@ class UsageAlert(BaseModel):
first_seen: datetime
last_seen: datetime
activity_path: str
notified_channels: tuple[UsageAlertChannel, ...] = ()
notification_attempts: int = 0
last_notification_attempt: datetime | None = None
next_notification_attempt: datetime | None = None
notification_error: Literal["delivery_failed", "transport_not_configured"] | None = None
class UsageAlertUpdate(BaseModel):

View file

@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule {
@@id([delivery_id, rule_id, source_id])
}
// Hourly samples of successful background daily-aggregate flushes, retained for 35 days.
model LiteLLM_UsageAlertFlush {
source String
hour DateTime
observed_at DateTime
@@id([source, hour])
}

View file

@ -0,0 +1,78 @@
from datetime import date, datetime, timezone
from typing import Final
import httpx
import pytest
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, activity_url
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule
_NOW: Final = datetime(2026, 10, 20, tzinfo=timezone.utc)
_RULE: Final = UsageAlertRule(
name="Request spike",
id="rule",
scope_type="key",
scope_id="reporting-key",
metric="requests",
direction="increase",
created_at=_NOW,
updated_at=_NOW,
created_by=None,
notification_channels=("slack",),
)
_ALERT: Final = UsageAlert(
id="incident",
rule_id="rule",
rule_name="Request spike",
scope_type="key",
scope_id="reporting-key",
metric="requests",
direction="increase",
status="open",
day=date(2026, 10, 20),
observed_value=1000,
baseline_value=200,
threshold=600,
first_seen=_NOW,
last_seen=_NOW,
baseline_start=date(2026, 10, 13),
baseline_end=date(2026, 10, 19),
baseline_sample_days=7,
activity_path="/api-keys?key=reporting-key&detail_tab=activity",
)
def test_activity_link_retains_dashboard_mount_and_entity_query() -> None:
assert activity_url(_ALERT, "https://gateway.example.com/") == (
"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity"
)
assert activity_url(_ALERT, None) == _ALERT.activity_path
@pytest.mark.asyncio
@pytest.mark.parametrize("status", [200, 503])
async def test_slack_ack_requires_successful_http_response(status: int, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com")
requests: Final[list[httpx.Request]] = []
def transport(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(status)
handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(transport))
slack: Final = SlackAlerting(
alerting=["slack"], default_webhook_url="https://hooks.example.com/test", async_http_handler=handler
)
delivery: Final = UsageAlertDelivery(slack)
try:
if status == 200:
assert await delivery.send(_ALERT, _RULE, "slack") is True
else:
with pytest.raises(httpx.HTTPStatusError):
await delivery.send(_ALERT, _RULE, "slack")
assert len(requests) == 1
assert b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content
finally:
await handler.close()

View file

@ -0,0 +1,155 @@
from datetime import date, datetime, timedelta, timezone
from typing import Final
import pytest
from litellm.proxy.spend_tracking.daily_usage_alerts import DailyComparison, compare_daily_usage
from litellm.proxy.spend_tracking.usage_alert_job import completed_day
from litellm.repositories.usage_alert_metrics import DailyAlertMetrics
from litellm.types.proxy.usage_alerts import UsageAlertRuleInput
_DAY: Final = date(2026, 10, 20)
def _row(day: date, requests: int = 200, spend: float = 20, ms: float = 100_000, timed: int = 200) -> DailyAlertMetrics:
return DailyAlertMetrics(
entity_id="key-a", day=day, spend=spend, requests=requests, errors=10, response_time_ms=ms, timed_requests=timed
)
@pytest.mark.parametrize(
"metric,observed,baseline", [("spend", 100, 20), ("requests", 1000, 200), ("latency", 2000, 500)]
)
def test_spikes_use_recorded_reference_days_and_weighted_latency(metric: str, observed: float, baseline: float) -> None:
rule: Final = UsageAlertRuleInput.model_validate(
{"name": "spike", "scope_type": "key", "metric": metric, "direction": "increase"}
)
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 3, 7))
current: Final = _row(_DAY, requests=1000, spend=100, ms=400_000)
assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(
3, observed, baseline, baseline * 3, True
)
def test_missing_target_is_zero_for_counts_but_never_zero_latency() -> None:
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
rule: Final = UsageAlertRuleInput(name="drop", scope_type="key", metric="requests", direction="decrease")
assert compare_daily_usage(rule, history, _DAY) == DailyComparison(3, 0, 200, 200 / 3, True)
assert compare_daily_usage(rule.model_copy(update={"metric": "latency"}), history, _DAY) is None
def test_low_volume_is_insufficient_and_recorded_zero_baseline_uses_absolute_floor() -> None:
rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase")
history: Final = tuple(_row(_DAY - timedelta(days=days), requests=10, spend=0) for days in (1, 2, 3))
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) is None
enough_requests: Final = tuple(row.model_copy(update={"requests": 200}) for row in history)
assert compare_daily_usage(rule, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison(
3, 100, 0, 0, True
)
floor: Final = rule.model_copy(update={"min_value": 100})
assert compare_daily_usage(floor, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison(
3, 100, 0, 100, False
)
def test_absolute_floor_suppresses_small_changes() -> None:
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
rule: Final = UsageAlertRuleInput(
name="spike", scope_type="key", metric="spend", direction="increase", min_value=100
)
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 60, False)
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, False)
def test_day_is_eligible_only_after_one_hour_utc_grace() -> None:
assert completed_day(datetime(2026, 10, 21, 0, 59, tzinfo=timezone.utc)) == date(2026, 10, 19)
assert completed_day(datetime(2026, 10, 21, 1, 0, tzinfo=timezone.utc)) == _DAY
def test_latency_weights_request_samples_instead_of_averaging_daily_averages() -> None:
rule: Final = UsageAlertRuleInput(
name="latency", scope_type="key", metric="latency", direction="increase", min_history_days=2
)
rows: Final = (
_row(_DAY - timedelta(days=1), requests=100, ms=100_000, timed=100),
_row(_DAY - timedelta(days=2), requests=400, ms=1_600_000, timed=400),
_row(_DAY, requests=100, ms=1_100_000, timed=100),
)
assert compare_daily_usage(rule, rows, _DAY) == DailyComparison(2, 11000, 3400, 10200, True)
def test_errors_are_failed_request_count_and_require_absolute_change_floor() -> None:
rule: Final = UsageAlertRuleInput(
name="errors", scope_type="key", metric="errors", direction="increase", min_value=10
)
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
current: Final = _row(_DAY).model_copy(update={"errors": 40})
assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(3, 40, 10, 30, True)
def test_a_value_at_the_configured_multiplier_alerts() -> None:
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase")
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, True)
def test_default_notifications_require_opt_in_and_do_not_duplicate_legacy_user_spend() -> None:
from litellm.proxy.spend_tracking.usage_alert_job import default_usage_rules
defaults: Final = default_usage_rules()
assert len(defaults) == 40
assert all(not rule.notification_channels for rule in defaults)
opted_in: Final = default_usage_rules(("slack",), legacy_user_spend_alerts=True)
quiet: Final = tuple(rule for rule in opted_in if not rule.notification_channels)
assert tuple((rule.scope_type, rule.metric, rule.direction) for rule in quiet) == (("user", "spend", "increase"),)
@pytest.mark.parametrize(
"scope,entity",
[("key", "unattributed"), ("key", "sk-example"), ("user", "default_user_id"), ("team", "litellm-dashboard")],
)
def test_sentinel_entities_cannot_create_alerts_with_unusable_activity_links(scope: str, entity: str) -> None:
from litellm.proxy.spend_tracking.usage_alert_job import _observations
from litellm.types.proxy.usage_alerts import UsageAlertRule
rule: Final = UsageAlertRule.model_validate(
{
"id": "rule",
"name": "spike",
"scope_type": scope,
"metric": "spend",
"direction": "increase",
"created_at": "2026-10-01T00:00:00Z",
"updated_at": "2026-10-01T00:00:00Z",
"created_by": None,
}
)
history: Final = tuple(
_row(_DAY - timedelta(days=days)).model_copy(update={"entity_id": entity}) for days in (1, 2, 3)
)
current: Final = _row(_DAY, spend=100).model_copy(update={"entity_id": entity})
assert _observations(rule, (*history, current), _DAY) == ()
def test_custom_entity_rule_replaces_only_the_matching_default() -> None:
from uuid import NAMESPACE_URL, uuid5
from litellm.proxy.spend_tracking.usage_alert_job import _suppressed_entities
from litellm.types.proxy.usage_alerts import UsageAlertRule
default: Final = UsageAlertRule(
id=str(uuid5(NAMESPACE_URL, "litellm:usage-alert:key:spend:increase")),
name="default",
scope_type="key",
metric="spend",
direction="increase",
created_at=datetime(2026, 10, 1, tzinfo=timezone.utc),
updated_at=datetime(2026, 10, 1, tzinfo=timezone.utc),
created_by=None,
)
custom: Final = default.model_copy(update={"id": "custom", "scope_id": "key-a"})
other_metric: Final = custom.model_copy(update={"id": "latency", "metric": "latency", "scope_id": "key-b"})
assert _suppressed_entities(default, (default, custom, other_metric)) == frozenset({"key-a"})
assert _suppressed_entities(custom, (default, custom, other_metric)) == frozenset()
wildcard: Final = custom.model_copy(update={"scope_id": None})
assert _suppressed_entities(default, (default, wildcard)) == frozenset({None})

View file

@ -7,13 +7,16 @@ Symbols pinned here:
from __future__ import annotations
import threading
from typing import Any
from typing import Any, TYPE_CHECKING
from unittest.mock import Mock
import pytest
from litellm.proxy.utils import send_email
if TYPE_CHECKING:
from tests.unit.proxy.utils.prisma_and_spend.conftest import InMemorySMTP
@pytest.fixture(autouse=True)
def _smtp_env(monkeypatch: pytest.MonkeyPatch) -> None:
@ -176,3 +179,12 @@ async def test_should_send_plain_text_and_html_as_multipart_alternative(monkeypa
assert tuple(part.get_content_type() for part in parts) == ("text/plain", "text/html")
assert parts[0].get_payload(decode=True).decode() == "Model retirement"
assert parts[1].get_payload(decode=True).decode() == "<p>Model retirement</p>"
@pytest.mark.asyncio
async def test_durable_alert_delivery_can_observe_smtp_failure(in_memory_smtp: InMemorySMTP) -> None:
in_memory_smtp.raise_on_send = RuntimeError("smtp unavailable")
with pytest.raises(RuntimeError, match="smtp unavailable"):
await send_email(receiver_email="alerts@example.com", subject="Daily usage", html="<p>Alert</p>",
raise_on_error=True)
assert in_memory_smtp.sent == []