mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat: evaluate daily usage alerts from existing aggregates
This commit is contained in:
parent
466305d294
commit
2de895ffee
18 changed files with 1011 additions and 4 deletions
|
|
@ -0,0 +1,6 @@
|
|||
CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertFlush" (
|
||||
source TEXT NOT NULL,
|
||||
hour TIMESTAMP(3) NOT NULL,
|
||||
observed_at TIMESTAMP(3) NOT NULL,
|
||||
PRIMARY KEY (source, hour)
|
||||
);
|
||||
|
|
@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule {
|
|||
|
||||
@@id([delivery_id, rule_id, source_id])
|
||||
}
|
||||
|
||||
// Hourly samples of successful background daily-aggregate flushes, retained for 35 days.
|
||||
model LiteLLM_UsageAlertFlush {
|
||||
source String
|
||||
hour DateTime
|
||||
observed_at DateTime
|
||||
|
||||
@@id([source, hour])
|
||||
}
|
||||
|
|
|
|||
88
litellm/integrations/SlackAlerting/usage_alerts.py
Normal file
88
litellm/integrations/SlackAlerting/usage_alerts.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
"""Deliver persisted usage incidents through the existing configured transports."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from html import escape
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.integrations.slack_alerting import AlertType
|
||||
from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertChannel, UsageAlertRule
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
|
||||
_URLS: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
def configured_destinations(slack: "SlackAlerting") -> tuple[str, ...]:
|
||||
return _URLS.validate_python(slack.alerting or ()) # pyright: ignore[reportUnknownMemberType] # validate legacy destination config
|
||||
|
||||
|
||||
def activity_url(alert: UsageAlert, base_url: str | None) -> str:
|
||||
if not base_url:
|
||||
return alert.activity_path
|
||||
parsed: Final = urlsplit(base_url)
|
||||
if parsed.scheme not in ("http", "https") or not parsed.netloc:
|
||||
return alert.activity_path
|
||||
return base_url.rstrip("/") + "/ui" + alert.activity_path
|
||||
|
||||
|
||||
def incident_message(alert: UsageAlert, url: str) -> str:
|
||||
scope: Final = "Gateway" if alert.scope_type == "deployment" else f"{alert.scope_type}: {alert.scope_id}"
|
||||
unit: Final = " ms" if alert.metric == "latency" else " USD" if alert.metric == "spend" else ""
|
||||
return (
|
||||
f"{alert.rule_name}\n{scope}\n{alert.day.isoformat()} UTC: "
|
||||
f"{alert.observed_value:,.2f}{unit}; baseline {alert.baseline_value:,.2f}{unit}; "
|
||||
f"threshold {alert.threshold:,.2f}{unit}.\nView activity: {url}"
|
||||
)
|
||||
|
||||
|
||||
class UsageAlertDelivery:
|
||||
def __init__(self, slack: "SlackAlerting") -> None:
|
||||
self._slack = slack
|
||||
|
||||
async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool:
|
||||
configured: Final = configured_destinations(self._slack)
|
||||
if channel not in configured:
|
||||
return False
|
||||
url: Final = activity_url(alert, os.getenv("PROXY_BASE_URL"))
|
||||
message: Final = incident_message(alert, url)
|
||||
if channel == "email":
|
||||
from litellm.proxy.utils import send_email
|
||||
|
||||
for recipient in rule.email_recipients:
|
||||
await send_email(
|
||||
receiver_email=recipient,
|
||||
subject=f"LiteLLM usage alert: {alert.rule_name}",
|
||||
html=(
|
||||
f"<p>{escape(message).replace(chr(10), '<br>')}</p>"
|
||||
f'<p><a href="{escape(url, quote=True)}">View activity</a></p>'
|
||||
),
|
||||
raise_on_error=True,
|
||||
)
|
||||
return bool(rule.email_recipients)
|
||||
overrides: Final = self._slack.alert_to_webhook_url or {}
|
||||
configured_url: Final = (
|
||||
overrides.get(AlertType.usage_alerts)
|
||||
or self._slack.default_webhook_url
|
||||
or os.getenv("SLACK_WEBHOOK_URL")
|
||||
or os.getenv("ALERTING_WEBHOOK_URL")
|
||||
)
|
||||
if not configured_url:
|
||||
return False
|
||||
urls: Final = (configured_url,) if isinstance(configured_url, str) else _URLS.validate_python(configured_url)
|
||||
for webhook in urls:
|
||||
await self._post(webhook, message)
|
||||
return True
|
||||
|
||||
async def _post(self, webhook: str, message: str) -> None:
|
||||
response: Final = await self._slack.async_http_handler.post( # pyright: ignore[reportUnknownMemberType] # legacy HTTP signature; supplied arguments are concrete
|
||||
url=webhook,
|
||||
headers={"Content-type": "application/json"},
|
||||
data=json.dumps({"text": message}),
|
||||
timeout=10,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
|
@ -665,6 +665,9 @@ class DBSpendUpdateWriter:
|
|||
self,
|
||||
redis_cache: RedisCache | None = None,
|
||||
):
|
||||
from litellm.repositories.usage_alert_health import UsageAlertFlushRecorder
|
||||
|
||||
self.usage_alert_flush_recorder = UsageAlertFlushRecorder()
|
||||
self.redis_cache = redis_cache
|
||||
self.redis_update_buffer = RedisUpdateBuffer(redis_cache=self.redis_cache)
|
||||
self.pod_lock_manager = PodLockManager()
|
||||
|
|
@ -1879,6 +1882,7 @@ class DBSpendUpdateWriter:
|
|||
daily_spend_transactions=daily_spend_update_transactions,
|
||||
)
|
||||
uncommitted.pop("daily_spend_update_transactions", None)
|
||||
await self.usage_alert_flush_recorder.record(prisma_client, "user")
|
||||
|
||||
if daily_team_spend_update_transactions is not None:
|
||||
await DBSpendUpdateWriter.update_daily_team_spend(
|
||||
|
|
@ -1888,6 +1892,7 @@ class DBSpendUpdateWriter:
|
|||
daily_spend_transactions=daily_team_spend_update_transactions,
|
||||
)
|
||||
uncommitted.pop("daily_team_spend_update_transactions", None)
|
||||
await self.usage_alert_flush_recorder.record(prisma_client, "team")
|
||||
|
||||
if daily_org_spend_update_transactions is not None:
|
||||
await DBSpendUpdateWriter.update_daily_org_spend(
|
||||
|
|
@ -1970,6 +1975,8 @@ class DBSpendUpdateWriter:
|
|||
)
|
||||
try:
|
||||
await asyncio.shield(commit_task)
|
||||
if entity_type in ("user", "team"):
|
||||
await self.usage_alert_flush_recorder.record(prisma_client, entity_type)
|
||||
except asyncio.CancelledError:
|
||||
if commit_started.is_set():
|
||||
queue.track_interrupted_commit(
|
||||
|
|
|
|||
|
|
@ -11678,6 +11678,21 @@ class ProxyStartupEvent:
|
|||
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
|
||||
)
|
||||
|
||||
from litellm.proxy.spend_tracking.usage_alert_job import USAGE_ALERT_JOB_ID, UsageAlertJob
|
||||
|
||||
usage_alert_job: Final = UsageAlertJob.from_prisma(prisma_client, proxy_logging_obj.slack_alerting_instance)
|
||||
scheduler.add_job(
|
||||
usage_alert_job.run,
|
||||
"interval",
|
||||
seconds=3600,
|
||||
next_run_time=datetime.now() + timedelta(seconds=batch_writing_interval + 10 + random.randint(0, 20)),
|
||||
id=USAGE_ALERT_JOB_ID,
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
|
||||
)
|
||||
|
||||
### TOOL DISCOVERY (own lock, same interval, never rides the spend commit) ###
|
||||
from litellm.proxy.utils import commit_tool_discoveries
|
||||
|
||||
|
|
|
|||
|
|
@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule {
|
|||
|
||||
@@id([delivery_id, rule_id, source_id])
|
||||
}
|
||||
|
||||
// Hourly samples of successful background daily-aggregate flushes, retained for 35 days.
|
||||
model LiteLLM_UsageAlertFlush {
|
||||
source String
|
||||
hour DateTime
|
||||
observed_at DateTime
|
||||
|
||||
@@id([source, hour])
|
||||
}
|
||||
|
|
|
|||
86
litellm/proxy/spend_tracking/daily_usage_alerts.py
Normal file
86
litellm/proxy/spend_tracking/daily_usage_alerts.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
"""Daily comparisons over existing aggregated counters, outside the request path."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, timedelta
|
||||
from typing import Final, Literal, Protocol
|
||||
|
||||
from litellm.repositories.usage_alert_metrics import DailyAlertMetrics
|
||||
|
||||
|
||||
class ComparisonRule(Protocol):
|
||||
@property
|
||||
def metric(self) -> Literal["spend", "requests", "latency", "errors"]: ...
|
||||
|
||||
@property
|
||||
def direction(self) -> Literal["increase", "decrease"]: ...
|
||||
|
||||
@property
|
||||
def multiplier(self) -> float: ...
|
||||
|
||||
@property
|
||||
def baseline_days(self) -> int: ...
|
||||
|
||||
@property
|
||||
def min_history_days(self) -> int: ...
|
||||
|
||||
@property
|
||||
def min_requests(self) -> int: ...
|
||||
|
||||
@property
|
||||
def min_value(self) -> float: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DailyComparison:
|
||||
baseline_sample_days: int
|
||||
observed: float
|
||||
baseline: float
|
||||
threshold: float
|
||||
breached: bool
|
||||
|
||||
|
||||
def _value(metric: str, rows: Sequence[DailyAlertMetrics], days: int) -> float | None:
|
||||
if metric == "latency":
|
||||
timed: Final = sum(row.timed_requests for row in rows)
|
||||
return sum(row.response_time_ms for row in rows) / timed if timed else None
|
||||
if metric == "spend":
|
||||
return sum(row.spend for row in rows) / days
|
||||
if metric == "errors":
|
||||
return sum(row.errors for row in rows) / days
|
||||
return sum(row.requests for row in rows) / days
|
||||
|
||||
|
||||
def compare_daily_usage(
|
||||
rule: ComparisonRule, rows: Sequence[DailyAlertMetrics], target: date
|
||||
) -> DailyComparison | None:
|
||||
"""None means insufficient evidence; it must not resolve an existing incident."""
|
||||
start: Final = target - timedelta(days=rule.baseline_days)
|
||||
history: Final = tuple(row for row in rows if start <= row.day < target)
|
||||
current: Final = tuple(row for row in rows if row.day == target)
|
||||
sample_days: Final = len({row.day for row in history})
|
||||
if sample_days < rule.min_history_days:
|
||||
return None
|
||||
baseline_requests: Final = sum(row.requests for row in history) / sample_days
|
||||
current_requests: Final = sum(row.requests for row in current)
|
||||
if baseline_requests < rule.min_requests:
|
||||
return None
|
||||
if rule.direction == "increase" and current_requests < rule.min_requests:
|
||||
return None
|
||||
if rule.metric == "latency":
|
||||
baseline_timed: Final = sum(row.timed_requests for row in history) / sample_days
|
||||
if baseline_timed < rule.min_requests or sum(row.timed_requests for row in current) < rule.min_requests:
|
||||
return None
|
||||
baseline: Final = _value(rule.metric, history, sample_days)
|
||||
observed: Final = _value(rule.metric, current, 1)
|
||||
if baseline is None or baseline < 0 or observed is None:
|
||||
return None
|
||||
if baseline == 0:
|
||||
if rule.direction == "decrease":
|
||||
return None
|
||||
return DailyComparison(sample_days, observed, 0, rule.min_value, observed > rule.min_value)
|
||||
threshold: Final = baseline * rule.multiplier if rule.direction == "increase" else baseline / rule.multiplier
|
||||
crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold
|
||||
return DailyComparison(
|
||||
sample_days, observed, baseline, threshold, crossed and abs(observed - baseline) >= rule.min_value
|
||||
)
|
||||
286
litellm/proxy/spend_tracking/usage_alert_job.py
Normal file
286
litellm/proxy/spend_tracking/usage_alert_job.py
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
"""Once-daily evaluation, polled hourly by one database-elected worker."""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Sequence
|
||||
from datetime import date, datetime, time, timedelta, timezone
|
||||
from itertools import groupby, product
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
Protocol,
|
||||
cast, # noqa: TID251 # generated Prisma delegate supplies the repository protocol
|
||||
)
|
||||
from uuid import NAMESPACE_URL, uuid4, uuid5
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, configured_destinations
|
||||
from litellm.proxy.spend_tracking.daily_usage_alerts import compare_daily_usage
|
||||
from litellm.repositories.usage_alert_health import UsageAlertHealthRepository
|
||||
from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, MetricsDatabase, UsageAlertMetricsRepository
|
||||
from litellm.repositories.usage_alerts import UsageAlertsRepository
|
||||
from litellm.types.integrations.slack_alerting import AlertType
|
||||
from litellm.types.proxy.usage_alerts import (
|
||||
UsageAlert,
|
||||
UsageAlertChannel,
|
||||
UsageAlertMetric,
|
||||
UsageAlertNotification,
|
||||
UsageAlertObservation,
|
||||
UsageAlertRule,
|
||||
UsageAlertRuleInput,
|
||||
UsageAlertScope,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
USAGE_ALERT_JOB_ID: Final = "daily_usage_alerts"
|
||||
_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("deployment", "key", "user", "team", "model")
|
||||
_METRICS: Final[tuple[UsageAlertMetric, ...]] = ("spend", "requests", "latency", "errors")
|
||||
_MIN_VALUES: Final = {"spend": 10, "requests": 100, "latency": 100, "errors": 10}
|
||||
|
||||
|
||||
def default_usage_rules(
|
||||
channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False
|
||||
) -> tuple[UsageAlertRuleInput, ...]:
|
||||
return tuple(
|
||||
UsageAlertRuleInput(
|
||||
name=f"Daily {scope} {metric} {direction}",
|
||||
scope_type=scope,
|
||||
metric=metric,
|
||||
direction=direction,
|
||||
min_value=_MIN_VALUES[metric],
|
||||
notification_channels=(
|
||||
()
|
||||
if legacy_user_spend_alerts and (scope, metric, direction) == ("user", "spend", "increase")
|
||||
else channels
|
||||
),
|
||||
)
|
||||
for scope, metric, direction in product(_SCOPES, _METRICS, ("increase", "decrease"))
|
||||
)
|
||||
|
||||
|
||||
def completed_day(now: datetime) -> date:
|
||||
"""A UTC day gets an hour for normal buffered writes before it is eligible."""
|
||||
return (now.astimezone(timezone.utc) - timedelta(hours=1)).date() - timedelta(days=1)
|
||||
|
||||
|
||||
def _has_activity_page(scope: UsageAlertScope, entity: str) -> bool:
|
||||
if not entity.strip():
|
||||
return False
|
||||
if scope == "key":
|
||||
return entity != "unattributed" and not entity.startswith("sk-")
|
||||
if scope == "user":
|
||||
return entity != "default_user_id"
|
||||
if scope == "team":
|
||||
return entity != "litellm-dashboard"
|
||||
return True
|
||||
|
||||
|
||||
def _observations(
|
||||
rule: UsageAlertRule,
|
||||
rows: Sequence[DailyAlertMetrics],
|
||||
target: date,
|
||||
suppressed: frozenset[str | None] = frozenset(),
|
||||
) -> tuple[UsageAlertObservation, ...]:
|
||||
def observation(entity: str, entity_rows: Sequence[DailyAlertMetrics]) -> UsageAlertObservation | None:
|
||||
if None in suppressed or entity in suppressed:
|
||||
return None
|
||||
if not _has_activity_page(rule.scope_type, entity):
|
||||
return None
|
||||
if rule.scope_id is not None and rule.scope_id != entity:
|
||||
return None
|
||||
comparison: Final = compare_daily_usage(rule, entity_rows, target)
|
||||
if comparison is None:
|
||||
return None
|
||||
identity: Final = None if rule.scope_type == "deployment" else entity
|
||||
return UsageAlertObservation(
|
||||
entity_id=identity,
|
||||
baseline_sample_days=comparison.baseline_sample_days,
|
||||
day=target,
|
||||
observed=comparison.observed,
|
||||
baseline=comparison.baseline,
|
||||
threshold=comparison.threshold,
|
||||
breached=comparison.breached,
|
||||
)
|
||||
|
||||
candidates: Final = tuple(
|
||||
observation(entity, tuple(group)) for entity, group in groupby(rows, lambda row: row.entity_id)
|
||||
)
|
||||
return tuple(item for item in candidates if item is not None)
|
||||
|
||||
|
||||
def _default_rule(rule: UsageAlertRule) -> bool:
|
||||
return rule.id == str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}"))
|
||||
|
||||
|
||||
def _suppressed_entities(rule: UsageAlertRule, enabled: tuple[UsageAlertRule, ...]) -> frozenset[str | None]:
|
||||
if not _default_rule(rule):
|
||||
return frozenset()
|
||||
matching: Final = tuple(
|
||||
other
|
||||
for other in enabled
|
||||
if (other.scope_type, other.metric, other.direction) == (rule.scope_type, rule.metric, rule.direction)
|
||||
)
|
||||
return frozenset(other.scope_id for other in matching if not _default_rule(other))
|
||||
|
||||
|
||||
class IncidentDelivery(Protocol):
|
||||
async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool: ...
|
||||
|
||||
|
||||
class UsageAlertJob:
|
||||
def __init__(
|
||||
self,
|
||||
store: UsageAlertsRepository,
|
||||
metrics: UsageAlertMetricsRepository,
|
||||
health: UsageAlertHealthRepository,
|
||||
delivery: IncidentDelivery,
|
||||
defaults: tuple[UsageAlertRuleInput, ...] | None = None,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._metrics = metrics
|
||||
self._health = health
|
||||
self._delivery = delivery
|
||||
self._defaults = default_usage_rules() if defaults is None else defaults
|
||||
self._defaults_registered = False
|
||||
|
||||
@classmethod
|
||||
def from_prisma(cls, prisma: "PrismaClient", slack: "SlackAlerting") -> "UsageAlertJob":
|
||||
database: Final = cast(MetricsDatabase, prisma.writer_db) # cast-ok: generated Prisma delegate
|
||||
return cls(
|
||||
UsageAlertsRepository.from_prisma(prisma),
|
||||
UsageAlertMetricsRepository(database),
|
||||
UsageAlertHealthRepository(database),
|
||||
UsageAlertDelivery(slack),
|
||||
default_usage_rules(
|
||||
("slack",)
|
||||
if "slack" in configured_destinations(slack) and AlertType.usage_alerts in slack.alert_types
|
||||
else (),
|
||||
legacy_user_spend_alerts=AlertType.user_spend_anomalies in slack.alert_types,
|
||||
),
|
||||
)
|
||||
|
||||
async def run(self, now: datetime | None = None) -> None:
|
||||
owner: Final = str(uuid4())
|
||||
if not await self._store.claim_job(owner):
|
||||
return
|
||||
try:
|
||||
await asyncio.wait_for(self._run(now or datetime.now(timezone.utc)), timeout=840)
|
||||
except asyncio.TimeoutError:
|
||||
verbose_proxy_logger.warning(
|
||||
"Usage alert evaluation exceeded its 14-minute lease budget; retrying next hour"
|
||||
)
|
||||
except Exception: # noqa: BLE001 # the scheduler retries; completed rule/day checkpoints survive a failed pass
|
||||
verbose_proxy_logger.exception("Daily usage alert evaluation failed")
|
||||
finally:
|
||||
await self._store.release_job(owner)
|
||||
|
||||
async def _rules(self) -> tuple[UsageAlertRule, ...]:
|
||||
pages: Final[list[UsageAlertRule]] = [] # mutable-ok: collect configuration pages
|
||||
offset = 0 # rebind-ok: configuration pagination cursor
|
||||
while page := await self._store.list_enabled_rules(limit=100, offset=offset):
|
||||
pages.extend(page)
|
||||
if len(page) < 100:
|
||||
return tuple(pages)
|
||||
offset += len(page)
|
||||
return tuple(pages)
|
||||
|
||||
async def _run(self, now: datetime) -> None:
|
||||
if not self._defaults_registered:
|
||||
await self._store.ensure_default_rules(self._defaults)
|
||||
self._defaults_registered = True
|
||||
target: Final = completed_day(now)
|
||||
rules: Final = await self._rules()
|
||||
due: Final = tuple([rule for rule in rules if not await self._store.is_day_complete(rule, target)])
|
||||
for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in _SCOPES):
|
||||
if scoped:
|
||||
await self._scope(scoped, target, rules)
|
||||
await self._notify()
|
||||
await self._store.prune()
|
||||
|
||||
async def _scope(
|
||||
self, rules: tuple[UsageAlertRule, ...], target: date, enabled: tuple[UsageAlertRule, ...]
|
||||
) -> None:
|
||||
scope: Final = rules[0].scope_type
|
||||
start: Final = datetime.combine(target, time(), tzinfo=timezone.utc)
|
||||
coverage: Final = await self._health.coverage("team" if scope == "team" else "user", start)
|
||||
eligible: Final = tuple(
|
||||
rule for rule in rules if coverage.after_grace and (rule.direction == "increase" or coverage.hours == 24)
|
||||
)
|
||||
suppressed: Final = {rule.id: _suppressed_entities(rule, enabled) for rule in eligible}
|
||||
for rule in rules:
|
||||
if rule not in eligible:
|
||||
await self._store.record_evaluation_status(rule, target, "insufficient_coverage")
|
||||
if not eligible:
|
||||
return
|
||||
evaluated: Final[set[str]] = set() # mutable-ok: at most one entry per configured rule across streaming pages
|
||||
async for rows in self._metrics.batches(scope, target, max(rule.baseline_days for rule in eligible)):
|
||||
evaluated.update(await self._page(eligible, rows, target, enabled))
|
||||
for rule in eligible:
|
||||
await self._store.finish_day(rule, target)
|
||||
await self._store.record_evaluation_status(
|
||||
rule, target, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history"
|
||||
)
|
||||
|
||||
async def _page(
|
||||
self,
|
||||
rules: tuple[UsageAlertRule, ...],
|
||||
rows: tuple[DailyAlertMetrics, ...],
|
||||
target: date,
|
||||
enabled: tuple[UsageAlertRule, ...],
|
||||
) -> tuple[str, ...]:
|
||||
entities: Final = (
|
||||
(None,) if rules[0].scope_type == "deployment" else tuple(dict.fromkeys(row.entity_id for row in rows))
|
||||
)
|
||||
active: Final = await self._store.breached_entities(tuple(rule.id for rule in rules), entities)
|
||||
return tuple(
|
||||
[
|
||||
rule.id
|
||||
for rule in rules
|
||||
if await self._rule_page(
|
||||
rule, rows, target, active.get(rule.id, frozenset()), _suppressed_entities(rule, enabled)
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
async def _rule_page(
|
||||
self,
|
||||
rule: UsageAlertRule,
|
||||
rows: tuple[DailyAlertMetrics, ...],
|
||||
target: date,
|
||||
active: frozenset[str | None],
|
||||
suppressed: frozenset[str | None],
|
||||
) -> bool:
|
||||
observations: Final = _observations(rule, rows, target, suppressed)
|
||||
changed: Final = tuple(item for item in observations if item.breached or item.entity_id in active)
|
||||
if changed:
|
||||
await self._store.record_observations(rule, changed)
|
||||
return bool(observations)
|
||||
|
||||
async def _deliver_channel(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> str | None:
|
||||
try:
|
||||
delivered: Final = await asyncio.wait_for(self._delivery.send(alert, rule, channel), timeout=30)
|
||||
if not delivered:
|
||||
return "transport_not_configured"
|
||||
await self._store.mark_notified(alert.id, channel)
|
||||
except Exception: # noqa: BLE001 # retry only failed channels; persisted acknowledgements survive restarts
|
||||
verbose_proxy_logger.warning("Usage alert delivery failed for incident %s via %s", alert.id, channel)
|
||||
return "delivery_failed"
|
||||
return None
|
||||
|
||||
async def _notify(self) -> None:
|
||||
while page := await self._store.pending_notifications(limit=100):
|
||||
for pending in page:
|
||||
await self._notify_incident(pending)
|
||||
|
||||
async def _notify_incident(self, pending: UsageAlertNotification) -> None:
|
||||
failures: Final = tuple(
|
||||
[
|
||||
await self._deliver_channel(pending.incident, pending.rule, channel)
|
||||
for channel in pending.pending_channels
|
||||
]
|
||||
)
|
||||
if any(failures):
|
||||
error: Final = "delivery_failed" if "delivery_failed" in failures else "transport_not_configured"
|
||||
await self._store.defer_notification(pending.incident.id, error)
|
||||
|
|
@ -8282,6 +8282,8 @@ async def send_email(
|
|||
html: str | None = None,
|
||||
text: str | None = None,
|
||||
message_id: str | None = None,
|
||||
*,
|
||||
raise_on_error: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
smtp_host,
|
||||
|
|
@ -8340,6 +8342,8 @@ async def send_email(
|
|||
)
|
||||
|
||||
except Exception as e:
|
||||
if raise_on_error:
|
||||
raise
|
||||
verbose_proxy_logger.exception("An error occurred while sending the email:" + str(e))
|
||||
return False
|
||||
return True
|
||||
|
|
|
|||
95
litellm/repositories/usage_alert_health.py
Normal file
95
litellm/repositories/usage_alert_health.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
"""Sample successful aggregate flushes without adding request-time telemetry."""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
Literal,
|
||||
TypeAlias,
|
||||
cast, # noqa: TID251 # generated Prisma delegate supplies the repository protocol
|
||||
)
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.repositories.usage_alert_metrics import MetricsDatabase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
FlushSource: TypeAlias = Literal["user", "team"]
|
||||
|
||||
|
||||
class FlushCoverage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
hours: int
|
||||
after_grace: bool
|
||||
|
||||
|
||||
_COVERAGE: Final = TypeAdapter(tuple[FlushCoverage, ...])
|
||||
|
||||
|
||||
class UsageAlertHealthRepository:
|
||||
def __init__(self, database: MetricsDatabase) -> None:
|
||||
self._database = database
|
||||
|
||||
async def record(self, source: FlushSource, now: datetime) -> bool:
|
||||
hour: Final = now.astimezone(timezone.utc).replace(minute=0, second=0, microsecond=0, tzinfo=None)
|
||||
rows: Final = await self._database.query_raw(
|
||||
"""INSERT INTO "LiteLLM_UsageAlertFlush" (source,hour,observed_at)
|
||||
SELECT $1,$2,$3 WHERE NOT EXISTS (
|
||||
SELECT 1 FROM "LiteLLM_SpendLogDelivery" WHERE state='pending' AND created_at<$2 LIMIT 1
|
||||
) ON CONFLICT (source,hour) DO UPDATE SET observed_at=EXCLUDED.observed_at RETURNING source""",
|
||||
source,
|
||||
hour,
|
||||
now.astimezone(timezone.utc).replace(tzinfo=None),
|
||||
)
|
||||
await self._database.query_raw(
|
||||
'DELETE FROM "LiteLLM_UsageAlertFlush" WHERE hour<$1 RETURNING source', hour - timedelta(days=35)
|
||||
)
|
||||
return bool(rows)
|
||||
|
||||
async def coverage(self, source: FlushSource, start: datetime) -> FlushCoverage:
|
||||
utc_start: Final = start.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
end: Final = utc_start + timedelta(days=1)
|
||||
rows: Final = _COVERAGE.validate_python(
|
||||
await self._database.query_raw(
|
||||
"""SELECT COUNT(*) FILTER(WHERE hour >= $2 AND hour < $3)::integer AS hours,
|
||||
COALESCE(BOOL_OR(observed_at >= $3 + INTERVAL '1 hour'),FALSE) AS after_grace
|
||||
FROM "LiteLLM_UsageAlertFlush" WHERE source=$1 AND hour >= $2""",
|
||||
source,
|
||||
utc_start,
|
||||
end,
|
||||
)
|
||||
)
|
||||
return rows[0] if rows else FlushCoverage(hours=0, after_grace=False)
|
||||
|
||||
|
||||
class UsageAlertFlushRecorder:
|
||||
"""At most one heartbeat attempt per source per worker per hour after success."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._user_hour: datetime | None = None
|
||||
self._team_hour: datetime | None = None
|
||||
|
||||
async def record(self, prisma_client: "PrismaClient", source: FlushSource) -> None:
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.utils import ProxyUpdateSpend
|
||||
|
||||
if ProxyUpdateSpend.disable_spend_updates():
|
||||
return
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
hour: Final = now.replace(minute=0, second=0, microsecond=0)
|
||||
prior: Final = self._user_hour if source == "user" else self._team_hour
|
||||
if prior == hour:
|
||||
return
|
||||
database: Final = cast(MetricsDatabase, prisma_client.writer_db) # cast-ok: generated Prisma delegate
|
||||
try:
|
||||
if not await UsageAlertHealthRepository(database).record(source, now):
|
||||
return
|
||||
except Exception: # noqa: BLE001 # observability must not fail or requeue a committed spend flush
|
||||
verbose_proxy_logger.warning("Could not record usage-alert aggregate freshness")
|
||||
return
|
||||
if source == "user":
|
||||
self._user_hour = hour
|
||||
else:
|
||||
self._team_hour = hour
|
||||
92
litellm/repositories/usage_alert_metrics.py
Normal file
92
litellm/repositories/usage_alert_metrics.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
"""Bounded reads of the daily rollups used by the Usage dashboard."""
|
||||
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from datetime import date, timedelta
|
||||
from typing import Final, Literal, Protocol, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.repositories.daily_activity_buckets import api_key_bucket, model_bucket
|
||||
|
||||
Scope: TypeAlias = Literal["deployment", "key", "user", "team", "model"]
|
||||
|
||||
|
||||
class DailyAlertMetrics(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
entity_id: str
|
||||
day: date
|
||||
spend: float
|
||||
requests: int
|
||||
errors: int
|
||||
response_time_ms: float
|
||||
timed_requests: int
|
||||
|
||||
|
||||
class MetricsDatabase(Protocol):
|
||||
async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ...
|
||||
|
||||
|
||||
_ROWS: Final = TypeAdapter(tuple[DailyAlertMetrics, ...])
|
||||
_BATCH_SIZE: Final = 500
|
||||
|
||||
|
||||
def _source(scope: Scope) -> tuple[str, str]:
|
||||
if scope == "team":
|
||||
return '"LiteLLM_DailyTeamSpend"', "team_id"
|
||||
if scope == "user":
|
||||
return '"LiteLLM_DailyUserSpend"', "user_id"
|
||||
if scope == "key":
|
||||
return '"LiteLLM_DailyUserSpend"', "api_key"
|
||||
if scope == "model":
|
||||
return '"LiteLLM_DailyUserSpend"', "COALESCE(NULLIF(model, ''), 'unknown-model')"
|
||||
return '"LiteLLM_DailyUserSpend"', "'deployment'::text"
|
||||
|
||||
|
||||
def _query(scope: Scope) -> str:
|
||||
table, entity = _source(scope)
|
||||
# Identifiers come only from the closed scope mapping; request values are parameters.
|
||||
return f"""
|
||||
WITH entities AS MATERIALIZED (
|
||||
SELECT DISTINCT {entity} AS entity_id FROM {table}
|
||||
WHERE date >= $1 AND date <= $2 AND {entity} >= COALESCE($3::text, '') AND ($3::text IS NULL OR {entity} <> $3)
|
||||
ORDER BY entity_id LIMIT $4
|
||||
)
|
||||
SELECT entities.entity_id, activity.date AS day,
|
||||
SUM(activity.spend) AS spend,
|
||||
SUM(activity.api_requests)::bigint AS requests,
|
||||
SUM(activity.failed_requests)::bigint AS errors,
|
||||
SUM(activity.total_response_time_ms) AS response_time_ms,
|
||||
SUM(activity.timed_requests)::bigint AS timed_requests
|
||||
FROM {table} activity JOIN entities ON {entity} = entities.entity_id
|
||||
WHERE activity.date >= $1 AND activity.date <= $2
|
||||
GROUP BY entities.entity_id, activity.date
|
||||
ORDER BY entities.entity_id, activity.date
|
||||
"""
|
||||
|
||||
|
||||
def _reporting_id(scope: Scope, row: DailyAlertMetrics) -> DailyAlertMetrics:
|
||||
if scope == "key":
|
||||
return row.model_copy(update={"entity_id": api_key_bucket(row.entity_id)})
|
||||
if scope == "model":
|
||||
return row.model_copy(update={"entity_id": model_bucket(row.entity_id)})
|
||||
return row
|
||||
|
||||
|
||||
class UsageAlertMetricsRepository:
|
||||
def __init__(self, database: MetricsDatabase) -> None:
|
||||
self._database = database
|
||||
|
||||
async def batches(
|
||||
self, scope: Scope, target: date, baseline_days: int
|
||||
) -> AsyncIterator[tuple[DailyAlertMetrics, ...]]:
|
||||
start: Final = (target - timedelta(days=baseline_days)).isoformat()
|
||||
query: Final = _query(scope)
|
||||
cursor: str | None = None # rebind-ok: keyset pagination retains only the current page
|
||||
while rows := _ROWS.validate_python(
|
||||
await self._database.query_raw(query, start, target.isoformat(), cursor, _BATCH_SIZE)
|
||||
):
|
||||
yield tuple(_reporting_id(scope, row) for row in rows)
|
||||
cursor = rows[-1].entity_id
|
||||
if scope == "deployment":
|
||||
return
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
import json
|
||||
from collections.abc import Mapping
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from itertools import groupby
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
|
|
@ -13,7 +14,7 @@ from typing import (
|
|||
from urllib.parse import urlencode
|
||||
from uuid import NAMESPACE_URL, uuid4, uuid5
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
|
||||
from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS
|
||||
|
|
@ -45,6 +46,12 @@ if TYPE_CHECKING:
|
|||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
|
||||
class _BreachedEntity(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
rule_id: str
|
||||
scope_id: str | None
|
||||
|
||||
|
||||
class AlertDatabase(Protocol):
|
||||
async def query_raw(self, query: str, *args: object) -> object: ...
|
||||
|
||||
|
|
@ -672,7 +679,9 @@ class UsageAlertsRepository:
|
|||
rows: Final = TypeAdapter(tuple[_Pending, ...]).validate_python(
|
||||
await self.database.query_raw(
|
||||
"""SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id
|
||||
WHERE i.status='open' AND r.enabled AND NOT r.deleted AND EXISTS (
|
||||
WHERE i.status='open' AND r.enabled AND NOT r.deleted
|
||||
AND COALESCE((i.record->>'next_notification_attempt')::timestamptz,'-infinity'::timestamptz)<=CURRENT_TIMESTAMP
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels)))
|
||||
ORDER BY i.updated_at,i.id LIMIT $1""",
|
||||
limit,
|
||||
|
|
@ -693,7 +702,16 @@ class UsageAlertsRepository:
|
|||
|
||||
async def mark_notified(self, incident_id: str, channel: UsageAlertChannel) -> None:
|
||||
await self.database.query_raw(
|
||||
'UPDATE "LiteLLM_UsageAlertIncident" SET notified_channels=array_append(notified_channels,$2::text) WHERE id=$1 AND NOT ($2::text=ANY(notified_channels)) RETURNING id',
|
||||
"""UPDATE "LiteLLM_UsageAlertIncident" i SET
|
||||
notified_channels=array_append(i.notified_channels,$2::text),
|
||||
record=i.record || jsonb_build_object('notified_channels',array_append(i.notified_channels,$2::text), 'last_notification_attempt',CURRENT_TIMESTAMP) ||
|
||||
CASE WHEN NOT EXISTS (
|
||||
SELECT 1 FROM "LiteLLM_UsageAlertRule" r,
|
||||
jsonb_array_elements_text(r.record->'notification_channels') channel
|
||||
WHERE r.id=i.rule_id AND NOT (channel=ANY(array_append(i.notified_channels,$2::text)))
|
||||
) THEN jsonb_build_object('notification_error',NULL,'next_notification_attempt',NULL)
|
||||
ELSE '{}'::jsonb END
|
||||
WHERE i.id=$1 AND NOT ($2::text=ANY(i.notified_channels)) RETURNING i.id""",
|
||||
incident_id,
|
||||
channel,
|
||||
)
|
||||
|
|
@ -718,3 +736,35 @@ class UsageAlertsRepository:
|
|||
)
|
||||
)
|
||||
return rows[0]["role"] if rows else None
|
||||
|
||||
async def breached_entities(
|
||||
self, rule_ids: tuple[str, ...], entity_ids: tuple[str | None, ...]
|
||||
) -> Mapping[str, frozenset[str | None]]:
|
||||
rows: Final = TypeAdapter(tuple[_BreachedEntity, ...]).validate_python(
|
||||
await self.database.query_raw(
|
||||
'SELECT DISTINCT rule_id,scope_id FROM "LiteLLM_UsageAlertIncident" '
|
||||
"WHERE rule_id=ANY($1::text[]) AND scope_key=ANY($2::text[]) AND breached ORDER BY rule_id",
|
||||
rule_ids,
|
||||
tuple(entity or "" for entity in entity_ids),
|
||||
)
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
rule_id: frozenset(row.scope_id for row in group)
|
||||
for rule_id, group in groupby(rows, lambda row: row.rule_id)
|
||||
}
|
||||
)
|
||||
|
||||
async def defer_notification(self, incident_id: str, error: str) -> None:
|
||||
await self.database.query_raw(
|
||||
"""UPDATE "LiteLLM_UsageAlertIncident" SET updated_at=CURRENT_TIMESTAMP,
|
||||
record=record || jsonb_build_object(
|
||||
'notification_error',$2::text,
|
||||
'notification_attempts',COALESCE((record->>'notification_attempts')::int,0)+1,
|
||||
'last_notification_attempt',CURRENT_TIMESTAMP,
|
||||
'next_notification_attempt',CURRENT_TIMESTAMP + make_interval(secs =>
|
||||
LEAST(86400,3600*power(2,LEAST(5,COALESCE((record->>'notification_attempts')::int,0)))::int))
|
||||
) WHERE id=$1 RETURNING id""",
|
||||
incident_id,
|
||||
error,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -177,6 +177,7 @@ class AlertType(str, Enum):
|
|||
failed_tracking_spend = "failed_tracking_spend"
|
||||
user_spend_thresholds = "user_spend_thresholds"
|
||||
user_spend_anomalies = "user_spend_anomalies"
|
||||
usage_alerts = "usage_alerts"
|
||||
|
||||
# Database alerts
|
||||
db_exceptions = "db_exceptions"
|
||||
|
|
|
|||
|
|
@ -85,6 +85,11 @@ class UsageAlert(BaseModel):
|
|||
first_seen: datetime
|
||||
last_seen: datetime
|
||||
activity_path: str
|
||||
notified_channels: tuple[UsageAlertChannel, ...] = ()
|
||||
notification_attempts: int = 0
|
||||
last_notification_attempt: datetime | None = None
|
||||
next_notification_attempt: datetime | None = None
|
||||
notification_error: Literal["delivery_failed", "transport_not_configured"] | None = None
|
||||
|
||||
|
||||
class UsageAlertUpdate(BaseModel):
|
||||
|
|
|
|||
|
|
@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule {
|
|||
|
||||
@@id([delivery_id, rule_id, source_id])
|
||||
}
|
||||
|
||||
// Hourly samples of successful background daily-aggregate flushes, retained for 35 days.
|
||||
model LiteLLM_UsageAlertFlush {
|
||||
source String
|
||||
hour DateTime
|
||||
observed_at DateTime
|
||||
|
||||
@@id([source, hour])
|
||||
}
|
||||
|
|
|
|||
78
tests/unit/integrations/SlackAlerting/test_usage_alerts.py
Normal file
78
tests/unit/integrations/SlackAlerting/test_usage_alerts.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
from datetime import date, datetime, timezone
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, activity_url
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule
|
||||
|
||||
_NOW: Final = datetime(2026, 10, 20, tzinfo=timezone.utc)
|
||||
_RULE: Final = UsageAlertRule(
|
||||
name="Request spike",
|
||||
id="rule",
|
||||
scope_type="key",
|
||||
scope_id="reporting-key",
|
||||
metric="requests",
|
||||
direction="increase",
|
||||
created_at=_NOW,
|
||||
updated_at=_NOW,
|
||||
created_by=None,
|
||||
notification_channels=("slack",),
|
||||
)
|
||||
_ALERT: Final = UsageAlert(
|
||||
id="incident",
|
||||
rule_id="rule",
|
||||
rule_name="Request spike",
|
||||
scope_type="key",
|
||||
scope_id="reporting-key",
|
||||
metric="requests",
|
||||
direction="increase",
|
||||
status="open",
|
||||
day=date(2026, 10, 20),
|
||||
observed_value=1000,
|
||||
baseline_value=200,
|
||||
threshold=600,
|
||||
first_seen=_NOW,
|
||||
last_seen=_NOW,
|
||||
baseline_start=date(2026, 10, 13),
|
||||
baseline_end=date(2026, 10, 19),
|
||||
baseline_sample_days=7,
|
||||
activity_path="/api-keys?key=reporting-key&detail_tab=activity",
|
||||
)
|
||||
|
||||
|
||||
def test_activity_link_retains_dashboard_mount_and_entity_query() -> None:
|
||||
assert activity_url(_ALERT, "https://gateway.example.com/") == (
|
||||
"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity"
|
||||
)
|
||||
assert activity_url(_ALERT, None) == _ALERT.activity_path
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", [200, 503])
|
||||
async def test_slack_ack_requires_successful_http_response(status: int, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com")
|
||||
requests: Final[list[httpx.Request]] = []
|
||||
|
||||
def transport(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(request)
|
||||
return httpx.Response(status)
|
||||
|
||||
handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(transport))
|
||||
slack: Final = SlackAlerting(
|
||||
alerting=["slack"], default_webhook_url="https://hooks.example.com/test", async_http_handler=handler
|
||||
)
|
||||
delivery: Final = UsageAlertDelivery(slack)
|
||||
try:
|
||||
if status == 200:
|
||||
assert await delivery.send(_ALERT, _RULE, "slack") is True
|
||||
else:
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
await delivery.send(_ALERT, _RULE, "slack")
|
||||
assert len(requests) == 1
|
||||
assert b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content
|
||||
finally:
|
||||
await handler.close()
|
||||
155
tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py
Normal file
155
tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.spend_tracking.daily_usage_alerts import DailyComparison, compare_daily_usage
|
||||
from litellm.proxy.spend_tracking.usage_alert_job import completed_day
|
||||
from litellm.repositories.usage_alert_metrics import DailyAlertMetrics
|
||||
from litellm.types.proxy.usage_alerts import UsageAlertRuleInput
|
||||
|
||||
_DAY: Final = date(2026, 10, 20)
|
||||
|
||||
|
||||
def _row(day: date, requests: int = 200, spend: float = 20, ms: float = 100_000, timed: int = 200) -> DailyAlertMetrics:
|
||||
return DailyAlertMetrics(
|
||||
entity_id="key-a", day=day, spend=spend, requests=requests, errors=10, response_time_ms=ms, timed_requests=timed
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"metric,observed,baseline", [("spend", 100, 20), ("requests", 1000, 200), ("latency", 2000, 500)]
|
||||
)
|
||||
def test_spikes_use_recorded_reference_days_and_weighted_latency(metric: str, observed: float, baseline: float) -> None:
|
||||
rule: Final = UsageAlertRuleInput.model_validate(
|
||||
{"name": "spike", "scope_type": "key", "metric": metric, "direction": "increase"}
|
||||
)
|
||||
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 3, 7))
|
||||
current: Final = _row(_DAY, requests=1000, spend=100, ms=400_000)
|
||||
assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(
|
||||
3, observed, baseline, baseline * 3, True
|
||||
)
|
||||
|
||||
|
||||
def test_missing_target_is_zero_for_counts_but_never_zero_latency() -> None:
|
||||
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
|
||||
rule: Final = UsageAlertRuleInput(name="drop", scope_type="key", metric="requests", direction="decrease")
|
||||
assert compare_daily_usage(rule, history, _DAY) == DailyComparison(3, 0, 200, 200 / 3, True)
|
||||
assert compare_daily_usage(rule.model_copy(update={"metric": "latency"}), history, _DAY) is None
|
||||
|
||||
|
||||
def test_low_volume_is_insufficient_and_recorded_zero_baseline_uses_absolute_floor() -> None:
|
||||
rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase")
|
||||
history: Final = tuple(_row(_DAY - timedelta(days=days), requests=10, spend=0) for days in (1, 2, 3))
|
||||
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) is None
|
||||
enough_requests: Final = tuple(row.model_copy(update={"requests": 200}) for row in history)
|
||||
assert compare_daily_usage(rule, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison(
|
||||
3, 100, 0, 0, True
|
||||
)
|
||||
floor: Final = rule.model_copy(update={"min_value": 100})
|
||||
assert compare_daily_usage(floor, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison(
|
||||
3, 100, 0, 100, False
|
||||
)
|
||||
|
||||
|
||||
def test_absolute_floor_suppresses_small_changes() -> None:
|
||||
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
|
||||
rule: Final = UsageAlertRuleInput(
|
||||
name="spike", scope_type="key", metric="spend", direction="increase", min_value=100
|
||||
)
|
||||
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 60, False)
|
||||
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, False)
|
||||
|
||||
|
||||
def test_day_is_eligible_only_after_one_hour_utc_grace() -> None:
|
||||
assert completed_day(datetime(2026, 10, 21, 0, 59, tzinfo=timezone.utc)) == date(2026, 10, 19)
|
||||
assert completed_day(datetime(2026, 10, 21, 1, 0, tzinfo=timezone.utc)) == _DAY
|
||||
|
||||
|
||||
def test_latency_weights_request_samples_instead_of_averaging_daily_averages() -> None:
|
||||
rule: Final = UsageAlertRuleInput(
|
||||
name="latency", scope_type="key", metric="latency", direction="increase", min_history_days=2
|
||||
)
|
||||
rows: Final = (
|
||||
_row(_DAY - timedelta(days=1), requests=100, ms=100_000, timed=100),
|
||||
_row(_DAY - timedelta(days=2), requests=400, ms=1_600_000, timed=400),
|
||||
_row(_DAY, requests=100, ms=1_100_000, timed=100),
|
||||
)
|
||||
assert compare_daily_usage(rule, rows, _DAY) == DailyComparison(2, 11000, 3400, 10200, True)
|
||||
|
||||
|
||||
def test_errors_are_failed_request_count_and_require_absolute_change_floor() -> None:
|
||||
rule: Final = UsageAlertRuleInput(
|
||||
name="errors", scope_type="key", metric="errors", direction="increase", min_value=10
|
||||
)
|
||||
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
|
||||
current: Final = _row(_DAY).model_copy(update={"errors": 40})
|
||||
assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(3, 40, 10, 30, True)
|
||||
|
||||
|
||||
def test_a_value_at_the_configured_multiplier_alerts() -> None:
|
||||
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
|
||||
rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase")
|
||||
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, True)
|
||||
|
||||
|
||||
def test_default_notifications_require_opt_in_and_do_not_duplicate_legacy_user_spend() -> None:
|
||||
from litellm.proxy.spend_tracking.usage_alert_job import default_usage_rules
|
||||
|
||||
defaults: Final = default_usage_rules()
|
||||
assert len(defaults) == 40
|
||||
assert all(not rule.notification_channels for rule in defaults)
|
||||
opted_in: Final = default_usage_rules(("slack",), legacy_user_spend_alerts=True)
|
||||
quiet: Final = tuple(rule for rule in opted_in if not rule.notification_channels)
|
||||
assert tuple((rule.scope_type, rule.metric, rule.direction) for rule in quiet) == (("user", "spend", "increase"),)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scope,entity",
|
||||
[("key", "unattributed"), ("key", "sk-example"), ("user", "default_user_id"), ("team", "litellm-dashboard")],
|
||||
)
|
||||
def test_sentinel_entities_cannot_create_alerts_with_unusable_activity_links(scope: str, entity: str) -> None:
|
||||
from litellm.proxy.spend_tracking.usage_alert_job import _observations
|
||||
from litellm.types.proxy.usage_alerts import UsageAlertRule
|
||||
|
||||
rule: Final = UsageAlertRule.model_validate(
|
||||
{
|
||||
"id": "rule",
|
||||
"name": "spike",
|
||||
"scope_type": scope,
|
||||
"metric": "spend",
|
||||
"direction": "increase",
|
||||
"created_at": "2026-10-01T00:00:00Z",
|
||||
"updated_at": "2026-10-01T00:00:00Z",
|
||||
"created_by": None,
|
||||
}
|
||||
)
|
||||
history: Final = tuple(
|
||||
_row(_DAY - timedelta(days=days)).model_copy(update={"entity_id": entity}) for days in (1, 2, 3)
|
||||
)
|
||||
current: Final = _row(_DAY, spend=100).model_copy(update={"entity_id": entity})
|
||||
assert _observations(rule, (*history, current), _DAY) == ()
|
||||
|
||||
|
||||
def test_custom_entity_rule_replaces_only_the_matching_default() -> None:
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
from litellm.proxy.spend_tracking.usage_alert_job import _suppressed_entities
|
||||
from litellm.types.proxy.usage_alerts import UsageAlertRule
|
||||
|
||||
default: Final = UsageAlertRule(
|
||||
id=str(uuid5(NAMESPACE_URL, "litellm:usage-alert:key:spend:increase")),
|
||||
name="default",
|
||||
scope_type="key",
|
||||
metric="spend",
|
||||
direction="increase",
|
||||
created_at=datetime(2026, 10, 1, tzinfo=timezone.utc),
|
||||
updated_at=datetime(2026, 10, 1, tzinfo=timezone.utc),
|
||||
created_by=None,
|
||||
)
|
||||
custom: Final = default.model_copy(update={"id": "custom", "scope_id": "key-a"})
|
||||
other_metric: Final = custom.model_copy(update={"id": "latency", "metric": "latency", "scope_id": "key-b"})
|
||||
assert _suppressed_entities(default, (default, custom, other_metric)) == frozenset({"key-a"})
|
||||
assert _suppressed_entities(custom, (default, custom, other_metric)) == frozenset()
|
||||
wildcard: Final = custom.model_copy(update={"scope_id": None})
|
||||
assert _suppressed_entities(default, (default, wildcard)) == frozenset({None})
|
||||
|
|
@ -7,13 +7,16 @@ Symbols pinned here:
|
|||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any
|
||||
from typing import Any, TYPE_CHECKING
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.utils import send_email
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tests.unit.proxy.utils.prisma_and_spend.conftest import InMemorySMTP
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _smtp_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
|
@ -176,3 +179,12 @@ async def test_should_send_plain_text_and_html_as_multipart_alternative(monkeypa
|
|||
assert tuple(part.get_content_type() for part in parts) == ("text/plain", "text/html")
|
||||
assert parts[0].get_payload(decode=True).decode() == "Model retirement"
|
||||
assert parts[1].get_payload(decode=True).decode() == "<p>Model retirement</p>"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_durable_alert_delivery_can_observe_smtp_failure(in_memory_smtp: InMemorySMTP) -> None:
|
||||
in_memory_smtp.raise_on_send = RuntimeError("smtp unavailable")
|
||||
with pytest.raises(RuntimeError, match="smtp unavailable"):
|
||||
await send_email(receiver_email="alerts@example.com", subject="Daily usage", html="<p>Alert</p>",
|
||||
raise_on_error=True)
|
||||
assert in_memory_smtp.sent == []
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue