mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
Merge 3bfc77ac18 into fe87b187c6
This commit is contained in:
commit
8244f5ef1e
10 changed files with 512 additions and 17 deletions
|
|
@ -1534,6 +1534,7 @@ KEY_ROTATION_JOB_NAME: Final = "litellm_key_rotation_job"
|
||||||
EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME: Final = "litellm_expired_ui_session_key_cleanup_job"
|
EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME: Final = "litellm_expired_ui_session_key_cleanup_job"
|
||||||
WEEKLY_SPEND_REPORT_JOB_ID: Final = "weekly_spend_report_job"
|
WEEKLY_SPEND_REPORT_JOB_ID: Final = "weekly_spend_report_job"
|
||||||
MONTHLY_SPEND_REPORT_JOB_ID: Final = "monthly_spend_report_job"
|
MONTHLY_SPEND_REPORT_JOB_ID: Final = "monthly_spend_report_job"
|
||||||
|
USER_SPEND_ALERTS_JOB_ID: Final = "user_spend_alerts_job"
|
||||||
PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job"
|
PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job"
|
||||||
SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report"
|
SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report"
|
||||||
SLACK_MODEL_DEPRECATION_LOCK_ID: Final = "slack_model_deprecation_warning"
|
SLACK_MODEL_DEPRECATION_LOCK_ID: Final = "slack_model_deprecation_warning"
|
||||||
|
|
|
||||||
|
|
@ -61,6 +61,7 @@ from .utils import process_slack_alerting_variables
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
|
from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
|
||||||
|
from litellm.proxy.utils import PrismaClient
|
||||||
from litellm.router import Router as _Router
|
from litellm.router import Router as _Router
|
||||||
|
|
||||||
Router = _Router
|
Router = _Router
|
||||||
|
|
@ -1897,6 +1898,69 @@ Model Info:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
verbose_proxy_logger.exception("Error sending weekly spend report %s", e)
|
verbose_proxy_logger.exception("Error sending weekly spend report %s", e)
|
||||||
|
|
||||||
|
async def send_user_spend_alerts(self, prisma_client: "PrismaClient | None" = None) -> None:
|
||||||
|
"""Check per-user daily/monthly spend thresholds and spend anomalies, alerting once per user per period."""
|
||||||
|
if self.alerting is None or "slack" not in self.alerting:
|
||||||
|
return
|
||||||
|
|
||||||
|
thresholds_enabled: Final = AlertType.user_spend_thresholds in self.alert_types
|
||||||
|
anomalies_enabled: Final = AlertType.user_spend_anomalies in self.alert_types
|
||||||
|
if not thresholds_enabled and not anomalies_enabled:
|
||||||
|
return
|
||||||
|
|
||||||
|
if prisma_client is None:
|
||||||
|
from litellm.proxy.proxy_server import prisma_client as global_prisma_client
|
||||||
|
|
||||||
|
prisma_client = global_prisma_client # rebind-ok: fall back to the proxy's global client
|
||||||
|
if prisma_client is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
from litellm.integrations.SlackAlerting.user_spend_alerts import (
|
||||||
|
evaluate_user_spend,
|
||||||
|
fetch_user_spend_rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
today: Final = datetime.datetime.now(datetime.timezone.utc).date()
|
||||||
|
rows: Final = await fetch_user_spend_rows(
|
||||||
|
prisma_client=prisma_client,
|
||||||
|
today=today,
|
||||||
|
baseline_days=self.alerting_args.spend_anomaly_baseline_days,
|
||||||
|
)
|
||||||
|
all_events: Final = tuple(
|
||||||
|
event
|
||||||
|
for row in rows
|
||||||
|
for event in evaluate_user_spend(
|
||||||
|
row=row,
|
||||||
|
args=self.alerting_args,
|
||||||
|
today=today,
|
||||||
|
thresholds_enabled=thresholds_enabled,
|
||||||
|
anomalies_enabled=anomalies_enabled,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
cached_flags: Final = await asyncio.gather(
|
||||||
|
*(self.internal_usage_cache.async_get_cache(key=event.cache_key) for event in all_events)
|
||||||
|
)
|
||||||
|
new_events: Final = tuple(event for event, cached in zip(all_events, cached_flags) if not cached)
|
||||||
|
for alert_type in (AlertType.user_spend_thresholds, AlertType.user_spend_anomalies):
|
||||||
|
typed_events = tuple(event for event in new_events if event.alert_type == alert_type)
|
||||||
|
if not typed_events:
|
||||||
|
continue
|
||||||
|
await self.send_alert(
|
||||||
|
message="\n\n".join(event.message for event in typed_events),
|
||||||
|
level="High",
|
||||||
|
alert_type=alert_type,
|
||||||
|
alerting_metadata={}, # mutable-ok: send_alert takes a dict payload
|
||||||
|
)
|
||||||
|
for event in typed_events:
|
||||||
|
await self.internal_usage_cache.async_set_cache(
|
||||||
|
key=event.cache_key,
|
||||||
|
value="SENT",
|
||||||
|
ttl=event.cache_ttl,
|
||||||
|
)
|
||||||
|
except Exception as e: # noqa: BLE001 # background job must not crash the scheduler
|
||||||
|
verbose_proxy_logger.exception("Error sending user spend alerts: %s", e)
|
||||||
|
|
||||||
async def send_fallback_stats_from_prometheus(self):
|
async def send_fallback_stats_from_prometheus(self):
|
||||||
"""
|
"""
|
||||||
Helper to send fallback statistics from prometheus server -> to slack
|
Helper to send fallback statistics from prometheus server -> to slack
|
||||||
|
|
|
||||||
139
litellm/integrations/SlackAlerting/user_spend_alerts.py
Normal file
139
litellm/integrations/SlackAlerting/user_spend_alerts.py
Normal file
|
|
@ -0,0 +1,139 @@
|
||||||
|
"""Per-user daily/monthly spend threshold alerts and spend anomaly detection."""
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING, Final, Literal
|
||||||
|
|
||||||
|
from pydantic import TypeAdapter
|
||||||
|
|
||||||
|
from litellm.constants import HOURS_IN_A_DAY
|
||||||
|
from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from litellm.proxy.utils import PrismaClient
|
||||||
|
|
||||||
|
DAY_SECONDS: Final = HOURS_IN_A_DAY * 60 * 60
|
||||||
|
MONTHLY_ALERT_TTL_SECONDS: Final = 32 * DAY_SECONDS
|
||||||
|
|
||||||
|
USER_SPEND_QUERY: Final = """
|
||||||
|
SELECT
|
||||||
|
user_id,
|
||||||
|
COALESCE(SUM(spend) FILTER (WHERE date = $1), 0)::float AS daily_spend,
|
||||||
|
COALESCE(SUM(spend) FILTER (WHERE date >= $2), 0)::float AS monthly_spend,
|
||||||
|
COALESCE(SUM(spend) FILTER (WHERE date >= $3 AND date < $1), 0)::float AS baseline_spend
|
||||||
|
FROM "LiteLLM_DailyUserSpend"
|
||||||
|
WHERE date >= LEAST($2, $3) AND user_id IS NOT NULL
|
||||||
|
GROUP BY user_id
|
||||||
|
HAVING COALESCE(SUM(spend) FILTER (WHERE date >= $2), 0) > 0
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class UserSpendRow:
|
||||||
|
user_id: str
|
||||||
|
daily_spend: float
|
||||||
|
monthly_spend: float
|
||||||
|
baseline_spend: float
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class UserSpendAlertEvent:
|
||||||
|
kind: Literal["daily_threshold", "monthly_threshold", "anomaly"]
|
||||||
|
alert_type: AlertType
|
||||||
|
message: str
|
||||||
|
cache_key: str
|
||||||
|
cache_ttl: int
|
||||||
|
|
||||||
|
|
||||||
|
USER_SPEND_ROWS_ADAPTER: Final = TypeAdapter(tuple[UserSpendRow, ...])
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_user_spend_rows(
|
||||||
|
prisma_client: "PrismaClient",
|
||||||
|
today: datetime.date,
|
||||||
|
baseline_days: int,
|
||||||
|
) -> tuple[UserSpendRow, ...]:
|
||||||
|
today_str: Final = today.strftime("%Y-%m-%d")
|
||||||
|
month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d")
|
||||||
|
baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d")
|
||||||
|
raw: Final = await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
|
||||||
|
return USER_SPEND_ROWS_ADAPTER.validate_python(raw)
|
||||||
|
|
||||||
|
|
||||||
|
def _daily_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None:
|
||||||
|
threshold: Final = args.daily_spend_per_user_threshold
|
||||||
|
if threshold is None or row.daily_spend < threshold:
|
||||||
|
return None
|
||||||
|
return UserSpendAlertEvent(
|
||||||
|
kind="daily_threshold",
|
||||||
|
alert_type=AlertType.user_spend_thresholds,
|
||||||
|
message=(
|
||||||
|
f"User Daily Spend Threshold Crossed:\n"
|
||||||
|
f"User: `{row.user_id}`\n"
|
||||||
|
f"Spend Today: `${row.daily_spend:.2f}`\n"
|
||||||
|
f"Daily Threshold: `${threshold:.2f}`"
|
||||||
|
),
|
||||||
|
cache_key=f"user_spend_alert_daily_{row.user_id}_{today_str}",
|
||||||
|
cache_ttl=DAY_SECONDS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _monthly_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, month_str: str) -> UserSpendAlertEvent | None:
|
||||||
|
threshold: Final = args.monthly_spend_per_user_threshold
|
||||||
|
if threshold is None or row.monthly_spend < threshold:
|
||||||
|
return None
|
||||||
|
return UserSpendAlertEvent(
|
||||||
|
kind="monthly_threshold",
|
||||||
|
alert_type=AlertType.user_spend_thresholds,
|
||||||
|
message=(
|
||||||
|
f"User Monthly Spend Threshold Crossed:\n"
|
||||||
|
f"User: `{row.user_id}`\n"
|
||||||
|
f"Spend This Month: `${row.monthly_spend:.2f}`\n"
|
||||||
|
f"Monthly Threshold: `${threshold:.2f}`"
|
||||||
|
),
|
||||||
|
cache_key=f"user_spend_alert_monthly_{row.user_id}_{month_str}",
|
||||||
|
cache_ttl=MONTHLY_ALERT_TTL_SECONDS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _anomaly_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None:
|
||||||
|
if row.daily_spend < args.spend_anomaly_min_spend:
|
||||||
|
return None
|
||||||
|
baseline_daily_avg: Final = row.baseline_spend / args.spend_anomaly_baseline_days
|
||||||
|
if row.baseline_spend > 0 and row.daily_spend <= args.spend_anomaly_multiplier * baseline_daily_avg:
|
||||||
|
return None
|
||||||
|
return UserSpendAlertEvent(
|
||||||
|
kind="anomaly",
|
||||||
|
alert_type=AlertType.user_spend_anomalies,
|
||||||
|
message=(
|
||||||
|
f"User Spend Anomaly Detected:\n"
|
||||||
|
f"User: `{row.user_id}`\n"
|
||||||
|
f"Spend Today: `${row.daily_spend:.2f}`\n"
|
||||||
|
f"Daily Average (last {args.spend_anomaly_baseline_days} days): `${baseline_daily_avg:.2f}`\n"
|
||||||
|
f"Trigger: spend above `{args.spend_anomaly_multiplier}x` the daily average "
|
||||||
|
f"(minimum `${args.spend_anomaly_min_spend:.2f}`)"
|
||||||
|
),
|
||||||
|
cache_key=f"user_spend_alert_anomaly_{row.user_id}_{today_str}",
|
||||||
|
cache_ttl=DAY_SECONDS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_user_spend(
|
||||||
|
row: UserSpendRow,
|
||||||
|
args: SlackAlertingArgs,
|
||||||
|
today: datetime.date,
|
||||||
|
thresholds_enabled: bool,
|
||||||
|
anomalies_enabled: bool,
|
||||||
|
) -> tuple[UserSpendAlertEvent, ...]:
|
||||||
|
today_str: Final = today.strftime("%Y-%m-%d")
|
||||||
|
month_str: Final = today.strftime("%Y-%m")
|
||||||
|
threshold_events: Final = (
|
||||||
|
(
|
||||||
|
_daily_threshold_event(row=row, args=args, today_str=today_str),
|
||||||
|
_monthly_threshold_event(row=row, args=args, month_str=month_str),
|
||||||
|
)
|
||||||
|
if thresholds_enabled
|
||||||
|
else ()
|
||||||
|
)
|
||||||
|
anomaly_events: Final = (_anomaly_event(row=row, args=args, today_str=today_str),) if anomalies_enabled else ()
|
||||||
|
return tuple(event for event in (*threshold_events, *anomaly_events) if event is not None)
|
||||||
|
|
@ -248,6 +248,7 @@ from litellm.constants import (
|
||||||
PROXY_BUDGET_RESCHEDULER_MAX_TIME,
|
PROXY_BUDGET_RESCHEDULER_MAX_TIME,
|
||||||
PROXY_BUDGET_RESCHEDULER_MIN_TIME,
|
PROXY_BUDGET_RESCHEDULER_MIN_TIME,
|
||||||
PROXY_CONFIG_RELOAD_INTERVAL_SECONDS,
|
PROXY_CONFIG_RELOAD_INTERVAL_SECONDS,
|
||||||
|
USER_SPEND_ALERTS_JOB_ID,
|
||||||
WEEKLY_SPEND_REPORT_JOB_ID,
|
WEEKLY_SPEND_REPORT_JOB_ID,
|
||||||
)
|
)
|
||||||
from litellm.exceptions import RejectedRequestError
|
from litellm.exceptions import RejectedRequestError
|
||||||
|
|
@ -9633,6 +9634,35 @@ class ProxyStartupEvent:
|
||||||
replace_existing=True,
|
replace_existing=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
slack_alerting_args: Final = proxy_logging_obj.slack_alerting_instance.alerting_args
|
||||||
|
user_spend_check_interval: Final = (
|
||||||
|
slack_alerting_args.user_spend_check_interval
|
||||||
|
if isinstance(slack_alerting_args, SlackAlertingArgs) # pyright: ignore[reportUnnecessaryIsInstance] # tests inject a mock slack_alerting_instance
|
||||||
|
else SlackAlertingArgs().user_spend_check_interval
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _scheduled_user_spend_alerts() -> None:
|
||||||
|
if (
|
||||||
|
await pod_lock_manager.acquire_lock(
|
||||||
|
cronjob_id=USER_SPEND_ALERTS_JOB_ID,
|
||||||
|
ttl=max(user_spend_check_interval - 60, 60),
|
||||||
|
allow_reentrant=False,
|
||||||
|
)
|
||||||
|
is False
|
||||||
|
):
|
||||||
|
return
|
||||||
|
await proxy_logging_obj.slack_alerting_instance.send_user_spend_alerts()
|
||||||
|
|
||||||
|
scheduler.add_job(
|
||||||
|
_scheduled_user_spend_alerts,
|
||||||
|
"interval",
|
||||||
|
seconds=user_spend_check_interval,
|
||||||
|
next_run_time=datetime.now() + timedelta(seconds=10 + random.randint(0, 60)),
|
||||||
|
id=USER_SPEND_ALERTS_JOB_ID,
|
||||||
|
replace_existing=True,
|
||||||
|
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
|
||||||
|
)
|
||||||
|
|
||||||
if os.getenv("PROMETHEUS_URL"):
|
if os.getenv("PROMETHEUS_URL"):
|
||||||
from zoneinfo import ZoneInfo
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
|
|
@ -14662,17 +14692,25 @@ async def alerting_settings(
|
||||||
alerting_args_dict = {}
|
alerting_args_dict = {}
|
||||||
alerting_values = None
|
alerting_values = None
|
||||||
|
|
||||||
allowed_args: Final = {
|
allowed_args: Final = MappingProxyType(
|
||||||
"slack_alerting": {"type": "Boolean"},
|
{
|
||||||
"daily_report_frequency": {"type": "Integer"},
|
"slack_alerting": "Boolean",
|
||||||
"report_check_interval": {"type": "Integer"},
|
"daily_report_frequency": "Integer",
|
||||||
"budget_alert_ttl": {"type": "Integer"},
|
"report_check_interval": "Integer",
|
||||||
"outage_alert_ttl": {"type": "Integer"},
|
"budget_alert_ttl": "Integer",
|
||||||
"region_outage_alert_ttl": {"type": "Integer"},
|
"outage_alert_ttl": "Integer",
|
||||||
"minor_outage_alert_threshold": {"type": "Integer"},
|
"region_outage_alert_ttl": "Integer",
|
||||||
"major_outage_alert_threshold": {"type": "Integer"},
|
"minor_outage_alert_threshold": "Integer",
|
||||||
"max_outage_alert_list_size": {"type": "Integer"},
|
"major_outage_alert_threshold": "Integer",
|
||||||
}
|
"max_outage_alert_list_size": "Integer",
|
||||||
|
"daily_spend_per_user_threshold": "Float",
|
||||||
|
"monthly_spend_per_user_threshold": "Float",
|
||||||
|
"spend_anomaly_multiplier": "Float",
|
||||||
|
"spend_anomaly_baseline_days": "Integer",
|
||||||
|
"spend_anomaly_min_spend": "Float",
|
||||||
|
"user_spend_check_interval": "Integer",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
_slack_alerting: Final[SlackAlerting] = proxy_logging_obj.slack_alerting_instance
|
_slack_alerting: Final[SlackAlerting] = proxy_logging_obj.slack_alerting_instance
|
||||||
_slack_alerting_args_dict: Final = _slack_alerting.alerting_args.model_dump()
|
_slack_alerting_args_dict: Final = _slack_alerting.alerting_args.model_dump()
|
||||||
|
|
@ -14687,7 +14725,7 @@ async def alerting_settings(
|
||||||
|
|
||||||
_response_obj = ConfigList(
|
_response_obj = ConfigList(
|
||||||
field_name="slack_alerting",
|
field_name="slack_alerting",
|
||||||
field_type=allowed_args["slack_alerting"]["type"],
|
field_type=allowed_args["slack_alerting"],
|
||||||
field_description="Enable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.",
|
field_description="Enable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.",
|
||||||
field_value=is_slack_enabled,
|
field_value=is_slack_enabled,
|
||||||
stored_in_db=True if alerting_values is not None else False,
|
stored_in_db=True if alerting_values is not None else False,
|
||||||
|
|
@ -14706,7 +14744,7 @@ async def alerting_settings(
|
||||||
|
|
||||||
_response_obj = ConfigList(
|
_response_obj = ConfigList(
|
||||||
field_name=field_name,
|
field_name=field_name,
|
||||||
field_type=allowed_args[field_name]["type"],
|
field_type=allowed_args[field_name],
|
||||||
field_description=field_info.description or "",
|
field_description=field_info.description or "",
|
||||||
field_value=_slack_alerting_args_dict.get(field_name, None),
|
field_value=_slack_alerting_args_dict.get(field_name, None),
|
||||||
stored_in_db=_stored_in_db,
|
stored_in_db=_stored_in_db,
|
||||||
|
|
|
||||||
|
|
@ -91,6 +91,40 @@ class SlackAlertingArgs(LiteLLMPydanticObjectBase):
|
||||||
default=False,
|
default=False,
|
||||||
description="If true, the alerting payload will be printed to the console.",
|
description="If true, the alerting payload will be printed to the console.",
|
||||||
)
|
)
|
||||||
|
daily_spend_per_user_threshold: float | None = Field(
|
||||||
|
default=None,
|
||||||
|
gt=0,
|
||||||
|
allow_inf_nan=False,
|
||||||
|
description="Alert when a user's spend for the current day (UTC) crosses this USD amount. Off by default.",
|
||||||
|
)
|
||||||
|
monthly_spend_per_user_threshold: float | None = Field(
|
||||||
|
default=None,
|
||||||
|
gt=0,
|
||||||
|
allow_inf_nan=False,
|
||||||
|
description="Alert when a user's spend for the current calendar month (UTC) crosses this USD amount. Off by default.",
|
||||||
|
)
|
||||||
|
spend_anomaly_multiplier: float = Field(
|
||||||
|
default=3.0,
|
||||||
|
gt=0,
|
||||||
|
allow_inf_nan=False,
|
||||||
|
description="Flag a user's spend as anomalous when today's spend exceeds this multiple of their trailing daily average.",
|
||||||
|
)
|
||||||
|
spend_anomaly_baseline_days: int = Field(
|
||||||
|
default=7,
|
||||||
|
ge=1,
|
||||||
|
description="Number of trailing days used to compute a user's daily average spend for anomaly detection.",
|
||||||
|
)
|
||||||
|
spend_anomaly_min_spend: float = Field(
|
||||||
|
default=10.0,
|
||||||
|
gt=0,
|
||||||
|
allow_inf_nan=False,
|
||||||
|
description="Minimum spend (USD) a user must reach today before an anomaly alert can fire. Reduces false positives.",
|
||||||
|
)
|
||||||
|
user_spend_check_interval: int = Field(
|
||||||
|
default=3600,
|
||||||
|
ge=60,
|
||||||
|
description="How often (in seconds) to check per-user spend thresholds and anomalies. Default is hourly.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DeploymentMetrics(LiteLLMPydanticObjectBase):
|
class DeploymentMetrics(LiteLLMPydanticObjectBase):
|
||||||
|
|
@ -138,6 +172,8 @@ class AlertType(str, Enum):
|
||||||
budget_alerts = "budget_alerts"
|
budget_alerts = "budget_alerts"
|
||||||
spend_reports = "spend_reports"
|
spend_reports = "spend_reports"
|
||||||
failed_tracking_spend = "failed_tracking_spend"
|
failed_tracking_spend = "failed_tracking_spend"
|
||||||
|
user_spend_thresholds = "user_spend_thresholds"
|
||||||
|
user_spend_anomalies = "user_spend_anomalies"
|
||||||
|
|
||||||
# Database alerts
|
# Database alerts
|
||||||
db_exceptions = "db_exceptions"
|
db_exceptions = "db_exceptions"
|
||||||
|
|
@ -182,6 +218,7 @@ DEFAULT_ALERT_TYPES: Final[list[AlertType]] = [
|
||||||
AlertType.budget_alerts,
|
AlertType.budget_alerts,
|
||||||
AlertType.spend_reports,
|
AlertType.spend_reports,
|
||||||
AlertType.failed_tracking_spend,
|
AlertType.failed_tracking_spend,
|
||||||
|
AlertType.user_spend_thresholds,
|
||||||
# Database alerts
|
# Database alerts
|
||||||
AlertType.db_exceptions,
|
AlertType.db_exceptions,
|
||||||
# Report alerts
|
# Report alerts
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,193 @@
|
||||||
|
import datetime
|
||||||
|
from typing import Final
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from pydantic import ValidationError
|
||||||
|
|
||||||
|
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||||
|
from litellm.integrations.SlackAlerting.user_spend_alerts import (
|
||||||
|
UserSpendRow,
|
||||||
|
evaluate_user_spend,
|
||||||
|
)
|
||||||
|
from litellm.types.integrations.slack_alerting import (
|
||||||
|
DEFAULT_ALERT_TYPES,
|
||||||
|
AlertType,
|
||||||
|
SlackAlertingArgs,
|
||||||
|
)
|
||||||
|
|
||||||
|
TODAY: Final = datetime.date(2026, 8, 15)
|
||||||
|
|
||||||
|
|
||||||
|
def _row(
|
||||||
|
daily_spend: float = 0.0,
|
||||||
|
monthly_spend: float = 0.0,
|
||||||
|
baseline_spend: float = 0.0,
|
||||||
|
) -> UserSpendRow:
|
||||||
|
return UserSpendRow(
|
||||||
|
user_id="user-1",
|
||||||
|
daily_spend=daily_spend,
|
||||||
|
monthly_spend=monthly_spend,
|
||||||
|
baseline_spend=baseline_spend,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _evaluate(row: UserSpendRow, args: SlackAlertingArgs, thresholds: bool = True, anomalies: bool = True):
|
||||||
|
return evaluate_user_spend(
|
||||||
|
row=row,
|
||||||
|
args=args,
|
||||||
|
today=TODAY,
|
||||||
|
thresholds_enabled=thresholds,
|
||||||
|
anomalies_enabled=anomalies,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_daily_threshold_crossed():
|
||||||
|
args: Final = SlackAlertingArgs(daily_spend_per_user_threshold=50.0, spend_anomaly_min_spend=1000.0)
|
||||||
|
events: Final = _evaluate(_row(daily_spend=75.0, monthly_spend=75.0), args)
|
||||||
|
assert [e.kind for e in events] == ["daily_threshold"]
|
||||||
|
assert "`$75.00`" in events[0].message
|
||||||
|
assert "`$50.00`" in events[0].message
|
||||||
|
assert events[0].alert_type == AlertType.user_spend_thresholds
|
||||||
|
assert events[0].cache_key == "user_spend_alert_daily_user-1_2026-08-15"
|
||||||
|
|
||||||
|
|
||||||
|
def test_daily_threshold_not_crossed():
|
||||||
|
args: Final = SlackAlertingArgs(daily_spend_per_user_threshold=50.0, spend_anomaly_min_spend=1000.0)
|
||||||
|
assert _evaluate(_row(daily_spend=49.99, monthly_spend=49.99), args) == ()
|
||||||
|
|
||||||
|
|
||||||
|
def test_thresholds_unset_by_default():
|
||||||
|
args: Final = SlackAlertingArgs(spend_anomaly_min_spend=1000.0)
|
||||||
|
assert _evaluate(_row(daily_spend=999.0, monthly_spend=999.0), args) == ()
|
||||||
|
|
||||||
|
|
||||||
|
def test_monthly_threshold_crossed():
|
||||||
|
args: Final = SlackAlertingArgs(monthly_spend_per_user_threshold=200.0, spend_anomaly_min_spend=1000.0)
|
||||||
|
events: Final = _evaluate(_row(daily_spend=5.0, monthly_spend=250.0), args)
|
||||||
|
assert [e.kind for e in events] == ["monthly_threshold"]
|
||||||
|
assert events[0].cache_key == "user_spend_alert_monthly_user-1_2026-08"
|
||||||
|
|
||||||
|
|
||||||
|
def test_thresholds_disabled_suppresses_threshold_events():
|
||||||
|
args: Final = SlackAlertingArgs(
|
||||||
|
daily_spend_per_user_threshold=50.0,
|
||||||
|
monthly_spend_per_user_threshold=200.0,
|
||||||
|
spend_anomaly_min_spend=1000.0,
|
||||||
|
)
|
||||||
|
assert _evaluate(_row(daily_spend=75.0, monthly_spend=250.0), args, thresholds=False) == ()
|
||||||
|
|
||||||
|
|
||||||
|
def test_anomaly_detected_above_multiple_of_baseline():
|
||||||
|
args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0)
|
||||||
|
events: Final = _evaluate(
|
||||||
|
_row(daily_spend=70.0, monthly_spend=100.0, baseline_spend=70.0), args
|
||||||
|
)
|
||||||
|
assert [e.kind for e in events] == ["anomaly"]
|
||||||
|
assert events[0].alert_type == AlertType.user_spend_anomalies
|
||||||
|
assert "`$10.00`" in events[0].message
|
||||||
|
assert events[0].cache_key == "user_spend_alert_anomaly_user-1_2026-08-15"
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_anomaly_within_baseline_multiple():
|
||||||
|
args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0)
|
||||||
|
assert (
|
||||||
|
_evaluate(_row(daily_spend=25.0, monthly_spend=100.0, baseline_spend=70.0), args) == ()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_anomaly_below_min_spend_floor():
|
||||||
|
args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0)
|
||||||
|
assert _evaluate(_row(daily_spend=9.0, monthly_spend=9.0, baseline_spend=0.1), args) == ()
|
||||||
|
|
||||||
|
|
||||||
|
def test_anomaly_for_new_user_without_baseline():
|
||||||
|
args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0)
|
||||||
|
events: Final = _evaluate(_row(daily_spend=15.0, monthly_spend=15.0), args)
|
||||||
|
assert [e.kind for e in events] == ["anomaly"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_sparse_baseline_averages_over_full_window():
|
||||||
|
args: Final = SlackAlertingArgs(
|
||||||
|
spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0, spend_anomaly_baseline_days=7
|
||||||
|
)
|
||||||
|
events: Final = _evaluate(_row(daily_spend=13.0, monthly_spend=20.0, baseline_spend=7.0), args)
|
||||||
|
assert [e.kind for e in events] == ["anomaly"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_anomalies_not_in_default_alert_types():
|
||||||
|
assert AlertType.user_spend_anomalies not in DEFAULT_ALERT_TYPES
|
||||||
|
assert AlertType.user_spend_thresholds in DEFAULT_ALERT_TYPES
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_config_rejected():
|
||||||
|
with pytest.raises(ValidationError, match="daily_spend_per_user_threshold"):
|
||||||
|
SlackAlertingArgs(daily_spend_per_user_threshold=0)
|
||||||
|
with pytest.raises(ValidationError, match="spend_anomaly_baseline_days"):
|
||||||
|
SlackAlertingArgs(spend_anomaly_baseline_days=0)
|
||||||
|
with pytest.raises(ValidationError, match="user_spend_check_interval"):
|
||||||
|
SlackAlertingArgs(user_spend_check_interval=10)
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_finite_config_rejected():
|
||||||
|
with pytest.raises(ValidationError, match="daily_spend_per_user_threshold"):
|
||||||
|
SlackAlertingArgs(daily_spend_per_user_threshold=float("inf"))
|
||||||
|
with pytest.raises(ValidationError, match="spend_anomaly_multiplier"):
|
||||||
|
SlackAlertingArgs(spend_anomaly_multiplier=float("nan"))
|
||||||
|
with pytest.raises(ValidationError, match="spend_anomaly_min_spend"):
|
||||||
|
SlackAlertingArgs(spend_anomaly_min_spend=float("inf"))
|
||||||
|
with pytest.raises(ValidationError, match="user_spend_check_interval"):
|
||||||
|
SlackAlertingArgs(user_spend_check_interval=float("inf"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_anomalies_disabled_suppresses_anomaly_events():
|
||||||
|
args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0)
|
||||||
|
assert _evaluate(_row(daily_spend=500.0, monthly_spend=500.0), args, anomalies=False) == ()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_send_user_spend_alerts_sends_and_dedupes():
|
||||||
|
slack_alerting: Final = SlackAlerting(
|
||||||
|
alerting=["slack"],
|
||||||
|
alerting_args={"daily_spend_per_user_threshold": 50.0, "spend_anomaly_min_spend": 1000.0},
|
||||||
|
)
|
||||||
|
mock_prisma: Final = AsyncMock()
|
||||||
|
mock_prisma.db.query_raw = AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
{
|
||||||
|
"user_id": "user-1",
|
||||||
|
"daily_spend": 75.0,
|
||||||
|
"monthly_spend": 75.0,
|
||||||
|
"baseline_spend": 0.0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"user_id": "user-2",
|
||||||
|
"daily_spend": 60.0,
|
||||||
|
"monthly_spend": 60.0,
|
||||||
|
"baseline_spend": 0.0,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
with patch.object(slack_alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert:
|
||||||
|
await slack_alerting.send_user_spend_alerts(prisma_client=mock_prisma)
|
||||||
|
assert mock_send_alert.call_count == 1
|
||||||
|
sent_kwargs: Final = mock_send_alert.call_args.kwargs
|
||||||
|
assert sent_kwargs["alert_type"] == AlertType.user_spend_thresholds
|
||||||
|
assert "User Daily Spend Threshold Crossed" in sent_kwargs["message"]
|
||||||
|
assert "`user-1`" in sent_kwargs["message"]
|
||||||
|
assert "`user-2`" in sent_kwargs["message"]
|
||||||
|
|
||||||
|
await slack_alerting.send_user_spend_alerts(prisma_client=mock_prisma)
|
||||||
|
assert mock_send_alert.call_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_send_user_spend_alerts_noop_when_alert_types_disabled():
|
||||||
|
slack_alerting: Final = SlackAlerting(
|
||||||
|
alerting=["slack"],
|
||||||
|
alert_types=[AlertType.budget_alerts],
|
||||||
|
alerting_args={"daily_spend_per_user_threshold": 50.0},
|
||||||
|
)
|
||||||
|
mock_prisma: Final = AsyncMock()
|
||||||
|
await slack_alerting.send_user_spend_alerts(prisma_client=mock_prisma)
|
||||||
|
mock_prisma.db.query_raw.assert_not_called()
|
||||||
|
|
@ -38,6 +38,14 @@ const SETTINGS: Setting[] = [
|
||||||
stored_in_db: null,
|
stored_in_db: null,
|
||||||
premium_field: false,
|
premium_field: false,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
field_name: "daily_spend_per_user_threshold",
|
||||||
|
field_description: "Daily spend threshold per user",
|
||||||
|
field_type: "Float",
|
||||||
|
field_value: 5.5,
|
||||||
|
stored_in_db: true,
|
||||||
|
premium_field: false,
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
const renderForm = (
|
const renderForm = (
|
||||||
|
|
@ -170,6 +178,19 @@ describe("DynamicForm change notifications", () => {
|
||||||
expect(handleInputChange).toHaveBeenCalledWith("daily_report_frequency", 128);
|
expect(handleInputChange).toHaveBeenCalledWith("daily_report_frequency", 128);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("renders a Float field as a decimal-friendly number input and reports changes as numbers", async () => {
|
||||||
|
const user = userEvent.setup();
|
||||||
|
const { handleInputChange } = renderForm();
|
||||||
|
|
||||||
|
const input = screen.getByDisplayValue("5.5");
|
||||||
|
expect(input).toHaveAttribute("type", "number");
|
||||||
|
expect(input).toHaveAttribute("step", "any");
|
||||||
|
|
||||||
|
await user.type(input, "1");
|
||||||
|
|
||||||
|
expect(handleInputChange).toHaveBeenCalledWith("daily_spend_per_user_threshold", 5.51);
|
||||||
|
});
|
||||||
|
|
||||||
it("reports a reset with the field name and its row index", async () => {
|
it("reports a reset with the field name and its row index", async () => {
|
||||||
const user = userEvent.setup();
|
const user = userEvent.setup();
|
||||||
const { handleResetField } = renderForm();
|
const { handleResetField } = renderForm();
|
||||||
|
|
@ -216,7 +237,7 @@ describe("DynamicForm presentation", () => {
|
||||||
|
|
||||||
expect(screen.getByText("daily_report_frequency")).toBeInTheDocument();
|
expect(screen.getByText("daily_report_frequency")).toBeInTheDocument();
|
||||||
expect(screen.getByText("How often the report runs")).toBeInTheDocument();
|
expect(screen.getByText("How often the report runs")).toBeInTheDocument();
|
||||||
expect(screen.getByText("In DB")).toBeInTheDocument();
|
expect(screen.getAllByText("In DB")).toHaveLength(2);
|
||||||
expect(screen.getByText("In Config")).toBeInTheDocument();
|
expect(screen.getByText("In Config")).toBeInTheDocument();
|
||||||
expect(screen.getByText("Not Set")).toBeInTheDocument();
|
expect(screen.getByText("Not Set")).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -63,11 +63,11 @@ const DynamicForm: React.FC<DynamicFormProps> = ({
|
||||||
};
|
};
|
||||||
|
|
||||||
const renderControl = (setting: AlertingSetting) => {
|
const renderControl = (setting: AlertingSetting) => {
|
||||||
if (setting.field_type === "Integer") {
|
if (setting.field_type === "Integer" || setting.field_type === "Float") {
|
||||||
return (
|
return (
|
||||||
<Input
|
<Input
|
||||||
type="number"
|
type="number"
|
||||||
step={1}
|
step={setting.field_type === "Integer" ? 1 : "any"}
|
||||||
value={setting.field_value ?? ""}
|
value={setting.field_value ?? ""}
|
||||||
onChange={(event) => handleNumericChange(setting, event.target.value)}
|
onChange={(event) => handleNumericChange(setting, event.target.value)}
|
||||||
/>
|
/>
|
||||||
|
|
|
||||||
|
|
@ -292,6 +292,8 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
|
||||||
llm_too_slow: "LLM Responses Too Slow",
|
llm_too_slow: "LLM Responses Too Slow",
|
||||||
llm_requests_hanging: "LLM Requests Hanging",
|
llm_requests_hanging: "LLM Requests Hanging",
|
||||||
budget_alerts: "Budget Alerts (API Keys, Users)",
|
budget_alerts: "Budget Alerts (API Keys, Users)",
|
||||||
|
user_spend_thresholds: "User Spend Thresholds (Daily/Monthly)",
|
||||||
|
user_spend_anomalies: "User Spend Anomaly Detection",
|
||||||
db_exceptions: "Database Exceptions (Read/Write)",
|
db_exceptions: "Database Exceptions (Read/Write)",
|
||||||
daily_reports: "Weekly/Monthly Spend Reports",
|
daily_reports: "Weekly/Monthly Spend Reports",
|
||||||
outage_alerts: "Outage Alerts",
|
outage_alerts: "Outage Alerts",
|
||||||
|
|
|
||||||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -22533,7 +22533,7 @@ export interface components {
|
||||||
* @description Enum for alert types and management event types
|
* @description Enum for alert types and management event types
|
||||||
* @enum {string}
|
* @enum {string}
|
||||||
*/
|
*/
|
||||||
AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted";
|
AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted";
|
||||||
/** AllowedVectorStoreIndexItem */
|
/** AllowedVectorStoreIndexItem */
|
||||||
AllowedVectorStoreIndexItem: {
|
AllowedVectorStoreIndexItem: {
|
||||||
/** Index Name */
|
/** Index Name */
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue