mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
feat(alerting): slack alerts for per-user daily/monthly spend thresholds and spend anomaly detection (#38438)
* feat(alerting): slack alerts for per-user daily/monthly spend thresholds and spend anomaly detection Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(alerting): use specific ValidationError matches in config rejection test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): tolerate mocked slack alerting args when scheduling user spend scan Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(alerting): reject non-finite values in user spend alert settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin <yassin@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
5988d93fed
commit
846900320e
12 changed files with 598 additions and 23 deletions
|
|
@ -1590,6 +1590,7 @@ KEY_ROTATION_JOB_NAME: Final = "litellm_key_rotation_job"
|
|||
EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME: Final = "litellm_expired_ui_session_key_cleanup_job"
|
||||
WEEKLY_SPEND_REPORT_JOB_ID: Final = "weekly_spend_report_job"
|
||||
MONTHLY_SPEND_REPORT_JOB_ID: Final = "monthly_spend_report_job"
|
||||
USER_SPEND_ALERTS_JOB_ID: Final = "user_spend_alerts_job"
|
||||
PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job"
|
||||
SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report"
|
||||
SLACK_MODEL_DEPRECATION_LOCK_ID: Final = "slack_model_deprecation_warning"
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ from .utils import process_slack_alerting_variables
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.router import Router as _Router
|
||||
|
||||
Router = _Router
|
||||
|
|
@ -1944,6 +1945,69 @@ Model Info:
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error sending weekly spend report %s", e)
|
||||
|
||||
async def send_user_spend_alerts(self, prisma_client: "PrismaClient | None" = None) -> None:
|
||||
"""Check per-user daily/monthly spend thresholds and spend anomalies, alerting once per user per period."""
|
||||
if self.alerting is None or "slack" not in self.alerting:
|
||||
return
|
||||
|
||||
thresholds_enabled: Final = AlertType.user_spend_thresholds in self.alert_types
|
||||
anomalies_enabled: Final = AlertType.user_spend_anomalies in self.alert_types
|
||||
if not thresholds_enabled and not anomalies_enabled:
|
||||
return
|
||||
|
||||
if prisma_client is None:
|
||||
from litellm.proxy.proxy_server import prisma_client as global_prisma_client
|
||||
|
||||
prisma_client = global_prisma_client # rebind-ok: fall back to the proxy's global client
|
||||
if prisma_client is None:
|
||||
return
|
||||
|
||||
from litellm.integrations.SlackAlerting.user_spend_alerts import (
|
||||
evaluate_user_spend,
|
||||
fetch_user_spend_rows,
|
||||
)
|
||||
|
||||
try:
|
||||
today: Final = datetime.datetime.now(datetime.timezone.utc).date()
|
||||
rows: Final = await fetch_user_spend_rows(
|
||||
prisma_client=prisma_client,
|
||||
today=today,
|
||||
baseline_days=self.alerting_args.spend_anomaly_baseline_days,
|
||||
)
|
||||
all_events: Final = tuple(
|
||||
event
|
||||
for row in rows
|
||||
for event in evaluate_user_spend(
|
||||
row=row,
|
||||
args=self.alerting_args,
|
||||
today=today,
|
||||
thresholds_enabled=thresholds_enabled,
|
||||
anomalies_enabled=anomalies_enabled,
|
||||
)
|
||||
)
|
||||
cached_flags: Final = await asyncio.gather(
|
||||
*(self.internal_usage_cache.async_get_cache(key=event.cache_key) for event in all_events)
|
||||
)
|
||||
new_events: Final = tuple(event for event, cached in zip(all_events, cached_flags) if not cached)
|
||||
for alert_type in (AlertType.user_spend_thresholds, AlertType.user_spend_anomalies):
|
||||
typed_events = tuple(event for event in new_events if event.alert_type == alert_type)
|
||||
if not typed_events:
|
||||
continue
|
||||
await self.send_alert(
|
||||
message="\n\n".join(event.message for event in typed_events),
|
||||
level="High",
|
||||
alert_type=alert_type,
|
||||
alerting_metadata={}, # mutable-ok: send_alert takes a dict payload
|
||||
)
|
||||
for event in typed_events:
|
||||
await self.internal_usage_cache.async_set_cache(
|
||||
key=event.cache_key,
|
||||
value="SENT",
|
||||
ttl=event.cache_ttl,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # background job must not crash the scheduler
|
||||
verbose_proxy_logger.exception("Error sending user spend alerts: %s", e)
|
||||
|
||||
async def send_fallback_stats_from_prometheus(self):
|
||||
"""
|
||||
Helper to send fallback statistics from prometheus server -> to slack
|
||||
|
|
|
|||
139
litellm/integrations/SlackAlerting/user_spend_alerts.py
Normal file
139
litellm/integrations/SlackAlerting/user_spend_alerts.py
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
"""Per-user daily/monthly spend threshold alerts and spend anomaly detection."""
|
||||
|
||||
import datetime
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.constants import HOURS_IN_A_DAY
|
||||
from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
DAY_SECONDS: Final = HOURS_IN_A_DAY * 60 * 60
|
||||
MONTHLY_ALERT_TTL_SECONDS: Final = 32 * DAY_SECONDS
|
||||
|
||||
USER_SPEND_QUERY: Final = """
|
||||
SELECT
|
||||
user_id,
|
||||
COALESCE(SUM(spend) FILTER (WHERE date = $1), 0)::float AS daily_spend,
|
||||
COALESCE(SUM(spend) FILTER (WHERE date >= $2), 0)::float AS monthly_spend,
|
||||
COALESCE(SUM(spend) FILTER (WHERE date >= $3 AND date < $1), 0)::float AS baseline_spend
|
||||
FROM "LiteLLM_DailyUserSpend"
|
||||
WHERE date >= LEAST($2, $3) AND user_id IS NOT NULL
|
||||
GROUP BY user_id
|
||||
HAVING COALESCE(SUM(spend) FILTER (WHERE date >= $2), 0) > 0
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UserSpendRow:
|
||||
user_id: str
|
||||
daily_spend: float
|
||||
monthly_spend: float
|
||||
baseline_spend: float
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UserSpendAlertEvent:
|
||||
kind: Literal["daily_threshold", "monthly_threshold", "anomaly"]
|
||||
alert_type: AlertType
|
||||
message: str
|
||||
cache_key: str
|
||||
cache_ttl: int
|
||||
|
||||
|
||||
USER_SPEND_ROWS_ADAPTER: Final = TypeAdapter(tuple[UserSpendRow, ...])
|
||||
|
||||
|
||||
async def fetch_user_spend_rows(
|
||||
prisma_client: "PrismaClient",
|
||||
today: datetime.date,
|
||||
baseline_days: int,
|
||||
) -> tuple[UserSpendRow, ...]:
|
||||
today_str: Final = today.strftime("%Y-%m-%d")
|
||||
month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d")
|
||||
baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d")
|
||||
raw: Final = await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
|
||||
return USER_SPEND_ROWS_ADAPTER.validate_python(raw)
|
||||
|
||||
|
||||
def _daily_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None:
|
||||
threshold: Final = args.daily_spend_per_user_threshold
|
||||
if threshold is None or row.daily_spend < threshold:
|
||||
return None
|
||||
return UserSpendAlertEvent(
|
||||
kind="daily_threshold",
|
||||
alert_type=AlertType.user_spend_thresholds,
|
||||
message=(
|
||||
f"User Daily Spend Threshold Crossed:\n"
|
||||
f"User: `{row.user_id}`\n"
|
||||
f"Spend Today: `${row.daily_spend:.2f}`\n"
|
||||
f"Daily Threshold: `${threshold:.2f}`"
|
||||
),
|
||||
cache_key=f"user_spend_alert_daily_{row.user_id}_{today_str}",
|
||||
cache_ttl=DAY_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def _monthly_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, month_str: str) -> UserSpendAlertEvent | None:
|
||||
threshold: Final = args.monthly_spend_per_user_threshold
|
||||
if threshold is None or row.monthly_spend < threshold:
|
||||
return None
|
||||
return UserSpendAlertEvent(
|
||||
kind="monthly_threshold",
|
||||
alert_type=AlertType.user_spend_thresholds,
|
||||
message=(
|
||||
f"User Monthly Spend Threshold Crossed:\n"
|
||||
f"User: `{row.user_id}`\n"
|
||||
f"Spend This Month: `${row.monthly_spend:.2f}`\n"
|
||||
f"Monthly Threshold: `${threshold:.2f}`"
|
||||
),
|
||||
cache_key=f"user_spend_alert_monthly_{row.user_id}_{month_str}",
|
||||
cache_ttl=MONTHLY_ALERT_TTL_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def _anomaly_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None:
|
||||
if row.daily_spend < args.spend_anomaly_min_spend:
|
||||
return None
|
||||
baseline_daily_avg: Final = row.baseline_spend / args.spend_anomaly_baseline_days
|
||||
if row.baseline_spend > 0 and row.daily_spend <= args.spend_anomaly_multiplier * baseline_daily_avg:
|
||||
return None
|
||||
return UserSpendAlertEvent(
|
||||
kind="anomaly",
|
||||
alert_type=AlertType.user_spend_anomalies,
|
||||
message=(
|
||||
f"User Spend Anomaly Detected:\n"
|
||||
f"User: `{row.user_id}`\n"
|
||||
f"Spend Today: `${row.daily_spend:.2f}`\n"
|
||||
f"Daily Average (last {args.spend_anomaly_baseline_days} days): `${baseline_daily_avg:.2f}`\n"
|
||||
f"Trigger: spend above `{args.spend_anomaly_multiplier}x` the daily average "
|
||||
f"(minimum `${args.spend_anomaly_min_spend:.2f}`)"
|
||||
),
|
||||
cache_key=f"user_spend_alert_anomaly_{row.user_id}_{today_str}",
|
||||
cache_ttl=DAY_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def evaluate_user_spend(
|
||||
row: UserSpendRow,
|
||||
args: SlackAlertingArgs,
|
||||
today: datetime.date,
|
||||
thresholds_enabled: bool,
|
||||
anomalies_enabled: bool,
|
||||
) -> tuple[UserSpendAlertEvent, ...]:
|
||||
today_str: Final = today.strftime("%Y-%m-%d")
|
||||
month_str: Final = today.strftime("%Y-%m")
|
||||
threshold_events: Final = (
|
||||
(
|
||||
_daily_threshold_event(row=row, args=args, today_str=today_str),
|
||||
_monthly_threshold_event(row=row, args=args, month_str=month_str),
|
||||
)
|
||||
if thresholds_enabled
|
||||
else ()
|
||||
)
|
||||
anomaly_events: Final = (_anomaly_event(row=row, args=args, today_str=today_str),) if anomalies_enabled else ()
|
||||
return tuple(event for event in (*threshold_events, *anomaly_events) if event is not None)
|
||||
|
|
@ -39,7 +39,7 @@ from typing import (
|
|||
import anyio
|
||||
import websockets
|
||||
import websockets.exceptions
|
||||
from pydantic import BaseModel, Json, JsonValue
|
||||
from pydantic import BaseModel, Json, JsonValue, ValidationError
|
||||
from typing_extensions import NotRequired, ReadOnly, assert_never
|
||||
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -253,6 +253,7 @@ from litellm.constants import (
|
|||
PROXY_BUDGET_RESCHEDULER_MAX_TIME,
|
||||
PROXY_BUDGET_RESCHEDULER_MIN_TIME,
|
||||
PROXY_CONFIG_RELOAD_INTERVAL_SECONDS,
|
||||
USER_SPEND_ALERTS_JOB_ID,
|
||||
WEEKLY_SPEND_REPORT_JOB_ID,
|
||||
)
|
||||
from litellm.exceptions import RejectedRequestError
|
||||
|
|
@ -9866,6 +9867,35 @@ class ProxyStartupEvent:
|
|||
replace_existing=True,
|
||||
)
|
||||
|
||||
slack_alerting_args: Final = proxy_logging_obj.slack_alerting_instance.alerting_args
|
||||
user_spend_check_interval: Final = (
|
||||
slack_alerting_args.user_spend_check_interval
|
||||
if isinstance(slack_alerting_args, SlackAlertingArgs) # pyright: ignore[reportUnnecessaryIsInstance] # tests inject a mock slack_alerting_instance
|
||||
else SlackAlertingArgs().user_spend_check_interval
|
||||
)
|
||||
|
||||
async def _scheduled_user_spend_alerts() -> None:
|
||||
if (
|
||||
await pod_lock_manager.acquire_lock(
|
||||
cronjob_id=USER_SPEND_ALERTS_JOB_ID,
|
||||
ttl=max(user_spend_check_interval - 60, 60),
|
||||
allow_reentrant=False,
|
||||
)
|
||||
is False
|
||||
):
|
||||
return
|
||||
await proxy_logging_obj.slack_alerting_instance.send_user_spend_alerts()
|
||||
|
||||
scheduler.add_job(
|
||||
_scheduled_user_spend_alerts,
|
||||
"interval",
|
||||
seconds=user_spend_check_interval,
|
||||
next_run_time=datetime.now(timezone.utc) + timedelta(seconds=10 + random.randint(0, 60)),
|
||||
id=USER_SPEND_ALERTS_JOB_ID,
|
||||
replace_existing=True,
|
||||
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
|
||||
)
|
||||
|
||||
if os.getenv("PROMETHEUS_URL"):
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
|
|
@ -14972,17 +15002,25 @@ async def alerting_settings(
|
|||
alerting_args_dict = {}
|
||||
alerting_values = None
|
||||
|
||||
allowed_args: Final = {
|
||||
"slack_alerting": {"type": "Boolean"},
|
||||
"daily_report_frequency": {"type": "Integer"},
|
||||
"report_check_interval": {"type": "Integer"},
|
||||
"budget_alert_ttl": {"type": "Integer"},
|
||||
"outage_alert_ttl": {"type": "Integer"},
|
||||
"region_outage_alert_ttl": {"type": "Integer"},
|
||||
"minor_outage_alert_threshold": {"type": "Integer"},
|
||||
"major_outage_alert_threshold": {"type": "Integer"},
|
||||
"max_outage_alert_list_size": {"type": "Integer"},
|
||||
}
|
||||
allowed_args: Final = MappingProxyType(
|
||||
{
|
||||
"slack_alerting": "Boolean",
|
||||
"daily_report_frequency": "Integer",
|
||||
"report_check_interval": "Integer",
|
||||
"budget_alert_ttl": "Integer",
|
||||
"outage_alert_ttl": "Integer",
|
||||
"region_outage_alert_ttl": "Integer",
|
||||
"minor_outage_alert_threshold": "Integer",
|
||||
"major_outage_alert_threshold": "Integer",
|
||||
"max_outage_alert_list_size": "Integer",
|
||||
"daily_spend_per_user_threshold": "Float",
|
||||
"monthly_spend_per_user_threshold": "Float",
|
||||
"spend_anomaly_multiplier": "Float",
|
||||
"spend_anomaly_baseline_days": "Integer",
|
||||
"spend_anomaly_min_spend": "Float",
|
||||
"user_spend_check_interval": "Integer",
|
||||
}
|
||||
)
|
||||
|
||||
_slack_alerting: Final[SlackAlerting] = proxy_logging_obj.slack_alerting_instance
|
||||
_slack_alerting_args_dict: Final = _slack_alerting.alerting_args.model_dump()
|
||||
|
|
@ -14997,7 +15035,7 @@ async def alerting_settings(
|
|||
|
||||
_response_obj = ConfigList(
|
||||
field_name="slack_alerting",
|
||||
field_type=allowed_args["slack_alerting"]["type"],
|
||||
field_type=allowed_args["slack_alerting"],
|
||||
field_description="Enable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.",
|
||||
field_value=is_slack_enabled,
|
||||
stored_in_db=True if alerting_values is not None else False,
|
||||
|
|
@ -15016,7 +15054,7 @@ async def alerting_settings(
|
|||
|
||||
_response_obj = ConfigList(
|
||||
field_name=field_name,
|
||||
field_type=allowed_args[field_name]["type"],
|
||||
field_type=allowed_args[field_name],
|
||||
field_description=field_info.description or "",
|
||||
field_value=_slack_alerting_args_dict.get(field_name, None),
|
||||
stored_in_db=_stored_in_db,
|
||||
|
|
@ -16444,6 +16482,16 @@ async def update_config_general_settings(
|
|||
detail={"error": f"Invalid type of field value={type(data.field_value)} passed in."},
|
||||
)
|
||||
|
||||
if data.field_name == "alerting_args":
|
||||
try:
|
||||
SlackAlertingArgs.model_validate(data.field_value)
|
||||
except ValidationError as e:
|
||||
errors: Final = "; ".join(f"{'.'.join(str(loc) for loc in err['loc'])}: {err['msg']}" for err in e.errors())
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": f"Invalid alerting_args: {errors}"},
|
||||
)
|
||||
|
||||
## get general settings from db
|
||||
db_general_settings: Final = await _config_param_table(prisma_client).find_first(
|
||||
where={"param_name": "general_settings"}
|
||||
|
|
|
|||
|
|
@ -91,6 +91,40 @@ class SlackAlertingArgs(LiteLLMPydanticObjectBase):
|
|||
default=False,
|
||||
description="If true, the alerting payload will be printed to the console.",
|
||||
)
|
||||
daily_spend_per_user_threshold: float | None = Field(
|
||||
default=None,
|
||||
gt=0,
|
||||
allow_inf_nan=False,
|
||||
description="Alert when a user's spend for the current day (UTC) crosses this USD amount. Off by default.",
|
||||
)
|
||||
monthly_spend_per_user_threshold: float | None = Field(
|
||||
default=None,
|
||||
gt=0,
|
||||
allow_inf_nan=False,
|
||||
description="Alert when a user's spend for the current calendar month (UTC) crosses this USD amount. Off by default.",
|
||||
)
|
||||
spend_anomaly_multiplier: float = Field(
|
||||
default=3.0,
|
||||
gt=0,
|
||||
allow_inf_nan=False,
|
||||
description="Flag a user's spend as anomalous when today's spend exceeds this multiple of their trailing daily average.",
|
||||
)
|
||||
spend_anomaly_baseline_days: int = Field(
|
||||
default=7,
|
||||
ge=1,
|
||||
description="Number of trailing days used to compute a user's daily average spend for anomaly detection.",
|
||||
)
|
||||
spend_anomaly_min_spend: float = Field(
|
||||
default=10.0,
|
||||
gt=0,
|
||||
allow_inf_nan=False,
|
||||
description="Minimum spend (USD) a user must reach today before an anomaly alert can fire. Reduces false positives.",
|
||||
)
|
||||
user_spend_check_interval: int = Field(
|
||||
default=3600,
|
||||
ge=60,
|
||||
description="How often (in seconds) to check per-user spend thresholds and anomalies. Default is hourly.",
|
||||
)
|
||||
|
||||
|
||||
class DeploymentMetrics(LiteLLMPydanticObjectBase):
|
||||
|
|
@ -138,6 +172,8 @@ class AlertType(str, Enum):
|
|||
budget_alerts = "budget_alerts"
|
||||
spend_reports = "spend_reports"
|
||||
failed_tracking_spend = "failed_tracking_spend"
|
||||
user_spend_thresholds = "user_spend_thresholds"
|
||||
user_spend_anomalies = "user_spend_anomalies"
|
||||
|
||||
# Database alerts
|
||||
db_exceptions = "db_exceptions"
|
||||
|
|
@ -182,6 +218,7 @@ DEFAULT_ALERT_TYPES: Final[list[AlertType]] = [
|
|||
AlertType.budget_alerts,
|
||||
AlertType.spend_reports,
|
||||
AlertType.failed_tracking_spend,
|
||||
AlertType.user_spend_thresholds,
|
||||
# Database alerts
|
||||
AlertType.db_exceptions,
|
||||
# Report alerts
|
||||
|
|
|
|||
|
|
@ -0,0 +1,193 @@
|
|||
import datetime
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.integrations.SlackAlerting.user_spend_alerts import (
|
||||
UserSpendRow,
|
||||
evaluate_user_spend,
|
||||
)
|
||||
from litellm.types.integrations.slack_alerting import (
|
||||
DEFAULT_ALERT_TYPES,
|
||||
AlertType,
|
||||
SlackAlertingArgs,
|
||||
)
|
||||
|
||||
TODAY: Final = datetime.date(2026, 8, 15)
|
||||
|
||||
|
||||
def _row(
|
||||
daily_spend: float = 0.0,
|
||||
monthly_spend: float = 0.0,
|
||||
baseline_spend: float = 0.0,
|
||||
) -> UserSpendRow:
|
||||
return UserSpendRow(
|
||||
user_id="user-1",
|
||||
daily_spend=daily_spend,
|
||||
monthly_spend=monthly_spend,
|
||||
baseline_spend=baseline_spend,
|
||||
)
|
||||
|
||||
|
||||
def _evaluate(row: UserSpendRow, args: SlackAlertingArgs, thresholds: bool = True, anomalies: bool = True):
|
||||
return evaluate_user_spend(
|
||||
row=row,
|
||||
args=args,
|
||||
today=TODAY,
|
||||
thresholds_enabled=thresholds,
|
||||
anomalies_enabled=anomalies,
|
||||
)
|
||||
|
||||
|
||||
def test_daily_threshold_crossed():
|
||||
args: Final = SlackAlertingArgs(daily_spend_per_user_threshold=50.0, spend_anomaly_min_spend=1000.0)
|
||||
events: Final = _evaluate(_row(daily_spend=75.0, monthly_spend=75.0), args)
|
||||
assert [e.kind for e in events] == ["daily_threshold"]
|
||||
assert "`$75.00`" in events[0].message
|
||||
assert "`$50.00`" in events[0].message
|
||||
assert events[0].alert_type == AlertType.user_spend_thresholds
|
||||
assert events[0].cache_key == "user_spend_alert_daily_user-1_2026-08-15"
|
||||
|
||||
|
||||
def test_daily_threshold_not_crossed():
|
||||
args: Final = SlackAlertingArgs(daily_spend_per_user_threshold=50.0, spend_anomaly_min_spend=1000.0)
|
||||
assert _evaluate(_row(daily_spend=49.99, monthly_spend=49.99), args) == ()
|
||||
|
||||
|
||||
def test_thresholds_unset_by_default():
|
||||
args: Final = SlackAlertingArgs(spend_anomaly_min_spend=1000.0)
|
||||
assert _evaluate(_row(daily_spend=999.0, monthly_spend=999.0), args) == ()
|
||||
|
||||
|
||||
def test_monthly_threshold_crossed():
|
||||
args: Final = SlackAlertingArgs(monthly_spend_per_user_threshold=200.0, spend_anomaly_min_spend=1000.0)
|
||||
events: Final = _evaluate(_row(daily_spend=5.0, monthly_spend=250.0), args)
|
||||
assert [e.kind for e in events] == ["monthly_threshold"]
|
||||
assert events[0].cache_key == "user_spend_alert_monthly_user-1_2026-08"
|
||||
|
||||
|
||||
def test_thresholds_disabled_suppresses_threshold_events():
|
||||
args: Final = SlackAlertingArgs(
|
||||
daily_spend_per_user_threshold=50.0,
|
||||
monthly_spend_per_user_threshold=200.0,
|
||||
spend_anomaly_min_spend=1000.0,
|
||||
)
|
||||
assert _evaluate(_row(daily_spend=75.0, monthly_spend=250.0), args, thresholds=False) == ()
|
||||
|
||||
|
||||
def test_anomaly_detected_above_multiple_of_baseline():
|
||||
args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0)
|
||||
events: Final = _evaluate(
|
||||
_row(daily_spend=70.0, monthly_spend=100.0, baseline_spend=70.0), args
|
||||
)
|
||||
assert [e.kind for e in events] == ["anomaly"]
|
||||
assert events[0].alert_type == AlertType.user_spend_anomalies
|
||||
assert "`$10.00`" in events[0].message
|
||||
assert events[0].cache_key == "user_spend_alert_anomaly_user-1_2026-08-15"
|
||||
|
||||
|
||||
def test_no_anomaly_within_baseline_multiple():
|
||||
args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0)
|
||||
assert (
|
||||
_evaluate(_row(daily_spend=25.0, monthly_spend=100.0, baseline_spend=70.0), args) == ()
|
||||
)
|
||||
|
||||
|
||||
def test_no_anomaly_below_min_spend_floor():
|
||||
args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0)
|
||||
assert _evaluate(_row(daily_spend=9.0, monthly_spend=9.0, baseline_spend=0.1), args) == ()
|
||||
|
||||
|
||||
def test_anomaly_for_new_user_without_baseline():
|
||||
args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0)
|
||||
events: Final = _evaluate(_row(daily_spend=15.0, monthly_spend=15.0), args)
|
||||
assert [e.kind for e in events] == ["anomaly"]
|
||||
|
||||
|
||||
def test_sparse_baseline_averages_over_full_window():
|
||||
args: Final = SlackAlertingArgs(
|
||||
spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0, spend_anomaly_baseline_days=7
|
||||
)
|
||||
events: Final = _evaluate(_row(daily_spend=13.0, monthly_spend=20.0, baseline_spend=7.0), args)
|
||||
assert [e.kind for e in events] == ["anomaly"]
|
||||
|
||||
|
||||
def test_anomalies_not_in_default_alert_types():
|
||||
assert AlertType.user_spend_anomalies not in DEFAULT_ALERT_TYPES
|
||||
assert AlertType.user_spend_thresholds in DEFAULT_ALERT_TYPES
|
||||
|
||||
|
||||
def test_invalid_config_rejected():
|
||||
with pytest.raises(ValidationError, match="daily_spend_per_user_threshold"):
|
||||
SlackAlertingArgs(daily_spend_per_user_threshold=0)
|
||||
with pytest.raises(ValidationError, match="spend_anomaly_baseline_days"):
|
||||
SlackAlertingArgs(spend_anomaly_baseline_days=0)
|
||||
with pytest.raises(ValidationError, match="user_spend_check_interval"):
|
||||
SlackAlertingArgs(user_spend_check_interval=10)
|
||||
|
||||
|
||||
def test_non_finite_config_rejected():
|
||||
with pytest.raises(ValidationError, match="daily_spend_per_user_threshold"):
|
||||
SlackAlertingArgs(daily_spend_per_user_threshold=float("inf"))
|
||||
with pytest.raises(ValidationError, match="spend_anomaly_multiplier"):
|
||||
SlackAlertingArgs(spend_anomaly_multiplier=float("nan"))
|
||||
with pytest.raises(ValidationError, match="spend_anomaly_min_spend"):
|
||||
SlackAlertingArgs(spend_anomaly_min_spend=float("inf"))
|
||||
with pytest.raises(ValidationError, match="user_spend_check_interval"):
|
||||
SlackAlertingArgs(user_spend_check_interval=float("inf"))
|
||||
|
||||
|
||||
def test_anomalies_disabled_suppresses_anomaly_events():
|
||||
args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0)
|
||||
assert _evaluate(_row(daily_spend=500.0, monthly_spend=500.0), args, anomalies=False) == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_user_spend_alerts_sends_and_dedupes():
|
||||
slack_alerting: Final = SlackAlerting(
|
||||
alerting=["slack"],
|
||||
alerting_args={"daily_spend_per_user_threshold": 50.0, "spend_anomaly_min_spend": 1000.0},
|
||||
)
|
||||
mock_prisma: Final = AsyncMock()
|
||||
mock_prisma.db.query_raw = AsyncMock(
|
||||
return_value=[
|
||||
{
|
||||
"user_id": "user-1",
|
||||
"daily_spend": 75.0,
|
||||
"monthly_spend": 75.0,
|
||||
"baseline_spend": 0.0,
|
||||
},
|
||||
{
|
||||
"user_id": "user-2",
|
||||
"daily_spend": 60.0,
|
||||
"monthly_spend": 60.0,
|
||||
"baseline_spend": 0.0,
|
||||
},
|
||||
]
|
||||
)
|
||||
with patch.object(slack_alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert:
|
||||
await slack_alerting.send_user_spend_alerts(prisma_client=mock_prisma)
|
||||
assert mock_send_alert.call_count == 1
|
||||
sent_kwargs: Final = mock_send_alert.call_args.kwargs
|
||||
assert sent_kwargs["alert_type"] == AlertType.user_spend_thresholds
|
||||
assert "User Daily Spend Threshold Crossed" in sent_kwargs["message"]
|
||||
assert "`user-1`" in sent_kwargs["message"]
|
||||
assert "`user-2`" in sent_kwargs["message"]
|
||||
|
||||
await slack_alerting.send_user_spend_alerts(prisma_client=mock_prisma)
|
||||
assert mock_send_alert.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_user_spend_alerts_noop_when_alert_types_disabled():
|
||||
slack_alerting: Final = SlackAlerting(
|
||||
alerting=["slack"],
|
||||
alert_types=[AlertType.budget_alerts],
|
||||
alerting_args={"daily_spend_per_user_threshold": 50.0},
|
||||
)
|
||||
mock_prisma: Final = AsyncMock()
|
||||
await slack_alerting.send_user_spend_alerts(prisma_client=mock_prisma)
|
||||
mock_prisma.db.query_raw.assert_not_called()
|
||||
|
|
@ -10445,6 +10445,75 @@ async def test_update_config_general_settings_emits_audit_log(monkeypatch):
|
|||
assert before["some_api_key"] != "sk-stored-secret"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_config_field_rejects_out_of_range_alerting_args(monkeypatch):
|
||||
"""Out-of-range alerting_args must be rejected at save time. If they land in the
|
||||
DB, SlackAlertingArgs raises during the config reload and alerting breaks."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
import litellm.proxy.proxy_server as proxy_server_module
|
||||
from litellm.proxy._types import ConfigFieldUpdate
|
||||
from litellm.proxy.proxy_server import update_config_general_settings
|
||||
|
||||
monkeypatch.setattr(proxy_server_module, "prisma_client", MagicMock())
|
||||
|
||||
admin = UserAPIKeyAuth(
|
||||
api_key="hashed-admin",
|
||||
user_id="admin-1",
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
)
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await update_config_general_settings(
|
||||
data=ConfigFieldUpdate(
|
||||
field_name="alerting_args",
|
||||
field_value={
|
||||
"daily_spend_per_user_threshold": -5.0,
|
||||
"user_spend_check_interval": 20,
|
||||
},
|
||||
config_type="general_settings",
|
||||
),
|
||||
user_api_key_dict=admin,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
error_msg = exc_info.value.detail["error"]
|
||||
assert "daily_spend_per_user_threshold" in error_msg
|
||||
assert "user_spend_check_interval" in error_msg
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_config_field_accepts_valid_alerting_args(monkeypatch):
|
||||
import litellm.proxy.proxy_server as proxy_server_module
|
||||
from litellm.proxy._types import ConfigFieldUpdate
|
||||
from litellm.proxy.proxy_server import update_config_general_settings
|
||||
|
||||
fake = _fake_prisma_with_config({})
|
||||
monkeypatch.setattr(proxy_server_module, "prisma_client", fake)
|
||||
monkeypatch.setattr(litellm, "store_audit_logs", False)
|
||||
|
||||
admin = UserAPIKeyAuth(
|
||||
api_key="hashed-admin",
|
||||
user_id="admin-1",
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
)
|
||||
await update_config_general_settings(
|
||||
data=ConfigFieldUpdate(
|
||||
field_name="alerting_args",
|
||||
field_value={
|
||||
"daily_spend_per_user_threshold": 5.0,
|
||||
"user_spend_check_interval": 60,
|
||||
},
|
||||
config_type="general_settings",
|
||||
),
|
||||
user_api_key_dict=admin,
|
||||
)
|
||||
|
||||
written = json.loads(fake.db.litellm_config.upsert.call_args.kwargs["data"]["update"]["param_value"])
|
||||
assert written["alerting_args"]["daily_spend_per_user_threshold"] == 5.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_config_general_settings_applies_ssrf_globals(monkeypatch):
|
||||
import litellm.proxy.proxy_server as proxy_server_module
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import React, { useState, useEffect } from "react";
|
|||
|
||||
import { alertingSettingsCall, updateConfigFieldSetting } from "../networking";
|
||||
import DynamicForm from "./dynamic_form";
|
||||
import { extractProxyErrorMessage } from "@/lib/http/client";
|
||||
import { toast } from "@/lib/toast";
|
||||
interface alertingSettingsItem {
|
||||
field_name: string;
|
||||
|
|
@ -43,7 +44,7 @@ const AlertingSettings: React.FC<AlertingSettingsProps> = ({ accessToken, premiu
|
|||
setAlertingSettings(updatedSettings);
|
||||
};
|
||||
|
||||
const handleSubmit = (formValues: Record<string, any>) => {
|
||||
const handleSubmit = async (formValues: Record<string, any>) => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -64,18 +65,18 @@ const AlertingSettings: React.FC<AlertingSettingsProps> = ({ accessToken, premiu
|
|||
const mergedFormValues = { ...formValues, ...initialFormValues };
|
||||
const { slack_alerting, ...alertingArgs } = mergedFormValues;
|
||||
try {
|
||||
updateConfigFieldSetting(accessToken, "alerting_args", alertingArgs);
|
||||
await updateConfigFieldSetting(accessToken, "alerting_args", alertingArgs);
|
||||
if (typeof slack_alerting === "boolean") {
|
||||
if (slack_alerting == true) {
|
||||
updateConfigFieldSetting(accessToken, "alerting", ["slack"]);
|
||||
await updateConfigFieldSetting(accessToken, "alerting", ["slack"]);
|
||||
} else {
|
||||
updateConfigFieldSetting(accessToken, "alerting", []);
|
||||
await updateConfigFieldSetting(accessToken, "alerting", []);
|
||||
}
|
||||
}
|
||||
// update value in state
|
||||
toast.success("Wait 10s for proxy to update.");
|
||||
} catch (error) {
|
||||
// do something
|
||||
toast.error(extractProxyErrorMessage(error));
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,14 @@ const SETTINGS: Setting[] = [
|
|||
stored_in_db: null,
|
||||
premium_field: false,
|
||||
},
|
||||
{
|
||||
field_name: "daily_spend_per_user_threshold",
|
||||
field_description: "Daily spend threshold per user",
|
||||
field_type: "Float",
|
||||
field_value: 5.5,
|
||||
stored_in_db: true,
|
||||
premium_field: false,
|
||||
},
|
||||
];
|
||||
|
||||
const renderForm = (
|
||||
|
|
@ -170,6 +178,19 @@ describe("DynamicForm change notifications", () => {
|
|||
expect(handleInputChange).toHaveBeenCalledWith("daily_report_frequency", 128);
|
||||
});
|
||||
|
||||
it("renders a Float field as a decimal-friendly number input and reports changes as numbers", async () => {
|
||||
const user = userEvent.setup();
|
||||
const { handleInputChange } = renderForm();
|
||||
|
||||
const input = screen.getByDisplayValue("5.5");
|
||||
expect(input).toHaveAttribute("type", "number");
|
||||
expect(input).toHaveAttribute("step", "any");
|
||||
|
||||
await user.type(input, "1");
|
||||
|
||||
expect(handleInputChange).toHaveBeenCalledWith("daily_spend_per_user_threshold", 5.51);
|
||||
});
|
||||
|
||||
it("reports a reset with the field name and its row index", async () => {
|
||||
const user = userEvent.setup();
|
||||
const { handleResetField } = renderForm();
|
||||
|
|
@ -216,7 +237,7 @@ describe("DynamicForm presentation", () => {
|
|||
|
||||
expect(screen.getByText("daily_report_frequency")).toBeInTheDocument();
|
||||
expect(screen.getByText("How often the report runs")).toBeInTheDocument();
|
||||
expect(screen.getByText("In DB")).toBeInTheDocument();
|
||||
expect(screen.getAllByText("In DB")).toHaveLength(2);
|
||||
expect(screen.getByText("In Config")).toBeInTheDocument();
|
||||
expect(screen.getByText("Not Set")).toBeInTheDocument();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -63,11 +63,11 @@ const DynamicForm: React.FC<DynamicFormProps> = ({
|
|||
};
|
||||
|
||||
const renderControl = (setting: AlertingSetting) => {
|
||||
if (setting.field_type === "Integer") {
|
||||
if (setting.field_type === "Integer" || setting.field_type === "Float") {
|
||||
return (
|
||||
<Input
|
||||
type="number"
|
||||
step={1}
|
||||
step={setting.field_type === "Integer" ? 1 : "any"}
|
||||
value={setting.field_value ?? ""}
|
||||
onChange={(event) => handleNumericChange(setting, event.target.value)}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -293,6 +293,8 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
|
|||
llm_too_slow: "LLM Responses Too Slow",
|
||||
llm_requests_hanging: "LLM Requests Hanging",
|
||||
budget_alerts: "Budget Alerts (API Keys, Users)",
|
||||
user_spend_thresholds: "User Spend Thresholds (Daily/Monthly)",
|
||||
user_spend_anomalies: "User Spend Anomaly Detection",
|
||||
db_exceptions: "Database Exceptions (Read/Write)",
|
||||
daily_reports: "Weekly/Monthly Spend Reports",
|
||||
outage_alerts: "Outage Alerts",
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -22962,7 +22962,7 @@ export interface components {
|
|||
* @description Enum for alert types and management event types
|
||||
* @enum {string}
|
||||
*/
|
||||
AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted";
|
||||
AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted";
|
||||
/** AllowedVectorStoreIndexItem */
|
||||
AllowedVectorStoreIndexItem: {
|
||||
/** Index Name */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue