From 570880f222e50b1b208dc6619736c8d9f7f1ca82 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 27 Aug 2026 01:12:30 +0000 Subject: [PATCH 1/6] feat(alerting): slack alerts for per-user daily/monthly spend thresholds and spend anomaly detection Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + .../SlackAlerting/slack_alerting.py | 64 +++++++ .../SlackAlerting/user_spend_alerts.py | 141 ++++++++++++++++ litellm/proxy/proxy_server.py | 33 ++++ litellm/types/integrations/slack_alerting.py | 28 +++ .../SlackAlerting/test_user_spend_alerts.py | 159 ++++++++++++++++++ .../dynamic_form.integration.test.tsx | 23 ++- .../src/components/alerting/dynamic_form.tsx | 4 +- .../src/components/settings.tsx | 2 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 10 files changed, 453 insertions(+), 4 deletions(-) create mode 100644 litellm/integrations/SlackAlerting/user_spend_alerts.py create mode 100644 tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py diff --git a/litellm/constants.py b/litellm/constants.py index 8713bd49f57..b7631ebc43c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1530,6 +1530,7 @@ KEY_ROTATION_JOB_NAME: Final = "litellm_key_rotation_job" EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME: Final = "litellm_expired_ui_session_key_cleanup_job" WEEKLY_SPEND_REPORT_JOB_ID: Final = "weekly_spend_report_job" MONTHLY_SPEND_REPORT_JOB_ID: Final = "monthly_spend_report_job" +USER_SPEND_ALERTS_JOB_ID: Final = "user_spend_alerts_job" PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job" SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report" SLACK_MODEL_DEPRECATION_LOCK_ID: Final = "slack_model_deprecation_warning" diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 65f4774a693..22b82dfc0ea 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -61,6 +61,7 @@ from .utils import process_slack_alerting_variables if TYPE_CHECKING: from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager + from litellm.proxy.utils import PrismaClient from litellm.router import Router as _Router Router = _Router @@ -1897,6 +1898,69 @@ Model Info: except Exception as e: verbose_proxy_logger.exception("Error sending weekly spend report %s", e) + async def send_user_spend_alerts(self, prisma_client: "PrismaClient | None" = None) -> None: + """Check per-user daily/monthly spend thresholds and spend anomalies, alerting once per user per period.""" + if self.alerting is None or "slack" not in self.alerting: + return + + thresholds_enabled: Final = AlertType.user_spend_thresholds in self.alert_types + anomalies_enabled: Final = AlertType.user_spend_anomalies in self.alert_types + if not thresholds_enabled and not anomalies_enabled: + return + + if prisma_client is None: + from litellm.proxy.proxy_server import prisma_client as global_prisma_client + + prisma_client = global_prisma_client # rebind-ok: fall back to the proxy's global client + if prisma_client is None: + return + + from litellm.integrations.SlackAlerting.user_spend_alerts import ( + evaluate_user_spend, + fetch_user_spend_rows, + ) + + try: + today: Final = datetime.datetime.now(datetime.timezone.utc).date() + rows: Final = await fetch_user_spend_rows( + prisma_client=prisma_client, + today=today, + baseline_days=self.alerting_args.spend_anomaly_baseline_days, + ) + all_events: Final = tuple( + event + for row in rows + for event in evaluate_user_spend( + row=row, + args=self.alerting_args, + today=today, + thresholds_enabled=thresholds_enabled, + anomalies_enabled=anomalies_enabled, + ) + ) + cached_flags: Final = await asyncio.gather( + *(self.internal_usage_cache.async_get_cache(key=event.cache_key) for event in all_events) + ) + new_events: Final = tuple(event for event, cached in zip(all_events, cached_flags) if not cached) + for alert_type in (AlertType.user_spend_thresholds, AlertType.user_spend_anomalies): + typed_events = tuple(event for event in new_events if event.alert_type == alert_type) + if not typed_events: + continue + await self.send_alert( + message="\n\n".join(event.message for event in typed_events), + level="High", + alert_type=alert_type, + alerting_metadata={}, + ) + for event in typed_events: + await self.internal_usage_cache.async_set_cache( + key=event.cache_key, + value="SENT", + ttl=event.cache_ttl, + ) + except Exception as e: # noqa: BLE001 # background job must not crash the scheduler + verbose_proxy_logger.exception("Error sending user spend alerts: %s", e) + async def send_fallback_stats_from_prometheus(self): """ Helper to send fallback statistics from prometheus server -> to slack diff --git a/litellm/integrations/SlackAlerting/user_spend_alerts.py b/litellm/integrations/SlackAlerting/user_spend_alerts.py new file mode 100644 index 00000000000..5a749c62299 --- /dev/null +++ b/litellm/integrations/SlackAlerting/user_spend_alerts.py @@ -0,0 +1,141 @@ +"""Per-user daily/monthly spend threshold alerts and spend anomaly detection.""" + +import datetime +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final, Literal + +from pydantic import TypeAdapter + +from litellm.constants import HOURS_IN_A_DAY +from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +DAY_SECONDS: Final = HOURS_IN_A_DAY * 60 * 60 +MONTHLY_ALERT_TTL_SECONDS: Final = 32 * DAY_SECONDS + +USER_SPEND_QUERY: Final = """ +SELECT + user_id, + COALESCE(SUM(spend) FILTER (WHERE date = $1), 0)::float AS daily_spend, + COALESCE(SUM(spend) FILTER (WHERE date >= $2), 0)::float AS monthly_spend, + COALESCE(SUM(spend) FILTER (WHERE date >= $3 AND date < $1), 0)::float AS baseline_spend, + COUNT(DISTINCT date) FILTER (WHERE date >= $3 AND date < $1 AND spend > 0)::int AS baseline_days +FROM "LiteLLM_DailyUserSpend" +WHERE date >= LEAST($2, $3) AND user_id IS NOT NULL +GROUP BY user_id +HAVING COALESCE(SUM(spend) FILTER (WHERE date >= $2), 0) > 0 +""" + + +@dataclass(frozen=True, slots=True) +class UserSpendRow: + user_id: str + daily_spend: float + monthly_spend: float + baseline_spend: float + baseline_days: int + + +@dataclass(frozen=True, slots=True) +class UserSpendAlertEvent: + kind: Literal["daily_threshold", "monthly_threshold", "anomaly"] + alert_type: AlertType + message: str + cache_key: str + cache_ttl: int + + +USER_SPEND_ROWS_ADAPTER: Final = TypeAdapter(tuple[UserSpendRow, ...]) + + +async def fetch_user_spend_rows( + prisma_client: "PrismaClient", + today: datetime.date, + baseline_days: int, +) -> tuple[UserSpendRow, ...]: + today_str: Final = today.strftime("%Y-%m-%d") + month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d") + baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d") + raw: Final = await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str) + return USER_SPEND_ROWS_ADAPTER.validate_python(raw) + + +def _daily_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None: + threshold: Final = args.daily_spend_per_user_threshold + if threshold is None or row.daily_spend < threshold: + return None + return UserSpendAlertEvent( + kind="daily_threshold", + alert_type=AlertType.user_spend_thresholds, + message=( + f"User Daily Spend Threshold Crossed:\n" + f"User: `{row.user_id}`\n" + f"Spend Today: `${row.daily_spend:.2f}`\n" + f"Daily Threshold: `${threshold:.2f}`" + ), + cache_key=f"user_spend_alert_daily_{row.user_id}_{today_str}", + cache_ttl=DAY_SECONDS, + ) + + +def _monthly_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, month_str: str) -> UserSpendAlertEvent | None: + threshold: Final = args.monthly_spend_per_user_threshold + if threshold is None or row.monthly_spend < threshold: + return None + return UserSpendAlertEvent( + kind="monthly_threshold", + alert_type=AlertType.user_spend_thresholds, + message=( + f"User Monthly Spend Threshold Crossed:\n" + f"User: `{row.user_id}`\n" + f"Spend This Month: `${row.monthly_spend:.2f}`\n" + f"Monthly Threshold: `${threshold:.2f}`" + ), + cache_key=f"user_spend_alert_monthly_{row.user_id}_{month_str}", + cache_ttl=MONTHLY_ALERT_TTL_SECONDS, + ) + + +def _anomaly_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None: + if row.daily_spend < args.spend_anomaly_min_spend: + return None + baseline_daily_avg: Final = row.baseline_spend / row.baseline_days if row.baseline_days > 0 else 0.0 + if row.baseline_days > 0 and row.daily_spend <= args.spend_anomaly_multiplier * baseline_daily_avg: + return None + return UserSpendAlertEvent( + kind="anomaly", + alert_type=AlertType.user_spend_anomalies, + message=( + f"User Spend Anomaly Detected:\n" + f"User: `{row.user_id}`\n" + f"Spend Today: `${row.daily_spend:.2f}`\n" + f"Daily Average (last {args.spend_anomaly_baseline_days} days): `${baseline_daily_avg:.2f}`\n" + f"Trigger: spend above `{args.spend_anomaly_multiplier}x` the daily average " + f"(minimum `${args.spend_anomaly_min_spend:.2f}`)" + ), + cache_key=f"user_spend_alert_anomaly_{row.user_id}_{today_str}", + cache_ttl=DAY_SECONDS, + ) + + +def evaluate_user_spend( + row: UserSpendRow, + args: SlackAlertingArgs, + today: datetime.date, + thresholds_enabled: bool, + anomalies_enabled: bool, +) -> tuple[UserSpendAlertEvent, ...]: + today_str: Final = today.strftime("%Y-%m-%d") + month_str: Final = today.strftime("%Y-%m") + threshold_events: Final = ( + ( + _daily_threshold_event(row=row, args=args, today_str=today_str), + _monthly_threshold_event(row=row, args=args, month_str=month_str), + ) + if thresholds_enabled + else () + ) + anomaly_events: Final = (_anomaly_event(row=row, args=args, today_str=today_str),) if anomalies_enabled else () + return tuple(event for event in (*threshold_events, *anomaly_events) if event is not None) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2cfe08fe332..9a3edd4b732 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -249,6 +249,7 @@ from litellm.constants import ( PROXY_BUDGET_RESCHEDULER_MIN_TIME, PROXY_CONFIG_RELOAD_INTERVAL_SECONDS, ROUTER_MODEL_NAME_RESPONSE_FIELD, + USER_SPEND_ALERTS_JOB_ID, WEEKLY_SPEND_REPORT_JOB_ID, ) from litellm.exceptions import RejectedRequestError @@ -9619,6 +9620,32 @@ class ProxyStartupEvent: replace_existing=True, ) + user_spend_check_interval: Final = ( + proxy_logging_obj.slack_alerting_instance.alerting_args.user_spend_check_interval + ) + + async def _scheduled_user_spend_alerts() -> None: + if ( + await pod_lock_manager.acquire_lock( + cronjob_id=USER_SPEND_ALERTS_JOB_ID, + ttl=max(user_spend_check_interval - 60, 60), + allow_reentrant=False, + ) + is False + ): + return + await proxy_logging_obj.slack_alerting_instance.send_user_spend_alerts() + + scheduler.add_job( + _scheduled_user_spend_alerts, + "interval", + seconds=user_spend_check_interval, + next_run_time=datetime.now() + timedelta(seconds=10 + random.randint(0, 60)), + id=USER_SPEND_ALERTS_JOB_ID, + replace_existing=True, + misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, + ) + if os.getenv("PROMETHEUS_URL"): from zoneinfo import ZoneInfo @@ -14658,6 +14685,12 @@ async def alerting_settings( "minor_outage_alert_threshold": {"type": "Integer"}, "major_outage_alert_threshold": {"type": "Integer"}, "max_outage_alert_list_size": {"type": "Integer"}, + "daily_spend_per_user_threshold": {"type": "Float"}, + "monthly_spend_per_user_threshold": {"type": "Float"}, + "spend_anomaly_multiplier": {"type": "Float"}, + "spend_anomaly_baseline_days": {"type": "Integer"}, + "spend_anomaly_min_spend": {"type": "Float"}, + "user_spend_check_interval": {"type": "Integer"}, } _slack_alerting: Final[SlackAlerting] = proxy_logging_obj.slack_alerting_instance diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py index b1b7bc3541a..4f890322a73 100644 --- a/litellm/types/integrations/slack_alerting.py +++ b/litellm/types/integrations/slack_alerting.py @@ -91,6 +91,30 @@ class SlackAlertingArgs(LiteLLMPydanticObjectBase): default=False, description="If true, the alerting payload will be printed to the console.", ) + daily_spend_per_user_threshold: float | None = Field( + default=None, + description="Alert when a user's spend for the current day (UTC) crosses this USD amount. Off by default.", + ) + monthly_spend_per_user_threshold: float | None = Field( + default=None, + description="Alert when a user's spend for the current calendar month (UTC) crosses this USD amount. Off by default.", + ) + spend_anomaly_multiplier: float = Field( + default=3.0, + description="Flag a user's spend as anomalous when today's spend exceeds this multiple of their trailing daily average.", + ) + spend_anomaly_baseline_days: int = Field( + default=7, + description="Number of trailing days used to compute a user's daily average spend for anomaly detection.", + ) + spend_anomaly_min_spend: float = Field( + default=10.0, + description="Minimum spend (USD) a user must reach today before an anomaly alert can fire. Reduces false positives.", + ) + user_spend_check_interval: int = Field( + default=3600, + description="How often (in seconds) to check per-user spend thresholds and anomalies. Default is hourly.", + ) class DeploymentMetrics(LiteLLMPydanticObjectBase): @@ -138,6 +162,8 @@ class AlertType(str, Enum): budget_alerts = "budget_alerts" spend_reports = "spend_reports" failed_tracking_spend = "failed_tracking_spend" + user_spend_thresholds = "user_spend_thresholds" + user_spend_anomalies = "user_spend_anomalies" # Database alerts db_exceptions = "db_exceptions" @@ -182,6 +208,8 @@ DEFAULT_ALERT_TYPES: Final[list[AlertType]] = [ AlertType.budget_alerts, AlertType.spend_reports, AlertType.failed_tracking_spend, + AlertType.user_spend_thresholds, + AlertType.user_spend_anomalies, # Database alerts AlertType.db_exceptions, # Report alerts diff --git a/tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py b/tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py new file mode 100644 index 00000000000..c58807c0ffc --- /dev/null +++ b/tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py @@ -0,0 +1,159 @@ +import datetime +from typing import Final +from unittest.mock import AsyncMock, patch + +import pytest + +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.integrations.SlackAlerting.user_spend_alerts import ( + UserSpendRow, + evaluate_user_spend, +) +from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs + +TODAY: Final = datetime.date(2026, 8, 15) + + +def _row( + daily_spend: float = 0.0, + monthly_spend: float = 0.0, + baseline_spend: float = 0.0, + baseline_days: int = 0, +) -> UserSpendRow: + return UserSpendRow( + user_id="user-1", + daily_spend=daily_spend, + monthly_spend=monthly_spend, + baseline_spend=baseline_spend, + baseline_days=baseline_days, + ) + + +def _evaluate(row: UserSpendRow, args: SlackAlertingArgs, thresholds: bool = True, anomalies: bool = True): + return evaluate_user_spend( + row=row, + args=args, + today=TODAY, + thresholds_enabled=thresholds, + anomalies_enabled=anomalies, + ) + + +def test_daily_threshold_crossed(): + args: Final = SlackAlertingArgs(daily_spend_per_user_threshold=50.0, spend_anomaly_min_spend=1000.0) + events: Final = _evaluate(_row(daily_spend=75.0, monthly_spend=75.0), args) + assert [e.kind for e in events] == ["daily_threshold"] + assert "`$75.00`" in events[0].message + assert "`$50.00`" in events[0].message + assert events[0].alert_type == AlertType.user_spend_thresholds + assert events[0].cache_key == "user_spend_alert_daily_user-1_2026-08-15" + + +def test_daily_threshold_not_crossed(): + args: Final = SlackAlertingArgs(daily_spend_per_user_threshold=50.0, spend_anomaly_min_spend=1000.0) + assert _evaluate(_row(daily_spend=49.99, monthly_spend=49.99), args) == () + + +def test_thresholds_unset_by_default(): + args: Final = SlackAlertingArgs(spend_anomaly_min_spend=1000.0) + assert _evaluate(_row(daily_spend=999.0, monthly_spend=999.0), args) == () + + +def test_monthly_threshold_crossed(): + args: Final = SlackAlertingArgs(monthly_spend_per_user_threshold=200.0, spend_anomaly_min_spend=1000.0) + events: Final = _evaluate(_row(daily_spend=5.0, monthly_spend=250.0), args) + assert [e.kind for e in events] == ["monthly_threshold"] + assert events[0].cache_key == "user_spend_alert_monthly_user-1_2026-08" + + +def test_thresholds_disabled_suppresses_threshold_events(): + args: Final = SlackAlertingArgs( + daily_spend_per_user_threshold=50.0, + monthly_spend_per_user_threshold=200.0, + spend_anomaly_min_spend=1000.0, + ) + assert _evaluate(_row(daily_spend=75.0, monthly_spend=250.0), args, thresholds=False) == () + + +def test_anomaly_detected_above_multiple_of_baseline(): + args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0) + events: Final = _evaluate( + _row(daily_spend=70.0, monthly_spend=100.0, baseline_spend=70.0, baseline_days=7), args + ) + assert [e.kind for e in events] == ["anomaly"] + assert events[0].alert_type == AlertType.user_spend_anomalies + assert "`$10.00`" in events[0].message # baseline daily average + assert events[0].cache_key == "user_spend_alert_anomaly_user-1_2026-08-15" + + +def test_no_anomaly_within_baseline_multiple(): + args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0) + assert ( + _evaluate(_row(daily_spend=25.0, monthly_spend=100.0, baseline_spend=70.0, baseline_days=7), args) == () + ) + + +def test_no_anomaly_below_min_spend_floor(): + args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0) + assert _evaluate(_row(daily_spend=9.0, monthly_spend=9.0, baseline_spend=0.1, baseline_days=1), args) == () + + +def test_anomaly_for_new_user_without_baseline(): + args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0) + events: Final = _evaluate(_row(daily_spend=15.0, monthly_spend=15.0), args) + assert [e.kind for e in events] == ["anomaly"] + + +def test_anomalies_disabled_suppresses_anomaly_events(): + args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0) + assert _evaluate(_row(daily_spend=500.0, monthly_spend=500.0), args, anomalies=False) == () + + +@pytest.mark.asyncio +async def test_send_user_spend_alerts_sends_and_dedupes(): + slack_alerting: Final = SlackAlerting( + alerting=["slack"], + alerting_args={"daily_spend_per_user_threshold": 50.0, "spend_anomaly_min_spend": 1000.0}, + ) + mock_prisma: Final = AsyncMock() + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + { + "user_id": "user-1", + "daily_spend": 75.0, + "monthly_spend": 75.0, + "baseline_spend": 0.0, + "baseline_days": 0, + }, + { + "user_id": "user-2", + "daily_spend": 60.0, + "monthly_spend": 60.0, + "baseline_spend": 0.0, + "baseline_days": 0, + }, + ] + ) + with patch.object(slack_alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert: + await slack_alerting.send_user_spend_alerts(prisma_client=mock_prisma) + assert mock_send_alert.call_count == 1 + sent_kwargs: Final = mock_send_alert.call_args.kwargs + assert sent_kwargs["alert_type"] == AlertType.user_spend_thresholds + assert "User Daily Spend Threshold Crossed" in sent_kwargs["message"] + assert "`user-1`" in sent_kwargs["message"] + assert "`user-2`" in sent_kwargs["message"] + + await slack_alerting.send_user_spend_alerts(prisma_client=mock_prisma) + assert mock_send_alert.call_count == 1 + + +@pytest.mark.asyncio +async def test_send_user_spend_alerts_noop_when_alert_types_disabled(): + slack_alerting: Final = SlackAlerting( + alerting=["slack"], + alert_types=[AlertType.budget_alerts], + alerting_args={"daily_spend_per_user_threshold": 50.0}, + ) + mock_prisma: Final = AsyncMock() + await slack_alerting.send_user_spend_alerts(prisma_client=mock_prisma) + mock_prisma.db.query_raw.assert_not_called() diff --git a/ui/litellm-dashboard/src/components/alerting/dynamic_form.integration.test.tsx b/ui/litellm-dashboard/src/components/alerting/dynamic_form.integration.test.tsx index f9b9765f59e..7764052173c 100644 --- a/ui/litellm-dashboard/src/components/alerting/dynamic_form.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/alerting/dynamic_form.integration.test.tsx @@ -38,6 +38,14 @@ const SETTINGS: Setting[] = [ stored_in_db: null, premium_field: false, }, + { + field_name: "daily_spend_per_user_threshold", + field_description: "Daily spend threshold per user", + field_type: "Float", + field_value: 5.5, + stored_in_db: true, + premium_field: false, + }, ]; const renderForm = ( @@ -170,6 +178,19 @@ describe("DynamicForm change notifications", () => { expect(handleInputChange).toHaveBeenCalledWith("daily_report_frequency", 128); }); + it("renders a Float field as a decimal-friendly number input and reports changes as numbers", async () => { + const user = userEvent.setup(); + const { handleInputChange } = renderForm(); + + const input = screen.getByDisplayValue("5.5"); + expect(input).toHaveAttribute("type", "number"); + expect(input).toHaveAttribute("step", "any"); + + await user.type(input, "1"); + + expect(handleInputChange).toHaveBeenCalledWith("daily_spend_per_user_threshold", 5.51); + }); + it("reports a reset with the field name and its row index", async () => { const user = userEvent.setup(); const { handleResetField } = renderForm(); @@ -216,7 +237,7 @@ describe("DynamicForm presentation", () => { expect(screen.getByText("daily_report_frequency")).toBeInTheDocument(); expect(screen.getByText("How often the report runs")).toBeInTheDocument(); - expect(screen.getByText("In DB")).toBeInTheDocument(); + expect(screen.getAllByText("In DB")).toHaveLength(2); expect(screen.getByText("In Config")).toBeInTheDocument(); expect(screen.getByText("Not Set")).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/alerting/dynamic_form.tsx b/ui/litellm-dashboard/src/components/alerting/dynamic_form.tsx index be5e76ab0df..42aa58ca0ea 100644 --- a/ui/litellm-dashboard/src/components/alerting/dynamic_form.tsx +++ b/ui/litellm-dashboard/src/components/alerting/dynamic_form.tsx @@ -63,11 +63,11 @@ const DynamicForm: React.FC = ({ }; const renderControl = (setting: AlertingSetting) => { - if (setting.field_type === "Integer") { + if (setting.field_type === "Integer" || setting.field_type === "Float") { return ( handleNumericChange(setting, event.target.value)} /> diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index e6337cef9bd..edc79359614 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -292,6 +292,8 @@ const Settings: React.FC = ({ accessToken, userRole, userID, llm_too_slow: "LLM Responses Too Slow", llm_requests_hanging: "LLM Requests Hanging", budget_alerts: "Budget Alerts (API Keys, Users)", + user_spend_thresholds: "User Spend Thresholds (Daily/Monthly)", + user_spend_anomalies: "User Spend Anomaly Detection", db_exceptions: "Database Exceptions (Read/Write)", daily_reports: "Weekly/Monthly Spend Reports", outage_alerts: "Outage Alerts", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 4ccf7b59fbc..7e48b8aa024 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -21809,7 +21809,7 @@ export interface components { * @description Enum for alert types and management event types * @enum {string} */ - AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted"; + AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted"; /** AllowedVectorStoreIndexItem */ AllowedVectorStoreIndexItem: { /** Index Name */ From 12a3f972f259560a7bb75895607daab7b063a315 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 27 Aug 2026 01:36:37 +0000 Subject: [PATCH 2/6] test(alerting): use specific ValidationError matches in config rejection test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integrations/SlackAlerting/test_user_spend_alerts.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py b/tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py index eaac300ec81..28f5d0bf7a2 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py @@ -3,6 +3,7 @@ from typing import Final from unittest.mock import AsyncMock, patch import pytest +from pydantic import ValidationError from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.SlackAlerting.user_spend_alerts import ( @@ -120,11 +121,11 @@ def test_anomalies_not_in_default_alert_types(): def test_invalid_config_rejected(): - with pytest.raises(ValueError): + with pytest.raises(ValidationError, match="daily_spend_per_user_threshold"): SlackAlertingArgs(daily_spend_per_user_threshold=0) - with pytest.raises(ValueError): + with pytest.raises(ValidationError, match="spend_anomaly_baseline_days"): SlackAlertingArgs(spend_anomaly_baseline_days=0) - with pytest.raises(ValueError): + with pytest.raises(ValidationError, match="user_spend_check_interval"): SlackAlertingArgs(user_spend_check_interval=10) From 96edff12d202b609b3b77a0d1d970a91c7a468cf Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 27 Aug 2026 02:18:46 +0000 Subject: [PATCH 3/6] fix(proxy): tolerate mocked slack alerting args when scheduling user spend scan Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 07c739ae8d2..288af7bc419 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9608,8 +9608,11 @@ class ProxyStartupEvent: replace_existing=True, ) + slack_alerting_args: Final = proxy_logging_obj.slack_alerting_instance.alerting_args user_spend_check_interval: Final = ( - proxy_logging_obj.slack_alerting_instance.alerting_args.user_spend_check_interval + slack_alerting_args.user_spend_check_interval + if isinstance(slack_alerting_args, SlackAlertingArgs) # pyright: ignore[reportUnnecessaryIsInstance] # tests inject a mock slack_alerting_instance + else SlackAlertingArgs().user_spend_check_interval ) async def _scheduled_user_spend_alerts() -> None: From 10f4e42605848fca83cb1803a278a23ebd15df63 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 27 Aug 2026 02:48:43 +0000 Subject: [PATCH 4/6] fix(alerting): reject non-finite values in user spend alert settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/types/integrations/slack_alerting.py | 4 ++++ .../SlackAlerting/test_user_spend_alerts.py | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py index 51eb5c8190e..64c0c530e9b 100644 --- a/litellm/types/integrations/slack_alerting.py +++ b/litellm/types/integrations/slack_alerting.py @@ -94,16 +94,19 @@ class SlackAlertingArgs(LiteLLMPydanticObjectBase): daily_spend_per_user_threshold: float | None = Field( default=None, gt=0, + allow_inf_nan=False, description="Alert when a user's spend for the current day (UTC) crosses this USD amount. Off by default.", ) monthly_spend_per_user_threshold: float | None = Field( default=None, gt=0, + allow_inf_nan=False, description="Alert when a user's spend for the current calendar month (UTC) crosses this USD amount. Off by default.", ) spend_anomaly_multiplier: float = Field( default=3.0, gt=0, + allow_inf_nan=False, description="Flag a user's spend as anomalous when today's spend exceeds this multiple of their trailing daily average.", ) spend_anomaly_baseline_days: int = Field( @@ -114,6 +117,7 @@ class SlackAlertingArgs(LiteLLMPydanticObjectBase): spend_anomaly_min_spend: float = Field( default=10.0, gt=0, + allow_inf_nan=False, description="Minimum spend (USD) a user must reach today before an anomaly alert can fire. Reduces false positives.", ) user_spend_check_interval: int = Field( diff --git a/tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py b/tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py index 28f5d0bf7a2..45e1acecec8 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_user_spend_alerts.py @@ -85,7 +85,7 @@ def test_anomaly_detected_above_multiple_of_baseline(): ) assert [e.kind for e in events] == ["anomaly"] assert events[0].alert_type == AlertType.user_spend_anomalies - assert "`$10.00`" in events[0].message # baseline daily average + assert "`$10.00`" in events[0].message assert events[0].cache_key == "user_spend_alert_anomaly_user-1_2026-08-15" @@ -129,6 +129,17 @@ def test_invalid_config_rejected(): SlackAlertingArgs(user_spend_check_interval=10) +def test_non_finite_config_rejected(): + with pytest.raises(ValidationError, match="daily_spend_per_user_threshold"): + SlackAlertingArgs(daily_spend_per_user_threshold=float("inf")) + with pytest.raises(ValidationError, match="spend_anomaly_multiplier"): + SlackAlertingArgs(spend_anomaly_multiplier=float("nan")) + with pytest.raises(ValidationError, match="spend_anomaly_min_spend"): + SlackAlertingArgs(spend_anomaly_min_spend=float("inf")) + with pytest.raises(ValidationError, match="user_spend_check_interval"): + SlackAlertingArgs(user_spend_check_interval=float("inf")) + + def test_anomalies_disabled_suppresses_anomaly_events(): args: Final = SlackAlertingArgs(spend_anomaly_multiplier=3.0, spend_anomaly_min_spend=10.0) assert _evaluate(_row(daily_spend=500.0, monthly_spend=500.0), args, anomalies=False) == () From 57231c7ad40004834a283e93cb59238061c7ead9 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 27 Aug 2026 20:04:02 +0000 Subject: [PATCH 5/6] fix(slack): annotate alert metadata payload Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/SlackAlerting/slack_alerting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 22b82dfc0ea..717407253b5 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1950,7 +1950,7 @@ Model Info: message="\n\n".join(event.message for event in typed_events), level="High", alert_type=alert_type, - alerting_metadata={}, + alerting_metadata={}, # mutable-ok: send_alert takes a dict payload ) for event in typed_events: await self.internal_usage_cache.async_set_cache( From 3bfc77ac18cffbbb171038fa2a950b760b616577 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 27 Aug 2026 20:04:02 +0000 Subject: [PATCH 6/6] refactor(proxy): freeze alert setting types Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 40 ++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 24b9b9bb817..fb18665b603 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14692,23 +14692,25 @@ async def alerting_settings( alerting_args_dict = {} alerting_values = None - allowed_args: Final = { - "slack_alerting": {"type": "Boolean"}, - "daily_report_frequency": {"type": "Integer"}, - "report_check_interval": {"type": "Integer"}, - "budget_alert_ttl": {"type": "Integer"}, - "outage_alert_ttl": {"type": "Integer"}, - "region_outage_alert_ttl": {"type": "Integer"}, - "minor_outage_alert_threshold": {"type": "Integer"}, - "major_outage_alert_threshold": {"type": "Integer"}, - "max_outage_alert_list_size": {"type": "Integer"}, - "daily_spend_per_user_threshold": {"type": "Float"}, - "monthly_spend_per_user_threshold": {"type": "Float"}, - "spend_anomaly_multiplier": {"type": "Float"}, - "spend_anomaly_baseline_days": {"type": "Integer"}, - "spend_anomaly_min_spend": {"type": "Float"}, - "user_spend_check_interval": {"type": "Integer"}, - } + allowed_args: Final = MappingProxyType( + { + "slack_alerting": "Boolean", + "daily_report_frequency": "Integer", + "report_check_interval": "Integer", + "budget_alert_ttl": "Integer", + "outage_alert_ttl": "Integer", + "region_outage_alert_ttl": "Integer", + "minor_outage_alert_threshold": "Integer", + "major_outage_alert_threshold": "Integer", + "max_outage_alert_list_size": "Integer", + "daily_spend_per_user_threshold": "Float", + "monthly_spend_per_user_threshold": "Float", + "spend_anomaly_multiplier": "Float", + "spend_anomaly_baseline_days": "Integer", + "spend_anomaly_min_spend": "Float", + "user_spend_check_interval": "Integer", + } + ) _slack_alerting: Final[SlackAlerting] = proxy_logging_obj.slack_alerting_instance _slack_alerting_args_dict: Final = _slack_alerting.alerting_args.model_dump() @@ -14723,7 +14725,7 @@ async def alerting_settings( _response_obj = ConfigList( field_name="slack_alerting", - field_type=allowed_args["slack_alerting"]["type"], + field_type=allowed_args["slack_alerting"], field_description="Enable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.", field_value=is_slack_enabled, stored_in_db=True if alerting_values is not None else False, @@ -14742,7 +14744,7 @@ async def alerting_settings( _response_obj = ConfigList( field_name=field_name, - field_type=allowed_args[field_name]["type"], + field_type=allowed_args[field_name], field_description=field_info.description or "", field_value=_slack_alerting_args_dict.get(field_name, None), stored_in_db=_stored_in_db,