From ea448a89513bfd6fcfcff58b4bfb23dc761d093e Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 17 Nov 2026 07:32:19 +0800 Subject: [PATCH] feat(alerts): add email labels, bulk actions and threshold editing (#52139) * feat(alerts): add bulk actions and enterprise thresholds * fix(alerts): accept fractional multipliers above one --- litellm/models/alert.py | 17 ++ litellm/models/performance_alert.py | 98 +++++++-- litellm/proxy/alerting_endpoints/alerts.py | 24 ++- litellm/repositories/alert_repository.py | 18 +- .../performance_alert_repository.py | 22 +- .../src/components/Alerts/AlertThresholds.tsx | 140 +++++++++++++ .../Alerts/Alerts.integration.test.tsx | 193 +++++++++++++++++- .../src/components/Alerts/Alerts.tsx | 27 ++- .../components/Alerts/BulkAlertActions.tsx | 104 ++++++++++ .../src/components/Alerts/ManageAlerts.tsx | 15 +- .../src/i18n/locales/en.json | 39 ++++ .../src/lib/http/schema.d.ts | 138 ++++++++++++- tests/integration/database/test_alerts.py | 76 +++++++ .../database/test_performance_alerts.py | 40 +++- tests/unit/models/test_performance_alert.py | 52 +++++ .../proxy/alerting_endpoints/test_alerts.py | 37 +++- 16 files changed, 993 insertions(+), 47 deletions(-) create mode 100644 litellm/ui/litellm-dashboard/src/components/Alerts/AlertThresholds.tsx create mode 100644 litellm/ui/litellm-dashboard/src/components/Alerts/BulkAlertActions.tsx diff --git a/litellm/models/alert.py b/litellm/models/alert.py index ae7463c92fc..eb5818e94e8 100644 --- a/litellm/models/alert.py +++ b/litellm/models/alert.py @@ -178,6 +178,21 @@ class AlertUpdate(BaseModel): status: AlertStatus = Field(description="Mark resolved, dismiss, or reopen the alert.") +class AlertBulkUpdate(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: Literal["resolved", "dismissed"] = Field(description="Status to apply to every matching open saved alert.") + type: AlertFilter | None = Field( + default=None, description="Only update this category, or all saved alerts if omitted." + ) + + +class AlertBulkResult(BaseModel): + model_config = ConfigDict(frozen=True) + + updated: int = Field(ge=0, description="Number of open saved alerts changed.") + + class AlertPage(BaseModel): model_config = ConfigDict(frozen=True) @@ -206,3 +221,5 @@ class AlertStore(Protocol): async def get(self, alert_id: int) -> Alert | None: ... async def set_status(self, alert_id: int, status: AlertStatus, actor: str) -> Alert | None: ... + + async def set_open_status(self, data: AlertBulkUpdate, actor: str) -> int: ... diff --git a/litellm/models/performance_alert.py b/litellm/models/performance_alert.py index 1cae91e5379..37f9d7dcbc8 100644 --- a/litellm/models/performance_alert.py +++ b/litellm/models/performance_alert.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from datetime import date, datetime, time, timedelta, timezone from typing import Final, Literal, Protocol, TypeAlias -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator from litellm.models.alert import MAX_ALERT_ID, AlertDetection, AlertType, MetricEvidence @@ -25,6 +25,49 @@ RELIABILITY_MIN_INCREASE: Final = 0.02 RELIABILITY_MIN_REQUESTS: Final = 500 +class AlertThreshold(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + multiplier: float = Field(gt=1, allow_inf_nan=False, strict=True, description="Minimum multiple of the baseline.") + minimum_increase: float = Field( + ge=0, + allow_inf_nan=False, + strict=True, + description="Minimum absolute increase above the baseline, in the metric's unit.", + ) + + +class PerformanceAlertThresholds(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + spend: AlertThreshold = Field( + default=AlertThreshold(multiplier=SPEND_MULTIPLIER, minimum_increase=SPEND_MIN_INCREASE_USD), + description="Spend increase in USD over the previous seven-day daily average.", + ) + traffic: AlertThreshold = Field( + default=AlertThreshold(multiplier=TRAFFIC_SPIKE_MULTIPLIER, minimum_increase=TRAFFIC_SPIKE_MIN_INCREASE), + description="Request increase over the previous seven-day daily average.", + ) + latency: AlertThreshold = Field( + default=AlertThreshold(multiplier=LATENCY_MULTIPLIER, minimum_increase=LATENCY_MIN_INCREASE_MS), + description="Average response-time increase in milliseconds over the previous seven-day weighted average.", + ) + reliability: AlertThreshold = Field( + default=AlertThreshold(multiplier=RELIABILITY_MULTIPLIER, minimum_increase=RELIABILITY_MIN_INCREASE), + description="Error-rate increase as a fraction, e.g. 0.02 is two percentage points.", + ) + + @field_validator("reliability") + @classmethod + def validate_error_rate(cls, value: AlertThreshold) -> AlertThreshold: + if value.minimum_increase > 1: + raise ValueError("Error-rate minimum increase cannot exceed 1") + return value + + +DEFAULT_THRESHOLDS: Final = PerformanceAlertThresholds() + + class ResourceWindow(BaseModel): """One resource's totals for the evaluated day, and summed over the seven days before it.""" @@ -89,6 +132,9 @@ class AlertMonitoring(BaseModel): alerts_used: int = Field(ge=0, le=FREE_ALERT_LIMIT, description="Lifetime free allowance consumed, capped at five.") alert_limit: int | None = Field(description="Free allowance, or null for Enterprise.") disabled_by_config: bool = Field(description="Whether performance_alerts_enabled is false in proxy settings.") + thresholds: PerformanceAlertThresholds = Field( + default=DEFAULT_THRESHOLDS, description="Effective built-in thresholds. Only Enterprise admins can edit them." + ) @property def active(self) -> bool: @@ -107,10 +153,19 @@ class ModelDeprecationMonitoringUpdate(BaseModel): model_deprecations_enabled: bool = Field(strict=True, description="Enable or disable model deprecation alerts.") +class AlertThresholdsUpdate(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + thresholds: PerformanceAlertThresholds = Field( + description="Replace the built-in performance thresholds. Enterprise only." + ) + + class AlertMonitoringStore(Protocol): async def monitoring(self) -> AlertMonitoring: ... async def set_enabled(self, enabled: bool) -> AlertMonitoring: ... async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring: ... + async def set_thresholds(self, thresholds: PerformanceAlertThresholds) -> AlertMonitoring: ... class PerformanceAlertStore(Protocol): @@ -181,58 +236,65 @@ def _alert( ) -def _spend_spike(window: ResourceWindow) -> tuple[float, float] | None: +def _spend_spike(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None: baseline: Final = window.base_spend / BASELINE_DAYS triggered: Final = ( window.active_days >= MIN_ACTIVE_DAYS and baseline > 0 - and window.spend >= SPEND_MULTIPLIER * baseline - and window.spend - baseline >= SPEND_MIN_INCREASE_USD + and window.spend >= threshold.multiplier * baseline + and window.spend - baseline >= threshold.minimum_increase ) return (window.spend, baseline) if triggered else None -def _traffic_spike(window: ResourceWindow) -> tuple[float, float] | None: +def _traffic_spike(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None: baseline: Final = window.base_requests / BASELINE_DAYS spike: Final = ( window.active_days >= MIN_ACTIVE_DAYS - and window.requests >= TRAFFIC_SPIKE_MULTIPLIER * baseline - and window.requests - baseline >= TRAFFIC_SPIKE_MIN_INCREASE + and window.requests > baseline + and window.requests >= threshold.multiplier * baseline + and window.requests - baseline >= threshold.minimum_increase ) return (window.requests, baseline) if spike else None -def _slower_responses(window: ResourceWindow) -> tuple[float, float] | None: +def _slower_responses(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None: current: Final = _ratio(window.response_ms, window.timed) baseline: Final = _ratio(window.base_response_ms, window.base_timed) triggered: Final = ( window.timed >= LATENCY_MIN_TIMED_REQUESTS and window.timed_days >= MIN_ACTIVE_DAYS and baseline > 0 - and current >= LATENCY_MULTIPLIER * baseline - and current - baseline >= LATENCY_MIN_INCREASE_MS + and current >= threshold.multiplier * baseline + and current - baseline >= threshold.minimum_increase ) return (current, baseline) if triggered else None -def _more_failures(window: ResourceWindow) -> tuple[float, float] | None: +def _more_failures(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None: current: Final = min(1.0, _ratio(window.failed, window.requests)) baseline: Final = min(1.0, _ratio(window.base_failed, window.base_requests)) triggered: Final = ( window.active_days >= MIN_ACTIVE_DAYS and window.requests >= RELIABILITY_MIN_REQUESTS - and current >= RELIABILITY_MULTIPLIER * baseline - and current - baseline >= RELIABILITY_MIN_INCREASE + and current > baseline + and current >= threshold.multiplier * baseline + and current - baseline >= threshold.minimum_increase ) return (current, baseline) if triggered else None -def detect(scope: PerformanceScope, window: ResourceWindow, day: date) -> tuple[AlertDetection, ...]: +def detect( + scope: PerformanceScope, + window: ResourceWindow, + day: date, + thresholds: PerformanceAlertThresholds = DEFAULT_THRESHOLDS, +) -> tuple[AlertDetection, ...]: checks: Final[tuple[tuple[AlertType, tuple[float, float] | None, int], ...]] = ( - ("spend", _spend_spike(window), window.requests), - ("traffic", _traffic_spike(window), window.requests), - ("latency", _slower_responses(window), window.timed), - ("reliability", _more_failures(window), window.requests), + ("spend", _spend_spike(window, thresholds.spend), window.requests), + ("traffic", _traffic_spike(window, thresholds.traffic), window.requests), + ("latency", _slower_responses(window, thresholds.latency), window.timed), + ("reliability", _more_failures(window, thresholds.reliability), window.requests), ) return tuple( _alert(scope, window, day, kind, found[0], found[1], sample_size) diff --git a/litellm/proxy/alerting_endpoints/alerts.py b/litellm/proxy/alerting_endpoints/alerts.py index 9c701208e0d..1cb7f66cefe 100644 --- a/litellm/proxy/alerting_endpoints/alerts.py +++ b/litellm/proxy/alerting_endpoints/alerts.py @@ -4,6 +4,8 @@ from fastapi import APIRouter, Depends, HTTPException, Path, Query from litellm.models.alert import ( Alert, + AlertBulkResult, + AlertBulkUpdate, AlertFilter, AlertID, AlertStatus, @@ -12,7 +14,12 @@ from litellm.models.alert import ( AlertView, AlertViewPage, ) -from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate, ModelDeprecationMonitoringUpdate +from litellm.models.performance_alert import ( + AlertMonitoring, + AlertMonitoringUpdate, + AlertThresholdsUpdate, + ModelDeprecationMonitoringUpdate, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.deps.repositories import AlertMonitoringRepositoryDep, AlertRepositoryDep @@ -60,6 +67,13 @@ async def list_alerts( return AlertViewPage(data=await store.enrich(page.data), next_cursor=page.next_cursor) +@router.patch("/alerts", response_model=AlertBulkResult, summary="Resolve or dismiss matching open alerts") +async def update_open_alerts(data: AlertBulkUpdate, user: Auth, repository: AlertRepositoryDep) -> AlertBulkResult: + """Update all matching open saved alerts, including unloaded pages. Only gateway admins can write.""" + store: Final = _authorized_store(user, repository, write=True) + return AlertBulkResult(updated=await store.set_open_status(data, user.user_id or "proxy-admin")) + + @router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read alert monitoring settings") async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -> AlertMonitoring: """Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin.""" @@ -71,12 +85,18 @@ async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) - @router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable alert monitoring") async def update_monitoring( - data: AlertMonitoringUpdate | ModelDeprecationMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep + data: AlertMonitoringUpdate | ModelDeprecationMonitoringUpdate | AlertThresholdsUpdate, + user: Auth, + repository: AlertMonitoringRepositoryDep, ) -> AlertMonitoring: """Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts.""" _authorize(user, write=True) if repository is None: raise HTTPException(status_code=503, detail="Alerts require a database connection") + if isinstance(data, AlertThresholdsUpdate): + if (await repository.monitoring()).alert_limit is not None: + raise HTTPException(status_code=403, detail="Custom alert thresholds require Enterprise") + return await repository.set_thresholds(data.thresholds) if isinstance(data, ModelDeprecationMonitoringUpdate): return await repository.set_model_deprecations_enabled(data.model_deprecations_enabled) if data.enabled and (await repository.monitoring()).disabled_by_config: diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py index cddafa761ad..3c38abc6509 100644 --- a/litellm/repositories/alert_repository.py +++ b/litellm/repositories/alert_repository.py @@ -18,6 +18,7 @@ from litellm._logging import verbose_proxy_logger from litellm.models.alert import ( METRIC_ALERT_TYPES, Alert, + AlertBulkUpdate, AlertDetection, AlertFilter, AlertHistoryPoint, @@ -36,6 +37,7 @@ if TYPE_CHECKING: class _HistoryTransaction(Protocol): async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... + async def execute_raw(self, query: str, *args: object) -> int: ... class _HistoryDatabase(Protocol): @@ -95,7 +97,7 @@ _HISTORY_SOURCES: Final[ "d.user_id", "api_requests", 'LEFT JOIN "LiteLLM_UserTable" m ON m.user_id = r.resource_id', - "m.user_alias", + "COALESCE(NULLIF(TRIM(m.user_email), ''), m.user_alias)", ), "key": ( '"LiteLLM_DailyUserSpend"', @@ -260,6 +262,20 @@ class AlertRepository: row: Final = await self._table.update(where={"id": alert_id}, data={"status": status, "updated_by": actor}) return None if row is None else _alert(row) + async def set_open_status(self, data: AlertBulkUpdate, actor: str) -> int: + async with self._database.tx(timeout=timedelta(seconds=10)) as transaction: + await transaction.query_raw("SELECT set_config('statement_timeout', '6s', true)") + return await transaction.execute_raw( + """UPDATE "LiteLLM_Alert" SET status = $1, updated_by = $2, + updated_at = now() AT TIME ZONE 'UTC' + WHERE status = 'open' AND + ($3::text IS NULL OR type = $3 OR ($3 = 'performance' AND type = ANY($4::text[])))""", + data.status, + actor, + data.type, + list(METRIC_ALERT_TYPES), + ) + async def enrich(self, alerts: tuple[Alert, ...]) -> tuple[AlertView, ...]: from prisma.errors import PrismaError diff --git a/litellm/repositories/performance_alert_repository.py b/litellm/repositories/performance_alert_repository.py index 2f60c18d172..e80c728ff7c 100644 --- a/litellm/repositories/performance_alert_repository.py +++ b/litellm/repositories/performance_alert_repository.py @@ -12,9 +12,11 @@ from typing_extensions import LiteralString from litellm.models.alert import METRIC_ALERT_TYPES from litellm.models.performance_alert import ( BASELINE_DAYS, + DEFAULT_THRESHOLDS, FREE_ALERT_LIMIT, AlertMonitoring, Claim, + PerformanceAlertThresholds, PerformanceScope, ResourceUnreadableError, ResourceWindow, @@ -196,9 +198,24 @@ class PerformanceAlertRepository: async with self._transaction() as transaction: monitoring: Final = await self._monitoring(transaction) return monitoring.model_copy( - update={"model_deprecations_enabled": await self._config.model_deprecations_enabled()} + update={ + "model_deprecations_enabled": await self._config.model_deprecations_enabled(), + "thresholds": await self._thresholds(), + } ) + async def _thresholds(self) -> PerformanceAlertThresholds: + if not self._enterprise: + return DEFAULT_THRESHOLDS + saved: Final = await self._config.get_param("performance_alert_thresholds") + return DEFAULT_THRESHOLDS if saved is None else PerformanceAlertThresholds.model_validate(saved.param_value) + + async def set_thresholds(self, thresholds: PerformanceAlertThresholds) -> AlertMonitoring: + if not self._enterprise: + raise PermissionError("Custom alert thresholds require Enterprise") + await self._config.set_param("performance_alert_thresholds", thresholds.model_dump(mode="json")) + return await self.monitoring() + async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring: await self._config.set_model_deprecations_enabled(enabled) return await self.monitoring() @@ -258,6 +275,7 @@ class PerformanceAlertRepository: raise async def _evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState | None: + thresholds: Final = await self._thresholds() async with self._transaction() as transaction: monitoring: Final = await self._monitoring(transaction, lock=True) if not monitoring.active: @@ -267,7 +285,7 @@ class PerformanceAlertRepository: ) saved: Final = await record_alerts( transaction, - tuple(chain.from_iterable(detect(state.scope, window, state.day) for window in windows)), + tuple(chain.from_iterable(detect(state.scope, window, state.day, thresholds) for window in windows)), None if self._enterprise else FREE_ALERT_LIMIT - monitoring.alerts_used, ) if saved: diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/AlertThresholds.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/AlertThresholds.tsx new file mode 100644 index 00000000000..e8077802408 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/AlertThresholds.tsx @@ -0,0 +1,140 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import type { components } from "@/lib/http/schema"; + +type Thresholds = components["schemas"]["PerformanceAlertThresholds"]; +const rules = ["spend", "traffic", "latency", "reliability"] as const; + +export default function AlertThresholds({ + thresholds, + enterprise, + readOnly, + pending, + save, +}: { + thresholds: Thresholds; + enterprise: boolean; + readOnly: boolean; + pending: boolean; + save: (thresholds: Thresholds) => Promise; +}) { + const { t, i18n } = useTranslation(); + const [editing, setEditing] = useState(false); + const [saved, setSaved] = useState(false); + const disabled = !enterprise || readOnly || pending; + const format = new Intl.NumberFormat(i18n.language, { maximumSignificantDigits: 15 }); + const minimum = (rule: (typeof rules)[number]) => + Number((thresholds[rule].minimum_increase * (rule === "reliability" ? 100 : 1)).toPrecision(15)); + async function submit(event: React.FormEvent) { + event.preventDefault(); + const values = new FormData(event.currentTarget); + const threshold = (rule: (typeof rules)[number]) => ({ + multiplier: Number(values.get(`${rule}-multiplier`)), + minimum_increase: Number(values.get(`${rule}-minimum`)) / (rule === "reliability" ? 100 : 1), + }); + setSaved(false); + try { + const updated = { + spend: threshold("spend"), + traffic: threshold("traffic"), + latency: threshold("latency"), + reliability: threshold("reliability"), + }; + await save(updated); + setEditing(false); + setSaved(true); + } catch { + setSaved(false); + } + } + return ( +
void submit(event)} onChange={() => setSaved(false)} className="space-y-4 py-4"> +
+

{t("alerts.thresholds.title")}

+

{t("alerts.thresholds.description")}

+ {!enterprise &&

{t("alerts.thresholds.enterpriseOnly")}

} +
+ {!editing && ( +
+ {rules.map((rule) => ( +
+
{t(`alerts.thresholds.rules.${rule}`)}
+
+ {t(`alerts.thresholds.summary.${rule}`, { + multiplier: format.format(thresholds[rule].multiplier), + minimum: format.format(minimum(rule)), + })} +
+
+ ))} +
+ )} + {editing && + rules.map((rule) => ( +
+ {t(`alerts.thresholds.rules.${rule}`)} +
+ + + event.currentTarget.setCustomValidity( + event.currentTarget.valueAsNumber <= 1 ? t("alerts.thresholds.multiplierInvalid") : "", + ) + } + /> +
+
+ + +
+
+ ))} +

{t("alerts.thresholds.safeguards")}

+ {enterprise && !readOnly && ( +
+ {editing ? ( + <> + + + + ) : ( + + )} + {saved && ( + + {t("alerts.thresholds.saved")} + + )} +
+ )} +
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx index 9233172f2d3..40a5c9b375e 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx @@ -1,6 +1,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import userEvent from "@testing-library/user-event"; -import { act, renderWithProviders, screen, testQueryClient, waitFor, within } from "../../../tests/test-utils"; +import { + act, + fireEvent, + renderWithProviders, + screen, + testQueryClient, + waitFor, + within, +} from "../../../tests/test-utils"; import Alerts from "./Alerts"; import type { SavedAlert } from "./alertPresentation"; @@ -49,12 +57,21 @@ const calls: { path: string; query: URLSearchParams; method: string; body: unkno let status = "open"; let rejectWrite = false; let rejectHistory = false; +let includeDeprecation = false; +let deprecationStatus = "open"; +const thresholds = { + spend: { multiplier: 3, minimum_increase: 50 }, + traffic: { multiplier: 10, minimum_increase: 1000 }, + latency: { multiplier: 3, minimum_increase: 500 }, + reliability: { multiplier: 3, minimum_increase: 0.02 }, +}; let monitoring = { model_deprecations_enabled: true, enabled: true, alerts_used: 0, alert_limit: 5 as number | null, disabled_by_config: false, + thresholds, }; let rejectMonitoring = false; function response(data: unknown, code = 200) { @@ -72,6 +89,50 @@ function promptCacheResponse(url: URL): Response | undefined { }); } +function deprecationResponse(path: string, method: string, body: { status?: string } | null) { + if (path === "/model/deprecations") + return response({ + deprecated: [], + imminent: includeDeprecation + ? [ + { + model_name: "example-model", + litellm_model: "example/model", + deprecation_date: "2026-12-01", + status: "imminent", + }, + ] + : [], + }); + if (path === "/user/daily/activity/aggregated") + return response({ + results: [{ date: "2026-11-16", breakdown: { models: { "example/model": { metrics: { api_requests: 10 } } } } }], + }); + if (path === "/alerting/model-deprecations/state") { + if (method === "PUT") deprecationStatus = body?.status ?? "open"; + const state = { + model: "example/model", + deprecation_date: "2026-12-01", + status: deprecationStatus, + last_seen_date: "2026-11-16", + last_seen_requests: 10, + }; + return response(state); + } + return undefined; +} + +function updateAlertResponse(url: URL, body: { status: string; type?: string }) { + if (rejectWrite) return response({ detail: "Unavailable" }, 503); + status = body.status; + if (url.pathname === "/alerts") { + if (!body.type || body.type === "prompt_cache") + cacheIncidents = cacheIncidents.map((incident) => ({ ...incident, status })); + return response({ updated: 30 }); + } + return response({ ...(url.pathname === `/alerts/${cacheIncident.id}` ? cacheIncident : alert), status }); +} + describe("Alerts", () => { beforeEach(() => { testQueryClient.clear(); @@ -79,6 +140,8 @@ describe("Alerts", () => { status = "open"; rejectWrite = false; rejectHistory = false; + includeDeprecation = false; + deprecationStatus = "open"; rejectMonitoring = false; monitoring = { model_deprecations_enabled: true, @@ -86,6 +149,7 @@ describe("Alerts", () => { alerts_used: 0, alert_limit: 5, disabled_by_config: false, + thresholds, }; role = "proxy_admin"; cacheIncidents = []; @@ -94,12 +158,13 @@ describe("Alerts", () => { vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { const request = input instanceof Request ? input : new Request(input); const url = new URL(request.url); - const body = request.method === "PATCH" ? await request.clone().json() : null; + const body = ["PATCH", "PUT"].includes(request.method) ? await request.clone().json() : null; const call = { path: url.pathname, query: url.searchParams, method: request.method, body }; calls.push(call); if (url.pathname === "/litellm/.well-known/litellm-ui-config") return response({}); if (url.pathname === "/management/v1/users/me") return response({ user_id: "example-admin", user_role: role }); - if (url.pathname === "/model/deprecations") return response({ deprecated: [], imminent: [] }); + const retirement = deprecationResponse(url.pathname, request.method, body); + if (retirement) return retirement; if (url.pathname === "/alerts/monitoring") { if (rejectMonitoring) return response({ detail: "Unavailable" }, 503); if (request.method === "PATCH") { @@ -108,11 +173,7 @@ describe("Alerts", () => { } return response(monitoring); } - if (request.method === "PATCH") { - if (rejectWrite) return response({ detail: "Unavailable" }, 503); - status = body.status; - return response({ ...(url.pathname === `/alerts/${cacheIncident.id}` ? cacheIncident : alert), status }); - } + if (request.method === "PATCH") return updateAlertResponse(url, body); const promptCache = promptCacheResponse(url); if (promptCache) return promptCache; if (url.pathname === `/alerts/${alert.id}`) @@ -296,6 +357,122 @@ describe("Alerts", () => { ); }); + it.each(["Resolve all", "Dismiss all"])("%s updates the selected category beyond the loaded page", async (action) => { + const user = userEvent.setup(); + renderWithProviders(, { searchParams: { type: "spend" } }); + await screen.findByRole("button", { name: /Spend on Research agent/ }); + await user.click(screen.getByRole("button", { name: action })); + const dialog = screen.getByRole("alertdialog"); + expect(dialog).toHaveTextContent("including results you have not loaded"); + await user.click(within(dialog).getByRole("button", { name: action })); + expect(await screen.findByText("No open alerts.")).toBeInTheDocument(); + expect(calls.filter((call) => call.method === "PATCH")).toEqual([ + expect.objectContaining({ + path: "/alerts", + body: { status: action === "Resolve all" ? "resolved" : "dismissed", type: "spend" }, + }), + ]); + }); + + it("keeps bulk failures retryable and leaves open alerts visible", async () => { + rejectWrite = true; + const user = userEvent.setup(); + renderWithProviders(); + await screen.findByRole("button", { name: /Spend on Research agent/ }); + await user.click(screen.getByRole("button", { name: "Dismiss all" })); + await user.click(within(screen.getByRole("alertdialog")).getByRole("button", { name: "Dismiss all" })); + expect(await screen.findByRole("alert")).toHaveTextContent("Some alerts could not be updated"); + rejectWrite = false; + await user.click(within(screen.getByRole("alertdialog")).getByRole("button", { name: "Dismiss all" })); + expect(await screen.findByText("No open alerts.")).toBeInTheDocument(); + }); + + it("resolves performance, prompt cache, and retirement alerts together in All alerts", async () => { + includeDeprecation = true; + cacheIncidents = [cacheIncident]; + const user = userEvent.setup(); + renderWithProviders(); + await screen.findByRole("button", { name: /example\/model retires/ }); + await user.click(screen.getByRole("button", { name: "Resolve all" })); + await user.click(within(screen.getByRole("alertdialog")).getByRole("button", { name: "Resolve all" })); + expect(await screen.findByText("No open alerts.")).toBeInTheDocument(); + expect(calls.filter((call) => ["PATCH", "PUT"].includes(call.method))).toEqual([ + expect.objectContaining({ path: "/alerts", body: { status: "resolved" } }), + expect.objectContaining({ + path: "/alerting/model-deprecations/state", + body: { + model: "example/model", + deprecation_date: "2026-12-01", + status: "resolved", + last_seen_date: "2026-11-16", + last_seen_requests: 10, + }, + }), + ]); + }); + + it("shows the current thresholds to free admins without an edit action", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(await screen.findByRole("button", { name: "Manage alerts" })); + expect(await screen.findByText("At least 3× the usual spend and $50 higher.")).toBeInTheDocument(); + expect(screen.getByText("Editing thresholds requires Enterprise.")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Edit thresholds" })).not.toBeInTheDocument(); + }); + + it("edits Enterprise thresholds with explicit units and saves the values", async () => { + monitoring.alert_limit = null; + monitoring.thresholds = { ...thresholds, latency: { multiplier: 1.005, minimum_increase: 500 } }; + const user = userEvent.setup(); + renderWithProviders(); + await user.click(await screen.findByRole("button", { name: "Manage alerts" })); + await user.click(await screen.findByRole("button", { name: "Edit thresholds" })); + const spend = screen.getByRole("group", { name: "Spend" }); + const multiplier = within(spend).getByRole("spinbutton", { name: "Times the usual value" }); + fireEvent.change(multiplier, { target: { value: "1" } }); + await user.click(screen.getByRole("button", { name: "Save thresholds" })); + expect(multiplier).toBeInvalid(); + expect(calls.filter((call) => call.method === "PATCH")).toEqual([]); + fireEvent.change(multiplier, { + target: { value: "4" }, + }); + fireEvent.change(screen.getByRole("spinbutton", { name: "Minimum increase ($)" }), { target: { value: "75" } }); + fireEvent.change(screen.getByRole("spinbutton", { name: "Minimum increase (percentage points)" }), { + target: { value: "3.5" }, + }); + await user.click(screen.getByRole("button", { name: "Save thresholds" })); + expect(await screen.findByText("Thresholds saved.")).toBeInTheDocument(); + expect(screen.getByText("At least 4× the usual spend and $75 higher.")).toBeInTheDocument(); + expect(screen.getByText("At least 3× the usual error rate and 3.5 percentage points higher.")).toBeInTheDocument(); + expect(calls.find((call) => call.method === "PATCH")).toMatchObject({ + path: "/alerts/monitoring", + body: { + thresholds: { + ...thresholds, + spend: { multiplier: 4, minimum_increase: 75 }, + latency: { multiplier: 1.005, minimum_increase: 500 }, + reliability: { multiplier: 3, minimum_increase: 0.035 }, + }, + }, + }); + await user.click(screen.getByRole("button", { name: "Edit thresholds" })); + expect(screen.getByRole("spinbutton", { name: "Minimum increase ($)" })).toHaveValue(75); + expect(screen.getByRole("spinbutton", { name: "Minimum increase (percentage points)" })).toHaveValue(3.5); + }); + + it("shows free thresholds without editing and hides bulk actions for viewers", async () => { + role = "proxy_admin_viewer"; + const user = userEvent.setup(); + renderWithProviders(); + await screen.findByRole("button", { name: /Spend on Research agent/ }); + expect(screen.queryByRole("button", { name: "Resolve all" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Dismiss all" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Manage alerts" })); + expect(await screen.findByText("At least 3× the usual spend and $50 higher.")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Edit thresholds" })).not.toBeInTheDocument(); + expect(screen.queryByRole("spinbutton")).not.toBeInTheDocument(); + }); + it("lets viewers read the evidence without write controls", async () => { role = "proxy_admin_viewer"; renderWithProviders(, { searchParams: { alert: alert.id } }); diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx index 38eda125699..2fec41cba0f 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx @@ -19,6 +19,7 @@ import AlertActions from "./AlertActions"; import AlertChart from "./AlertChart"; import AlertDetail from "./AlertDetail"; import ManageAlerts from "./ManageAlerts"; +import BulkAlertActions from "./BulkAlertActions"; const parsers = { alert: parseAsString, @@ -119,6 +120,8 @@ function AlertsInbox({ auth }: { auth: ReturnType }) { const pagedLists = [...(showPromptCache ? [promptCache] : []), ...(showPerformance ? [list] : [])]; const loading = pagedLists.some((query) => query.isLoading) || deprecations.isLoading; const hasError = pagedLists.some((query) => query.isError) || (showDeprecations && deprecations.isError); + const refreshing = pagedLists.some((query) => query.isFetching) || deprecations.isFetching; + const bulkUnavailable = hasError || items.length === 0; function refresh() { void client.invalidateQueries({ queryKey: ["alert-monitoring"] }); pagedLists.forEach((query) => void query.refetch()); @@ -140,7 +143,7 @@ function AlertsInbox({ auth }: { auth: ReturnType }) { ))} -
+
- +
+ + +
{showDeprecations && deprecations.isError && ( void deprecations.refetch()}>{t("alerts.unified.deprecationsError")} diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/BulkAlertActions.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/BulkAlertActions.tsx new file mode 100644 index 00000000000..93cafa99f36 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/BulkAlertActions.tsx @@ -0,0 +1,104 @@ +import { useState } from "react"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/components/ui/button"; +import { + AlertDialog, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogCancel, +} from "@/components/ui/alert-dialog"; +import { backgroundFetchClient } from "@/lib/http/api"; +import type { DeprecationAlert } from "@/components/ModelDeprecationAlerts/deprecationAlerts"; +import { AlertStatus, effectiveAlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; +import type { AlertFilter } from "./alertPresentation"; +import { TYPE_LABEL_KEYS } from "./alertPresentation"; + +type BulkStatus = "resolved" | "dismissed"; + +export default function BulkAlertActions({ + filter, + state, + readOnly, + deprecations, + disabled, +}: { + filter: AlertFilter; + state: AlertStatus; + readOnly: boolean; + deprecations: DeprecationAlert[] | undefined; + disabled: boolean; +}) { + const { t } = useTranslation(); + const client = useQueryClient(); + const [status, setStatus] = useState(null); + const mutation = useMutation({ + mutationFn: async (next: BulkStatus) => { + if (filter !== "model-deprecations") { + const { data } = await backgroundFetchClient.PATCH("/alerts", { + body: { status: next, type: filter === "all" ? undefined : filter }, + }); + if (!data) throw new Error("Could not update alerts"); + } + for (const alert of deprecations ?? []) { + if (effectiveAlertStatus(alert) !== AlertStatus.Open) continue; + const { data } = await backgroundFetchClient.PUT("/alerting/model-deprecations/state", { + body: { + model: alert.usageModel, + deprecation_date: alert.model.deprecation_date, + status: next, + last_seen_date: alert.lastUsedDate, + last_seen_requests: alert.lastDayRequests, + }, + }); + if (!data) throw new Error("Could not update model deprecation alerts"); + } + }, + onSuccess: () => setStatus(null), + onSettled: () => + Promise.all(["alerts", "alert", "deprecationAlerts"].map((key) => client.invalidateQueries({ queryKey: [key] }))), + }); + function choose(next: BulkStatus) { + mutation.reset(); + setStatus(next); + } + if (readOnly || state !== AlertStatus.Open) return null; + return ( + <> + + + !open && !mutation.isPending && setStatus(null)}> + + + + {t(status === "resolved" ? "alerts.bulk.resolve" : "alerts.bulk.dismiss")} + + + {t("alerts.bulk.description", { type: t(TYPE_LABEL_KEYS[filter]) })} + + + {mutation.isError && ( +

+ {t("alerts.bulk.error")} +

+ )} + + {t("alerts.bulk.cancel")} + + +
+
+ + ); +} diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/ManageAlerts.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/ManageAlerts.tsx index 9959ce19050..a9d5178de46 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/ManageAlerts.tsx +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/ManageAlerts.tsx @@ -15,6 +15,7 @@ import { Switch } from "@/components/ui/switch"; import { backgroundFetchClient } from "@/lib/http/api"; import { getProxyBaseUrl } from "@/components/networking"; import type { components } from "@/lib/http/schema"; +import AlertThresholds from "./AlertThresholds"; function statusMessage(data: components["schemas"]["AlertMonitoring"]) { if (data.disabled_by_config) return "alerts.monitoring.disabledByConfig"; @@ -41,7 +42,10 @@ export default function ManageAlerts({ scope, readOnly }: { scope: string; readO const query = useQuery(queryOptions); const mutation = useMutation({ mutationFn: async ( - body: components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"], + body: + | components["schemas"]["AlertMonitoringUpdate"] + | components["schemas"]["ModelDeprecationMonitoringUpdate"] + | components["schemas"]["AlertThresholdsUpdate"], ) => { const { data } = await backgroundFetchClient.PATCH("/alerts/monitoring", { body }); if (!data) throw new Error("Could not save monitoring settings"); @@ -64,7 +68,7 @@ export default function ManageAlerts({ scope, readOnly }: { scope: string; readO {t("alerts.monitoring.manage")} - + {t("alerts.monitoring.manage")} {t("alerts.monitoring.description")} @@ -125,6 +129,13 @@ export default function ManageAlerts({ scope, readOnly }: { scope: string; readO aria-describedby="performance-monitoring-status" />
+ mutation.mutateAsync({ thresholds })} + /> )} {mutation.isError && ( diff --git a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json index 8f755b63373..2942e00fea8 100644 --- a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json @@ -6096,6 +6096,45 @@ "upgrade": "Upgrade to Enterprise", "loadError": "Could not load monitoring settings.", "saveError": "Could not save monitoring settings. Try again." + }, + "bulk": { + "resolve": "Resolve all", + "dismiss": "Dismiss all", + "cancel": "Cancel", + "saving": "Updating alerts…", + "description": "This updates all open alerts matching “{{type}}”, including results you have not loaded. You can reopen individual alerts later.", + "error": "Some alerts could not be updated. Saved changes were kept. Try again to update the remaining open alerts." + }, + "thresholds": { + "title": "Performance alert thresholds", + "description": "Usual values come from the previous seven days. Both conditions must be met.", + "enterpriseOnly": "Editing thresholds requires Enterprise.", + "edit": "Edit thresholds", + "cancel": "Cancel", + "multiplier": "Times the usual value", + "multiplierInvalid": "Use a multiplier greater than 1.", + "rules": { + "spend": "Spend", + "traffic": "Traffic", + "latency": "Latency", + "reliability": "Error rate" + }, + "minimum": { + "spend": "Minimum increase ($)", + "traffic": "Minimum increase (requests)", + "latency": "Minimum increase (ms)", + "reliability": "Minimum increase (percentage points)" + }, + "summary": { + "spend": "At least {{multiplier}}× the usual spend and ${{minimum}} higher.", + "traffic": "At least {{multiplier}}× the usual requests and {{minimum}} more requests.", + "latency": "At least {{multiplier}}× the usual latency and {{minimum}} ms higher.", + "reliability": "At least {{multiplier}}× the usual error rate and {{minimum}} percentage points higher." + }, + "safeguards": "Requires four active baseline days. Latency also needs 100 timed requests; error rate needs 500 requests. Changes apply to future checks.", + "save": "Save thresholds", + "saving": "Saving…", + "saved": "Thresholds saved." } }, "caching": { diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts index ada43316510..5acfacd2dfc 100644 --- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1019,7 +1019,11 @@ export interface paths { delete?: never; options?: never; head?: never; - patch?: never; + /** + * Resolve or dismiss matching open alerts + * @description Update all matching open saved alerts, including unloaded pages. Only gateway admins can write. + */ + patch: operations["update_open_alerts_alerts_patch"]; trace?: never; }; "/alerts/monitoring": { @@ -27750,6 +27754,28 @@ export interface components { */ window_start: string; }; + /** AlertBulkResult */ + AlertBulkResult: { + /** + * Updated + * @description Number of open saved alerts changed. + */ + updated: number; + }; + /** AlertBulkUpdate */ + AlertBulkUpdate: { + /** + * Status + * @description Status to apply to every matching open saved alert. + * @enum {string} + */ + status: "resolved" | "dismissed"; + /** + * Type + * @description Only update this category, or all saved alerts if omitted. + */ + type?: ("spend" | "traffic" | "latency" | "reliability") | ("model_deprecation" | "prompt_cache") | "performance" | null; + }; /** AlertHistoryPoint */ AlertHistoryPoint: { /** @@ -27792,6 +27818,28 @@ export interface components { * @default true */ model_deprecations_enabled: boolean; + /** + * @description Effective built-in thresholds. Only Enterprise admins can edit them. + * @default { + * "latency": { + * "minimum_increase": 500, + * "multiplier": 3 + * }, + * "reliability": { + * "minimum_increase": 0.02, + * "multiplier": 3 + * }, + * "spend": { + * "minimum_increase": 50, + * "multiplier": 3 + * }, + * "traffic": { + * "minimum_increase": 1000, + * "multiplier": 10 + * } + * } + */ + thresholds: components["schemas"]["PerformanceAlertThresholds"]; }; /** AlertMonitoringUpdate */ AlertMonitoringUpdate: { @@ -27801,6 +27849,24 @@ export interface components { */ enabled: boolean; }; + /** AlertThreshold */ + AlertThreshold: { + /** + * Minimum Increase + * @description Minimum absolute increase above the baseline, in the metric's unit. + */ + minimum_increase: number; + /** + * Multiplier + * @description Minimum multiple of the baseline. + */ + multiplier: number; + }; + /** AlertThresholdsUpdate */ + AlertThresholdsUpdate: { + /** @description Replace the built-in performance thresholds. Enterprise only. */ + thresholds: components["schemas"]["PerformanceAlertThresholds"]; + }; /** * AlertType * @description Enum for alert types and management event types @@ -43769,6 +43835,41 @@ export interface components { /** User Id */ user_id: string; }; + /** PerformanceAlertThresholds */ + PerformanceAlertThresholds: { + /** + * @description Average response-time increase in milliseconds over the previous seven-day weighted average. + * @default { + * "minimum_increase": 500, + * "multiplier": 3 + * } + */ + latency: components["schemas"]["AlertThreshold"]; + /** + * @description Error-rate increase as a fraction, e.g. 0.02 is two percentage points. + * @default { + * "minimum_increase": 0.02, + * "multiplier": 3 + * } + */ + reliability: components["schemas"]["AlertThreshold"]; + /** + * @description Spend increase in USD over the previous seven-day daily average. + * @default { + * "minimum_increase": 50, + * "multiplier": 3 + * } + */ + spend: components["schemas"]["AlertThreshold"]; + /** + * @description Request increase over the previous seven-day daily average. + * @default { + * "minimum_increase": 1000, + * "multiplier": 10 + * } + */ + traffic: components["schemas"]["AlertThreshold"]; + }; /** * PiiAction * @enum {string} @@ -56356,6 +56457,39 @@ export interface operations { }; }; }; + update_open_alerts_alerts_patch: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AlertBulkUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertBulkResult"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_monitoring_alerts_monitoring_get: { parameters: { query?: never; @@ -56385,7 +56519,7 @@ export interface operations { }; requestBody: { content: { - "application/json": components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"]; + "application/json": components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"] | components["schemas"]["AlertThresholdsUpdate"]; }; }; responses: { diff --git a/tests/integration/database/test_alerts.py b/tests/integration/database/test_alerts.py index e15a24133a6..e8874ee215a 100644 --- a/tests/integration/database/test_alerts.py +++ b/tests/integration/database/test_alerts.py @@ -336,6 +336,82 @@ async def test_daily_history_matches_scope_and_weighted_metrics( write_rows('DELETE FROM "LiteLLM_Alert" WHERE resource_id=%s', (resource,)) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("email", "alias", "expected"), + ( + ("person@example.com", "Example user", "person@example.com"), + ("person@example.com", None, "person@example.com"), + (None, "Example user", "Example user"), + ("", "Example user", "Example user"), + (" ", "Example user", "Example user"), + (None, None, None), + ), +) +async def test_user_alerts_display_email_in_list_and_detail( + alerts: Alerts, gateway: Gateway, email: str | None, alias: str | None, expected: str | None +) -> None: + write_rows( + 'INSERT INTO "LiteLLM_UserTable" (user_id, user_email, user_alias) VALUES (%s, %s, %s)', + (alerts.resource_id, email, alias), + ) + try: + saved: Final = await alerts.repository.record(alerts.detection().model_copy(update={"resource_type": "user"})) + detail: Final = gateway.request("GET", f"/alerts/{saved.id}") + assert detail.status_code == 200, detail.text + view: Final = AlertView.model_validate_json(detail.content) + assert view.resource_name == expected + assert view.resource_id == alerts.resource_id + assert view.evidence == saved.evidence + listed: Final = gateway.request("GET", "/alerts", params={"type": "spend", "limit": "1"}) + assert listed.status_code == 200, listed.text + assert AlertViewPage.model_validate_json(listed.content).data == (view,) + finally: + write_rows('DELETE FROM "LiteLLM_UserTable" WHERE user_id=%s', (alerts.resource_id,)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", ("resolved", "dismissed")) +async def test_bulk_status_updates_all_matching_open_pages_and_preserves_other_alerts( + alerts: Alerts, gateway: Gateway, status: str +) -> None: + saved: Final = tuple([await alerts.repository.record(alerts.detection(day=day)) for day in range(30)]) + other: Final = await alerts.repository.record(alerts.detection("traffic")) + already_resolved: Final = await alerts.repository.set_status(int(saved[0].id), "resolved", "previous-admin") + response: Final = gateway.request("PATCH", "/alerts", {"status": status, "type": "spend"}) + assert response.status_code == 200, response.text + assert response.json() == {"updated": 29} + for alert in saved[1:]: + changed: Final = await alerts.repository.get(int(alert.id)) + assert changed is not None + assert changed.status == status + assert changed.updated_by + assert changed.evidence == alert.evidence + assert await alerts.repository.get(int(saved[0].id)) == already_resolved + assert await alerts.repository.get(int(other.id)) == other + repeat: Final = gateway.request("PATCH", "/alerts", {"status": status, "type": "spend"}) + assert repeat.status_code == 200, repeat.text + assert repeat.json() == {"updated": 0} + + +@pytest.mark.asyncio +async def test_bulk_performance_filter_leaves_prompt_cache_until_all_alerts_are_updated( + alerts: Alerts, gateway: Gateway +) -> None: + spend: Final = await alerts.repository.record(alerts.detection()) + cache: Final = await alerts.repository.record(alerts.detection("prompt_cache")) + performance: Final = gateway.request("PATCH", "/alerts", {"status": "resolved", "type": "performance"}) + assert performance.status_code == 200, performance.text + resolved: Final = await alerts.repository.get(int(spend.id)) + assert resolved is not None and resolved.status == "resolved" + assert await alerts.repository.get(int(cache.id)) == cache + all_alerts: Final = gateway.request("PATCH", "/alerts", {"status": "dismissed"}) + assert all_alerts.status_code == 200, all_alerts.text + dismissed: Final = await alerts.repository.get(int(cache.id)) + assert dismissed is not None and dismissed.status == "dismissed" + assert await alerts.repository.get(int(spend.id)) == resolved + + @pytest.mark.asyncio async def test_type_filter_applies_before_cursor_limit(alerts: Alerts, gateway: Gateway) -> None: oldest: Final = await alerts.repository.record(alerts.detection("traffic")) diff --git a/tests/integration/database/test_performance_alerts.py b/tests/integration/database/test_performance_alerts.py index d1602554076..8f0205294e9 100644 --- a/tests/integration/database/test_performance_alerts.py +++ b/tests/integration/database/test_performance_alerts.py @@ -14,7 +14,13 @@ from litellm_proxy_extras.request_log_indexes import ensure_request_log_indexes from psycopg import sql from litellm.models.alert import AlertDetection, DeprecationEvidence, PromptCacheEvidence -from litellm.models.performance_alert import FREE_ALERT_LIMIT, SweepOwnershipLostError, SweepState +from litellm.models.performance_alert import ( + DEFAULT_THRESHOLDS, + FREE_ALERT_LIMIT, + PerformanceAlertThresholds, + SweepOwnershipLostError, + SweepState, +) from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.spend_tracking.performance_alerts import run_performance_alerts from litellm.proxy.utils import PrismaClient, ProxyLogging @@ -145,6 +151,38 @@ async def fixture() -> AsyncIterator[Fixture]: await client.disconnect() +@pytest.mark.asyncio +async def test_enterprise_thresholds_persist_and_change_detection_but_free_uses_defaults(fixture: Fixture) -> None: + from litellm.repositories.config_repository import ConfigRepository + + config: Final = ConfigRepository(fixture.client, use_writer=True) + previous: Final = await config.get_param("performance_alert_thresholds") + custom: Final = PerformanceAlertThresholds.model_validate({"spend": {"multiplier": 2, "minimum_increase": 20}}) + enterprise: Final = PerformanceAlertRepository(fixture.client, enterprise=True) + free: Final = PerformanceAlertRepository(fixture.client, enterprise=False) + resource: Final = f"{fixture.prefix}-custom-threshold" + _seed("team", resource, Day(spend=10), Day(spend=40)) + try: + assert (await enterprise.set_thresholds(custom)).thresholds == custom + restarted: Final = PerformanceAlertRepository(fixture.client, enterprise=True) + assert (await restarted.monitoring()).thresholds == custom + assert (await free.monitoring()).thresholds == DEFAULT_THRESHOLDS + with pytest.raises(PermissionError, match="Enterprise"): + await free.set_thresholds(custom) + claim: Final = await free.claim(timedelta(0)) + assert claim is not None + state: Final = SweepState(day=DAY, scope="team", cursor=fixture.prefix) + await free.evaluate_page(claim.owner, state, 25) + assert fixture.saved() == {} + await restarted.evaluate_page(claim.owner, state, 25) + assert fixture.saved() == {(resource, "spend"): ("team", "open")} + finally: + await config.set_param( + "performance_alert_thresholds", + previous.param_value if previous else DEFAULT_THRESHOLDS.model_dump(mode="json"), + ) + + @pytest.mark.asyncio async def test_a_full_sweep_saves_one_alert_per_real_change_and_nothing_for_normal_or_new_usage( fixture: Fixture, diff --git a/tests/unit/models/test_performance_alert.py b/tests/unit/models/test_performance_alert.py index 3d76b2bed79..d4a15ade89d 100644 --- a/tests/unit/models/test_performance_alert.py +++ b/tests/unit/models/test_performance_alert.py @@ -2,10 +2,13 @@ from datetime import date, datetime, timedelta, timezone from typing import Final import pytest +from pydantic import ValidationError from litellm.models.alert import AlertDetection, MetricEvidence from litellm.models.performance_alert import ( SCOPES, + AlertThreshold, + PerformanceAlertThresholds, PerformanceScope, ResourceWindow, SweepState, @@ -46,6 +49,55 @@ def test_steady_usage_raises_nothing() -> None: assert _types() == () +@pytest.mark.parametrize( + ("kind", "changes", "minimum", "too_high"), + ( + ("spend", {"spend": 40}, 20, 31), + ("traffic", {"requests": 2000}, 500, 1001), + ("latency", {"response_ms": 300_000}, 150, 201), + ("reliability", {"failed": 20}, 0.005, 0.011), + ), +) +def test_custom_multiplier_and_minimum_both_control_each_existing_rule( + kind: str, changes: dict[str, object], minimum: float, too_high: float +) -> None: + window: Final = _window(**changes) + assert detect("team", window, DAY) == () + thresholds: Final = PerformanceAlertThresholds.model_validate( + {kind: {"multiplier": 2, "minimum_increase": minimum}} + ) + detected: Final = detect("team", window, DAY, thresholds) + assert tuple(alert.type for alert in detected) == (kind,) + for threshold in ( + AlertThreshold(multiplier=5, minimum_increase=minimum), + AlertThreshold(multiplier=2, minimum_increase=too_high), + ): + assert detect("team", window, DAY, thresholds.model_copy(update={kind: threshold})) == () + + +@pytest.mark.parametrize( + "settings", + ( + {"spend": {"multiplier": 1, "minimum_increase": 1}}, + {"spend": {"multiplier": True, "minimum_increase": 1}}, + {"latency": {"multiplier": 2, "minimum_increase": -1}}, + {"traffic": {"multiplier": float("inf"), "minimum_increase": 1}}, + {"reliability": {"multiplier": 2, "minimum_increase": 1.01}}, + {"new_rule": {"multiplier": 2, "minimum_increase": 1}}, + ), +) +def test_thresholds_reject_invalid_values_and_new_rules(settings: dict[str, object]) -> None: + with pytest.raises(ValidationError): + PerformanceAlertThresholds.model_validate(settings) + + +def test_zero_minimum_does_not_report_a_zero_error_rate_as_an_increase() -> None: + thresholds: Final = PerformanceAlertThresholds.model_validate( + {"reliability": {"multiplier": 2, "minimum_increase": 0}} + ) + assert detect("team", _window(failed=0, base_failed=0), DAY, thresholds) == () + + @pytest.mark.parametrize("scope", SCOPES) @pytest.mark.parametrize("requests", (0, 100)) def test_quiet_or_idle_resources_do_not_raise_traffic_alerts(scope: PerformanceScope, requests: int) -> None: diff --git a/tests/unit/proxy/alerting_endpoints/test_alerts.py b/tests/unit/proxy/alerting_endpoints/test_alerts.py index 332d62b4e04..98724bfbebe 100644 --- a/tests/unit/proxy/alerting_endpoints/test_alerts.py +++ b/tests/unit/proxy/alerting_endpoints/test_alerts.py @@ -8,6 +8,7 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from pydantic import BaseModel, ConfigDict +from litellm.models.performance_alert import DEFAULT_THRESHOLDS from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.alerting_endpoints.alerts import router from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -73,7 +74,9 @@ def client(app: FastAPI) -> Iterator[TestClient]: @pytest.mark.parametrize( "role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.TEAM, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) ) -@pytest.mark.parametrize("method,path", (("GET", "/alerts"), ("GET", "/alerts/5"), ("PATCH", "/alerts/5"))) +@pytest.mark.parametrize( + "method,path", (("GET", "/alerts"), ("GET", "/alerts/5"), ("PATCH", "/alerts/5"), ("PATCH", "/alerts")) +) def test_permissions_precede_database_access( app: FastAPI, table: MagicMock, role: LitellmUserRoles, method: str, path: str ) -> None: @@ -142,6 +145,7 @@ def test_deprecation_preference_updates_without_changing_performance(app: FastAP "alerts_used": 3, "alert_limit": 5, "disabled_by_config": False, + "thresholds": DEFAULT_THRESHOLDS.model_dump(), } prisma.writer_db.litellm_config.upsert.assert_awaited_once_with( where={"param_name": "model_deprecation_alerts_enabled"}, @@ -171,11 +175,42 @@ def test_config_opt_out_cannot_be_enabled_from_the_api(app: FastAPI) -> None: "alert_limit": 5, "disabled_by_config": True, "model_deprecations_enabled": True, + "thresholds": DEFAULT_THRESHOLDS.model_dump(), } assert response.status_code == 409, response.text transaction.execute_raw.assert_not_awaited() +@pytest.mark.parametrize("enterprise", (False, True)) +@pytest.mark.parametrize("role", (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)) +def test_only_enterprise_admins_can_save_thresholds(app: FastAPI, enterprise: bool, role: LitellmUserRoles) -> None: + prisma: Final = MagicMock() + prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=None) + prisma.writer_db.litellm_config.upsert = AsyncMock() + transaction: Final = prisma.writer_db.tx.return_value.__aenter__.return_value + transaction.query_raw = AsyncMock(return_value=[{"enabled": True, "alerts_used": 0}]) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role) + app.dependency_overrides[get_alert_monitoring_repository] = lambda: PerformanceAlertRepository( + prisma, enterprise=enterprise + ) + with TestClient(app) as client: + response: Final = client.patch( + "/alerts/monitoring", json={"thresholds": {"spend": {"multiplier": 2, "minimum_increase": 20}}} + ) + if enterprise and role == LitellmUserRoles.PROXY_ADMIN: + assert response.status_code == 200, response.text + prisma.writer_db.litellm_config.upsert.assert_awaited_once() + else: + assert response.status_code == 403, response.text + prisma.writer_db.litellm_config.upsert.assert_not_awaited() + + +@pytest.mark.parametrize("body", ({"status": "open"}, {"status": "resolved", "type": "unknown"}, {})) +def test_bulk_update_rejects_reopening_or_unknown_filters(client: TestClient, body: object) -> None: + response: Final = client.patch("/alerts", json=body) + assert response.status_code == 422, response.text + + @pytest.mark.parametrize("alert_id", ("0", "-1", "01", "1.0", "oops", "9223372036854775808", "1" * 20)) @pytest.mark.parametrize("method", ("GET", "PATCH")) def test_invalid_id_never_accesses_database(client: TestClient, table: MagicMock, alert_id: str, method: str) -> None: