feat(alerts): add email labels, bulk actions and threshold editing (#52139)

* feat(alerts): add bulk actions and enterprise thresholds

* fix(alerts): accept fractional multipliers above one
This commit is contained in:
moe-berri 2026-11-17 07:32:19 +08:00 • committed by GitHub
parent ea987e7856
commit ea448a8951
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 993 additions and 47 deletions

View file

@ -178,6 +178,21 @@ class AlertUpdate(BaseModel):
status: AlertStatus = Field(description="Mark resolved, dismiss, or reopen the alert.")
class AlertBulkUpdate(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
status: Literal["resolved", "dismissed"] = Field(description="Status to apply to every matching open saved alert.")
type: AlertFilter | None = Field(
default=None, description="Only update this category, or all saved alerts if omitted."
)
class AlertBulkResult(BaseModel):
model_config = ConfigDict(frozen=True)
updated: int = Field(ge=0, description="Number of open saved alerts changed.")
class AlertPage(BaseModel):
model_config = ConfigDict(frozen=True)
@ -206,3 +221,5 @@ class AlertStore(Protocol):
async def get(self, alert_id: int) -> Alert | None: ...
async def set_status(self, alert_id: int, status: AlertStatus, actor: str) -> Alert | None: ...
async def set_open_status(self, data: AlertBulkUpdate, actor: str) -> int: ...

View file

@ -3,7 +3,7 @@ from dataclasses import dataclass
from datetime import date, datetime, time, timedelta, timezone
from typing import Final, Literal, Protocol, TypeAlias
from pydantic import BaseModel, ConfigDict, Field
from pydantic import BaseModel, ConfigDict, Field, field_validator
from litellm.models.alert import MAX_ALERT_ID, AlertDetection, AlertType, MetricEvidence
@ -25,6 +25,49 @@ RELIABILITY_MIN_INCREASE: Final = 0.02
RELIABILITY_MIN_REQUESTS: Final = 500
class AlertThreshold(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
multiplier: float = Field(gt=1, allow_inf_nan=False, strict=True, description="Minimum multiple of the baseline.")
minimum_increase: float = Field(
ge=0,
allow_inf_nan=False,
strict=True,
description="Minimum absolute increase above the baseline, in the metric's unit.",
)
class PerformanceAlertThresholds(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
spend: AlertThreshold = Field(
default=AlertThreshold(multiplier=SPEND_MULTIPLIER, minimum_increase=SPEND_MIN_INCREASE_USD),
description="Spend increase in USD over the previous seven-day daily average.",
)
traffic: AlertThreshold = Field(
default=AlertThreshold(multiplier=TRAFFIC_SPIKE_MULTIPLIER, minimum_increase=TRAFFIC_SPIKE_MIN_INCREASE),
description="Request increase over the previous seven-day daily average.",
)
latency: AlertThreshold = Field(
default=AlertThreshold(multiplier=LATENCY_MULTIPLIER, minimum_increase=LATENCY_MIN_INCREASE_MS),
description="Average response-time increase in milliseconds over the previous seven-day weighted average.",
)
reliability: AlertThreshold = Field(
default=AlertThreshold(multiplier=RELIABILITY_MULTIPLIER, minimum_increase=RELIABILITY_MIN_INCREASE),
description="Error-rate increase as a fraction, e.g. 0.02 is two percentage points.",
)
@field_validator("reliability")
@classmethod
def validate_error_rate(cls, value: AlertThreshold) -> AlertThreshold:
if value.minimum_increase > 1:
raise ValueError("Error-rate minimum increase cannot exceed 1")
return value
DEFAULT_THRESHOLDS: Final = PerformanceAlertThresholds()
class ResourceWindow(BaseModel):
"""One resource's totals for the evaluated day, and summed over the seven days before it."""
@ -89,6 +132,9 @@ class AlertMonitoring(BaseModel):
alerts_used: int = Field(ge=0, le=FREE_ALERT_LIMIT, description="Lifetime free allowance consumed, capped at five.")
alert_limit: int | None = Field(description="Free allowance, or null for Enterprise.")
disabled_by_config: bool = Field(description="Whether performance_alerts_enabled is false in proxy settings.")
thresholds: PerformanceAlertThresholds = Field(
default=DEFAULT_THRESHOLDS, description="Effective built-in thresholds. Only Enterprise admins can edit them."
)
@property
def active(self) -> bool:
@ -107,10 +153,19 @@ class ModelDeprecationMonitoringUpdate(BaseModel):
model_deprecations_enabled: bool = Field(strict=True, description="Enable or disable model deprecation alerts.")
class AlertThresholdsUpdate(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
thresholds: PerformanceAlertThresholds = Field(
description="Replace the built-in performance thresholds. Enterprise only."
)
class AlertMonitoringStore(Protocol):
async def monitoring(self) -> AlertMonitoring: ...
async def set_enabled(self, enabled: bool) -> AlertMonitoring: ...
async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring: ...
async def set_thresholds(self, thresholds: PerformanceAlertThresholds) -> AlertMonitoring: ...
class PerformanceAlertStore(Protocol):
@ -181,58 +236,65 @@ def _alert(
)
def _spend_spike(window: ResourceWindow) -> tuple[float, float] | None:
def _spend_spike(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None:
baseline: Final = window.base_spend / BASELINE_DAYS
triggered: Final = (
window.active_days >= MIN_ACTIVE_DAYS
and baseline > 0
and window.spend >= SPEND_MULTIPLIER * baseline
and window.spend - baseline >= SPEND_MIN_INCREASE_USD
and window.spend >= threshold.multiplier * baseline
and window.spend - baseline >= threshold.minimum_increase
)
return (window.spend, baseline) if triggered else None
def _traffic_spike(window: ResourceWindow) -> tuple[float, float] | None:
def _traffic_spike(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None:
baseline: Final = window.base_requests / BASELINE_DAYS
spike: Final = (
window.active_days >= MIN_ACTIVE_DAYS
and window.requests >= TRAFFIC_SPIKE_MULTIPLIER * baseline
and window.requests - baseline >= TRAFFIC_SPIKE_MIN_INCREASE
and window.requests > baseline
and window.requests >= threshold.multiplier * baseline
and window.requests - baseline >= threshold.minimum_increase
)
return (window.requests, baseline) if spike else None
def _slower_responses(window: ResourceWindow) -> tuple[float, float] | None:
def _slower_responses(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None:
current: Final = _ratio(window.response_ms, window.timed)
baseline: Final = _ratio(window.base_response_ms, window.base_timed)
triggered: Final = (
window.timed >= LATENCY_MIN_TIMED_REQUESTS
and window.timed_days >= MIN_ACTIVE_DAYS
and baseline > 0
and current >= LATENCY_MULTIPLIER * baseline
and current - baseline >= LATENCY_MIN_INCREASE_MS
and current >= threshold.multiplier * baseline
and current - baseline >= threshold.minimum_increase
)
return (current, baseline) if triggered else None
def _more_failures(window: ResourceWindow) -> tuple[float, float] | None:
def _more_failures(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None:
current: Final = min(1.0, _ratio(window.failed, window.requests))
baseline: Final = min(1.0, _ratio(window.base_failed, window.base_requests))
triggered: Final = (
window.active_days >= MIN_ACTIVE_DAYS
and window.requests >= RELIABILITY_MIN_REQUESTS
and current >= RELIABILITY_MULTIPLIER * baseline
and current - baseline >= RELIABILITY_MIN_INCREASE
and current > baseline
and current >= threshold.multiplier * baseline
and current - baseline >= threshold.minimum_increase
)
return (current, baseline) if triggered else None
def detect(scope: PerformanceScope, window: ResourceWindow, day: date) -> tuple[AlertDetection, ...]:
def detect(
scope: PerformanceScope,
window: ResourceWindow,
day: date,
thresholds: PerformanceAlertThresholds = DEFAULT_THRESHOLDS,
) -> tuple[AlertDetection, ...]:
checks: Final[tuple[tuple[AlertType, tuple[float, float] | None, int], ...]] = (
("spend", _spend_spike(window), window.requests),
("traffic", _traffic_spike(window), window.requests),
("latency", _slower_responses(window), window.timed),
("reliability", _more_failures(window), window.requests),
("spend", _spend_spike(window, thresholds.spend), window.requests),
("traffic", _traffic_spike(window, thresholds.traffic), window.requests),
("latency", _slower_responses(window, thresholds.latency), window.timed),
("reliability", _more_failures(window, thresholds.reliability), window.requests),
)
return tuple(
_alert(scope, window, day, kind, found[0], found[1], sample_size)

View file

@ -4,6 +4,8 @@ from fastapi import APIRouter, Depends, HTTPException, Path, Query
from litellm.models.alert import (
Alert,
AlertBulkResult,
AlertBulkUpdate,
AlertFilter,
AlertID,
AlertStatus,
@ -12,7 +14,12 @@ from litellm.models.alert import (
AlertView,
AlertViewPage,
)
from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate, ModelDeprecationMonitoringUpdate
from litellm.models.performance_alert import (
AlertMonitoring,
AlertMonitoringUpdate,
AlertThresholdsUpdate,
ModelDeprecationMonitoringUpdate,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.deps.repositories import AlertMonitoringRepositoryDep, AlertRepositoryDep
@ -60,6 +67,13 @@ async def list_alerts(
return AlertViewPage(data=await store.enrich(page.data), next_cursor=page.next_cursor)
@router.patch("/alerts", response_model=AlertBulkResult, summary="Resolve or dismiss matching open alerts")
async def update_open_alerts(data: AlertBulkUpdate, user: Auth, repository: AlertRepositoryDep) -> AlertBulkResult:
"""Update all matching open saved alerts, including unloaded pages. Only gateway admins can write."""
store: Final = _authorized_store(user, repository, write=True)
return AlertBulkResult(updated=await store.set_open_status(data, user.user_id or "proxy-admin"))
@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read alert monitoring settings")
async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -> AlertMonitoring:
"""Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin."""
@ -71,12 +85,18 @@ async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -
@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable alert monitoring")
async def update_monitoring(
data: AlertMonitoringUpdate | ModelDeprecationMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep
data: AlertMonitoringUpdate | ModelDeprecationMonitoringUpdate | AlertThresholdsUpdate,
user: Auth,
repository: AlertMonitoringRepositoryDep,
) -> AlertMonitoring:
"""Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts."""
_authorize(user, write=True)
if repository is None:
raise HTTPException(status_code=503, detail="Alerts require a database connection")
if isinstance(data, AlertThresholdsUpdate):
if (await repository.monitoring()).alert_limit is not None:
raise HTTPException(status_code=403, detail="Custom alert thresholds require Enterprise")
return await repository.set_thresholds(data.thresholds)
if isinstance(data, ModelDeprecationMonitoringUpdate):
return await repository.set_model_deprecations_enabled(data.model_deprecations_enabled)
if data.enabled and (await repository.monitoring()).disabled_by_config:

View file

@ -18,6 +18,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.models.alert import (
METRIC_ALERT_TYPES,
Alert,
AlertBulkUpdate,
AlertDetection,
AlertFilter,
AlertHistoryPoint,
@ -36,6 +37,7 @@ if TYPE_CHECKING:
class _HistoryTransaction(Protocol):
async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ...
async def execute_raw(self, query: str, *args: object) -> int: ...
class _HistoryDatabase(Protocol):
@ -95,7 +97,7 @@ _HISTORY_SOURCES: Final[
"d.user_id",
"api_requests",
'LEFT JOIN "LiteLLM_UserTable" m ON m.user_id = r.resource_id',
"m.user_alias",
"COALESCE(NULLIF(TRIM(m.user_email), ''), m.user_alias)",
),
"key": (
'"LiteLLM_DailyUserSpend"',
@ -260,6 +262,20 @@ class AlertRepository:
row: Final = await self._table.update(where={"id": alert_id}, data={"status": status, "updated_by": actor})
return None if row is None else _alert(row)
async def set_open_status(self, data: AlertBulkUpdate, actor: str) -> int:
async with self._database.tx(timeout=timedelta(seconds=10)) as transaction:
await transaction.query_raw("SELECT set_config('statement_timeout', '6s', true)")
return await transaction.execute_raw(
"""UPDATE "LiteLLM_Alert" SET status = $1, updated_by = $2,
updated_at = now() AT TIME ZONE 'UTC'
WHERE status = 'open' AND
($3::text IS NULL OR type = $3 OR ($3 = 'performance' AND type = ANY($4::text[])))""",
data.status,
actor,
data.type,
list(METRIC_ALERT_TYPES),
)
async def enrich(self, alerts: tuple[Alert, ...]) -> tuple[AlertView, ...]:
from prisma.errors import PrismaError

View file

@ -12,9 +12,11 @@ from typing_extensions import LiteralString
from litellm.models.alert import METRIC_ALERT_TYPES
from litellm.models.performance_alert import (
BASELINE_DAYS,
DEFAULT_THRESHOLDS,
FREE_ALERT_LIMIT,
AlertMonitoring,
Claim,
PerformanceAlertThresholds,
PerformanceScope,
ResourceUnreadableError,
ResourceWindow,
@ -196,9 +198,24 @@ class PerformanceAlertRepository:
async with self._transaction() as transaction:
monitoring: Final = await self._monitoring(transaction)
return monitoring.model_copy(
update={"model_deprecations_enabled": await self._config.model_deprecations_enabled()}
update={
"model_deprecations_enabled": await self._config.model_deprecations_enabled(),
"thresholds": await self._thresholds(),
}
)
async def _thresholds(self) -> PerformanceAlertThresholds:
if not self._enterprise:
return DEFAULT_THRESHOLDS
saved: Final = await self._config.get_param("performance_alert_thresholds")
return DEFAULT_THRESHOLDS if saved is None else PerformanceAlertThresholds.model_validate(saved.param_value)
async def set_thresholds(self, thresholds: PerformanceAlertThresholds) -> AlertMonitoring:
if not self._enterprise:
raise PermissionError("Custom alert thresholds require Enterprise")
await self._config.set_param("performance_alert_thresholds", thresholds.model_dump(mode="json"))
return await self.monitoring()
async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring:
await self._config.set_model_deprecations_enabled(enabled)
return await self.monitoring()
@ -258,6 +275,7 @@ class PerformanceAlertRepository:
raise
async def _evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState | None:
thresholds: Final = await self._thresholds()
async with self._transaction() as transaction:
monitoring: Final = await self._monitoring(transaction, lock=True)
if not monitoring.active:
@ -267,7 +285,7 @@ class PerformanceAlertRepository:
)
saved: Final = await record_alerts(
transaction,
tuple(chain.from_iterable(detect(state.scope, window, state.day) for window in windows)),
tuple(chain.from_iterable(detect(state.scope, window, state.day, thresholds) for window in windows)),
None if self._enterprise else FREE_ALERT_LIMIT - monitoring.alerts_used,
)
if saved:

View file

@ -0,0 +1,140 @@
import { useState } from "react";
import { useTranslation } from "react-i18next";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import type { components } from "@/lib/http/schema";
type Thresholds = components["schemas"]["PerformanceAlertThresholds"];
const rules = ["spend", "traffic", "latency", "reliability"] as const;
export default function AlertThresholds({
thresholds,
enterprise,
readOnly,
pending,
save,
}: {
thresholds: Thresholds;
enterprise: boolean;
readOnly: boolean;
pending: boolean;
save: (thresholds: Thresholds) => Promise<unknown>;
}) {
const { t, i18n } = useTranslation();
const [editing, setEditing] = useState(false);
const [saved, setSaved] = useState(false);
const disabled = !enterprise || readOnly || pending;
const format = new Intl.NumberFormat(i18n.language, { maximumSignificantDigits: 15 });
const minimum = (rule: (typeof rules)[number]) =>
Number((thresholds[rule].minimum_increase * (rule === "reliability" ? 100 : 1)).toPrecision(15));
async function submit(event: React.FormEvent<HTMLFormElement>) {
event.preventDefault();
const values = new FormData(event.currentTarget);
const threshold = (rule: (typeof rules)[number]) => ({
multiplier: Number(values.get(`${rule}-multiplier`)),
minimum_increase: Number(values.get(`${rule}-minimum`)) / (rule === "reliability" ? 100 : 1),
});
setSaved(false);
try {
const updated = {
spend: threshold("spend"),
traffic: threshold("traffic"),
latency: threshold("latency"),
reliability: threshold("reliability"),
};
await save(updated);
setEditing(false);
setSaved(true);
} catch {
setSaved(false);
}
}
return (
<form onSubmit={(event) => void submit(event)} onChange={() => setSaved(false)} className="space-y-4 py-4">
<div className="space-y-1">
<h3 className="text-sm font-semibold">{t("alerts.thresholds.title")}</h3>
<p className="text-sm text-muted-foreground">{t("alerts.thresholds.description")}</p>
{!enterprise && <p className="text-sm text-muted-foreground">{t("alerts.thresholds.enterpriseOnly")}</p>}
</div>
{!editing && (
<dl className="space-y-3 text-sm">
{rules.map((rule) => (
<div key={rule} className="space-y-1">
<dt className="font-medium">{t(`alerts.thresholds.rules.${rule}`)}</dt>
<dd className="tabular-nums text-muted-foreground">
{t(`alerts.thresholds.summary.${rule}`, {
multiplier: format.format(thresholds[rule].multiplier),
minimum: format.format(minimum(rule)),
})}
</dd>
</div>
))}
</dl>
)}
{editing &&
rules.map((rule) => (
<fieldset key={rule} disabled={disabled} className="grid grid-cols-2 gap-3">
<legend className="mb-2 text-sm font-medium">{t(`alerts.thresholds.rules.${rule}`)}</legend>
<div className="space-y-1">
<label htmlFor={`${rule}-multiplier`} className="text-xs text-muted-foreground">
{t("alerts.thresholds.multiplier")}
</label>
<Input
id={`${rule}-multiplier`}
name={`${rule}-multiplier`}
type="number"
min="1"
step="any"
required
defaultValue={thresholds[rule].multiplier}
onChange={(event) =>
event.currentTarget.setCustomValidity(
event.currentTarget.valueAsNumber <= 1 ? t("alerts.thresholds.multiplierInvalid") : "",
)
}
/>
</div>
<div className="space-y-1">
<label htmlFor={`${rule}-minimum`} className="text-xs text-muted-foreground">
{t(`alerts.thresholds.minimum.${rule}`)}
</label>
<Input
id={`${rule}-minimum`}
name={`${rule}-minimum`}
type="number"
min="0"
step="any"
required
max={rule === "reliability" ? 100 : undefined}
defaultValue={minimum(rule)}
/>
</div>
</fieldset>
))}
<p className="text-xs text-muted-foreground">{t("alerts.thresholds.safeguards")}</p>
{enterprise && !readOnly && (
<div className="flex items-center gap-3">
{editing ? (
<>
<Button type="submit" size="sm" disabled={pending}>
{t(pending ? "alerts.thresholds.saving" : "alerts.thresholds.save")}
</Button>
<Button type="button" variant="ghost" size="sm" disabled={pending} onClick={() => setEditing(false)}>
{t("alerts.thresholds.cancel")}
</Button>
</>
) : (
<Button type="button" variant="outline" size="sm" onClick={() => setEditing(true)}>
{t("alerts.thresholds.edit")}
</Button>
)}
{saved && (
<span role="status" className="text-sm text-muted-foreground">
{t("alerts.thresholds.saved")}
</span>
)}
</div>
)}
</form>
);
}

View file

@ -1,6 +1,14 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import userEvent from "@testing-library/user-event";
import { act, renderWithProviders, screen, testQueryClient, waitFor, within } from "../../../tests/test-utils";
import {
act,
fireEvent,
renderWithProviders,
screen,
testQueryClient,
waitFor,
within,
} from "../../../tests/test-utils";
import Alerts from "./Alerts";
import type { SavedAlert } from "./alertPresentation";
@ -49,12 +57,21 @@ const calls: { path: string; query: URLSearchParams; method: string; body: unkno
let status = "open";
let rejectWrite = false;
let rejectHistory = false;
let includeDeprecation = false;
let deprecationStatus = "open";
const thresholds = {
spend: { multiplier: 3, minimum_increase: 50 },
traffic: { multiplier: 10, minimum_increase: 1000 },
latency: { multiplier: 3, minimum_increase: 500 },
reliability: { multiplier: 3, minimum_increase: 0.02 },
};
let monitoring = {
model_deprecations_enabled: true,
enabled: true,
alerts_used: 0,
alert_limit: 5 as number | null,
disabled_by_config: false,
thresholds,
};
let rejectMonitoring = false;
function response(data: unknown, code = 200) {
@ -72,6 +89,50 @@ function promptCacheResponse(url: URL): Response | undefined {
});
}
function deprecationResponse(path: string, method: string, body: { status?: string } | null) {
if (path === "/model/deprecations")
return response({
deprecated: [],
imminent: includeDeprecation
? [
{
model_name: "example-model",
litellm_model: "example/model",
deprecation_date: "2026-12-01",
status: "imminent",
},
]
: [],
});
if (path === "/user/daily/activity/aggregated")
return response({
results: [{ date: "2026-11-16", breakdown: { models: { "example/model": { metrics: { api_requests: 10 } } } } }],
});
if (path === "/alerting/model-deprecations/state") {
if (method === "PUT") deprecationStatus = body?.status ?? "open";
const state = {
model: "example/model",
deprecation_date: "2026-12-01",
status: deprecationStatus,
last_seen_date: "2026-11-16",
last_seen_requests: 10,
};
return response(state);
}
return undefined;
}
function updateAlertResponse(url: URL, body: { status: string; type?: string }) {
if (rejectWrite) return response({ detail: "Unavailable" }, 503);
status = body.status;
if (url.pathname === "/alerts") {
if (!body.type || body.type === "prompt_cache")
cacheIncidents = cacheIncidents.map((incident) => ({ ...incident, status }));
return response({ updated: 30 });
}
return response({ ...(url.pathname === `/alerts/${cacheIncident.id}` ? cacheIncident : alert), status });
}
describe("Alerts", () => {
beforeEach(() => {
testQueryClient.clear();
@ -79,6 +140,8 @@ describe("Alerts", () => {
status = "open";
rejectWrite = false;
rejectHistory = false;
includeDeprecation = false;
deprecationStatus = "open";
rejectMonitoring = false;
monitoring = {
model_deprecations_enabled: true,
@ -86,6 +149,7 @@ describe("Alerts", () => {
alerts_used: 0,
alert_limit: 5,
disabled_by_config: false,
thresholds,
};
role = "proxy_admin";
cacheIncidents = [];
@ -94,12 +158,13 @@ describe("Alerts", () => {
vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
const request = input instanceof Request ? input : new Request(input);
const url = new URL(request.url);
const body = request.method === "PATCH" ? await request.clone().json() : null;
const body = ["PATCH", "PUT"].includes(request.method) ? await request.clone().json() : null;
const call = { path: url.pathname, query: url.searchParams, method: request.method, body };
calls.push(call);
if (url.pathname === "/litellm/.well-known/litellm-ui-config") return response({});
if (url.pathname === "/management/v1/users/me") return response({ user_id: "example-admin", user_role: role });
if (url.pathname === "/model/deprecations") return response({ deprecated: [], imminent: [] });
const retirement = deprecationResponse(url.pathname, request.method, body);
if (retirement) return retirement;
if (url.pathname === "/alerts/monitoring") {
if (rejectMonitoring) return response({ detail: "Unavailable" }, 503);
if (request.method === "PATCH") {
@ -108,11 +173,7 @@ describe("Alerts", () => {
}
return response(monitoring);
}
if (request.method === "PATCH") {
if (rejectWrite) return response({ detail: "Unavailable" }, 503);
status = body.status;
return response({ ...(url.pathname === `/alerts/${cacheIncident.id}` ? cacheIncident : alert), status });
}
if (request.method === "PATCH") return updateAlertResponse(url, body);
const promptCache = promptCacheResponse(url);
if (promptCache) return promptCache;
if (url.pathname === `/alerts/${alert.id}`)
@ -296,6 +357,122 @@ describe("Alerts", () => {
);
});
it.each(["Resolve all", "Dismiss all"])("%s updates the selected category beyond the loaded page", async (action) => {
const user = userEvent.setup();
renderWithProviders(<Alerts />, { searchParams: { type: "spend" } });
await screen.findByRole("button", { name: /Spend on Research agent/ });
await user.click(screen.getByRole("button", { name: action }));
const dialog = screen.getByRole("alertdialog");
expect(dialog).toHaveTextContent("including results you have not loaded");
await user.click(within(dialog).getByRole("button", { name: action }));
expect(await screen.findByText("No open alerts.")).toBeInTheDocument();
expect(calls.filter((call) => call.method === "PATCH")).toEqual([
expect.objectContaining({
path: "/alerts",
body: { status: action === "Resolve all" ? "resolved" : "dismissed", type: "spend" },
}),
]);
});
it("keeps bulk failures retryable and leaves open alerts visible", async () => {
rejectWrite = true;
const user = userEvent.setup();
renderWithProviders(<Alerts />);
await screen.findByRole("button", { name: /Spend on Research agent/ });
await user.click(screen.getByRole("button", { name: "Dismiss all" }));
await user.click(within(screen.getByRole("alertdialog")).getByRole("button", { name: "Dismiss all" }));
expect(await screen.findByRole("alert")).toHaveTextContent("Some alerts could not be updated");
rejectWrite = false;
await user.click(within(screen.getByRole("alertdialog")).getByRole("button", { name: "Dismiss all" }));
expect(await screen.findByText("No open alerts.")).toBeInTheDocument();
});
it("resolves performance, prompt cache, and retirement alerts together in All alerts", async () => {
includeDeprecation = true;
cacheIncidents = [cacheIncident];
const user = userEvent.setup();
renderWithProviders(<Alerts />);
await screen.findByRole("button", { name: /example\/model retires/ });
await user.click(screen.getByRole("button", { name: "Resolve all" }));
await user.click(within(screen.getByRole("alertdialog")).getByRole("button", { name: "Resolve all" }));
expect(await screen.findByText("No open alerts.")).toBeInTheDocument();
expect(calls.filter((call) => ["PATCH", "PUT"].includes(call.method))).toEqual([
expect.objectContaining({ path: "/alerts", body: { status: "resolved" } }),
expect.objectContaining({
path: "/alerting/model-deprecations/state",
body: {
model: "example/model",
deprecation_date: "2026-12-01",
status: "resolved",
last_seen_date: "2026-11-16",
last_seen_requests: 10,
},
}),
]);
});
it("shows the current thresholds to free admins without an edit action", async () => {
const user = userEvent.setup();
renderWithProviders(<Alerts />);
await user.click(await screen.findByRole("button", { name: "Manage alerts" }));
expect(await screen.findByText("At least 3× the usual spend and $50 higher.")).toBeInTheDocument();
expect(screen.getByText("Editing thresholds requires Enterprise.")).toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Edit thresholds" })).not.toBeInTheDocument();
});
it("edits Enterprise thresholds with explicit units and saves the values", async () => {
monitoring.alert_limit = null;
monitoring.thresholds = { ...thresholds, latency: { multiplier: 1.005, minimum_increase: 500 } };
const user = userEvent.setup();
renderWithProviders(<Alerts />);
await user.click(await screen.findByRole("button", { name: "Manage alerts" }));
await user.click(await screen.findByRole("button", { name: "Edit thresholds" }));
const spend = screen.getByRole("group", { name: "Spend" });
const multiplier = within(spend).getByRole("spinbutton", { name: "Times the usual value" });
fireEvent.change(multiplier, { target: { value: "1" } });
await user.click(screen.getByRole("button", { name: "Save thresholds" }));
expect(multiplier).toBeInvalid();
expect(calls.filter((call) => call.method === "PATCH")).toEqual([]);
fireEvent.change(multiplier, {
target: { value: "4" },
});
fireEvent.change(screen.getByRole("spinbutton", { name: "Minimum increase ($)" }), { target: { value: "75" } });
fireEvent.change(screen.getByRole("spinbutton", { name: "Minimum increase (percentage points)" }), {
target: { value: "3.5" },
});
await user.click(screen.getByRole("button", { name: "Save thresholds" }));
expect(await screen.findByText("Thresholds saved.")).toBeInTheDocument();
expect(screen.getByText("At least 4× the usual spend and $75 higher.")).toBeInTheDocument();
expect(screen.getByText("At least 3× the usual error rate and 3.5 percentage points higher.")).toBeInTheDocument();
expect(calls.find((call) => call.method === "PATCH")).toMatchObject({
path: "/alerts/monitoring",
body: {
thresholds: {
...thresholds,
spend: { multiplier: 4, minimum_increase: 75 },
latency: { multiplier: 1.005, minimum_increase: 500 },
reliability: { multiplier: 3, minimum_increase: 0.035 },
},
},
});
await user.click(screen.getByRole("button", { name: "Edit thresholds" }));
expect(screen.getByRole("spinbutton", { name: "Minimum increase ($)" })).toHaveValue(75);
expect(screen.getByRole("spinbutton", { name: "Minimum increase (percentage points)" })).toHaveValue(3.5);
});
it("shows free thresholds without editing and hides bulk actions for viewers", async () => {
role = "proxy_admin_viewer";
const user = userEvent.setup();
renderWithProviders(<Alerts />);
await screen.findByRole("button", { name: /Spend on Research agent/ });
expect(screen.queryByRole("button", { name: "Resolve all" })).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Dismiss all" })).not.toBeInTheDocument();
await user.click(screen.getByRole("button", { name: "Manage alerts" }));
expect(await screen.findByText("At least 3× the usual spend and $50 higher.")).toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Edit thresholds" })).not.toBeInTheDocument();
expect(screen.queryByRole("spinbutton")).not.toBeInTheDocument();
});
it("lets viewers read the evidence without write controls", async () => {
role = "proxy_admin_viewer";
renderWithProviders(<Alerts />, { searchParams: { alert: alert.id } });

View file

@ -19,6 +19,7 @@ import AlertActions from "./AlertActions";
import AlertChart from "./AlertChart";
import AlertDetail from "./AlertDetail";
import ManageAlerts from "./ManageAlerts";
import BulkAlertActions from "./BulkAlertActions";
const parsers = {
alert: parseAsString,
@ -119,6 +120,8 @@ function AlertsInbox({ auth }: { auth: ReturnType<typeof useAuthorized> }) {
const pagedLists = [...(showPromptCache ? [promptCache] : []), ...(showPerformance ? [list] : [])];
const loading = pagedLists.some((query) => query.isLoading) || deprecations.isLoading;
const hasError = pagedLists.some((query) => query.isError) || (showDeprecations && deprecations.isError);
const refreshing = pagedLists.some((query) => query.isFetching) || deprecations.isFetching;
const bulkUnavailable = hasError || items.length === 0;
function refresh() {
void client.invalidateQueries({ queryKey: ["alert-monitoring"] });
pagedLists.forEach((query) => void query.refetch());
@ -140,7 +143,7 @@ function AlertsInbox({ auth }: { auth: ReturnType<typeof useAuthorized> }) {
))}
</PageTabsList>
</PageTabs>
<div className="flex items-center justify-between gap-3">
<div className="flex flex-wrap items-center justify-between gap-3">
<Select value={params.type} onValueChange={(value) => void setParams({ type: value as AlertFilter })}>
<SelectTrigger className="w-52" aria-label={t("alerts.unified.typeFilter")}>
<SelectValue>{t(TYPE_LABEL_KEYS[params.type])}</SelectValue>
@ -153,15 +156,19 @@ function AlertsInbox({ auth }: { auth: ReturnType<typeof useAuthorized> }) {
))}
</SelectContent>
</Select>
<Button
variant="outline"
size="sm"
disabled={pagedLists.some((query) => query.isFetching) || deprecations.isFetching}
onClick={refresh}
>
<RefreshCw className="size-3.5" />
{t("alerts.unified.refresh")}
</Button>
<div className="flex flex-wrap items-center gap-2">
<BulkAlertActions
filter={params.type}
state={params.state}
readOnly={Boolean(auth.isViewOnly)}
deprecations={deprecations.data}
disabled={bulkUnavailable || refreshing}
/>
<Button variant="outline" size="sm" disabled={refreshing} onClick={refresh}>
<RefreshCw className="size-3.5" />
{t("alerts.unified.refresh")}
</Button>
</div>
</div>
{showDeprecations && deprecations.isError && (
<LoadError retry={() => void deprecations.refetch()}>{t("alerts.unified.deprecationsError")}</LoadError>

View file

@ -0,0 +1,104 @@
import { useState } from "react";
import { useMutation, useQueryClient } from "@tanstack/react-query";
import { useTranslation } from "react-i18next";
import { Button } from "@/components/ui/button";
import {
AlertDialog,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
AlertDialogCancel,
} from "@/components/ui/alert-dialog";
import { backgroundFetchClient } from "@/lib/http/api";
import type { DeprecationAlert } from "@/components/ModelDeprecationAlerts/deprecationAlerts";
import { AlertStatus, effectiveAlertStatus } from "@/components/ModelDeprecationAlerts/migrationState";
import type { AlertFilter } from "./alertPresentation";
import { TYPE_LABEL_KEYS } from "./alertPresentation";
type BulkStatus = "resolved" | "dismissed";
export default function BulkAlertActions({
filter,
state,
readOnly,
deprecations,
disabled,
}: {
filter: AlertFilter;
state: AlertStatus;
readOnly: boolean;
deprecations: DeprecationAlert[] | undefined;
disabled: boolean;
}) {
const { t } = useTranslation();
const client = useQueryClient();
const [status, setStatus] = useState<BulkStatus | null>(null);
const mutation = useMutation({
mutationFn: async (next: BulkStatus) => {
if (filter !== "model-deprecations") {
const { data } = await backgroundFetchClient.PATCH("/alerts", {
body: { status: next, type: filter === "all" ? undefined : filter },
});
if (!data) throw new Error("Could not update alerts");
}
for (const alert of deprecations ?? []) {
if (effectiveAlertStatus(alert) !== AlertStatus.Open) continue;
const { data } = await backgroundFetchClient.PUT("/alerting/model-deprecations/state", {
body: {
model: alert.usageModel,
deprecation_date: alert.model.deprecation_date,
status: next,
last_seen_date: alert.lastUsedDate,
last_seen_requests: alert.lastDayRequests,
},
});
if (!data) throw new Error("Could not update model deprecation alerts");
}
},
onSuccess: () => setStatus(null),
onSettled: () =>
Promise.all(["alerts", "alert", "deprecationAlerts"].map((key) => client.invalidateQueries({ queryKey: [key] }))),
});
function choose(next: BulkStatus) {
mutation.reset();
setStatus(next);
}
if (readOnly || state !== AlertStatus.Open) return null;
return (
<>
<Button variant="outline" size="sm" disabled={disabled || mutation.isPending} onClick={() => choose("resolved")}>
{t("alerts.bulk.resolve")}
</Button>
<Button variant="outline" size="sm" disabled={disabled || mutation.isPending} onClick={() => choose("dismissed")}>
{t("alerts.bulk.dismiss")}
</Button>
<AlertDialog open={status !== null} onOpenChange={(open) => !open && !mutation.isPending && setStatus(null)}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>
{t(status === "resolved" ? "alerts.bulk.resolve" : "alerts.bulk.dismiss")}
</AlertDialogTitle>
<AlertDialogDescription>
{t("alerts.bulk.description", { type: t(TYPE_LABEL_KEYS[filter]) })}
</AlertDialogDescription>
</AlertDialogHeader>
{mutation.isError && (
<p role="alert" className="text-sm text-destructive">
{t("alerts.bulk.error")}
</p>
)}
<AlertDialogFooter>
<AlertDialogCancel disabled={mutation.isPending}>{t("alerts.bulk.cancel")}</AlertDialogCancel>
<Button disabled={mutation.isPending} onClick={() => status && mutation.mutate(status)}>
{mutation.isPending
? t("alerts.bulk.saving")
: t(status === "resolved" ? "alerts.bulk.resolve" : "alerts.bulk.dismiss")}
</Button>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</>
);
}

View file

@ -15,6 +15,7 @@ import { Switch } from "@/components/ui/switch";
import { backgroundFetchClient } from "@/lib/http/api";
import { getProxyBaseUrl } from "@/components/networking";
import type { components } from "@/lib/http/schema";
import AlertThresholds from "./AlertThresholds";
function statusMessage(data: components["schemas"]["AlertMonitoring"]) {
if (data.disabled_by_config) return "alerts.monitoring.disabledByConfig";
@ -41,7 +42,10 @@ export default function ManageAlerts({ scope, readOnly }: { scope: string; readO
const query = useQuery(queryOptions);
const mutation = useMutation({
mutationFn: async (
body: components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"],
body:
| components["schemas"]["AlertMonitoringUpdate"]
| components["schemas"]["ModelDeprecationMonitoringUpdate"]
| components["schemas"]["AlertThresholdsUpdate"],
) => {
const { data } = await backgroundFetchClient.PATCH("/alerts/monitoring", { body });
if (!data) throw new Error("Could not save monitoring settings");
@ -64,7 +68,7 @@ export default function ManageAlerts({ scope, readOnly }: { scope: string; readO
<Settings className="size-4" aria-hidden />
{t("alerts.monitoring.manage")}
</DialogTrigger>
<DialogContent>
<DialogContent className="max-h-[90vh] overflow-y-auto sm:max-w-xl">
<DialogHeader>
<DialogTitle>{t("alerts.monitoring.manage")}</DialogTitle>
<DialogDescription>{t("alerts.monitoring.description")}</DialogDescription>
@ -125,6 +129,13 @@ export default function ManageAlerts({ scope, readOnly }: { scope: string; readO
aria-describedby="performance-monitoring-status"
/>
</div>
<AlertThresholds
thresholds={data.thresholds}
enterprise={data.alert_limit === null}
readOnly={readOnly}
pending={mutation.isPending}
save={(thresholds) => mutation.mutateAsync({ thresholds })}
/>
</div>
)}
{mutation.isError && (

View file

@ -6096,6 +6096,45 @@
"upgrade": "Upgrade to Enterprise",
"loadError": "Could not load monitoring settings.",
"saveError": "Could not save monitoring settings. Try again."
},
"bulk": {
"resolve": "Resolve all",
"dismiss": "Dismiss all",
"cancel": "Cancel",
"saving": "Updating alerts…",
"description": "This updates all open alerts matching “{{type}}”, including results you have not loaded. You can reopen individual alerts later.",
"error": "Some alerts could not be updated. Saved changes were kept. Try again to update the remaining open alerts."
},
"thresholds": {
"title": "Performance alert thresholds",
"description": "Usual values come from the previous seven days. Both conditions must be met.",
"enterpriseOnly": "Editing thresholds requires Enterprise.",
"edit": "Edit thresholds",
"cancel": "Cancel",
"multiplier": "Times the usual value",
"multiplierInvalid": "Use a multiplier greater than 1.",
"rules": {
"spend": "Spend",
"traffic": "Traffic",
"latency": "Latency",
"reliability": "Error rate"
},
"minimum": {
"spend": "Minimum increase ($)",
"traffic": "Minimum increase (requests)",
"latency": "Minimum increase (ms)",
"reliability": "Minimum increase (percentage points)"
},
"summary": {
"spend": "At least {{multiplier}}× the usual spend and ${{minimum}} higher.",
"traffic": "At least {{multiplier}}× the usual requests and {{minimum}} more requests.",
"latency": "At least {{multiplier}}× the usual latency and {{minimum}} ms higher.",
"reliability": "At least {{multiplier}}× the usual error rate and {{minimum}} percentage points higher."
},
"safeguards": "Requires four active baseline days. Latency also needs 100 timed requests; error rate needs 500 requests. Changes apply to future checks.",
"save": "Save thresholds",
"saving": "Saving…",
"saved": "Thresholds saved."
}
},
"caching": {

View file

@ -1019,7 +1019,11 @@ export interface paths {
delete?: never;
options?: never;
head?: never;
patch?: never;
/**
* Resolve or dismiss matching open alerts
* @description Update all matching open saved alerts, including unloaded pages. Only gateway admins can write.
*/
patch: operations["update_open_alerts_alerts_patch"];
trace?: never;
};
"/alerts/monitoring": {
@ -27750,6 +27754,28 @@ export interface components {
*/
window_start: string;
};
/** AlertBulkResult */
AlertBulkResult: {
/**
* Updated
* @description Number of open saved alerts changed.
*/
updated: number;
};
/** AlertBulkUpdate */
AlertBulkUpdate: {
/**
* Status
* @description Status to apply to every matching open saved alert.
* @enum {string}
*/
status: "resolved" | "dismissed";
/**
* Type
* @description Only update this category, or all saved alerts if omitted.
*/
type?: ("spend" | "traffic" | "latency" | "reliability") | ("model_deprecation" | "prompt_cache") | "performance" | null;
};
/** AlertHistoryPoint */
AlertHistoryPoint: {
/**
@ -27792,6 +27818,28 @@ export interface components {
* @default true
*/
model_deprecations_enabled: boolean;
/**
* @description Effective built-in thresholds. Only Enterprise admins can edit them.
* @default {
* "latency": {
* "minimum_increase": 500,
* "multiplier": 3
* },
* "reliability": {
* "minimum_increase": 0.02,
* "multiplier": 3
* },
* "spend": {
* "minimum_increase": 50,
* "multiplier": 3
* },
* "traffic": {
* "minimum_increase": 1000,
* "multiplier": 10
* }
* }
*/
thresholds: components["schemas"]["PerformanceAlertThresholds"];
};
/** AlertMonitoringUpdate */
AlertMonitoringUpdate: {
@ -27801,6 +27849,24 @@ export interface components {
*/
enabled: boolean;
};
/** AlertThreshold */
AlertThreshold: {
/**
* Minimum Increase
* @description Minimum absolute increase above the baseline, in the metric's unit.
*/
minimum_increase: number;
/**
* Multiplier
* @description Minimum multiple of the baseline.
*/
multiplier: number;
};
/** AlertThresholdsUpdate */
AlertThresholdsUpdate: {
/** @description Replace the built-in performance thresholds. Enterprise only. */
thresholds: components["schemas"]["PerformanceAlertThresholds"];
};
/**
* AlertType
* @description Enum for alert types and management event types
@ -43769,6 +43835,41 @@ export interface components {
/** User Id */
user_id: string;
};
/** PerformanceAlertThresholds */
PerformanceAlertThresholds: {
/**
* @description Average response-time increase in milliseconds over the previous seven-day weighted average.
* @default {
* "minimum_increase": 500,
* "multiplier": 3
* }
*/
latency: components["schemas"]["AlertThreshold"];
/**
* @description Error-rate increase as a fraction, e.g. 0.02 is two percentage points.
* @default {
* "minimum_increase": 0.02,
* "multiplier": 3
* }
*/
reliability: components["schemas"]["AlertThreshold"];
/**
* @description Spend increase in USD over the previous seven-day daily average.
* @default {
* "minimum_increase": 50,
* "multiplier": 3
* }
*/
spend: components["schemas"]["AlertThreshold"];
/**
* @description Request increase over the previous seven-day daily average.
* @default {
* "minimum_increase": 1000,
* "multiplier": 10
* }
*/
traffic: components["schemas"]["AlertThreshold"];
};
/**
* PiiAction
* @enum {string}
@ -56356,6 +56457,39 @@ export interface operations {
};
};
};
update_open_alerts_alerts_patch: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["AlertBulkUpdate"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["AlertBulkResult"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_monitoring_alerts_monitoring_get: {
parameters: {
query?: never;
@ -56385,7 +56519,7 @@ export interface operations {
};
requestBody: {
content: {
"application/json": components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"];
"application/json": components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"] | components["schemas"]["AlertThresholdsUpdate"];
};
};
responses: {

View file

@ -336,6 +336,82 @@ async def test_daily_history_matches_scope_and_weighted_metrics(
write_rows('DELETE FROM "LiteLLM_Alert" WHERE resource_id=%s', (resource,))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("email", "alias", "expected"),
(
("person@example.com", "Example user", "person@example.com"),
("person@example.com", None, "person@example.com"),
(None, "Example user", "Example user"),
("", "Example user", "Example user"),
(" ", "Example user", "Example user"),
(None, None, None),
),
)
async def test_user_alerts_display_email_in_list_and_detail(
alerts: Alerts, gateway: Gateway, email: str | None, alias: str | None, expected: str | None
) -> None:
write_rows(
'INSERT INTO "LiteLLM_UserTable" (user_id, user_email, user_alias) VALUES (%s, %s, %s)',
(alerts.resource_id, email, alias),
)
try:
saved: Final = await alerts.repository.record(alerts.detection().model_copy(update={"resource_type": "user"}))
detail: Final = gateway.request("GET", f"/alerts/{saved.id}")
assert detail.status_code == 200, detail.text
view: Final = AlertView.model_validate_json(detail.content)
assert view.resource_name == expected
assert view.resource_id == alerts.resource_id
assert view.evidence == saved.evidence
listed: Final = gateway.request("GET", "/alerts", params={"type": "spend", "limit": "1"})
assert listed.status_code == 200, listed.text
assert AlertViewPage.model_validate_json(listed.content).data == (view,)
finally:
write_rows('DELETE FROM "LiteLLM_UserTable" WHERE user_id=%s', (alerts.resource_id,))
@pytest.mark.asyncio
@pytest.mark.parametrize("status", ("resolved", "dismissed"))
async def test_bulk_status_updates_all_matching_open_pages_and_preserves_other_alerts(
alerts: Alerts, gateway: Gateway, status: str
) -> None:
saved: Final = tuple([await alerts.repository.record(alerts.detection(day=day)) for day in range(30)])
other: Final = await alerts.repository.record(alerts.detection("traffic"))
already_resolved: Final = await alerts.repository.set_status(int(saved[0].id), "resolved", "previous-admin")
response: Final = gateway.request("PATCH", "/alerts", {"status": status, "type": "spend"})
assert response.status_code == 200, response.text
assert response.json() == {"updated": 29}
for alert in saved[1:]:
changed: Final = await alerts.repository.get(int(alert.id))
assert changed is not None
assert changed.status == status
assert changed.updated_by
assert changed.evidence == alert.evidence
assert await alerts.repository.get(int(saved[0].id)) == already_resolved
assert await alerts.repository.get(int(other.id)) == other
repeat: Final = gateway.request("PATCH", "/alerts", {"status": status, "type": "spend"})
assert repeat.status_code == 200, repeat.text
assert repeat.json() == {"updated": 0}
@pytest.mark.asyncio
async def test_bulk_performance_filter_leaves_prompt_cache_until_all_alerts_are_updated(
alerts: Alerts, gateway: Gateway
) -> None:
spend: Final = await alerts.repository.record(alerts.detection())
cache: Final = await alerts.repository.record(alerts.detection("prompt_cache"))
performance: Final = gateway.request("PATCH", "/alerts", {"status": "resolved", "type": "performance"})
assert performance.status_code == 200, performance.text
resolved: Final = await alerts.repository.get(int(spend.id))
assert resolved is not None and resolved.status == "resolved"
assert await alerts.repository.get(int(cache.id)) == cache
all_alerts: Final = gateway.request("PATCH", "/alerts", {"status": "dismissed"})
assert all_alerts.status_code == 200, all_alerts.text
dismissed: Final = await alerts.repository.get(int(cache.id))
assert dismissed is not None and dismissed.status == "dismissed"
assert await alerts.repository.get(int(spend.id)) == resolved
@pytest.mark.asyncio
async def test_type_filter_applies_before_cursor_limit(alerts: Alerts, gateway: Gateway) -> None:
oldest: Final = await alerts.repository.record(alerts.detection("traffic"))

View file

@ -14,7 +14,13 @@ from litellm_proxy_extras.request_log_indexes import ensure_request_log_indexes
from psycopg import sql
from litellm.models.alert import AlertDetection, DeprecationEvidence, PromptCacheEvidence
from litellm.models.performance_alert import FREE_ALERT_LIMIT, SweepOwnershipLostError, SweepState
from litellm.models.performance_alert import (
DEFAULT_THRESHOLDS,
FREE_ALERT_LIMIT,
PerformanceAlertThresholds,
SweepOwnershipLostError,
SweepState,
)
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.spend_tracking.performance_alerts import run_performance_alerts
from litellm.proxy.utils import PrismaClient, ProxyLogging
@ -145,6 +151,38 @@ async def fixture() -> AsyncIterator[Fixture]:
await client.disconnect()
@pytest.mark.asyncio
async def test_enterprise_thresholds_persist_and_change_detection_but_free_uses_defaults(fixture: Fixture) -> None:
from litellm.repositories.config_repository import ConfigRepository
config: Final = ConfigRepository(fixture.client, use_writer=True)
previous: Final = await config.get_param("performance_alert_thresholds")
custom: Final = PerformanceAlertThresholds.model_validate({"spend": {"multiplier": 2, "minimum_increase": 20}})
enterprise: Final = PerformanceAlertRepository(fixture.client, enterprise=True)
free: Final = PerformanceAlertRepository(fixture.client, enterprise=False)
resource: Final = f"{fixture.prefix}-custom-threshold"
_seed("team", resource, Day(spend=10), Day(spend=40))
try:
assert (await enterprise.set_thresholds(custom)).thresholds == custom
restarted: Final = PerformanceAlertRepository(fixture.client, enterprise=True)
assert (await restarted.monitoring()).thresholds == custom
assert (await free.monitoring()).thresholds == DEFAULT_THRESHOLDS
with pytest.raises(PermissionError, match="Enterprise"):
await free.set_thresholds(custom)
claim: Final = await free.claim(timedelta(0))
assert claim is not None
state: Final = SweepState(day=DAY, scope="team", cursor=fixture.prefix)
await free.evaluate_page(claim.owner, state, 25)
assert fixture.saved() == {}
await restarted.evaluate_page(claim.owner, state, 25)
assert fixture.saved() == {(resource, "spend"): ("team", "open")}
finally:
await config.set_param(
"performance_alert_thresholds",
previous.param_value if previous else DEFAULT_THRESHOLDS.model_dump(mode="json"),
)
@pytest.mark.asyncio
async def test_a_full_sweep_saves_one_alert_per_real_change_and_nothing_for_normal_or_new_usage(
fixture: Fixture,

View file

@ -2,10 +2,13 @@ from datetime import date, datetime, timedelta, timezone
from typing import Final
import pytest
from pydantic import ValidationError
from litellm.models.alert import AlertDetection, MetricEvidence
from litellm.models.performance_alert import (
SCOPES,
AlertThreshold,
PerformanceAlertThresholds,
PerformanceScope,
ResourceWindow,
SweepState,
@ -46,6 +49,55 @@ def test_steady_usage_raises_nothing() -> None:
assert _types() == ()
@pytest.mark.parametrize(
("kind", "changes", "minimum", "too_high"),
(
("spend", {"spend": 40}, 20, 31),
("traffic", {"requests": 2000}, 500, 1001),
("latency", {"response_ms": 300_000}, 150, 201),
("reliability", {"failed": 20}, 0.005, 0.011),
),
)
def test_custom_multiplier_and_minimum_both_control_each_existing_rule(
kind: str, changes: dict[str, object], minimum: float, too_high: float
) -> None:
window: Final = _window(**changes)
assert detect("team", window, DAY) == ()
thresholds: Final = PerformanceAlertThresholds.model_validate(
{kind: {"multiplier": 2, "minimum_increase": minimum}}
)
detected: Final = detect("team", window, DAY, thresholds)
assert tuple(alert.type for alert in detected) == (kind,)
for threshold in (
AlertThreshold(multiplier=5, minimum_increase=minimum),
AlertThreshold(multiplier=2, minimum_increase=too_high),
):
assert detect("team", window, DAY, thresholds.model_copy(update={kind: threshold})) == ()
@pytest.mark.parametrize(
"settings",
(
{"spend": {"multiplier": 1, "minimum_increase": 1}},
{"spend": {"multiplier": True, "minimum_increase": 1}},
{"latency": {"multiplier": 2, "minimum_increase": -1}},
{"traffic": {"multiplier": float("inf"), "minimum_increase": 1}},
{"reliability": {"multiplier": 2, "minimum_increase": 1.01}},
{"new_rule": {"multiplier": 2, "minimum_increase": 1}},
),
)
def test_thresholds_reject_invalid_values_and_new_rules(settings: dict[str, object]) -> None:
with pytest.raises(ValidationError):
PerformanceAlertThresholds.model_validate(settings)
def test_zero_minimum_does_not_report_a_zero_error_rate_as_an_increase() -> None:
thresholds: Final = PerformanceAlertThresholds.model_validate(
{"reliability": {"multiplier": 2, "minimum_increase": 0}}
)
assert detect("team", _window(failed=0, base_failed=0), DAY, thresholds) == ()
@pytest.mark.parametrize("scope", SCOPES)
@pytest.mark.parametrize("requests", (0, 100))
def test_quiet_or_idle_resources_do_not_raise_traffic_alerts(scope: PerformanceScope, requests: int) -> None:

View file

@ -8,6 +8,7 @@ from fastapi import FastAPI
from fastapi.testclient import TestClient
from pydantic import BaseModel, ConfigDict
from litellm.models.performance_alert import DEFAULT_THRESHOLDS
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.alerting_endpoints.alerts import router
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@ -73,7 +74,9 @@ def client(app: FastAPI) -> Iterator[TestClient]:
@pytest.mark.parametrize(
"role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.TEAM, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)
)
@pytest.mark.parametrize("method,path", (("GET", "/alerts"), ("GET", "/alerts/5"), ("PATCH", "/alerts/5")))
@pytest.mark.parametrize(
"method,path", (("GET", "/alerts"), ("GET", "/alerts/5"), ("PATCH", "/alerts/5"), ("PATCH", "/alerts"))
)
def test_permissions_precede_database_access(
app: FastAPI, table: MagicMock, role: LitellmUserRoles, method: str, path: str
) -> None:
@ -142,6 +145,7 @@ def test_deprecation_preference_updates_without_changing_performance(app: FastAP
"alerts_used": 3,
"alert_limit": 5,
"disabled_by_config": False,
"thresholds": DEFAULT_THRESHOLDS.model_dump(),
}
prisma.writer_db.litellm_config.upsert.assert_awaited_once_with(
where={"param_name": "model_deprecation_alerts_enabled"},
@ -171,11 +175,42 @@ def test_config_opt_out_cannot_be_enabled_from_the_api(app: FastAPI) -> None:
"alert_limit": 5,
"disabled_by_config": True,
"model_deprecations_enabled": True,
"thresholds": DEFAULT_THRESHOLDS.model_dump(),
}
assert response.status_code == 409, response.text
transaction.execute_raw.assert_not_awaited()
@pytest.mark.parametrize("enterprise", (False, True))
@pytest.mark.parametrize("role", (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY))
def test_only_enterprise_admins_can_save_thresholds(app: FastAPI, enterprise: bool, role: LitellmUserRoles) -> None:
prisma: Final = MagicMock()
prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
prisma.writer_db.litellm_config.upsert = AsyncMock()
transaction: Final = prisma.writer_db.tx.return_value.__aenter__.return_value
transaction.query_raw = AsyncMock(return_value=[{"enabled": True, "alerts_used": 0}])
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role)
app.dependency_overrides[get_alert_monitoring_repository] = lambda: PerformanceAlertRepository(
prisma, enterprise=enterprise
)
with TestClient(app) as client:
response: Final = client.patch(
"/alerts/monitoring", json={"thresholds": {"spend": {"multiplier": 2, "minimum_increase": 20}}}
)
if enterprise and role == LitellmUserRoles.PROXY_ADMIN:
assert response.status_code == 200, response.text
prisma.writer_db.litellm_config.upsert.assert_awaited_once()
else:
assert response.status_code == 403, response.text
prisma.writer_db.litellm_config.upsert.assert_not_awaited()
@pytest.mark.parametrize("body", ({"status": "open"}, {"status": "resolved", "type": "unknown"}, {}))
def test_bulk_update_rejects_reopening_or_unknown_filters(client: TestClient, body: object) -> None:
response: Final = client.patch("/alerts", json=body)
assert response.status_code == 422, response.text
@pytest.mark.parametrize("alert_id", ("0", "-1", "01", "1.0", "oops", "9223372036854775808", "1" * 20))
@pytest.mark.parametrize("method", ("GET", "PATCH"))
def test_invalid_id_never_accesses_database(client: TestClient, table: MagicMock, alert_id: str, method: str) -> None: