mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
feat(alerts): add email labels, bulk actions and threshold editing (#52139)
* feat(alerts): add bulk actions and enterprise thresholds * fix(alerts): accept fractional multipliers above one
This commit is contained in:
parent
ea987e7856
commit
ea448a8951
16 changed files with 993 additions and 47 deletions
|
|
@ -178,6 +178,21 @@ class AlertUpdate(BaseModel):
|
|||
status: AlertStatus = Field(description="Mark resolved, dismiss, or reopen the alert.")
|
||||
|
||||
|
||||
class AlertBulkUpdate(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
status: Literal["resolved", "dismissed"] = Field(description="Status to apply to every matching open saved alert.")
|
||||
type: AlertFilter | None = Field(
|
||||
default=None, description="Only update this category, or all saved alerts if omitted."
|
||||
)
|
||||
|
||||
|
||||
class AlertBulkResult(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
updated: int = Field(ge=0, description="Number of open saved alerts changed.")
|
||||
|
||||
|
||||
class AlertPage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
|
@ -206,3 +221,5 @@ class AlertStore(Protocol):
|
|||
async def get(self, alert_id: int) -> Alert | None: ...
|
||||
|
||||
async def set_status(self, alert_id: int, status: AlertStatus, actor: str) -> Alert | None: ...
|
||||
|
||||
async def set_open_status(self, data: AlertBulkUpdate, actor: str) -> int: ...
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from dataclasses import dataclass
|
|||
from datetime import date, datetime, time, timedelta, timezone
|
||||
from typing import Final, Literal, Protocol, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
from litellm.models.alert import MAX_ALERT_ID, AlertDetection, AlertType, MetricEvidence
|
||||
|
||||
|
|
@ -25,6 +25,49 @@ RELIABILITY_MIN_INCREASE: Final = 0.02
|
|||
RELIABILITY_MIN_REQUESTS: Final = 500
|
||||
|
||||
|
||||
class AlertThreshold(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
multiplier: float = Field(gt=1, allow_inf_nan=False, strict=True, description="Minimum multiple of the baseline.")
|
||||
minimum_increase: float = Field(
|
||||
ge=0,
|
||||
allow_inf_nan=False,
|
||||
strict=True,
|
||||
description="Minimum absolute increase above the baseline, in the metric's unit.",
|
||||
)
|
||||
|
||||
|
||||
class PerformanceAlertThresholds(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
spend: AlertThreshold = Field(
|
||||
default=AlertThreshold(multiplier=SPEND_MULTIPLIER, minimum_increase=SPEND_MIN_INCREASE_USD),
|
||||
description="Spend increase in USD over the previous seven-day daily average.",
|
||||
)
|
||||
traffic: AlertThreshold = Field(
|
||||
default=AlertThreshold(multiplier=TRAFFIC_SPIKE_MULTIPLIER, minimum_increase=TRAFFIC_SPIKE_MIN_INCREASE),
|
||||
description="Request increase over the previous seven-day daily average.",
|
||||
)
|
||||
latency: AlertThreshold = Field(
|
||||
default=AlertThreshold(multiplier=LATENCY_MULTIPLIER, minimum_increase=LATENCY_MIN_INCREASE_MS),
|
||||
description="Average response-time increase in milliseconds over the previous seven-day weighted average.",
|
||||
)
|
||||
reliability: AlertThreshold = Field(
|
||||
default=AlertThreshold(multiplier=RELIABILITY_MULTIPLIER, minimum_increase=RELIABILITY_MIN_INCREASE),
|
||||
description="Error-rate increase as a fraction, e.g. 0.02 is two percentage points.",
|
||||
)
|
||||
|
||||
@field_validator("reliability")
|
||||
@classmethod
|
||||
def validate_error_rate(cls, value: AlertThreshold) -> AlertThreshold:
|
||||
if value.minimum_increase > 1:
|
||||
raise ValueError("Error-rate minimum increase cannot exceed 1")
|
||||
return value
|
||||
|
||||
|
||||
DEFAULT_THRESHOLDS: Final = PerformanceAlertThresholds()
|
||||
|
||||
|
||||
class ResourceWindow(BaseModel):
|
||||
"""One resource's totals for the evaluated day, and summed over the seven days before it."""
|
||||
|
||||
|
|
@ -89,6 +132,9 @@ class AlertMonitoring(BaseModel):
|
|||
alerts_used: int = Field(ge=0, le=FREE_ALERT_LIMIT, description="Lifetime free allowance consumed, capped at five.")
|
||||
alert_limit: int | None = Field(description="Free allowance, or null for Enterprise.")
|
||||
disabled_by_config: bool = Field(description="Whether performance_alerts_enabled is false in proxy settings.")
|
||||
thresholds: PerformanceAlertThresholds = Field(
|
||||
default=DEFAULT_THRESHOLDS, description="Effective built-in thresholds. Only Enterprise admins can edit them."
|
||||
)
|
||||
|
||||
@property
|
||||
def active(self) -> bool:
|
||||
|
|
@ -107,10 +153,19 @@ class ModelDeprecationMonitoringUpdate(BaseModel):
|
|||
model_deprecations_enabled: bool = Field(strict=True, description="Enable or disable model deprecation alerts.")
|
||||
|
||||
|
||||
class AlertThresholdsUpdate(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
thresholds: PerformanceAlertThresholds = Field(
|
||||
description="Replace the built-in performance thresholds. Enterprise only."
|
||||
)
|
||||
|
||||
|
||||
class AlertMonitoringStore(Protocol):
|
||||
async def monitoring(self) -> AlertMonitoring: ...
|
||||
async def set_enabled(self, enabled: bool) -> AlertMonitoring: ...
|
||||
async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring: ...
|
||||
async def set_thresholds(self, thresholds: PerformanceAlertThresholds) -> AlertMonitoring: ...
|
||||
|
||||
|
||||
class PerformanceAlertStore(Protocol):
|
||||
|
|
@ -181,58 +236,65 @@ def _alert(
|
|||
)
|
||||
|
||||
|
||||
def _spend_spike(window: ResourceWindow) -> tuple[float, float] | None:
|
||||
def _spend_spike(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None:
|
||||
baseline: Final = window.base_spend / BASELINE_DAYS
|
||||
triggered: Final = (
|
||||
window.active_days >= MIN_ACTIVE_DAYS
|
||||
and baseline > 0
|
||||
and window.spend >= SPEND_MULTIPLIER * baseline
|
||||
and window.spend - baseline >= SPEND_MIN_INCREASE_USD
|
||||
and window.spend >= threshold.multiplier * baseline
|
||||
and window.spend - baseline >= threshold.minimum_increase
|
||||
)
|
||||
return (window.spend, baseline) if triggered else None
|
||||
|
||||
|
||||
def _traffic_spike(window: ResourceWindow) -> tuple[float, float] | None:
|
||||
def _traffic_spike(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None:
|
||||
baseline: Final = window.base_requests / BASELINE_DAYS
|
||||
spike: Final = (
|
||||
window.active_days >= MIN_ACTIVE_DAYS
|
||||
and window.requests >= TRAFFIC_SPIKE_MULTIPLIER * baseline
|
||||
and window.requests - baseline >= TRAFFIC_SPIKE_MIN_INCREASE
|
||||
and window.requests > baseline
|
||||
and window.requests >= threshold.multiplier * baseline
|
||||
and window.requests - baseline >= threshold.minimum_increase
|
||||
)
|
||||
return (window.requests, baseline) if spike else None
|
||||
|
||||
|
||||
def _slower_responses(window: ResourceWindow) -> tuple[float, float] | None:
|
||||
def _slower_responses(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None:
|
||||
current: Final = _ratio(window.response_ms, window.timed)
|
||||
baseline: Final = _ratio(window.base_response_ms, window.base_timed)
|
||||
triggered: Final = (
|
||||
window.timed >= LATENCY_MIN_TIMED_REQUESTS
|
||||
and window.timed_days >= MIN_ACTIVE_DAYS
|
||||
and baseline > 0
|
||||
and current >= LATENCY_MULTIPLIER * baseline
|
||||
and current - baseline >= LATENCY_MIN_INCREASE_MS
|
||||
and current >= threshold.multiplier * baseline
|
||||
and current - baseline >= threshold.minimum_increase
|
||||
)
|
||||
return (current, baseline) if triggered else None
|
||||
|
||||
|
||||
def _more_failures(window: ResourceWindow) -> tuple[float, float] | None:
|
||||
def _more_failures(window: ResourceWindow, threshold: AlertThreshold) -> tuple[float, float] | None:
|
||||
current: Final = min(1.0, _ratio(window.failed, window.requests))
|
||||
baseline: Final = min(1.0, _ratio(window.base_failed, window.base_requests))
|
||||
triggered: Final = (
|
||||
window.active_days >= MIN_ACTIVE_DAYS
|
||||
and window.requests >= RELIABILITY_MIN_REQUESTS
|
||||
and current >= RELIABILITY_MULTIPLIER * baseline
|
||||
and current - baseline >= RELIABILITY_MIN_INCREASE
|
||||
and current > baseline
|
||||
and current >= threshold.multiplier * baseline
|
||||
and current - baseline >= threshold.minimum_increase
|
||||
)
|
||||
return (current, baseline) if triggered else None
|
||||
|
||||
|
||||
def detect(scope: PerformanceScope, window: ResourceWindow, day: date) -> tuple[AlertDetection, ...]:
|
||||
def detect(
|
||||
scope: PerformanceScope,
|
||||
window: ResourceWindow,
|
||||
day: date,
|
||||
thresholds: PerformanceAlertThresholds = DEFAULT_THRESHOLDS,
|
||||
) -> tuple[AlertDetection, ...]:
|
||||
checks: Final[tuple[tuple[AlertType, tuple[float, float] | None, int], ...]] = (
|
||||
("spend", _spend_spike(window), window.requests),
|
||||
("traffic", _traffic_spike(window), window.requests),
|
||||
("latency", _slower_responses(window), window.timed),
|
||||
("reliability", _more_failures(window), window.requests),
|
||||
("spend", _spend_spike(window, thresholds.spend), window.requests),
|
||||
("traffic", _traffic_spike(window, thresholds.traffic), window.requests),
|
||||
("latency", _slower_responses(window, thresholds.latency), window.timed),
|
||||
("reliability", _more_failures(window, thresholds.reliability), window.requests),
|
||||
)
|
||||
return tuple(
|
||||
_alert(scope, window, day, kind, found[0], found[1], sample_size)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ from fastapi import APIRouter, Depends, HTTPException, Path, Query
|
|||
|
||||
from litellm.models.alert import (
|
||||
Alert,
|
||||
AlertBulkResult,
|
||||
AlertBulkUpdate,
|
||||
AlertFilter,
|
||||
AlertID,
|
||||
AlertStatus,
|
||||
|
|
@ -12,7 +14,12 @@ from litellm.models.alert import (
|
|||
AlertView,
|
||||
AlertViewPage,
|
||||
)
|
||||
from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate, ModelDeprecationMonitoringUpdate
|
||||
from litellm.models.performance_alert import (
|
||||
AlertMonitoring,
|
||||
AlertMonitoringUpdate,
|
||||
AlertThresholdsUpdate,
|
||||
ModelDeprecationMonitoringUpdate,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.deps.repositories import AlertMonitoringRepositoryDep, AlertRepositoryDep
|
||||
|
|
@ -60,6 +67,13 @@ async def list_alerts(
|
|||
return AlertViewPage(data=await store.enrich(page.data), next_cursor=page.next_cursor)
|
||||
|
||||
|
||||
@router.patch("/alerts", response_model=AlertBulkResult, summary="Resolve or dismiss matching open alerts")
|
||||
async def update_open_alerts(data: AlertBulkUpdate, user: Auth, repository: AlertRepositoryDep) -> AlertBulkResult:
|
||||
"""Update all matching open saved alerts, including unloaded pages. Only gateway admins can write."""
|
||||
store: Final = _authorized_store(user, repository, write=True)
|
||||
return AlertBulkResult(updated=await store.set_open_status(data, user.user_id or "proxy-admin"))
|
||||
|
||||
|
||||
@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read alert monitoring settings")
|
||||
async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -> AlertMonitoring:
|
||||
"""Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin."""
|
||||
|
|
@ -71,12 +85,18 @@ async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -
|
|||
|
||||
@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable alert monitoring")
|
||||
async def update_monitoring(
|
||||
data: AlertMonitoringUpdate | ModelDeprecationMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep
|
||||
data: AlertMonitoringUpdate | ModelDeprecationMonitoringUpdate | AlertThresholdsUpdate,
|
||||
user: Auth,
|
||||
repository: AlertMonitoringRepositoryDep,
|
||||
) -> AlertMonitoring:
|
||||
"""Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts."""
|
||||
_authorize(user, write=True)
|
||||
if repository is None:
|
||||
raise HTTPException(status_code=503, detail="Alerts require a database connection")
|
||||
if isinstance(data, AlertThresholdsUpdate):
|
||||
if (await repository.monitoring()).alert_limit is not None:
|
||||
raise HTTPException(status_code=403, detail="Custom alert thresholds require Enterprise")
|
||||
return await repository.set_thresholds(data.thresholds)
|
||||
if isinstance(data, ModelDeprecationMonitoringUpdate):
|
||||
return await repository.set_model_deprecations_enabled(data.model_deprecations_enabled)
|
||||
if data.enabled and (await repository.monitoring()).disabled_by_config:
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm.models.alert import (
|
||||
METRIC_ALERT_TYPES,
|
||||
Alert,
|
||||
AlertBulkUpdate,
|
||||
AlertDetection,
|
||||
AlertFilter,
|
||||
AlertHistoryPoint,
|
||||
|
|
@ -36,6 +37,7 @@ if TYPE_CHECKING:
|
|||
|
||||
class _HistoryTransaction(Protocol):
|
||||
async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ...
|
||||
async def execute_raw(self, query: str, *args: object) -> int: ...
|
||||
|
||||
|
||||
class _HistoryDatabase(Protocol):
|
||||
|
|
@ -95,7 +97,7 @@ _HISTORY_SOURCES: Final[
|
|||
"d.user_id",
|
||||
"api_requests",
|
||||
'LEFT JOIN "LiteLLM_UserTable" m ON m.user_id = r.resource_id',
|
||||
"m.user_alias",
|
||||
"COALESCE(NULLIF(TRIM(m.user_email), ''), m.user_alias)",
|
||||
),
|
||||
"key": (
|
||||
'"LiteLLM_DailyUserSpend"',
|
||||
|
|
@ -260,6 +262,20 @@ class AlertRepository:
|
|||
row: Final = await self._table.update(where={"id": alert_id}, data={"status": status, "updated_by": actor})
|
||||
return None if row is None else _alert(row)
|
||||
|
||||
async def set_open_status(self, data: AlertBulkUpdate, actor: str) -> int:
|
||||
async with self._database.tx(timeout=timedelta(seconds=10)) as transaction:
|
||||
await transaction.query_raw("SELECT set_config('statement_timeout', '6s', true)")
|
||||
return await transaction.execute_raw(
|
||||
"""UPDATE "LiteLLM_Alert" SET status = $1, updated_by = $2,
|
||||
updated_at = now() AT TIME ZONE 'UTC'
|
||||
WHERE status = 'open' AND
|
||||
($3::text IS NULL OR type = $3 OR ($3 = 'performance' AND type = ANY($4::text[])))""",
|
||||
data.status,
|
||||
actor,
|
||||
data.type,
|
||||
list(METRIC_ALERT_TYPES),
|
||||
)
|
||||
|
||||
async def enrich(self, alerts: tuple[Alert, ...]) -> tuple[AlertView, ...]:
|
||||
from prisma.errors import PrismaError
|
||||
|
||||
|
|
|
|||
|
|
@ -12,9 +12,11 @@ from typing_extensions import LiteralString
|
|||
from litellm.models.alert import METRIC_ALERT_TYPES
|
||||
from litellm.models.performance_alert import (
|
||||
BASELINE_DAYS,
|
||||
DEFAULT_THRESHOLDS,
|
||||
FREE_ALERT_LIMIT,
|
||||
AlertMonitoring,
|
||||
Claim,
|
||||
PerformanceAlertThresholds,
|
||||
PerformanceScope,
|
||||
ResourceUnreadableError,
|
||||
ResourceWindow,
|
||||
|
|
@ -196,9 +198,24 @@ class PerformanceAlertRepository:
|
|||
async with self._transaction() as transaction:
|
||||
monitoring: Final = await self._monitoring(transaction)
|
||||
return monitoring.model_copy(
|
||||
update={"model_deprecations_enabled": await self._config.model_deprecations_enabled()}
|
||||
update={
|
||||
"model_deprecations_enabled": await self._config.model_deprecations_enabled(),
|
||||
"thresholds": await self._thresholds(),
|
||||
}
|
||||
)
|
||||
|
||||
async def _thresholds(self) -> PerformanceAlertThresholds:
|
||||
if not self._enterprise:
|
||||
return DEFAULT_THRESHOLDS
|
||||
saved: Final = await self._config.get_param("performance_alert_thresholds")
|
||||
return DEFAULT_THRESHOLDS if saved is None else PerformanceAlertThresholds.model_validate(saved.param_value)
|
||||
|
||||
async def set_thresholds(self, thresholds: PerformanceAlertThresholds) -> AlertMonitoring:
|
||||
if not self._enterprise:
|
||||
raise PermissionError("Custom alert thresholds require Enterprise")
|
||||
await self._config.set_param("performance_alert_thresholds", thresholds.model_dump(mode="json"))
|
||||
return await self.monitoring()
|
||||
|
||||
async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring:
|
||||
await self._config.set_model_deprecations_enabled(enabled)
|
||||
return await self.monitoring()
|
||||
|
|
@ -258,6 +275,7 @@ class PerformanceAlertRepository:
|
|||
raise
|
||||
|
||||
async def _evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState | None:
|
||||
thresholds: Final = await self._thresholds()
|
||||
async with self._transaction() as transaction:
|
||||
monitoring: Final = await self._monitoring(transaction, lock=True)
|
||||
if not monitoring.active:
|
||||
|
|
@ -267,7 +285,7 @@ class PerformanceAlertRepository:
|
|||
)
|
||||
saved: Final = await record_alerts(
|
||||
transaction,
|
||||
tuple(chain.from_iterable(detect(state.scope, window, state.day) for window in windows)),
|
||||
tuple(chain.from_iterable(detect(state.scope, window, state.day, thresholds) for window in windows)),
|
||||
None if self._enterprise else FREE_ALERT_LIMIT - monitoring.alerts_used,
|
||||
)
|
||||
if saved:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,140 @@
|
|||
import { useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import type { components } from "@/lib/http/schema";
|
||||
|
||||
type Thresholds = components["schemas"]["PerformanceAlertThresholds"];
|
||||
const rules = ["spend", "traffic", "latency", "reliability"] as const;
|
||||
|
||||
export default function AlertThresholds({
|
||||
thresholds,
|
||||
enterprise,
|
||||
readOnly,
|
||||
pending,
|
||||
save,
|
||||
}: {
|
||||
thresholds: Thresholds;
|
||||
enterprise: boolean;
|
||||
readOnly: boolean;
|
||||
pending: boolean;
|
||||
save: (thresholds: Thresholds) => Promise<unknown>;
|
||||
}) {
|
||||
const { t, i18n } = useTranslation();
|
||||
const [editing, setEditing] = useState(false);
|
||||
const [saved, setSaved] = useState(false);
|
||||
const disabled = !enterprise || readOnly || pending;
|
||||
const format = new Intl.NumberFormat(i18n.language, { maximumSignificantDigits: 15 });
|
||||
const minimum = (rule: (typeof rules)[number]) =>
|
||||
Number((thresholds[rule].minimum_increase * (rule === "reliability" ? 100 : 1)).toPrecision(15));
|
||||
async function submit(event: React.FormEvent<HTMLFormElement>) {
|
||||
event.preventDefault();
|
||||
const values = new FormData(event.currentTarget);
|
||||
const threshold = (rule: (typeof rules)[number]) => ({
|
||||
multiplier: Number(values.get(`${rule}-multiplier`)),
|
||||
minimum_increase: Number(values.get(`${rule}-minimum`)) / (rule === "reliability" ? 100 : 1),
|
||||
});
|
||||
setSaved(false);
|
||||
try {
|
||||
const updated = {
|
||||
spend: threshold("spend"),
|
||||
traffic: threshold("traffic"),
|
||||
latency: threshold("latency"),
|
||||
reliability: threshold("reliability"),
|
||||
};
|
||||
await save(updated);
|
||||
setEditing(false);
|
||||
setSaved(true);
|
||||
} catch {
|
||||
setSaved(false);
|
||||
}
|
||||
}
|
||||
return (
|
||||
<form onSubmit={(event) => void submit(event)} onChange={() => setSaved(false)} className="space-y-4 py-4">
|
||||
<div className="space-y-1">
|
||||
<h3 className="text-sm font-semibold">{t("alerts.thresholds.title")}</h3>
|
||||
<p className="text-sm text-muted-foreground">{t("alerts.thresholds.description")}</p>
|
||||
{!enterprise && <p className="text-sm text-muted-foreground">{t("alerts.thresholds.enterpriseOnly")}</p>}
|
||||
</div>
|
||||
{!editing && (
|
||||
<dl className="space-y-3 text-sm">
|
||||
{rules.map((rule) => (
|
||||
<div key={rule} className="space-y-1">
|
||||
<dt className="font-medium">{t(`alerts.thresholds.rules.${rule}`)}</dt>
|
||||
<dd className="tabular-nums text-muted-foreground">
|
||||
{t(`alerts.thresholds.summary.${rule}`, {
|
||||
multiplier: format.format(thresholds[rule].multiplier),
|
||||
minimum: format.format(minimum(rule)),
|
||||
})}
|
||||
</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
)}
|
||||
{editing &&
|
||||
rules.map((rule) => (
|
||||
<fieldset key={rule} disabled={disabled} className="grid grid-cols-2 gap-3">
|
||||
<legend className="mb-2 text-sm font-medium">{t(`alerts.thresholds.rules.${rule}`)}</legend>
|
||||
<div className="space-y-1">
|
||||
<label htmlFor={`${rule}-multiplier`} className="text-xs text-muted-foreground">
|
||||
{t("alerts.thresholds.multiplier")}
|
||||
</label>
|
||||
<Input
|
||||
id={`${rule}-multiplier`}
|
||||
name={`${rule}-multiplier`}
|
||||
type="number"
|
||||
min="1"
|
||||
step="any"
|
||||
required
|
||||
defaultValue={thresholds[rule].multiplier}
|
||||
onChange={(event) =>
|
||||
event.currentTarget.setCustomValidity(
|
||||
event.currentTarget.valueAsNumber <= 1 ? t("alerts.thresholds.multiplierInvalid") : "",
|
||||
)
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-1">
|
||||
<label htmlFor={`${rule}-minimum`} className="text-xs text-muted-foreground">
|
||||
{t(`alerts.thresholds.minimum.${rule}`)}
|
||||
</label>
|
||||
<Input
|
||||
id={`${rule}-minimum`}
|
||||
name={`${rule}-minimum`}
|
||||
type="number"
|
||||
min="0"
|
||||
step="any"
|
||||
required
|
||||
max={rule === "reliability" ? 100 : undefined}
|
||||
defaultValue={minimum(rule)}
|
||||
/>
|
||||
</div>
|
||||
</fieldset>
|
||||
))}
|
||||
<p className="text-xs text-muted-foreground">{t("alerts.thresholds.safeguards")}</p>
|
||||
{enterprise && !readOnly && (
|
||||
<div className="flex items-center gap-3">
|
||||
{editing ? (
|
||||
<>
|
||||
<Button type="submit" size="sm" disabled={pending}>
|
||||
{t(pending ? "alerts.thresholds.saving" : "alerts.thresholds.save")}
|
||||
</Button>
|
||||
<Button type="button" variant="ghost" size="sm" disabled={pending} onClick={() => setEditing(false)}>
|
||||
{t("alerts.thresholds.cancel")}
|
||||
</Button>
|
||||
</>
|
||||
) : (
|
||||
<Button type="button" variant="outline" size="sm" onClick={() => setEditing(true)}>
|
||||
{t("alerts.thresholds.edit")}
|
||||
</Button>
|
||||
)}
|
||||
{saved && (
|
||||
<span role="status" className="text-sm text-muted-foreground">
|
||||
{t("alerts.thresholds.saved")}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</form>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,6 +1,14 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { act, renderWithProviders, screen, testQueryClient, waitFor, within } from "../../../tests/test-utils";
|
||||
import {
|
||||
act,
|
||||
fireEvent,
|
||||
renderWithProviders,
|
||||
screen,
|
||||
testQueryClient,
|
||||
waitFor,
|
||||
within,
|
||||
} from "../../../tests/test-utils";
|
||||
import Alerts from "./Alerts";
|
||||
import type { SavedAlert } from "./alertPresentation";
|
||||
|
||||
|
|
@ -49,12 +57,21 @@ const calls: { path: string; query: URLSearchParams; method: string; body: unkno
|
|||
let status = "open";
|
||||
let rejectWrite = false;
|
||||
let rejectHistory = false;
|
||||
let includeDeprecation = false;
|
||||
let deprecationStatus = "open";
|
||||
const thresholds = {
|
||||
spend: { multiplier: 3, minimum_increase: 50 },
|
||||
traffic: { multiplier: 10, minimum_increase: 1000 },
|
||||
latency: { multiplier: 3, minimum_increase: 500 },
|
||||
reliability: { multiplier: 3, minimum_increase: 0.02 },
|
||||
};
|
||||
let monitoring = {
|
||||
model_deprecations_enabled: true,
|
||||
enabled: true,
|
||||
alerts_used: 0,
|
||||
alert_limit: 5 as number | null,
|
||||
disabled_by_config: false,
|
||||
thresholds,
|
||||
};
|
||||
let rejectMonitoring = false;
|
||||
function response(data: unknown, code = 200) {
|
||||
|
|
@ -72,6 +89,50 @@ function promptCacheResponse(url: URL): Response | undefined {
|
|||
});
|
||||
}
|
||||
|
||||
function deprecationResponse(path: string, method: string, body: { status?: string } | null) {
|
||||
if (path === "/model/deprecations")
|
||||
return response({
|
||||
deprecated: [],
|
||||
imminent: includeDeprecation
|
||||
? [
|
||||
{
|
||||
model_name: "example-model",
|
||||
litellm_model: "example/model",
|
||||
deprecation_date: "2026-12-01",
|
||||
status: "imminent",
|
||||
},
|
||||
]
|
||||
: [],
|
||||
});
|
||||
if (path === "/user/daily/activity/aggregated")
|
||||
return response({
|
||||
results: [{ date: "2026-11-16", breakdown: { models: { "example/model": { metrics: { api_requests: 10 } } } } }],
|
||||
});
|
||||
if (path === "/alerting/model-deprecations/state") {
|
||||
if (method === "PUT") deprecationStatus = body?.status ?? "open";
|
||||
const state = {
|
||||
model: "example/model",
|
||||
deprecation_date: "2026-12-01",
|
||||
status: deprecationStatus,
|
||||
last_seen_date: "2026-11-16",
|
||||
last_seen_requests: 10,
|
||||
};
|
||||
return response(state);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function updateAlertResponse(url: URL, body: { status: string; type?: string }) {
|
||||
if (rejectWrite) return response({ detail: "Unavailable" }, 503);
|
||||
status = body.status;
|
||||
if (url.pathname === "/alerts") {
|
||||
if (!body.type || body.type === "prompt_cache")
|
||||
cacheIncidents = cacheIncidents.map((incident) => ({ ...incident, status }));
|
||||
return response({ updated: 30 });
|
||||
}
|
||||
return response({ ...(url.pathname === `/alerts/${cacheIncident.id}` ? cacheIncident : alert), status });
|
||||
}
|
||||
|
||||
describe("Alerts", () => {
|
||||
beforeEach(() => {
|
||||
testQueryClient.clear();
|
||||
|
|
@ -79,6 +140,8 @@ describe("Alerts", () => {
|
|||
status = "open";
|
||||
rejectWrite = false;
|
||||
rejectHistory = false;
|
||||
includeDeprecation = false;
|
||||
deprecationStatus = "open";
|
||||
rejectMonitoring = false;
|
||||
monitoring = {
|
||||
model_deprecations_enabled: true,
|
||||
|
|
@ -86,6 +149,7 @@ describe("Alerts", () => {
|
|||
alerts_used: 0,
|
||||
alert_limit: 5,
|
||||
disabled_by_config: false,
|
||||
thresholds,
|
||||
};
|
||||
role = "proxy_admin";
|
||||
cacheIncidents = [];
|
||||
|
|
@ -94,12 +158,13 @@ describe("Alerts", () => {
|
|||
vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
|
||||
const request = input instanceof Request ? input : new Request(input);
|
||||
const url = new URL(request.url);
|
||||
const body = request.method === "PATCH" ? await request.clone().json() : null;
|
||||
const body = ["PATCH", "PUT"].includes(request.method) ? await request.clone().json() : null;
|
||||
const call = { path: url.pathname, query: url.searchParams, method: request.method, body };
|
||||
calls.push(call);
|
||||
if (url.pathname === "/litellm/.well-known/litellm-ui-config") return response({});
|
||||
if (url.pathname === "/management/v1/users/me") return response({ user_id: "example-admin", user_role: role });
|
||||
if (url.pathname === "/model/deprecations") return response({ deprecated: [], imminent: [] });
|
||||
const retirement = deprecationResponse(url.pathname, request.method, body);
|
||||
if (retirement) return retirement;
|
||||
if (url.pathname === "/alerts/monitoring") {
|
||||
if (rejectMonitoring) return response({ detail: "Unavailable" }, 503);
|
||||
if (request.method === "PATCH") {
|
||||
|
|
@ -108,11 +173,7 @@ describe("Alerts", () => {
|
|||
}
|
||||
return response(monitoring);
|
||||
}
|
||||
if (request.method === "PATCH") {
|
||||
if (rejectWrite) return response({ detail: "Unavailable" }, 503);
|
||||
status = body.status;
|
||||
return response({ ...(url.pathname === `/alerts/${cacheIncident.id}` ? cacheIncident : alert), status });
|
||||
}
|
||||
if (request.method === "PATCH") return updateAlertResponse(url, body);
|
||||
const promptCache = promptCacheResponse(url);
|
||||
if (promptCache) return promptCache;
|
||||
if (url.pathname === `/alerts/${alert.id}`)
|
||||
|
|
@ -296,6 +357,122 @@ describe("Alerts", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it.each(["Resolve all", "Dismiss all"])("%s updates the selected category beyond the loaded page", async (action) => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />, { searchParams: { type: "spend" } });
|
||||
await screen.findByRole("button", { name: /Spend on Research agent/ });
|
||||
await user.click(screen.getByRole("button", { name: action }));
|
||||
const dialog = screen.getByRole("alertdialog");
|
||||
expect(dialog).toHaveTextContent("including results you have not loaded");
|
||||
await user.click(within(dialog).getByRole("button", { name: action }));
|
||||
expect(await screen.findByText("No open alerts.")).toBeInTheDocument();
|
||||
expect(calls.filter((call) => call.method === "PATCH")).toEqual([
|
||||
expect.objectContaining({
|
||||
path: "/alerts",
|
||||
body: { status: action === "Resolve all" ? "resolved" : "dismissed", type: "spend" },
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps bulk failures retryable and leaves open alerts visible", async () => {
|
||||
rejectWrite = true;
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
await screen.findByRole("button", { name: /Spend on Research agent/ });
|
||||
await user.click(screen.getByRole("button", { name: "Dismiss all" }));
|
||||
await user.click(within(screen.getByRole("alertdialog")).getByRole("button", { name: "Dismiss all" }));
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("Some alerts could not be updated");
|
||||
rejectWrite = false;
|
||||
await user.click(within(screen.getByRole("alertdialog")).getByRole("button", { name: "Dismiss all" }));
|
||||
expect(await screen.findByText("No open alerts.")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("resolves performance, prompt cache, and retirement alerts together in All alerts", async () => {
|
||||
includeDeprecation = true;
|
||||
cacheIncidents = [cacheIncident];
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
await screen.findByRole("button", { name: /example\/model retires/ });
|
||||
await user.click(screen.getByRole("button", { name: "Resolve all" }));
|
||||
await user.click(within(screen.getByRole("alertdialog")).getByRole("button", { name: "Resolve all" }));
|
||||
expect(await screen.findByText("No open alerts.")).toBeInTheDocument();
|
||||
expect(calls.filter((call) => ["PATCH", "PUT"].includes(call.method))).toEqual([
|
||||
expect.objectContaining({ path: "/alerts", body: { status: "resolved" } }),
|
||||
expect.objectContaining({
|
||||
path: "/alerting/model-deprecations/state",
|
||||
body: {
|
||||
model: "example/model",
|
||||
deprecation_date: "2026-12-01",
|
||||
status: "resolved",
|
||||
last_seen_date: "2026-11-16",
|
||||
last_seen_requests: 10,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("shows the current thresholds to free admins without an edit action", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
await user.click(await screen.findByRole("button", { name: "Manage alerts" }));
|
||||
expect(await screen.findByText("At least 3× the usual spend and $50 higher.")).toBeInTheDocument();
|
||||
expect(screen.getByText("Editing thresholds requires Enterprise.")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Edit thresholds" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("edits Enterprise thresholds with explicit units and saves the values", async () => {
|
||||
monitoring.alert_limit = null;
|
||||
monitoring.thresholds = { ...thresholds, latency: { multiplier: 1.005, minimum_increase: 500 } };
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
await user.click(await screen.findByRole("button", { name: "Manage alerts" }));
|
||||
await user.click(await screen.findByRole("button", { name: "Edit thresholds" }));
|
||||
const spend = screen.getByRole("group", { name: "Spend" });
|
||||
const multiplier = within(spend).getByRole("spinbutton", { name: "Times the usual value" });
|
||||
fireEvent.change(multiplier, { target: { value: "1" } });
|
||||
await user.click(screen.getByRole("button", { name: "Save thresholds" }));
|
||||
expect(multiplier).toBeInvalid();
|
||||
expect(calls.filter((call) => call.method === "PATCH")).toEqual([]);
|
||||
fireEvent.change(multiplier, {
|
||||
target: { value: "4" },
|
||||
});
|
||||
fireEvent.change(screen.getByRole("spinbutton", { name: "Minimum increase ($)" }), { target: { value: "75" } });
|
||||
fireEvent.change(screen.getByRole("spinbutton", { name: "Minimum increase (percentage points)" }), {
|
||||
target: { value: "3.5" },
|
||||
});
|
||||
await user.click(screen.getByRole("button", { name: "Save thresholds" }));
|
||||
expect(await screen.findByText("Thresholds saved.")).toBeInTheDocument();
|
||||
expect(screen.getByText("At least 4× the usual spend and $75 higher.")).toBeInTheDocument();
|
||||
expect(screen.getByText("At least 3× the usual error rate and 3.5 percentage points higher.")).toBeInTheDocument();
|
||||
expect(calls.find((call) => call.method === "PATCH")).toMatchObject({
|
||||
path: "/alerts/monitoring",
|
||||
body: {
|
||||
thresholds: {
|
||||
...thresholds,
|
||||
spend: { multiplier: 4, minimum_increase: 75 },
|
||||
latency: { multiplier: 1.005, minimum_increase: 500 },
|
||||
reliability: { multiplier: 3, minimum_increase: 0.035 },
|
||||
},
|
||||
},
|
||||
});
|
||||
await user.click(screen.getByRole("button", { name: "Edit thresholds" }));
|
||||
expect(screen.getByRole("spinbutton", { name: "Minimum increase ($)" })).toHaveValue(75);
|
||||
expect(screen.getByRole("spinbutton", { name: "Minimum increase (percentage points)" })).toHaveValue(3.5);
|
||||
});
|
||||
|
||||
it("shows free thresholds without editing and hides bulk actions for viewers", async () => {
|
||||
role = "proxy_admin_viewer";
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
await screen.findByRole("button", { name: /Spend on Research agent/ });
|
||||
expect(screen.queryByRole("button", { name: "Resolve all" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Dismiss all" })).not.toBeInTheDocument();
|
||||
await user.click(screen.getByRole("button", { name: "Manage alerts" }));
|
||||
expect(await screen.findByText("At least 3× the usual spend and $50 higher.")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Edit thresholds" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("spinbutton")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("lets viewers read the evidence without write controls", async () => {
|
||||
role = "proxy_admin_viewer";
|
||||
renderWithProviders(<Alerts />, { searchParams: { alert: alert.id } });
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import AlertActions from "./AlertActions";
|
|||
import AlertChart from "./AlertChart";
|
||||
import AlertDetail from "./AlertDetail";
|
||||
import ManageAlerts from "./ManageAlerts";
|
||||
import BulkAlertActions from "./BulkAlertActions";
|
||||
|
||||
const parsers = {
|
||||
alert: parseAsString,
|
||||
|
|
@ -119,6 +120,8 @@ function AlertsInbox({ auth }: { auth: ReturnType<typeof useAuthorized> }) {
|
|||
const pagedLists = [...(showPromptCache ? [promptCache] : []), ...(showPerformance ? [list] : [])];
|
||||
const loading = pagedLists.some((query) => query.isLoading) || deprecations.isLoading;
|
||||
const hasError = pagedLists.some((query) => query.isError) || (showDeprecations && deprecations.isError);
|
||||
const refreshing = pagedLists.some((query) => query.isFetching) || deprecations.isFetching;
|
||||
const bulkUnavailable = hasError || items.length === 0;
|
||||
function refresh() {
|
||||
void client.invalidateQueries({ queryKey: ["alert-monitoring"] });
|
||||
pagedLists.forEach((query) => void query.refetch());
|
||||
|
|
@ -140,7 +143,7 @@ function AlertsInbox({ auth }: { auth: ReturnType<typeof useAuthorized> }) {
|
|||
))}
|
||||
</PageTabsList>
|
||||
</PageTabs>
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<Select value={params.type} onValueChange={(value) => void setParams({ type: value as AlertFilter })}>
|
||||
<SelectTrigger className="w-52" aria-label={t("alerts.unified.typeFilter")}>
|
||||
<SelectValue>{t(TYPE_LABEL_KEYS[params.type])}</SelectValue>
|
||||
|
|
@ -153,15 +156,19 @@ function AlertsInbox({ auth }: { auth: ReturnType<typeof useAuthorized> }) {
|
|||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={pagedLists.some((query) => query.isFetching) || deprecations.isFetching}
|
||||
onClick={refresh}
|
||||
>
|
||||
<RefreshCw className="size-3.5" />
|
||||
{t("alerts.unified.refresh")}
|
||||
</Button>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<BulkAlertActions
|
||||
filter={params.type}
|
||||
state={params.state}
|
||||
readOnly={Boolean(auth.isViewOnly)}
|
||||
deprecations={deprecations.data}
|
||||
disabled={bulkUnavailable || refreshing}
|
||||
/>
|
||||
<Button variant="outline" size="sm" disabled={refreshing} onClick={refresh}>
|
||||
<RefreshCw className="size-3.5" />
|
||||
{t("alerts.unified.refresh")}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
{showDeprecations && deprecations.isError && (
|
||||
<LoadError retry={() => void deprecations.refetch()}>{t("alerts.unified.deprecationsError")}</LoadError>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,104 @@
|
|||
import { useState } from "react";
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
AlertDialog,
|
||||
AlertDialogContent,
|
||||
AlertDialogDescription,
|
||||
AlertDialogFooter,
|
||||
AlertDialogHeader,
|
||||
AlertDialogTitle,
|
||||
AlertDialogCancel,
|
||||
} from "@/components/ui/alert-dialog";
|
||||
import { backgroundFetchClient } from "@/lib/http/api";
|
||||
import type { DeprecationAlert } from "@/components/ModelDeprecationAlerts/deprecationAlerts";
|
||||
import { AlertStatus, effectiveAlertStatus } from "@/components/ModelDeprecationAlerts/migrationState";
|
||||
import type { AlertFilter } from "./alertPresentation";
|
||||
import { TYPE_LABEL_KEYS } from "./alertPresentation";
|
||||
|
||||
type BulkStatus = "resolved" | "dismissed";
|
||||
|
||||
export default function BulkAlertActions({
|
||||
filter,
|
||||
state,
|
||||
readOnly,
|
||||
deprecations,
|
||||
disabled,
|
||||
}: {
|
||||
filter: AlertFilter;
|
||||
state: AlertStatus;
|
||||
readOnly: boolean;
|
||||
deprecations: DeprecationAlert[] | undefined;
|
||||
disabled: boolean;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const client = useQueryClient();
|
||||
const [status, setStatus] = useState<BulkStatus | null>(null);
|
||||
const mutation = useMutation({
|
||||
mutationFn: async (next: BulkStatus) => {
|
||||
if (filter !== "model-deprecations") {
|
||||
const { data } = await backgroundFetchClient.PATCH("/alerts", {
|
||||
body: { status: next, type: filter === "all" ? undefined : filter },
|
||||
});
|
||||
if (!data) throw new Error("Could not update alerts");
|
||||
}
|
||||
for (const alert of deprecations ?? []) {
|
||||
if (effectiveAlertStatus(alert) !== AlertStatus.Open) continue;
|
||||
const { data } = await backgroundFetchClient.PUT("/alerting/model-deprecations/state", {
|
||||
body: {
|
||||
model: alert.usageModel,
|
||||
deprecation_date: alert.model.deprecation_date,
|
||||
status: next,
|
||||
last_seen_date: alert.lastUsedDate,
|
||||
last_seen_requests: alert.lastDayRequests,
|
||||
},
|
||||
});
|
||||
if (!data) throw new Error("Could not update model deprecation alerts");
|
||||
}
|
||||
},
|
||||
onSuccess: () => setStatus(null),
|
||||
onSettled: () =>
|
||||
Promise.all(["alerts", "alert", "deprecationAlerts"].map((key) => client.invalidateQueries({ queryKey: [key] }))),
|
||||
});
|
||||
function choose(next: BulkStatus) {
|
||||
mutation.reset();
|
||||
setStatus(next);
|
||||
}
|
||||
if (readOnly || state !== AlertStatus.Open) return null;
|
||||
return (
|
||||
<>
|
||||
<Button variant="outline" size="sm" disabled={disabled || mutation.isPending} onClick={() => choose("resolved")}>
|
||||
{t("alerts.bulk.resolve")}
|
||||
</Button>
|
||||
<Button variant="outline" size="sm" disabled={disabled || mutation.isPending} onClick={() => choose("dismissed")}>
|
||||
{t("alerts.bulk.dismiss")}
|
||||
</Button>
|
||||
<AlertDialog open={status !== null} onOpenChange={(open) => !open && !mutation.isPending && setStatus(null)}>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>
|
||||
{t(status === "resolved" ? "alerts.bulk.resolve" : "alerts.bulk.dismiss")}
|
||||
</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
{t("alerts.bulk.description", { type: t(TYPE_LABEL_KEYS[filter]) })}
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
{mutation.isError && (
|
||||
<p role="alert" className="text-sm text-destructive">
|
||||
{t("alerts.bulk.error")}
|
||||
</p>
|
||||
)}
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel disabled={mutation.isPending}>{t("alerts.bulk.cancel")}</AlertDialogCancel>
|
||||
<Button disabled={mutation.isPending} onClick={() => status && mutation.mutate(status)}>
|
||||
{mutation.isPending
|
||||
? t("alerts.bulk.saving")
|
||||
: t(status === "resolved" ? "alerts.bulk.resolve" : "alerts.bulk.dismiss")}
|
||||
</Button>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
@ -15,6 +15,7 @@ import { Switch } from "@/components/ui/switch";
|
|||
import { backgroundFetchClient } from "@/lib/http/api";
|
||||
import { getProxyBaseUrl } from "@/components/networking";
|
||||
import type { components } from "@/lib/http/schema";
|
||||
import AlertThresholds from "./AlertThresholds";
|
||||
|
||||
function statusMessage(data: components["schemas"]["AlertMonitoring"]) {
|
||||
if (data.disabled_by_config) return "alerts.monitoring.disabledByConfig";
|
||||
|
|
@ -41,7 +42,10 @@ export default function ManageAlerts({ scope, readOnly }: { scope: string; readO
|
|||
const query = useQuery(queryOptions);
|
||||
const mutation = useMutation({
|
||||
mutationFn: async (
|
||||
body: components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"],
|
||||
body:
|
||||
| components["schemas"]["AlertMonitoringUpdate"]
|
||||
| components["schemas"]["ModelDeprecationMonitoringUpdate"]
|
||||
| components["schemas"]["AlertThresholdsUpdate"],
|
||||
) => {
|
||||
const { data } = await backgroundFetchClient.PATCH("/alerts/monitoring", { body });
|
||||
if (!data) throw new Error("Could not save monitoring settings");
|
||||
|
|
@ -64,7 +68,7 @@ export default function ManageAlerts({ scope, readOnly }: { scope: string; readO
|
|||
<Settings className="size-4" aria-hidden />
|
||||
{t("alerts.monitoring.manage")}
|
||||
</DialogTrigger>
|
||||
<DialogContent>
|
||||
<DialogContent className="max-h-[90vh] overflow-y-auto sm:max-w-xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle>{t("alerts.monitoring.manage")}</DialogTitle>
|
||||
<DialogDescription>{t("alerts.monitoring.description")}</DialogDescription>
|
||||
|
|
@ -125,6 +129,13 @@ export default function ManageAlerts({ scope, readOnly }: { scope: string; readO
|
|||
aria-describedby="performance-monitoring-status"
|
||||
/>
|
||||
</div>
|
||||
<AlertThresholds
|
||||
thresholds={data.thresholds}
|
||||
enterprise={data.alert_limit === null}
|
||||
readOnly={readOnly}
|
||||
pending={mutation.isPending}
|
||||
save={(thresholds) => mutation.mutateAsync({ thresholds })}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{mutation.isError && (
|
||||
|
|
|
|||
|
|
@ -6096,6 +6096,45 @@
|
|||
"upgrade": "Upgrade to Enterprise",
|
||||
"loadError": "Could not load monitoring settings.",
|
||||
"saveError": "Could not save monitoring settings. Try again."
|
||||
},
|
||||
"bulk": {
|
||||
"resolve": "Resolve all",
|
||||
"dismiss": "Dismiss all",
|
||||
"cancel": "Cancel",
|
||||
"saving": "Updating alerts…",
|
||||
"description": "This updates all open alerts matching “{{type}}”, including results you have not loaded. You can reopen individual alerts later.",
|
||||
"error": "Some alerts could not be updated. Saved changes were kept. Try again to update the remaining open alerts."
|
||||
},
|
||||
"thresholds": {
|
||||
"title": "Performance alert thresholds",
|
||||
"description": "Usual values come from the previous seven days. Both conditions must be met.",
|
||||
"enterpriseOnly": "Editing thresholds requires Enterprise.",
|
||||
"edit": "Edit thresholds",
|
||||
"cancel": "Cancel",
|
||||
"multiplier": "Times the usual value",
|
||||
"multiplierInvalid": "Use a multiplier greater than 1.",
|
||||
"rules": {
|
||||
"spend": "Spend",
|
||||
"traffic": "Traffic",
|
||||
"latency": "Latency",
|
||||
"reliability": "Error rate"
|
||||
},
|
||||
"minimum": {
|
||||
"spend": "Minimum increase ($)",
|
||||
"traffic": "Minimum increase (requests)",
|
||||
"latency": "Minimum increase (ms)",
|
||||
"reliability": "Minimum increase (percentage points)"
|
||||
},
|
||||
"summary": {
|
||||
"spend": "At least {{multiplier}}× the usual spend and ${{minimum}} higher.",
|
||||
"traffic": "At least {{multiplier}}× the usual requests and {{minimum}} more requests.",
|
||||
"latency": "At least {{multiplier}}× the usual latency and {{minimum}} ms higher.",
|
||||
"reliability": "At least {{multiplier}}× the usual error rate and {{minimum}} percentage points higher."
|
||||
},
|
||||
"safeguards": "Requires four active baseline days. Latency also needs 100 timed requests; error rate needs 500 requests. Changes apply to future checks.",
|
||||
"save": "Save thresholds",
|
||||
"saving": "Saving…",
|
||||
"saved": "Thresholds saved."
|
||||
}
|
||||
},
|
||||
"caching": {
|
||||
|
|
|
|||
138
litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
138
litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -1019,7 +1019,11 @@ export interface paths {
|
|||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
/**
|
||||
* Resolve or dismiss matching open alerts
|
||||
* @description Update all matching open saved alerts, including unloaded pages. Only gateway admins can write.
|
||||
*/
|
||||
patch: operations["update_open_alerts_alerts_patch"];
|
||||
trace?: never;
|
||||
};
|
||||
"/alerts/monitoring": {
|
||||
|
|
@ -27750,6 +27754,28 @@ export interface components {
|
|||
*/
|
||||
window_start: string;
|
||||
};
|
||||
/** AlertBulkResult */
|
||||
AlertBulkResult: {
|
||||
/**
|
||||
* Updated
|
||||
* @description Number of open saved alerts changed.
|
||||
*/
|
||||
updated: number;
|
||||
};
|
||||
/** AlertBulkUpdate */
|
||||
AlertBulkUpdate: {
|
||||
/**
|
||||
* Status
|
||||
* @description Status to apply to every matching open saved alert.
|
||||
* @enum {string}
|
||||
*/
|
||||
status: "resolved" | "dismissed";
|
||||
/**
|
||||
* Type
|
||||
* @description Only update this category, or all saved alerts if omitted.
|
||||
*/
|
||||
type?: ("spend" | "traffic" | "latency" | "reliability") | ("model_deprecation" | "prompt_cache") | "performance" | null;
|
||||
};
|
||||
/** AlertHistoryPoint */
|
||||
AlertHistoryPoint: {
|
||||
/**
|
||||
|
|
@ -27792,6 +27818,28 @@ export interface components {
|
|||
* @default true
|
||||
*/
|
||||
model_deprecations_enabled: boolean;
|
||||
/**
|
||||
* @description Effective built-in thresholds. Only Enterprise admins can edit them.
|
||||
* @default {
|
||||
* "latency": {
|
||||
* "minimum_increase": 500,
|
||||
* "multiplier": 3
|
||||
* },
|
||||
* "reliability": {
|
||||
* "minimum_increase": 0.02,
|
||||
* "multiplier": 3
|
||||
* },
|
||||
* "spend": {
|
||||
* "minimum_increase": 50,
|
||||
* "multiplier": 3
|
||||
* },
|
||||
* "traffic": {
|
||||
* "minimum_increase": 1000,
|
||||
* "multiplier": 10
|
||||
* }
|
||||
* }
|
||||
*/
|
||||
thresholds: components["schemas"]["PerformanceAlertThresholds"];
|
||||
};
|
||||
/** AlertMonitoringUpdate */
|
||||
AlertMonitoringUpdate: {
|
||||
|
|
@ -27801,6 +27849,24 @@ export interface components {
|
|||
*/
|
||||
enabled: boolean;
|
||||
};
|
||||
/** AlertThreshold */
|
||||
AlertThreshold: {
|
||||
/**
|
||||
* Minimum Increase
|
||||
* @description Minimum absolute increase above the baseline, in the metric's unit.
|
||||
*/
|
||||
minimum_increase: number;
|
||||
/**
|
||||
* Multiplier
|
||||
* @description Minimum multiple of the baseline.
|
||||
*/
|
||||
multiplier: number;
|
||||
};
|
||||
/** AlertThresholdsUpdate */
|
||||
AlertThresholdsUpdate: {
|
||||
/** @description Replace the built-in performance thresholds. Enterprise only. */
|
||||
thresholds: components["schemas"]["PerformanceAlertThresholds"];
|
||||
};
|
||||
/**
|
||||
* AlertType
|
||||
* @description Enum for alert types and management event types
|
||||
|
|
@ -43769,6 +43835,41 @@ export interface components {
|
|||
/** User Id */
|
||||
user_id: string;
|
||||
};
|
||||
/** PerformanceAlertThresholds */
|
||||
PerformanceAlertThresholds: {
|
||||
/**
|
||||
* @description Average response-time increase in milliseconds over the previous seven-day weighted average.
|
||||
* @default {
|
||||
* "minimum_increase": 500,
|
||||
* "multiplier": 3
|
||||
* }
|
||||
*/
|
||||
latency: components["schemas"]["AlertThreshold"];
|
||||
/**
|
||||
* @description Error-rate increase as a fraction, e.g. 0.02 is two percentage points.
|
||||
* @default {
|
||||
* "minimum_increase": 0.02,
|
||||
* "multiplier": 3
|
||||
* }
|
||||
*/
|
||||
reliability: components["schemas"]["AlertThreshold"];
|
||||
/**
|
||||
* @description Spend increase in USD over the previous seven-day daily average.
|
||||
* @default {
|
||||
* "minimum_increase": 50,
|
||||
* "multiplier": 3
|
||||
* }
|
||||
*/
|
||||
spend: components["schemas"]["AlertThreshold"];
|
||||
/**
|
||||
* @description Request increase over the previous seven-day daily average.
|
||||
* @default {
|
||||
* "minimum_increase": 1000,
|
||||
* "multiplier": 10
|
||||
* }
|
||||
*/
|
||||
traffic: components["schemas"]["AlertThreshold"];
|
||||
};
|
||||
/**
|
||||
* PiiAction
|
||||
* @enum {string}
|
||||
|
|
@ -56356,6 +56457,39 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
update_open_alerts_alerts_patch: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["AlertBulkUpdate"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["AlertBulkResult"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_monitoring_alerts_monitoring_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -56385,7 +56519,7 @@ export interface operations {
|
|||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"];
|
||||
"application/json": components["schemas"]["AlertMonitoringUpdate"] | components["schemas"]["ModelDeprecationMonitoringUpdate"] | components["schemas"]["AlertThresholdsUpdate"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
|
|
|
|||
|
|
@ -336,6 +336,82 @@ async def test_daily_history_matches_scope_and_weighted_metrics(
|
|||
write_rows('DELETE FROM "LiteLLM_Alert" WHERE resource_id=%s', (resource,))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("email", "alias", "expected"),
|
||||
(
|
||||
("person@example.com", "Example user", "person@example.com"),
|
||||
("person@example.com", None, "person@example.com"),
|
||||
(None, "Example user", "Example user"),
|
||||
("", "Example user", "Example user"),
|
||||
(" ", "Example user", "Example user"),
|
||||
(None, None, None),
|
||||
),
|
||||
)
|
||||
async def test_user_alerts_display_email_in_list_and_detail(
|
||||
alerts: Alerts, gateway: Gateway, email: str | None, alias: str | None, expected: str | None
|
||||
) -> None:
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_UserTable" (user_id, user_email, user_alias) VALUES (%s, %s, %s)',
|
||||
(alerts.resource_id, email, alias),
|
||||
)
|
||||
try:
|
||||
saved: Final = await alerts.repository.record(alerts.detection().model_copy(update={"resource_type": "user"}))
|
||||
detail: Final = gateway.request("GET", f"/alerts/{saved.id}")
|
||||
assert detail.status_code == 200, detail.text
|
||||
view: Final = AlertView.model_validate_json(detail.content)
|
||||
assert view.resource_name == expected
|
||||
assert view.resource_id == alerts.resource_id
|
||||
assert view.evidence == saved.evidence
|
||||
listed: Final = gateway.request("GET", "/alerts", params={"type": "spend", "limit": "1"})
|
||||
assert listed.status_code == 200, listed.text
|
||||
assert AlertViewPage.model_validate_json(listed.content).data == (view,)
|
||||
finally:
|
||||
write_rows('DELETE FROM "LiteLLM_UserTable" WHERE user_id=%s', (alerts.resource_id,))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", ("resolved", "dismissed"))
|
||||
async def test_bulk_status_updates_all_matching_open_pages_and_preserves_other_alerts(
|
||||
alerts: Alerts, gateway: Gateway, status: str
|
||||
) -> None:
|
||||
saved: Final = tuple([await alerts.repository.record(alerts.detection(day=day)) for day in range(30)])
|
||||
other: Final = await alerts.repository.record(alerts.detection("traffic"))
|
||||
already_resolved: Final = await alerts.repository.set_status(int(saved[0].id), "resolved", "previous-admin")
|
||||
response: Final = gateway.request("PATCH", "/alerts", {"status": status, "type": "spend"})
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"updated": 29}
|
||||
for alert in saved[1:]:
|
||||
changed: Final = await alerts.repository.get(int(alert.id))
|
||||
assert changed is not None
|
||||
assert changed.status == status
|
||||
assert changed.updated_by
|
||||
assert changed.evidence == alert.evidence
|
||||
assert await alerts.repository.get(int(saved[0].id)) == already_resolved
|
||||
assert await alerts.repository.get(int(other.id)) == other
|
||||
repeat: Final = gateway.request("PATCH", "/alerts", {"status": status, "type": "spend"})
|
||||
assert repeat.status_code == 200, repeat.text
|
||||
assert repeat.json() == {"updated": 0}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_performance_filter_leaves_prompt_cache_until_all_alerts_are_updated(
|
||||
alerts: Alerts, gateway: Gateway
|
||||
) -> None:
|
||||
spend: Final = await alerts.repository.record(alerts.detection())
|
||||
cache: Final = await alerts.repository.record(alerts.detection("prompt_cache"))
|
||||
performance: Final = gateway.request("PATCH", "/alerts", {"status": "resolved", "type": "performance"})
|
||||
assert performance.status_code == 200, performance.text
|
||||
resolved: Final = await alerts.repository.get(int(spend.id))
|
||||
assert resolved is not None and resolved.status == "resolved"
|
||||
assert await alerts.repository.get(int(cache.id)) == cache
|
||||
all_alerts: Final = gateway.request("PATCH", "/alerts", {"status": "dismissed"})
|
||||
assert all_alerts.status_code == 200, all_alerts.text
|
||||
dismissed: Final = await alerts.repository.get(int(cache.id))
|
||||
assert dismissed is not None and dismissed.status == "dismissed"
|
||||
assert await alerts.repository.get(int(spend.id)) == resolved
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_type_filter_applies_before_cursor_limit(alerts: Alerts, gateway: Gateway) -> None:
|
||||
oldest: Final = await alerts.repository.record(alerts.detection("traffic"))
|
||||
|
|
|
|||
|
|
@ -14,7 +14,13 @@ from litellm_proxy_extras.request_log_indexes import ensure_request_log_indexes
|
|||
from psycopg import sql
|
||||
|
||||
from litellm.models.alert import AlertDetection, DeprecationEvidence, PromptCacheEvidence
|
||||
from litellm.models.performance_alert import FREE_ALERT_LIMIT, SweepOwnershipLostError, SweepState
|
||||
from litellm.models.performance_alert import (
|
||||
DEFAULT_THRESHOLDS,
|
||||
FREE_ALERT_LIMIT,
|
||||
PerformanceAlertThresholds,
|
||||
SweepOwnershipLostError,
|
||||
SweepState,
|
||||
)
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.spend_tracking.performance_alerts import run_performance_alerts
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
|
|
@ -145,6 +151,38 @@ async def fixture() -> AsyncIterator[Fixture]:
|
|||
await client.disconnect()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enterprise_thresholds_persist_and_change_detection_but_free_uses_defaults(fixture: Fixture) -> None:
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
|
||||
config: Final = ConfigRepository(fixture.client, use_writer=True)
|
||||
previous: Final = await config.get_param("performance_alert_thresholds")
|
||||
custom: Final = PerformanceAlertThresholds.model_validate({"spend": {"multiplier": 2, "minimum_increase": 20}})
|
||||
enterprise: Final = PerformanceAlertRepository(fixture.client, enterprise=True)
|
||||
free: Final = PerformanceAlertRepository(fixture.client, enterprise=False)
|
||||
resource: Final = f"{fixture.prefix}-custom-threshold"
|
||||
_seed("team", resource, Day(spend=10), Day(spend=40))
|
||||
try:
|
||||
assert (await enterprise.set_thresholds(custom)).thresholds == custom
|
||||
restarted: Final = PerformanceAlertRepository(fixture.client, enterprise=True)
|
||||
assert (await restarted.monitoring()).thresholds == custom
|
||||
assert (await free.monitoring()).thresholds == DEFAULT_THRESHOLDS
|
||||
with pytest.raises(PermissionError, match="Enterprise"):
|
||||
await free.set_thresholds(custom)
|
||||
claim: Final = await free.claim(timedelta(0))
|
||||
assert claim is not None
|
||||
state: Final = SweepState(day=DAY, scope="team", cursor=fixture.prefix)
|
||||
await free.evaluate_page(claim.owner, state, 25)
|
||||
assert fixture.saved() == {}
|
||||
await restarted.evaluate_page(claim.owner, state, 25)
|
||||
assert fixture.saved() == {(resource, "spend"): ("team", "open")}
|
||||
finally:
|
||||
await config.set_param(
|
||||
"performance_alert_thresholds",
|
||||
previous.param_value if previous else DEFAULT_THRESHOLDS.model_dump(mode="json"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_full_sweep_saves_one_alert_per_real_change_and_nothing_for_normal_or_new_usage(
|
||||
fixture: Fixture,
|
||||
|
|
|
|||
|
|
@ -2,10 +2,13 @@ from datetime import date, datetime, timedelta, timezone
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.models.alert import AlertDetection, MetricEvidence
|
||||
from litellm.models.performance_alert import (
|
||||
SCOPES,
|
||||
AlertThreshold,
|
||||
PerformanceAlertThresholds,
|
||||
PerformanceScope,
|
||||
ResourceWindow,
|
||||
SweepState,
|
||||
|
|
@ -46,6 +49,55 @@ def test_steady_usage_raises_nothing() -> None:
|
|||
assert _types() == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kind", "changes", "minimum", "too_high"),
|
||||
(
|
||||
("spend", {"spend": 40}, 20, 31),
|
||||
("traffic", {"requests": 2000}, 500, 1001),
|
||||
("latency", {"response_ms": 300_000}, 150, 201),
|
||||
("reliability", {"failed": 20}, 0.005, 0.011),
|
||||
),
|
||||
)
|
||||
def test_custom_multiplier_and_minimum_both_control_each_existing_rule(
|
||||
kind: str, changes: dict[str, object], minimum: float, too_high: float
|
||||
) -> None:
|
||||
window: Final = _window(**changes)
|
||||
assert detect("team", window, DAY) == ()
|
||||
thresholds: Final = PerformanceAlertThresholds.model_validate(
|
||||
{kind: {"multiplier": 2, "minimum_increase": minimum}}
|
||||
)
|
||||
detected: Final = detect("team", window, DAY, thresholds)
|
||||
assert tuple(alert.type for alert in detected) == (kind,)
|
||||
for threshold in (
|
||||
AlertThreshold(multiplier=5, minimum_increase=minimum),
|
||||
AlertThreshold(multiplier=2, minimum_increase=too_high),
|
||||
):
|
||||
assert detect("team", window, DAY, thresholds.model_copy(update={kind: threshold})) == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"settings",
|
||||
(
|
||||
{"spend": {"multiplier": 1, "minimum_increase": 1}},
|
||||
{"spend": {"multiplier": True, "minimum_increase": 1}},
|
||||
{"latency": {"multiplier": 2, "minimum_increase": -1}},
|
||||
{"traffic": {"multiplier": float("inf"), "minimum_increase": 1}},
|
||||
{"reliability": {"multiplier": 2, "minimum_increase": 1.01}},
|
||||
{"new_rule": {"multiplier": 2, "minimum_increase": 1}},
|
||||
),
|
||||
)
|
||||
def test_thresholds_reject_invalid_values_and_new_rules(settings: dict[str, object]) -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
PerformanceAlertThresholds.model_validate(settings)
|
||||
|
||||
|
||||
def test_zero_minimum_does_not_report_a_zero_error_rate_as_an_increase() -> None:
|
||||
thresholds: Final = PerformanceAlertThresholds.model_validate(
|
||||
{"reliability": {"multiplier": 2, "minimum_increase": 0}}
|
||||
)
|
||||
assert detect("team", _window(failed=0, base_failed=0), DAY, thresholds) == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scope", SCOPES)
|
||||
@pytest.mark.parametrize("requests", (0, 100))
|
||||
def test_quiet_or_idle_resources_do_not_raise_traffic_alerts(scope: PerformanceScope, requests: int) -> None:
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from fastapi import FastAPI
|
|||
from fastapi.testclient import TestClient
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from litellm.models.performance_alert import DEFAULT_THRESHOLDS
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.alerting_endpoints.alerts import router
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
|
@ -73,7 +74,9 @@ def client(app: FastAPI) -> Iterator[TestClient]:
|
|||
@pytest.mark.parametrize(
|
||||
"role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.TEAM, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)
|
||||
)
|
||||
@pytest.mark.parametrize("method,path", (("GET", "/alerts"), ("GET", "/alerts/5"), ("PATCH", "/alerts/5")))
|
||||
@pytest.mark.parametrize(
|
||||
"method,path", (("GET", "/alerts"), ("GET", "/alerts/5"), ("PATCH", "/alerts/5"), ("PATCH", "/alerts"))
|
||||
)
|
||||
def test_permissions_precede_database_access(
|
||||
app: FastAPI, table: MagicMock, role: LitellmUserRoles, method: str, path: str
|
||||
) -> None:
|
||||
|
|
@ -142,6 +145,7 @@ def test_deprecation_preference_updates_without_changing_performance(app: FastAP
|
|||
"alerts_used": 3,
|
||||
"alert_limit": 5,
|
||||
"disabled_by_config": False,
|
||||
"thresholds": DEFAULT_THRESHOLDS.model_dump(),
|
||||
}
|
||||
prisma.writer_db.litellm_config.upsert.assert_awaited_once_with(
|
||||
where={"param_name": "model_deprecation_alerts_enabled"},
|
||||
|
|
@ -171,11 +175,42 @@ def test_config_opt_out_cannot_be_enabled_from_the_api(app: FastAPI) -> None:
|
|||
"alert_limit": 5,
|
||||
"disabled_by_config": True,
|
||||
"model_deprecations_enabled": True,
|
||||
"thresholds": DEFAULT_THRESHOLDS.model_dump(),
|
||||
}
|
||||
assert response.status_code == 409, response.text
|
||||
transaction.execute_raw.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enterprise", (False, True))
|
||||
@pytest.mark.parametrize("role", (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY))
|
||||
def test_only_enterprise_admins_can_save_thresholds(app: FastAPI, enterprise: bool, role: LitellmUserRoles) -> None:
|
||||
prisma: Final = MagicMock()
|
||||
prisma.writer_db.litellm_config.find_unique = AsyncMock(return_value=None)
|
||||
prisma.writer_db.litellm_config.upsert = AsyncMock()
|
||||
transaction: Final = prisma.writer_db.tx.return_value.__aenter__.return_value
|
||||
transaction.query_raw = AsyncMock(return_value=[{"enabled": True, "alerts_used": 0}])
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role)
|
||||
app.dependency_overrides[get_alert_monitoring_repository] = lambda: PerformanceAlertRepository(
|
||||
prisma, enterprise=enterprise
|
||||
)
|
||||
with TestClient(app) as client:
|
||||
response: Final = client.patch(
|
||||
"/alerts/monitoring", json={"thresholds": {"spend": {"multiplier": 2, "minimum_increase": 20}}}
|
||||
)
|
||||
if enterprise and role == LitellmUserRoles.PROXY_ADMIN:
|
||||
assert response.status_code == 200, response.text
|
||||
prisma.writer_db.litellm_config.upsert.assert_awaited_once()
|
||||
else:
|
||||
assert response.status_code == 403, response.text
|
||||
prisma.writer_db.litellm_config.upsert.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", ({"status": "open"}, {"status": "resolved", "type": "unknown"}, {}))
|
||||
def test_bulk_update_rejects_reopening_or_unknown_filters(client: TestClient, body: object) -> None:
|
||||
response: Final = client.patch("/alerts", json=body)
|
||||
assert response.status_code == 422, response.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alert_id", ("0", "-1", "01", "1.0", "oops", "9223372036854775808", "1" * 20))
|
||||
@pytest.mark.parametrize("method", ("GET", "PATCH"))
|
||||
def test_invalid_id_never_accesses_database(client: TestClient, table: MagicMock, alert_id: str, method: str) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue