scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

refactor(alerts): align dashboard routes and add notification retry

Browse source
This commit is contained in:
moe-berri 2026-11-08 16:07:27 +08:00
commit 9469979906
64 changed files with 2560 additions and 3044 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
backend/routes/allowlist.py
View file

@ -110,7 +110,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/v1/rust_control_plane/",
# Alerting / email / IP allowlist
"/alerting/",
"/issues/",
"/alerts/",
"/email/",
"/add/allowed_ip",
"/delete/allowed_ip",
@ -154,7 +154,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
{
"/",
"/routes",
"/issues",
"/alerts",
"/lens",
"/openapi.json",
"/docs",

18
litellm/integrations/SlackAlerting/usage_alerts.py
View file

@ -15,8 +15,8 @@ from pydantic import TypeAdapter
from litellm.models.issues import IssueStore
from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.proxy.issues import IssueDeliveryBinding, IssueDeliveryInput, IssuePendingDelivery
from litellm.types.proxy.usage_alerts import UsageAlert
from litellm.types.proxy.issues import AlertDeliveryBinding, AlertDeliveryInput, AlertPendingDelivery
from litellm.types.proxy.usage_alerts import PerformanceSignal
if TYPE_CHECKING:
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
@ -28,7 +28,7 @@ def configured_destinations(slack: "SlackAlerting") -> tuple[str, ...]:
return _URLS.validate_python(slack.alerting or ()) # pyright: ignore[reportUnknownMemberType] # validate legacy destination config
def activity_url(alert: UsageAlert, base_url: str | None) -> str:
def activity_url(alert: PerformanceSignal, base_url: str | None) -> str:
if not base_url:
return alert.activity_path
parsed: Final = urlsplit(base_url)
@ -37,7 +37,7 @@ def activity_url(alert: UsageAlert, base_url: str | None) -> str:
return base_url.rstrip("/") + "/ui" + alert.activity_path
def incident_message(alert: UsageAlert, url: str | None) -> str:
def incident_message(alert: PerformanceSignal, url: str | None) -> str:
scope: Final = "Gateway" if alert.scope_type == "deployment" else f"{alert.scope_type}: {alert.scope_id}"
unit: Final = (
" ms"
@ -65,14 +65,14 @@ class UsageAlertDelivery:
self._slack = slack
self._store = store
async def send_issue(self, pending: IssuePendingDelivery) -> bool:
async def send_issue(self, pending: AlertPendingDelivery) -> bool:
urls: Final = self._webhooks() if pending.delivery.channel == "slack" else ()
if len(urls) > 1 and pending.delivery.slack_destination is None:
if self._store is None:
raise RuntimeError("Multiple alert destinations require a delivery store")
bindings: Final = (
IssueDeliveryBinding(
delivery=IssueDeliveryInput(
AlertDeliveryBinding(
delivery=AlertDeliveryInput(
id=str(uuid5(NAMESPACE_URL, f"{pending.delivery.id}:{destination}")),
issue_id=pending.issue.id,
channel="slack",
@ -88,7 +88,7 @@ class UsageAlertDelivery:
while batch := tuple(islice(bindings, 10)):
await self._store.queue_deliveries(batch)
return True
path: Final = "/alerts?" + urlencode({"issue": pending.issue.id})
path: Final = "/alerts?" + urlencode({"alert": pending.issue.id})
issue_url: Final = activity_url(
pending.alert.model_copy(update={"activity_path": path}), os.getenv("PROXY_BASE_URL")
)
@ -103,7 +103,7 @@ class UsageAlertDelivery:
async def _send(
self,
pending: IssuePendingDelivery,
pending: AlertPendingDelivery,
urls: tuple[str, ...],
message: str,
subject: str,

90
litellm/models/issues.py
View file

@ -3,23 +3,23 @@ from typing import Literal, Protocol
from litellm.models.team import LiteLLM_TeamTable
from litellm.types.proxy.issues import (
Issue,
IssueActivityResponse,
IssueCohort,
IssueDeliveryBinding,
IssueFamily,
IssueFamilyList,
IssueFamilySetting,
IssueFamilyUpdate,
IssueImpact,
IssueInput,
IssueKind,
IssueList,
IssuePendingDelivery,
IssueProjectionSignal,
IssueScope,
IssueSignal,
IssueSignalList,
Alert,
AlertActivityResponse,
AlertCohort,
AlertDeliveryBinding,
AlertFamily,
AlertFamilyList,
AlertFamilySetting,
AlertFamilyUpdate,
AlertImpact,
AlertInput,
AlertKind,
AlertList,
AlertPendingDelivery,
AlertProjectionSignal,
AlertScope,
AlertSignal,
AlertSignalList,
)
from litellm.types.proxy.management_endpoints.common_daily_activity import (
DailyActivityKeyPageResponse,
@ -30,20 +30,20 @@ from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus
class IssueStore(Protocol):
async def get_issue(self, issue_id: str) -> Issue | None: ...
async def get_issue(self, issue_id: str) -> Alert | None: ...
async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: ...
async def list_issues(
self,
kind: IssueKind | None = None,
family: IssueFamily | None = None,
kind: AlertKind | None = None,
family: AlertFamily | None = None,
status: UsageAlertStatus | None = None,
scope_type: IssueScope | None = None,
scope_type: AlertScope | None = None,
scope_id: str | None = None,
limit: int = 50,
offset: int = 0,
) -> IssueList: ...
) -> AlertList: ...
async def set_status(
self,
@ -51,42 +51,42 @@ class IssueStore(Protocol):
status: UsageAlertStatus,
expected_status: UsageAlertStatus,
actor: str | None = None,
authorized_scopes: tuple[IssueCohort, ...] | None = None,
) -> Issue | None: ...
authorized_scopes: tuple[AlertCohort, ...] | None = None,
) -> Alert | None: ...
async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]: ...
async def status_scopes(self, issue_id: str) -> tuple[AlertCohort, ...]: ...
async def upsert_issues(
self,
inputs: tuple[IssueInput, ...],
inputs: tuple[AlertInput, ...],
reopen: bool = False,
status: UsageAlertStatus | None = None,
) -> tuple[Issue, ...]: ...
) -> tuple[Alert, ...]: ...
async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: ...
async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Alert, ...]: ...
async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: ...
async def add_signal_batch(self, signals: tuple[AlertSignal, ...]) -> None: ...
async def list_signals(
self, issue_id: str, limit: int = 50, offset: int = 0, visible_scope: IssueCohort | None = None
) -> IssueSignalList: ...
self, issue_id: str, limit: int = 50, offset: int = 0, visible_scope: AlertCohort | None = None
) -> AlertSignalList: ...
async def active_signals(
self,
day: date,
limit: int = 500,
cursor: str | None = None,
scope_type: IssueScope | None = None,
scope_type: AlertScope | None = None,
metric: UsageAlertMetric | None = None,
scope_ids: tuple[str, ...] | None = None,
completed_only: bool = False,
) -> tuple[IssueProjectionSignal, ...]: ...
) -> tuple[AlertProjectionSignal, ...]: ...
async def active_resource_issue_ids(self, cursor: str | None = None) -> tuple[str, ...]: ...
async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: ...
async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: ...
async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Alert, ...]: ...
async def recover_issue_ids(self, ids: tuple[str, ...], observed_at: datetime) -> int: ...
@ -96,17 +96,17 @@ class IssueStore(Protocol):
async def finish_deprecation_projection(self, observed_at: datetime) -> int: ...
async def family_settings(self) -> IssueFamilyList: ...
async def family_settings(self) -> AlertFamilyList: ...
async def update_family(
self, family: PerformanceIssueFamily, data: IssueFamilyUpdate
) -> IssueFamilySetting | None: ...
self, family: PerformanceIssueFamily, data: AlertFamilyUpdate
) -> AlertFamilySetting | None: ...
async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: ...
async def queue_deliveries(self, bindings: tuple[AlertDeliveryBinding, ...]) -> None: ...
async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: ...
async def pending_deliveries(self, limit: int = 100) -> tuple[AlertPendingDelivery, ...]: ...
async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: ...
async def pending_delivery(self, delivery_id: str) -> AlertPendingDelivery | None: ...
async def reserve_delivery(self, delivery_id: str) -> bool: ...
@ -121,15 +121,15 @@ class IssueStore(Protocol):
class IssueActivityStore(Protocol):
async def daily_activity(
self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20
) -> IssueActivityResponse: ...
self, cohort: AlertCohort, start: date, end: date, model_limit: int = 20
) -> AlertActivityResponse: ...
async def keys(
self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int
) -> DailyActivityKeyPageResponse: ...
async def users(
self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int
) -> DailyActivityUserPageResponse: ...
async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: ...
async def impact(self, cohort: AlertCohort, start: date, end: date) -> AlertImpact: ...

48
litellm/models/usage_alerts.py
View file

@ -2,35 +2,32 @@ from datetime import date
from typing import Protocol
from litellm.types.proxy.usage_alerts import (
UsageAlert,
UsageAlertGroupList,
UsageAlertGroupUpdate,
UsageAlertGroupUpdateResult,
UsageAlertList,
AlertModelList,
AlertRule,
AlertRuleInput,
AlertRuleList,
PerformanceSignal,
PerformanceSignalList,
UsageAlertMetric,
UsageAlertModelList,
UsageAlertRule,
UsageAlertRuleInput,
UsageAlertRuleKind,
UsageAlertRuleList,
UsageAlertScope,
UsageAlertStatus,
)
class UsageAlertStore(Protocol):
async def get_rule(self, rule_id: str) -> UsageAlertRule | None: ...
async def get_rule(self, rule_id: str) -> AlertRule | None: ...
async def save_rule(
self,
data: UsageAlertRuleInput,
data: AlertRuleInput,
created_by: str | None,
existing: UsageAlertRule | None = None,
existing: AlertRule | None = None,
*,
overrides_defaults: bool = False,
) -> UsageAlertRule | None: ...
) -> AlertRule | None: ...
async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: ...
async def delete_rule(self, rule_id: str, *, existing: AlertRule | None = None) -> bool: ...
async def list_rules(
self,
@ -39,9 +36,9 @@ class UsageAlertStore(Protocol):
limit: int = 50,
offset: int = 0,
kind: UsageAlertRuleKind = "all",
) -> UsageAlertRuleList: ...
) -> AlertRuleList: ...
async def get_alert(self, alert_id: str) -> UsageAlert | None: ...
async def get_alert(self, alert_id: str) -> PerformanceSignal | None: ...
async def list_alerts(
self,
@ -53,24 +50,11 @@ class UsageAlertStore(Protocol):
end_date: date | None = None,
limit: int = 50,
offset: int = 0,
) -> UsageAlertList: ...
) -> PerformanceSignalList: ...
async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None: ...
async def set_status(self, alert_id: str, status: UsageAlertStatus) -> PerformanceSignal | None: ...
async def list_groups(
self,
scope_type: UsageAlertScope | None = None,
scope_id: str | None = None,
status: UsageAlertStatus | None = None,
start_date: date | None = None,
end_date: date | None = None,
limit: int = 50,
offset: int = 0,
) -> UsageAlertGroupList: ...
async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult: ...
async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList: ...
async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> AlertModelList: ...
async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None: ...

87
litellm/proxy/_types.py
View file

@ -595,21 +595,20 @@ class LiteLLMRoutes(enum.Enum):
)
llm_api_routes_without_mcp = sorted(set(llm_api_routes) - set(mcp_inference_routes))
info_routes = [
"/usage/alert-groups",
"/usage/alert-models",
"/usage/alert-rules",
"/usage/alert-rules/{rule_id}",
"/usage/alerts",
"/usage/alerts/{alert_id}",
"/issues",
"/issues/families",
"/issues/families/{family}",
"/issues/{issue_id}",
"/issues/{issue_id}/signals",
"/issues/{issue_id}/activity",
"/issues/{issue_id}/keys",
"/issues/{issue_id}/users",
"/issues/{issue_id}/impact",
"/alerts/models",
"/alerts/rules",
"/alerts/rules/{rule_id}",
"/alerts/signals",
"/alerts/signals/{signal_id}",
"/alerts",
"/alerts/defaults",
"/alerts/defaults/{family}",
"/alerts/{alert_id}",
"/alerts/{alert_id}/signals",
"/alerts/{alert_id}/activity",
"/alerts/{alert_id}/keys",
"/alerts/{alert_id}/users",
"/alerts/{alert_id}/impact",
"/key/info",
"/key/health",
"/team/info",
@ -677,21 +676,20 @@ class LiteLLMRoutes(enum.Enum):
management_routes = (
[
"/usage/alert-groups",
"/usage/alert-models",
"/usage/alert-rules",
"/usage/alert-rules/{rule_id}",
"/usage/alerts",
"/usage/alerts/{alert_id}",
"/issues",
"/issues/families",
"/issues/families/{family}",
"/issues/{issue_id}",
"/issues/{issue_id}/signals",
"/issues/{issue_id}/activity",
"/issues/{issue_id}/keys",
"/issues/{issue_id}/users",
"/issues/{issue_id}/impact",
"/alerts/models",
"/alerts/rules",
"/alerts/rules/{rule_id}",
"/alerts/signals",
"/alerts/signals/{signal_id}",
"/alerts",
"/alerts/defaults",
"/alerts/defaults/{family}",
"/alerts/{alert_id}",
"/alerts/{alert_id}/signals",
"/alerts/{alert_id}/activity",
"/alerts/{alert_id}/keys",
"/alerts/{alert_id}/users",
"/alerts/{alert_id}/impact",
# user
"/user/new",
"/management/v1/users/bulk",
@ -903,21 +901,20 @@ class LiteLLMRoutes(enum.Enum):
)
usage_alert_routes = (
"/usage/alert-groups",
"/usage/alert-models",
"/usage/alert-rules",
"/usage/alert-rules/{rule_id}",
"/usage/alerts",
"/usage/alerts/{alert_id}",
"/issues",
"/issues/families",
"/issues/families/{family}",
"/issues/{issue_id}",
"/issues/{issue_id}/signals",
"/issues/{issue_id}/activity",
"/issues/{issue_id}/keys",
"/issues/{issue_id}/users",
"/issues/{issue_id}/impact",
"/alerts/models",
"/alerts/rules",
"/alerts/rules/{rule_id}",
"/alerts/signals",
"/alerts/signals/{signal_id}",
"/alerts",
"/alerts/defaults",
"/alerts/defaults/{family}",
"/alerts/{alert_id}",
"/alerts/{alert_id}/signals",
"/alerts/{alert_id}/activity",
"/alerts/{alert_id}/keys",
"/alerts/{alert_id}/users",
"/alerts/{alert_id}/impact",
)
self_managed_routes = [

162
litellm/proxy/alerting_endpoints/issues.py
View file

@ -1,4 +1,4 @@
"""Unified issue inbox and investigations backed by existing daily aggregates."""
"""Alerts and their supporting Activity backed by existing daily aggregates."""
from collections.abc import Generator
from contextlib import contextmanager
@ -23,19 +23,19 @@ from litellm.proxy.management_endpoints.common_utils import team_member_has_perm
from litellm.repositories.analytics_budget import AnalyticsCapacityError
from litellm.types.proxy.auth.user_roles import LitellmUserRoles
from litellm.types.proxy.issues import (
Issue,
IssueActivityResponse,
IssueCohort,
IssueFamily,
IssueFamilyList,
IssueFamilySetting,
IssueFamilyUpdate,
IssueImpact,
IssueKind,
IssueList,
IssueScope,
IssueSignalList,
IssueUpdate,
Alert,
AlertActivityResponse,
AlertCohort,
AlertFamily,
AlertFamilyList,
AlertFamilySetting,
AlertFamilyUpdate,
AlertImpact,
AlertKind,
AlertList,
AlertScope,
AlertSignalList,
AlertUpdate,
)
from litellm.types.proxy.management_endpoints.common_daily_activity import (
DailyActivityKeyPageResponse,
@ -44,18 +44,18 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import (
from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily
from litellm.types.proxy.usage_alerts import UsageAlertStatus
router: Final = APIRouter(prefix="/issues", tags=["issues"])
router: Final = APIRouter(prefix="/alerts", tags=["alerts"])
def require_issues(repository: IssueStore | None) -> IssueStore:
if repository is None:
raise HTTPException(status_code=503, detail="Issues require a database connection")
raise HTTPException(status_code=503, detail="Alerts require a database connection")
return repository
def require_activity(repository: IssueActivityStore | None) -> IssueActivityStore:
if repository is None:
raise HTTPException(status_code=503, detail="Issue Activity requires a database connection")
raise HTTPException(status_code=503, detail="Alert Activity requires a database connection")
return repository
@ -68,16 +68,16 @@ def activity_budget() -> Generator[None]:
async def authorized_issue(
issue_id: str,
alert_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
*,
write: bool = False,
) -> Issue:
issue: Final = await require_issues(repository).get_issue(issue_id)
) -> Alert:
issue: Final = await require_issues(repository).get_issue(alert_id)
if issue is None:
raise HTTPException(status_code=404, detail="Issue not found")
raise HTTPException(status_code=404, detail="Alert not found")
await require_scope(
user,
require_repository(alerts),
@ -88,7 +88,7 @@ async def authorized_issue(
return issue
async def can_view_activity(issue: Issue, user: User, repository: IssueStore) -> bool:
async def can_view_activity(issue: Alert, user: User, repository: IssueStore) -> bool:
if issue.cohort.scope_type != "team" or user.user_role in (
LitellmUserRoles.PROXY_ADMIN,
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
@ -102,14 +102,14 @@ async def can_view_activity(issue: Issue, user: User, repository: IssueStore) ->
)
async def require_activity_access(issue: Issue, user: User, repository: IssuesRepositoryDep) -> None:
async def require_activity_access(issue: Alert, user: User, repository: IssuesRepositoryDep) -> None:
if not await can_view_activity(issue, user, require_issues(repository)):
raise HTTPException(status_code=403, detail="Full team Activity permission is required for this investigation")
raise HTTPException(status_code=403, detail="Full team Activity permission is required for this alert")
async def require_status_scopes(
issue: Issue, user: User, repository: IssueStore, alerts: UsageAlertsRepositoryDep
) -> tuple[IssueCohort, ...] | None:
issue: Alert, user: User, repository: IssueStore, alerts: UsageAlertsRepositoryDep
) -> tuple[AlertCohort, ...] | None:
store: Final = require_repository(alerts)
await require_scope(
user,
@ -130,7 +130,7 @@ async def require_status_scopes(
return scopes
def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]:
def activity_range(issue: Alert, start_date: date | None, end_date: date | None) -> tuple[date, date]:
start: Final = start_date or issue.range_start
end: Final = end_date or issue.range_end
if start > end or (end - start).days > 365:
@ -138,44 +138,44 @@ def activity_range(issue: Issue, start_date: date | None, end_date: date | None)
return start, end
@router.get("", response_model=IssueList)
async def list_issues(
@router.get("", response_model=AlertList)
async def list_alerts(
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
kind: IssueKind | None = None,
family: IssueFamily | None = None,
kind: AlertKind | None = None,
family: AlertFamily | None = None,
status: UsageAlertStatus | None = None,
scope_type: IssueScope | None = None,
scope_type: AlertScope | None = None,
scope_id: ScopeId = None,
limit: Limit = 50,
offset: Offset = 0,
) -> IssueList:
) -> AlertList:
await require_scope(
user, require_repository(alerts), "model" if scope_type == "provider" else scope_type, scope_id, write=False
)
return await require_issues(repository).list_issues(kind, family, status, scope_type, scope_id, limit, offset)
@router.get("/families", response_model=IssueFamilyList)
async def list_issue_families(
@router.get("/defaults", response_model=AlertFamilyList)
async def list_alert_defaults(
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
) -> IssueFamilyList:
) -> AlertFamilyList:
await require_scope(user, require_repository(alerts), "deployment", None, write=False)
return await require_issues(repository).family_settings()
@router.patch("/families/{family}", response_model=IssueFamilySetting)
async def update_issue_family(
@router.patch("/defaults/{family}", response_model=AlertFamilySetting)
async def update_alert_default(
family: PerformanceIssueFamily,
data: IssueFamilyUpdate,
data: AlertFamilyUpdate,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
) -> IssueFamilySetting:
require_write_access(user, f"/issues/families/{family}")
) -> AlertFamilySetting:
require_write_access(user, f"/alerts/defaults/{family}")
await require_scope(user, require_repository(alerts), "deployment", None, write=True)
try:
result: Final = await require_issues(repository).update_family(family, data)
@ -188,17 +188,17 @@ async def update_issue_family(
return result
@router.get("/{issue_id}", response_model=Issue)
async def get_issue(
issue_id: str,
@router.get("/{alert_id}", response_model=Alert)
async def get_alert(
alert_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
) -> Issue:
issue: Final = await authorized_issue(issue_id, user, repository, alerts)
) -> Alert:
issue: Final = await authorized_issue(alert_id, user, repository, alerts)
activity_allowed: Final = await can_view_activity(issue, user, require_issues(repository))
try:
require_write_access(user, f"/issues/{issue_id}")
require_write_access(user, f"/alerts/{alert_id}")
await require_status_scopes(issue, user, require_issues(repository), alerts)
except HTTPException as exc:
if exc.status_code != 403:
@ -207,37 +207,37 @@ async def get_issue(
return issue.model_copy(update={"can_update_status": True, "can_view_activity": activity_allowed})
@router.patch("/{issue_id}", response_model=Issue)
async def update_issue(
issue_id: str,
data: IssueUpdate,
@router.patch("/{alert_id}", response_model=Alert)
async def update_alert(
alert_id: str,
data: AlertUpdate,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
) -> Issue:
require_write_access(user, f"/issues/{issue_id}")
issue: Final = await authorized_issue(issue_id, user, repository, alerts)
) -> Alert:
require_write_access(user, f"/alerts/{alert_id}")
issue: Final = await authorized_issue(alert_id, user, repository, alerts)
scopes: Final = await require_status_scopes(issue, user, require_issues(repository), alerts)
if issue.kind == "model_deprecation" and not issue.active and data.status == "open":
raise HTTPException(status_code=409, detail="This retirement issue reopens when the model receives new traffic")
raise HTTPException(status_code=409, detail="This retirement alert reopens when the model receives new traffic")
changed: Final = await require_issues(repository).set_status(
issue_id, data.status, data.expected_status, user.user_id, authorized_scopes=scopes
alert_id, data.status, data.expected_status, user.user_id, authorized_scopes=scopes
)
if changed is None:
raise HTTPException(status_code=409, detail="Issue status changed; reload and retry")
raise HTTPException(status_code=409, detail="Alert status changed; reload and retry")
return changed
@router.get("/{issue_id}/signals", response_model=IssueSignalList)
async def list_issue_signals(
issue_id: str,
@router.get("/{alert_id}/signals", response_model=AlertSignalList)
async def list_alert_signals(
alert_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
limit: Limit = 50,
offset: Offset = 0,
) -> IssueSignalList:
issue: Final = await authorized_issue(issue_id, user, repository, alerts)
) -> AlertSignalList:
issue: Final = await authorized_issue(alert_id, user, repository, alerts)
visible_scope: Final = (
None
if user.user_role
@ -247,12 +247,12 @@ async def list_issue_signals(
)
else issue.cohort
)
return await require_issues(repository).list_signals(issue_id, limit, offset, visible_scope)
return await require_issues(repository).list_signals(alert_id, limit, offset, visible_scope)
@router.get("/{issue_id}/activity", response_model=IssueActivityResponse)
async def get_issue_activity(
issue_id: str,
@router.get("/{alert_id}/activity", response_model=AlertActivityResponse)
async def get_alert_activity(
alert_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
@ -260,17 +260,17 @@ async def get_issue_activity(
start_date: date | None = None,
end_date: date | None = None,
model_limit: Limit = 20,
) -> IssueActivityResponse:
issue: Final = await authorized_issue(issue_id, user, repository, alerts)
) -> AlertActivityResponse:
issue: Final = await authorized_issue(alert_id, user, repository, alerts)
await require_activity_access(issue, user, repository)
start, end = activity_range(issue, start_date, end_date)
with activity_budget():
return await require_activity(activity).daily_activity(issue.cohort, start, end, model_limit)
@router.get("/{issue_id}/keys", response_model=DailyActivityKeyPageResponse)
async def get_issue_keys(
issue_id: str,
@router.get("/{alert_id}/keys", response_model=DailyActivityKeyPageResponse)
async def get_alert_keys(
alert_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
@ -280,16 +280,16 @@ async def get_issue_keys(
limit: Limit = 20,
offset: Offset = 0,
) -> DailyActivityKeyPageResponse:
issue: Final = await authorized_issue(issue_id, user, repository, alerts)
issue: Final = await authorized_issue(alert_id, user, repository, alerts)
await require_activity_access(issue, user, repository)
start, end = activity_range(issue, start_date, end_date)
with activity_budget():
return await require_activity(activity).keys(issue.cohort, start, end, limit, offset)
@router.get("/{issue_id}/users", response_model=DailyActivityUserPageResponse)
async def get_issue_users(
issue_id: str,
@router.get("/{alert_id}/users", response_model=DailyActivityUserPageResponse)
async def get_alert_users(
alert_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
@ -299,7 +299,7 @@ async def get_issue_users(
limit: Limit = 20,
offset: Offset = 0,
) -> DailyActivityUserPageResponse:
issue: Final = await authorized_issue(issue_id, user, repository, alerts)
issue: Final = await authorized_issue(alert_id, user, repository, alerts)
await require_activity_access(issue, user, repository)
if issue.cohort.scope_type == "team":
raise HTTPException(status_code=422, detail="Historical team aggregates do not retain user membership")
@ -308,17 +308,17 @@ async def get_issue_users(
return await require_activity(activity).users(issue.cohort, start, end, limit, offset)
@router.get("/{issue_id}/impact", response_model=IssueImpact)
async def get_issue_impact(
issue_id: str,
@router.get("/{alert_id}/impact", response_model=AlertImpact)
async def get_alert_impact(
alert_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
activity: IssueActivityRepositoryDep,
start_date: date | None = None,
end_date: date | None = None,
) -> IssueImpact:
issue: Final = await authorized_issue(issue_id, user, repository, alerts)
) -> AlertImpact:
issue: Final = await authorized_issue(alert_id, user, repository, alerts)
await require_activity_access(issue, user, repository)
start, end = activity_range(issue, start_date, end_date)
with activity_budget():

127
litellm/proxy/alerting_endpoints/usage_alerts.py
View file

@ -1,4 +1,4 @@
"""Public management API for daily usage alert rules and incidents."""
"""Performance rules and read-only detection evidence."""
from datetime import date
from typing import Annotated, Final, TypeAlias
@ -14,28 +14,24 @@ from litellm.proxy.deps.repositories import UsageAlertsRepositoryDep
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
from litellm.types.proxy.auth.user_roles import LitellmUserRoles
from litellm.types.proxy.usage_alerts import (
UsageAlert,
UsageAlertGroupList,
UsageAlertGroupUpdate,
UsageAlertGroupUpdateResult,
UsageAlertList,
AlertModelList,
AlertRule,
AlertRuleInput,
AlertRuleList,
PerformanceSignal,
PerformanceSignalList,
UsageAlertMetric,
UsageAlertModelList,
UsageAlertRule,
UsageAlertRuleInput,
UsageAlertRuleKind,
UsageAlertRuleList,
UsageAlertScope,
UsageAlertStatus,
UsageAlertUpdate,
)
router: Final = APIRouter(prefix="/usage", tags=["usage alerts"])
router: Final = APIRouter(prefix="/alerts", tags=["alerts"])
def require_repository(repository: UsageAlertStore | None) -> UsageAlertStore:
if repository is None:
raise HTTPException(status_code=503, detail="Usage alerts require a database connection")
raise HTTPException(status_code=503, detail="Alert signals require a database connection")
return repository
@ -71,9 +67,7 @@ def require_write_access(user: UserAPIKeyAuth, route: str) -> None:
raise HTTPException(status_code=403, detail="This credential does not grant alert write access")
def require_notification_access(
user: UserAPIKeyAuth, data: UsageAlertRuleInput, existing: UsageAlertRule | None = None
) -> None:
def require_notification_access(user: UserAPIKeyAuth, data: AlertRuleInput, existing: AlertRule | None = None) -> None:
if user.user_role == LitellmUserRoles.PROXY_ADMIN:
return
if data.notification_channels or data.email_recipients:
@ -97,16 +91,16 @@ async def require_scope(
if user.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY:
if not write:
return
raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts")
raise HTTPException(status_code=403, detail="Read-only users cannot manage alerts")
if user.user_role not in (
LitellmUserRoles.INTERNAL_USER,
LitellmUserRoles.INTERNAL_USER_VIEW_ONLY,
LitellmUserRoles.ORG_ADMIN,
LitellmUserRoles.TEAM,
):
raise HTTPException(status_code=403, detail="Usage alerts require a management role")
raise HTTPException(status_code=403, detail="Alert signals require a management role")
if write and user.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY:
raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts")
raise HTTPException(status_code=403, detail="Read-only users cannot manage alerts")
if scope_id is None:
raise HTTPException(status_code=403, detail="Specify a scope you can access")
if scope_type == "user" and user.user_id is not None and scope_id == user.user_id:
@ -128,8 +122,8 @@ async def require_scope(
raise HTTPException(status_code=403, detail="You cannot access this alert scope")
@router.get("/alert-rules", response_model=UsageAlertRuleList)
async def list_usage_alert_rules(
@router.get("/rules", response_model=AlertRuleList)
async def list_alert_rules(
user: User,
repository: Repository,
scope_type: UsageAlertScope | None = None,
@ -137,59 +131,30 @@ async def list_usage_alert_rules(
limit: Limit = 50,
offset: Offset = 0,
kind: UsageAlertRuleKind = "all",
) -> UsageAlertRuleList:
) -> AlertRuleList:
store: Final = require_repository(repository)
await require_scope(user, store, scope_type, scope_id, write=False)
return await store.list_rules(scope_type, scope_id, limit, offset, kind)
@router.get("/alert-groups", response_model=UsageAlertGroupList)
async def list_usage_alert_groups(
user: User,
repository: Repository,
scope_type: UsageAlertScope | None = None,
scope_id: ScopeId = None,
status: UsageAlertStatus | None = None,
start_date: date | None = None,
end_date: date | None = None,
limit: Limit = 50,
offset: Offset = 0,
) -> UsageAlertGroupList:
store: Final = require_repository(repository)
await require_scope(user, store, scope_type, scope_id, write=False)
if start_date is not None and end_date is not None and start_date > end_date:
raise HTTPException(status_code=422, detail="start_date must not be after end_date")
return await store.list_groups(scope_type, scope_id, status, start_date, end_date, limit, offset)
@router.patch("/alert-groups", response_model=UsageAlertGroupUpdateResult)
async def update_usage_alert_group(
data: UsageAlertGroupUpdate, user: User, repository: Repository
) -> UsageAlertGroupUpdateResult:
store: Final = require_repository(repository)
require_write_access(user, "/usage/alert-groups")
await require_scope(user, store, data.scope_type, data.scope_id, write=True)
return await store.set_group_status(data)
@router.get("/alert-models", response_model=UsageAlertModelList)
async def list_usage_alert_models(
@router.get("/models", response_model=AlertModelList)
async def list_alert_models(
user: User,
repository: Repository,
search: ScopeId = None,
cursor: ScopeId = None,
limit: Limit = 20,
) -> UsageAlertModelList:
) -> AlertModelList:
store: Final = require_repository(repository)
await require_scope(user, store, "model", None, write=False)
return await store.list_reported_models(search, cursor, limit)
@router.post("/alert-rules", response_model=UsageAlertRule, status_code=201)
async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, repository: Repository) -> UsageAlertRule:
"""Create a daily usage alert rule for a scope the caller can manage."""
@router.post("/rules", response_model=AlertRule, status_code=201)
async def create_alert_rule(data: AlertRuleInput, user: User, repository: Repository) -> AlertRule:
"""Create a daily performance rule for a scope the caller can manage."""
store: Final = require_repository(repository)
require_write_access(user, "/usage/alert-rules")
require_write_access(user, "/alerts/rules")
await require_scope(user, store, data.scope_type, data.scope_id, write=True)
require_notification_access(user, data)
saved: Final = await store.save_rule(
@ -200,8 +165,8 @@ async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, reposit
return saved
@router.get("/alert-rules/{rule_id}", response_model=UsageAlertRule)
async def get_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> UsageAlertRule:
@router.get("/rules/{rule_id}", response_model=AlertRule)
async def get_alert_rule(rule_id: str, user: User, repository: Repository) -> AlertRule:
store: Final = require_repository(repository)
rule: Final = await store.get_rule(rule_id)
if rule is None:
@ -210,12 +175,10 @@ async def get_usage_alert_rule(rule_id: str, user: User, repository: Repository)
return rule
@router.put("/alert-rules/{rule_id}", response_model=UsageAlertRule)
async def update_usage_alert_rule(
rule_id: str, data: UsageAlertRuleInput, user: User, repository: Repository
) -> UsageAlertRule:
@router.put("/rules/{rule_id}", response_model=AlertRule)
async def update_alert_rule(rule_id: str, data: AlertRuleInput, user: User, repository: Repository) -> AlertRule:
store: Final = require_repository(repository)
require_write_access(user, f"/usage/alert-rules/{rule_id}")
require_write_access(user, f"/alerts/rules/{rule_id}")
rule: Final = await store.get_rule(rule_id)
if rule is None:
raise HTTPException(status_code=404, detail="Alert rule not found")
@ -237,10 +200,10 @@ async def update_usage_alert_rule(
return saved
@router.delete("/alert-rules/{rule_id}", status_code=204)
async def delete_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> Response:
@router.delete("/rules/{rule_id}", status_code=204)
async def delete_alert_rule(rule_id: str, user: User, repository: Repository) -> Response:
store: Final = require_repository(repository)
require_write_access(user, f"/usage/alert-rules/{rule_id}")
require_write_access(user, f"/alerts/rules/{rule_id}")
rule: Final = await store.get_rule(rule_id)
if rule is None:
raise HTTPException(status_code=404, detail="Alert rule not found")
@ -251,8 +214,8 @@ async def delete_usage_alert_rule(rule_id: str, user: User, repository: Reposito
return Response(status_code=204)
@router.get("/alerts", response_model=UsageAlertList)
async def list_usage_alerts(
@router.get("/signals", response_model=PerformanceSignalList)
async def list_performance_signals(
user: User,
repository: Repository,
scope_type: UsageAlertScope | None = None,
@ -263,7 +226,7 @@ async def list_usage_alerts(
end_date: date | None = None,
limit: Limit = 50,
offset: Offset = 0,
) -> UsageAlertList:
) -> PerformanceSignalList:
store: Final = require_repository(repository)
await require_scope(user, store, scope_type, scope_id, write=False)
if start_date is not None and end_date is not None and start_date > end_date:
@ -271,25 +234,11 @@ async def list_usage_alerts(
return await store.list_alerts(scope_type, scope_id, metric, status, start_date, end_date, limit, offset)
@router.get("/alerts/{alert_id}", response_model=UsageAlert)
async def get_usage_alert(alert_id: str, user: User, repository: Repository) -> UsageAlert:
@router.get("/signals/{signal_id}", response_model=PerformanceSignal)
async def get_performance_signal(signal_id: str, user: User, repository: Repository) -> PerformanceSignal:
store: Final = require_repository(repository)
alert: Final = await store.get_alert(alert_id)
alert: Final = await store.get_alert(signal_id)
if alert is None:
raise HTTPException(status_code=404, detail="Usage alert not found")
raise HTTPException(status_code=404, detail="Alert signal not found")
await require_scope(user, store, alert.scope_type, alert.scope_id, write=False)
return alert
@router.patch("/alerts/{alert_id}", response_model=UsageAlert)
async def update_usage_alert(alert_id: str, data: UsageAlertUpdate, user: User, repository: Repository) -> UsageAlert:
store: Final = require_repository(repository)
require_write_access(user, f"/usage/alerts/{alert_id}")
alert: Final = await store.get_alert(alert_id)
if alert is None:
raise HTTPException(status_code=404, detail="Usage alert not found")
await require_scope(user, store, alert.scope_type, alert.scope_id, write=True)
updated: Final = await store.set_status(alert_id, data.status)
if updated is None:
raise HTTPException(status_code=404, detail="Usage alert not found")
return updated

16
litellm/proxy/db/db_spend_update_writer.py
View file

@ -1986,19 +1986,21 @@ class DBSpendUpdateWriter:
) -> None:
transactions: Final = await queue.flush_and_get_aggregated_daily_spend_update_transactions()
commit_started: Final = asyncio.Event()
commit_task: Final = _start_daily_spend_commit(
commit_started,
lambda: commit(
async def commit_and_record_freshness() -> None:
await commit(
n_retry_times=n_retry_times,
prisma_client=prisma_client,
proxy_logging_obj=proxy_logging_obj,
daily_spend_transactions=cast(dict[str, _DailySpendTransactionT], transactions),
),
)
try:
await asyncio.shield(commit_task)
)
commit_started.set()
if entity_type in ("user", "team"):
await self.usage_alert_flush_recorder.record(prisma_client, entity_type)
commit_task: Final = _start_daily_spend_commit(commit_started, commit_and_record_freshness)
try:
await asyncio.shield(commit_task)
except asyncio.CancelledError:
if commit_started.is_set():
queue.track_interrupted_commit(

4
litellm/proxy/spend_tracking/daily_usage_alerts.py
View file

@ -8,7 +8,7 @@ from typing import Final, Literal, Protocol
from litellm.repositories.usage_alert_metrics import DailyAlertMetrics
from litellm.types.proxy.usage_alert_defaults import is_default_usage_rule
from litellm.types.proxy.usage_alerts import UsageAlertRule
from litellm.types.proxy.usage_alerts import AlertRule
class ComparisonRule(Protocol):
@ -76,7 +76,7 @@ def compare_daily_usage(
if (
rule.metric == "requests"
and rule.direction == "decrease"
and isinstance(rule, UsageAlertRule)
and isinstance(rule, AlertRule)
and is_default_usage_rule(rule)
and any(
sum(row.requests for row in group) < rule.min_requests

20
litellm/proxy/spend_tracking/deprecation_issues.py
View file

@ -22,7 +22,7 @@ from litellm.repositories.analytics_budget import (
)
from litellm.repositories.config_repository import BoundedConfigRepository
from litellm.repositories.usage_alert_metrics import UsageAlertMetricsRepository
from litellm.types.proxy.issues import IssueCohort, IssueDeprecation, IssueImpact, IssueInput, IssueSignal
from litellm.types.proxy.issues import AlertCohort, AlertDeprecation, AlertImpact, AlertInput, AlertSignal
from litellm.types.proxy.model_deprecation import (
DeprecationAlertState,
ModelDeprecationInfo,
@ -84,7 +84,7 @@ def deprecation_issue(
usage: ModelDeprecationUsage,
state: DeprecationAlertState,
now: datetime,
) -> IssueInput:
) -> AlertInput:
today: Final = now.astimezone(timezone.utc).date()
model: Final = candidate.model
name: Final = model.model_name or usage.model
@ -93,21 +93,21 @@ def deprecation_issue(
if model.deprecation_date < today
else f"{name} retires on {model.deprecation_date.isoformat()}"
)
return IssueInput(
return AlertInput(
id=f"deprecation:{candidate.state_key.removeprefix('model_deprecation_alert:')}",
cohort_key=candidate.state_key,
kind="model_deprecation",
family="retirement",
title=title,
cohort=IssueCohort(scope_type="model", scope_id=usage.model),
cohort=AlertCohort(scope_type="model", scope_id=usage.model),
observed_on=usage.last_used_date,
observed_at=now,
range_start=today - timedelta(days=29),
range_end=today,
grouping_reason="model_deprecation",
explanation="This model has recorded requests in the last 30 UTC dates and is retiring within 30 days or has retired.",
impact=IssueImpact(keys=usage.keys, users=usage.users, teams=usage.teams, models=1, requests=usage.requests),
deprecation=IssueDeprecation(
impact=AlertImpact(keys=usage.keys, users=usage.users, teams=usage.teams, models=1, requests=usage.requests),
deprecation=AlertDeprecation(
model=model,
usage_model=usage.model,
requests=usage.requests,
@ -124,8 +124,8 @@ def used_deprecation_issues(
usage: Mapping[str, ModelDeprecationUsage],
states: Mapping[str, DeprecationAlertState],
now: datetime,
) -> tuple[IssueInput, ...]:
def build(candidate: DeprecationCandidate) -> IssueInput:
) -> tuple[AlertInput, ...]:
def build(candidate: DeprecationCandidate) -> AlertInput:
state: Final = states.get(candidate.state_key) or DeprecationAlertState(
model=candidate.usage_model, deprecation_date=candidate.model.deprecation_date
)
@ -175,8 +175,8 @@ class DeprecationIssueProjector:
saved: Final = await self._issues.upsert_issues(issues)
await self._issues.add_signal_batch(
tuple(
IssueSignal(
issue_id=issue.id, source="model_deprecation", source_id=issue.id, deprecation=issue.deprecation
AlertSignal(
alert_id=issue.id, source="model_deprecation", source_id=issue.id, deprecation=issue.deprecation
)
for issue in saved
)

98
litellm/proxy/spend_tracking/performance_issues.py
View file

@ -23,17 +23,17 @@ from litellm.repositories.usage_alert_metrics import (
)
from litellm.repositories.usage_alerts import UsageAlertsRepository
from litellm.types.proxy.issues import (
Issue,
IssueCohort,
IssueComparison,
IssueDeliveryBinding,
IssueDeliveryInput,
IssueImpact,
IssueInput,
IssuePendingDelivery,
IssueProjectionSignal,
IssueSignal,
IssueSignalGroupingReason,
Alert,
AlertCohort,
AlertComparison,
AlertDeliveryBinding,
AlertDeliveryInput,
AlertImpact,
AlertInput,
AlertPendingDelivery,
AlertProjectionSignal,
AlertSignal,
AlertSignalGroupingReason,
)
from litellm.types.proxy.usage_alert_defaults import (
PerformanceIssueFamily,
@ -41,12 +41,12 @@ from litellm.types.proxy.usage_alert_defaults import (
is_default_usage_rule,
)
from litellm.types.proxy.usage_alerts import (
AlertRule,
UsageAlertDirection,
UsageAlertMetric,
UsageAlertMetricPoint,
UsageAlertProjectionCursor,
UsageAlertProjectionWork,
UsageAlertRule,
UsageAlertScope,
)
@ -54,30 +54,30 @@ _SCOPES: Final[tuple[UsageAlertScope, ...]] = ("key", "user", "team", "model", "
_METRICS: Final[tuple[UsageAlertMetric, ...]] = ("spend", "latency", "error_rate", "errors", "requests")
class IssueDelivery(Protocol):
async def send_issue(self, pending: IssuePendingDelivery) -> bool: ...
class AlertDelivery(Protocol):
async def send_issue(self, pending: AlertPendingDelivery) -> bool: ...
@dataclass(frozen=True, slots=True)
class _Candidate:
source: IssueProjectionSignal
source: AlertProjectionSignal
comparison: DailyComparison
rows: tuple[DailyAlertMetrics, ...]
impact: IssueImpact
impact: AlertImpact
window: IssueMetricWindow
@dataclass(frozen=True, slots=True)
class _Target:
issue: IssueInput
reason: IssueSignalGroupingReason = "same_resource"
issue: AlertInput
reason: AlertSignalGroupingReason = "same_resource"
explanation: str = ""
@dataclass(frozen=True, slots=True)
class _ProviderGroup:
evidence: ProviderLatencyEvidence
rule: UsageAlertRule
rule: AlertRule
def _family(metric: UsageAlertMetric) -> PerformanceIssueFamily:
@ -92,7 +92,7 @@ def _cohort_key(family: str, scope: str, entity: str | None, direction: UsageAle
return str(uuid5(NAMESPACE_URL, f"litellm:issue:{family}:{scope}:{entity}:{direction}"))
def _window(source: IssueProjectionSignal, day: date) -> IssueMetricWindow:
def _window(source: AlertProjectionSignal, day: date) -> IssueMetricWindow:
return IssueMetricWindow(
source.alert.id, source.alert.scope_id or "deployment", day - timedelta(days=source.rule.baseline_days), day
)
@ -109,8 +109,8 @@ def _history(candidate: _Candidate, metric: UsageAlertMetric, observed: float) -
)
def _comparison(candidate: _Candidate) -> IssueComparison:
return IssueComparison(
def _comparison(candidate: _Candidate) -> AlertComparison:
return AlertComparison(
metric=candidate.source.rule.metric,
direction=candidate.source.rule.direction,
observed_value=candidate.comparison.observed,
@ -123,11 +123,11 @@ def _comparison(candidate: _Candidate) -> IssueComparison:
)
def _request_context(candidate: _Candidate) -> IssueComparison:
def _request_context(candidate: _Candidate) -> AlertComparison:
history: Final = tuple(row for row in candidate.rows if row.day < candidate.window.day)
baseline: Final = sum(row.requests for row in history) / len({row.day for row in history})
observed: Final = sum(row.requests for row in candidate.rows if row.day == candidate.window.day)
return IssueComparison(
return AlertComparison(
metric="requests",
direction="increase" if observed >= baseline else "decrease",
observed_value=observed,
@ -139,7 +139,7 @@ def _request_context(candidate: _Candidate) -> IssueComparison:
)
def _resource_issue(candidate: _Candidate, now: datetime) -> IssueInput:
def _resource_issue(candidate: _Candidate, now: datetime) -> AlertInput:
rule: Final = candidate.source.rule
family: Final = _family(rule.metric)
cohort_key: Final = _cohort_key(family, rule.scope_type, candidate.source.alert.scope_id, rule.direction)
@ -150,7 +150,7 @@ def _resource_issue(candidate: _Candidate, now: datetime) -> IssueInput:
"errors": "Failed requests",
"requests": "Traffic",
}[rule.metric]
return IssueInput(
return AlertInput(
id=str(
uuid5(NAMESPACE_URL, f"{cohort_key}:{candidate.source.alert.id}:{candidate.window.day}:{rule.updated_at}")
),
@ -158,7 +158,7 @@ def _resource_issue(candidate: _Candidate, now: datetime) -> IssueInput:
kind="performance",
family=family,
title=f"{title} {'increased' if rule.direction == 'increase' else 'decreased'}",
cohort=IssueCohort(scope_type=rule.scope_type, scope_id=candidate.source.alert.scope_id),
cohort=AlertCohort(scope_type=rule.scope_type, scope_id=candidate.source.alert.scope_id),
observed_on=candidate.window.day,
observed_at=now,
range_start=candidate.window.start,
@ -186,7 +186,7 @@ def _volume_explains_spend(traffic: _Candidate, spend: _Candidate) -> bool:
return 0.8 <= cost_ratio / request_ratio <= 1.2
def _bindings(candidate: _Candidate, issue_id: str) -> tuple[IssueDeliveryBinding, ...]:
def _bindings(candidate: _Candidate, issue_id: str) -> tuple[AlertDeliveryBinding, ...]:
rule: Final = candidate.source.rule
destinations: Final = (
tuple(("email", address) for address in rule.email_recipients) if "email" in rule.notification_channels else ()
@ -196,8 +196,8 @@ def _bindings(candidate: _Candidate, issue_id: str) -> tuple[IssueDeliveryBindin
*((("slack", "configured-slack-route"),) if "slack" in rule.notification_channels else ()),
)
return tuple(
IssueDeliveryBinding(
delivery=IssueDeliveryInput(
AlertDeliveryBinding(
delivery=AlertDeliveryInput(
id=str(uuid5(NAMESPACE_URL, f"{issue_id}:{channel}:{destination}")),
issue_id=issue_id,
channel="email" if channel == "email" else "slack",
@ -217,13 +217,13 @@ class PerformanceIssueProjector:
store: IssueStore,
metrics: UsageAlertMetricsRepository,
providers: ProviderLatencyRepository,
delivery: IssueDelivery,
delivery: AlertDelivery,
) -> None:
self._store = store
self._metrics = metrics
self._providers = providers
self._delivery = delivery
self._provider_rule: UsageAlertRule | None = None
self._provider_rule: AlertRule | None = None
self._provider_cache: OrderedDict[str, _ProviderGroup | None] = OrderedDict() # mutable-ok: max 32 providers
async def run_pending(
@ -300,7 +300,7 @@ class PerformanceIssueProjector:
)
async def _candidates(
self, sources: tuple[IssueProjectionSignal, ...], scope: UsageAlertScope, day: date
self, sources: tuple[AlertProjectionSignal, ...], scope: UsageAlertScope, day: date
) -> tuple[_Candidate, ...]:
eligible: Final = sources
windows: Final = tuple(_window(source, day) for source in eligible)
@ -318,7 +318,7 @@ class PerformanceIssueProjector:
)
@staticmethod
def _candidate(source: IssueProjectionSignal, days: tuple[IssueMetricDay, ...], day: date) -> _Candidate | None:
def _candidate(source: AlertProjectionSignal, days: tuple[IssueMetricDay, ...], day: date) -> _Candidate | None:
rows: Final = tuple(item.metrics for item in days)
comparison: Final = compare_daily_usage(source.rule, rows, day)
if comparison is None or not comparison.breached:
@ -327,7 +327,7 @@ class PerformanceIssueProjector:
source,
comparison,
rows,
IssueImpact(
AlertImpact(
keys=days[0].keys if days else None,
users=days[0].users if days else None,
teams=1 if source.rule.scope_type == "team" else None,
@ -339,7 +339,7 @@ class PerformanceIssueProjector:
async def _project_page(
self,
sources: tuple[IssueProjectionSignal, ...],
sources: tuple[AlertProjectionSignal, ...],
scope: UsageAlertScope,
day: date,
now: datetime,
@ -542,7 +542,7 @@ class PerformanceIssueProjector:
)
@staticmethod
def _same_policy(left: UsageAlertRule, right: UsageAlertRule) -> bool:
def _same_policy(left: AlertRule, right: AlertRule) -> bool:
return (left.baseline_days, left.min_history_days, left.min_requests, left.multiplier, left.min_value) == (
right.baseline_days,
right.min_history_days,
@ -579,21 +579,21 @@ class PerformanceIssueProjector:
async def _prior_provider_page(
self,
existing: tuple[Issue, ...],
existing: tuple[Alert, ...],
day: date,
rule: UsageAlertRule,
rule: AlertRule,
now: datetime,
) -> tuple[tuple[Issue, bool, ProviderLatencyEvidence | None], ...]:
) -> tuple[tuple[Alert, bool, ProviderLatencyEvidence | None], ...]:
states: Final = tuple([await self._prior_provider(issue, day, rule) for issue in existing])
await self._store.recover_issue_ids(tuple(issue.id for issue, recovered, _ in states if recovered), now)
return states
async def _prior_provider(
self,
issue: Issue,
issue: Alert,
day: date,
rule: UsageAlertRule,
) -> tuple[Issue, bool, ProviderLatencyEvidence | None]:
rule: AlertRule,
) -> tuple[Alert, bool, ProviderLatencyEvidence | None]:
if await self._providers.cohort_recovered(issue.cohort.model_pairs, day, rule):
return issue, True, None
evidence: Final = await self._providers.latency_cohort(issue.cohort.model_pairs, day, rule)
@ -665,7 +665,7 @@ class PerformanceIssueProjector:
return None
key: Final = _cohort_key("latency", "provider", evidence.cohort.scope_id, "increase")
return _Target(
IssueInput(
AlertInput(
id=str(
uuid5(
NAMESPACE_URL, f"{key}:{item.source.alert.id}:{item.window.day}:{item.source.rule.updated_at}"
@ -701,11 +701,11 @@ class PerformanceIssueProjector:
saved: Final = {item.cohort_key: item for item in await self._store.upsert_issues(inputs)}
await self._store.add_signal_batch(
tuple(
IssueSignal(
issue_id=saved[targets[candidate.source.alert.id].issue.cohort_key].id,
source="usage_alert",
AlertSignal(
alert_id=saved[targets[candidate.source.alert.id].issue.cohort_key].id,
source="performance",
source_id=candidate.source.alert.id,
usage_alert=candidate.source.alert,
performance=candidate.source.alert,
grouping_reason=targets[candidate.source.alert.id].reason,
explanation=targets[candidate.source.alert.id].explanation,
)
@ -729,7 +729,7 @@ class PerformanceIssueProjector:
return
await self._notify_one(item, deadline)
async def _notify_one(self, item: IssuePendingDelivery, deadline: float) -> None:
async def _notify_one(self, item: AlertPendingDelivery, deadline: float) -> None:
current: Final = await self._store.pending_delivery(item.delivery.id)
if current is None:
return

23
litellm/proxy/spend_tracking/usage_alert_job.py
View file

@ -31,11 +31,11 @@ from litellm.repositories.usage_alerts import UsageAlertsRepository
from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.proxy.usage_alert_defaults import USAGE_ALERT_SCOPES, default_usage_rules
from litellm.types.proxy.usage_alerts import (
AlertRule,
AlertRuleInput,
UsageAlertEvaluationStatus,
UsageAlertObservation,
UsageAlertProgress,
UsageAlertRule,
UsageAlertRuleInput,
UsageAlertScope,
UsageAlertWorkCursor,
)
@ -72,7 +72,7 @@ class _Observations:
def _observations(
rule: UsageAlertRule,
rule: AlertRule,
rows: Sequence[DailyAlertMetrics],
target: date,
suppressed: frozenset[str | None] = frozenset(),
@ -117,7 +117,7 @@ class UsageAlertJob:
store: UsageAlertsRepository,
metrics: UsageAlertMetricsRepository,
health: UsageAlertHealthRepository,
defaults: tuple[UsageAlertRuleInput, ...] | None = None,
defaults: tuple[AlertRuleInput, ...] | None = None,
*,
deprecation_projection: Callable[[datetime], Awaitable[None]] | None = None,
performance_projection: PerformanceIssueProjector,
@ -221,7 +221,7 @@ class UsageAlertJob:
await self._store.prune()
@staticmethod
def _position(rule: UsageAlertRule, target: date) -> UsageAlertProgress:
def _position(rule: AlertRule, target: date) -> UsageAlertProgress:
if rule.evaluation_progress is not None and rule.evaluation_progress.revision == rule.updated_at:
return rule.evaluation_progress
return UsageAlertProgress(day=target, revision=rule.updated_at)
@ -275,15 +275,14 @@ class UsageAlertJob:
return cursor.advance(None), advanced, True
return cursor.advance(selected[-1].id), advanced, False
async def _attempt_scope_page(self, rules: tuple[UsageAlertRule, ...], target: date, wildcard: bool) -> bool:
async def _attempt_scope_page(self, rules: tuple[AlertRule, ...], target: date, wildcard: bool) -> bool:
try:
return await self._scope_page(rules, target, wildcard)
except AnalyticsCapacityError:
await self._store.save_progress(tuple((rule, self._position(rule, target)) for rule in rules))
verbose_proxy_logger.info("Usage alert page deferred within its database budget")
return False
async def _scope_page(self, rules: tuple[UsageAlertRule, ...], target: date, wildcard: bool) -> bool:
async def _scope_page(self, rules: tuple[AlertRule, ...], target: date, wildcard: bool) -> bool:
scope: Final = rules[0].scope_type
start: Final = datetime.combine(target, time(), tzinfo=timezone.utc)
coverage: Final = await self._health.coverage("team" if scope == "team" else "user", start)
@ -354,7 +353,7 @@ class UsageAlertJob:
async def _record_evaluations(
self,
evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...],
evaluations: tuple[tuple[AlertRule, UsageAlertEvaluationStatus], ...],
target: date,
*,
completed: bool,
@ -364,7 +363,7 @@ class UsageAlertJob:
async def _page(
self,
rules: Mapping[str | None, tuple[UsageAlertRule, ...]],
rules: Mapping[str | None, tuple[AlertRule, ...]],
rows: tuple[DailyAlertMetrics, ...],
target: date,
suppressed: Mapping[str, frozenset[str | None]],
@ -386,7 +385,7 @@ class UsageAlertJob:
async def _rule_chunk(
self,
selected: tuple[UsageAlertRule, ...],
selected: tuple[AlertRule, ...],
entities: tuple[str | None, ...],
rows: tuple[DailyAlertMetrics, ...],
by_entity: Mapping[str, tuple[DailyAlertMetrics, ...]],
@ -410,7 +409,7 @@ class UsageAlertJob:
async def _rule_page(
self,
rule: UsageAlertRule,
rule: AlertRule,
rows: tuple[DailyAlertMetrics, ...],
target: date,
active: frozenset[str | None],

22
litellm/repositories/issue_activity.py
View file

@ -1,4 +1,4 @@
"""Issue investigations reuse the same aggregate filters as entity Activity."""
"""Alert investigations reuse the same aggregate filters as entity Activity."""
from collections.abc import Mapping
from dataclasses import asdict, replace
@ -15,9 +15,9 @@ from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_
from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase, DailyActivityRepository
from litellm.repositories.daily_activity_sql import PRISMA_TO_PG_TABLE, build_where_clause
from litellm.types.proxy.issues import (
IssueActivityResponse,
IssueCohort,
IssueImpact,
AlertActivityResponse,
AlertCohort,
AlertImpact,
)
from litellm.types.proxy.management_endpoints.common_daily_activity import (
DailyActivityKeyPageResponse,
@ -51,7 +51,7 @@ class _UserLabel(BaseModel, frozen=True):
user_email: str | None
def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyActivityScope:
def issue_activity_scope(cohort: AlertCohort, start: date, end: date) -> DailyActivityScope:
if start > end or (end - start).days > 365:
raise ValueError("Choose an ordered date range of at most 366 days")
return DailyActivityScope(
@ -83,14 +83,14 @@ class IssueActivityRepository:
return cls(daily_activity_repository(prisma, query_database=database), database)
async def daily_activity(
self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20
) -> IssueActivityResponse:
self, cohort: AlertCohort, start: date, end: date, model_limit: int = 20
) -> AlertActivityResponse:
return await BoundedDailyActivityRepository(self.database).aggregate(
issue_activity_scope(cohort, start, end), model_limit=model_limit if cohort.scope_type == "key" else None
)
async def keys(
self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int
) -> DailyActivityKeyPageResponse:
if not 1 <= limit <= 100 or offset < 0:
raise ValueError("Read between one and 100 keys with a nonnegative offset")
@ -113,7 +113,7 @@ class IssueActivityRepository:
)
async def users(
self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int
) -> DailyActivityUserPageResponse:
if not 1 <= limit <= 100 or offset < 0:
raise ValueError("Read between one and 100 users with a nonnegative offset")
@ -206,7 +206,7 @@ class IssueActivityRepository:
)
return rows[0]
async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact:
async def impact(self, cohort: AlertCohort, start: date, end: date) -> AlertImpact:
scope: Final = issue_activity_scope(cohort, start, end)
counts: Final = await self._counts(scope)
teams: Final = (
@ -214,7 +214,7 @@ class IssueActivityRepository:
if cohort.scope_type not in ("user", "team")
else None
)
return IssueImpact(
return AlertImpact(
keys=counts.keys,
models=counts.models,
requests=counts.requests,

28
litellm/repositories/issue_families.py
View file

@ -10,10 +10,10 @@ from pydantic import BaseModel, TypeAdapter
from litellm.repositories.usage_alerts import AlertDatabase
from litellm.types.proxy.issues import (
IssueFamilyList,
IssueFamilyPolicy,
IssueFamilySetting,
IssueFamilyUpdate,
AlertFamilyList,
AlertFamilyPolicy,
AlertFamilySetting,
AlertFamilyUpdate,
)
from litellm.types.proxy.usage_alert_defaults import (
DEFAULT_USAGE_POLICIES,
@ -22,11 +22,11 @@ from litellm.types.proxy.usage_alert_defaults import (
PerformanceIssueFamily,
default_usage_rule_id,
)
from litellm.types.proxy.usage_alerts import UsageAlertEvaluationCounts, UsageAlertRule
from litellm.types.proxy.usage_alerts import AlertRule, UsageAlertEvaluationCounts
class _RuleRow(BaseModel, frozen=True):
record: UsageAlertRule
record: AlertRule
class _Count(BaseModel, frozen=True):
@ -37,14 +37,14 @@ class _ProjectionState(BaseModel, frozen=True):
pending: bool
def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) -> IssueFamilySetting:
def _setting(family: PerformanceIssueFamily, rules: tuple[AlertRule, ...]) -> AlertFamilySetting:
policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family)
groups: Final = tuple(
(policy, tuple(rule for rule in rules if rule.metric == policy.metric and rule.direction == policy.direction))
for policy in policies
)
values: Final = tuple(
IssueFamilyPolicy(
AlertFamilyPolicy(
direction=policy.direction,
enabled=group[0].enabled if group and len({rule.enabled for rule in group}) == 1 else None,
multiplier=group[0].multiplier if group and len({rule.multiplier for rule in group}) == 1 else None,
@ -69,7 +69,7 @@ def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...])
for policy, group in groups
)
family_rules: Final = tuple(chain.from_iterable(group for _, group in groups))
return IssueFamilySetting(
return AlertFamilySetting(
family=family,
enabled=family_rules[0].enabled if family_rules and len({r.enabled for r in family_rules}) == 1 else None,
total_rules=len(family_rules),
@ -81,7 +81,7 @@ def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...])
)
async def family_settings(database: AlertDatabase) -> IssueFamilyList:
async def family_settings(database: AlertDatabase) -> AlertFamilyList:
rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python(
await database.query_raw(
'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted',
@ -94,7 +94,7 @@ async def family_settings(database: AlertDatabase) -> IssueFamilyList:
WHERE id LIKE 'performance-dirty:%' AND NOT completed LIMIT 1) AS pending"""
)
)
return IssueFamilyList(
return AlertFamilyList(
data=tuple(
_setting(family, tuple(row.record for row in rows))
for family in ("spend", "latency", "reliability", "traffic")
@ -106,8 +106,8 @@ async def family_settings(database: AlertDatabase) -> IssueFamilyList:
async def update_family(
database: AlertDatabase,
family: PerformanceIssueFamily,
data: IssueFamilyUpdate,
) -> IssueFamilySetting | None:
data: AlertFamilyUpdate,
) -> AlertFamilySetting | None:
policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family)
if any(update.direction not in {policy.direction for policy in policies} for update in data.policies):
raise ValueError("This family does not support that direction")
@ -126,7 +126,7 @@ async def update_family(
policy.direction: policy.model_dump(exclude_none=True, exclude={"direction"}) for policy in data.policies
}
records: Final = tuple(
UsageAlertRule.model_validate(
AlertRule.model_validate(
{
**row.record.model_dump(),
**({"enabled": data.enabled} if data.enabled is not None else {}),

150
litellm/repositories/issues.py
View file

@ -14,21 +14,21 @@ from litellm.models.team import LiteLLM_TeamTable
from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key
from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository
from litellm.types.proxy.issues import (
Issue,
IssueCohort,
IssueDeliveryBinding,
IssueFamily,
IssueFamilyList,
IssueFamilySetting,
IssueFamilyUpdate,
IssueInput,
IssueKind,
IssueList,
IssuePendingDelivery,
IssueProjectionSignal,
IssueScope,
IssueSignal,
IssueSignalList,
Alert,
AlertCohort,
AlertDeliveryBinding,
AlertFamily,
AlertFamilyList,
AlertFamilySetting,
AlertFamilyUpdate,
AlertInput,
AlertKind,
AlertList,
AlertPendingDelivery,
AlertProjectionSignal,
AlertScope,
AlertSignal,
AlertSignalList,
)
from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS, PerformanceIssueFamily
from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus
@ -38,11 +38,11 @@ if TYPE_CHECKING:
class _IssueRow(BaseModel, frozen=True):
record: Issue
record: Alert
class _SignalRow(BaseModel, frozen=True):
record: IssueSignal
record: AlertSignal
class _Count(BaseModel, frozen=True):
@ -82,8 +82,8 @@ def _utc(value: datetime) -> str:
return value.astimezone(timezone.utc).replace(tzinfo=None).isoformat()
def _issue_payload(data: IssueInput, status: UsageAlertStatus | None) -> Mapping[str, object]:
initial: Final = Issue.model_validate(
def _issue_payload(data: AlertInput, status: UsageAlertStatus | None) -> Mapping[str, object]:
initial: Final = Alert.model_validate(
{
**data.model_dump(),
"status": status or "open",
@ -110,10 +110,10 @@ class IssuesRepository:
async def upsert_issues(
self,
inputs: tuple[IssueInput, ...],
inputs: tuple[AlertInput, ...],
reopen: bool = False,
status: UsageAlertStatus | None = None,
) -> tuple[Issue, ...]:
) -> tuple[Alert, ...]:
if not inputs:
return ()
if len(inputs) > 500 or len(frozenset(item.id for item in inputs)) != len(inputs):
@ -155,7 +155,7 @@ class IssuesRepository:
raise ValueError("An issue ID already belongs to another cohort")
return tuple(row.record for row in rows)
async def get_issue(self, issue_id: str) -> Issue | None:
async def get_issue(self, issue_id: str) -> Alert | None:
rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
await self.database.query_raw(f"SELECT {_RECORD} AS record {_JOIN} WHERE i.id=$1", issue_id)
)
@ -182,14 +182,14 @@ class IssuesRepository:
async def list_issues(
self,
kind: IssueKind | None = None,
family: IssueFamily | None = None,
kind: AlertKind | None = None,
family: AlertFamily | None = None,
status: UsageAlertStatus | None = None,
scope_type: IssueScope | None = None,
scope_type: AlertScope | None = None,
scope_id: str | None = None,
limit: int = 50,
offset: int = 0,
) -> IssueList:
) -> AlertList:
where: Final = f"""($1::text IS NULL OR i.kind=$1) AND ($2::text IS NULL OR i.family=$2)
AND ($3::text IS NULL OR ({_STATUS})=$3) AND ($4::text IS NULL OR i.scope_type=$4)
AND ($5::text IS NULL OR i.scope_id=$5)"""
@ -212,14 +212,14 @@ class IssuesRepository:
offset,
)
)
return IssueList(
return AlertList(
data=await self._labels(tuple(row.record for row in rows)),
total=count[0].count,
limit=len(rows) if rows and offset + len(rows) < count[0].count else limit,
offset=offset,
)
async def _labels(self, issues: tuple[Issue, ...]) -> tuple[Issue, ...]:
async def _labels(self, issues: tuple[Alert, ...]) -> tuple[Alert, ...]:
if not issues:
return ()
labels: Final = await UsageAlertsRepository(self.database).resource_labels(
@ -230,7 +230,7 @@ class IssuesRepository:
for issue in issues
)
async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]:
async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Alert, ...]:
if len(cohort_keys) > 500:
raise ValueError("Read at most 500 issue cohorts per batch")
rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
@ -249,8 +249,8 @@ class IssuesRepository:
status: UsageAlertStatus,
expected_status: UsageAlertStatus,
actor: str | None = None,
authorized_scopes: tuple[IssueCohort, ...] | None = None,
) -> Issue | None:
authorized_scopes: tuple[AlertCohort, ...] | None = None,
) -> Alert | None:
if authorized_scopes is not None and len(authorized_scopes) > 10:
raise ValueError("Authorize at most 10 alert source scopes")
changed: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python(
@ -261,8 +261,8 @@ class IssuesRepository:
AND ($5::jsonb IS NULL OR NOT EXISTS (
SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id AND NOT EXISTS (
SELECT 1 FROM jsonb_to_recordset($5::jsonb) AS x(scope_type text,scope_id text)
WHERE s.source='usage_alert' AND x.scope_type=s.record->'usage_alert'->>'scope_type'
AND x.scope_id IS NOT DISTINCT FROM s.record->'usage_alert'->>'scope_id')))
WHERE s.source='performance' AND x.scope_type=s.record->'performance'->>'scope_type'
AND x.scope_id IS NOT DISTINCT FROM s.record->'performance'->>'scope_id')))
), ack AS (
INSERT INTO "LiteLLM_Config" AS prior (param_name,param_value)
SELECT ack_key,jsonb_build_object('model',record->'deprecation'->>'usage_model',
@ -292,18 +292,18 @@ class IssuesRepository:
)
return await self.get_issue(issue_id) if changed else None
async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]:
return TypeAdapter(tuple[IssueCohort, ...]).validate_python(
async def status_scopes(self, issue_id: str) -> tuple[AlertCohort, ...]:
return TypeAdapter(tuple[AlertCohort, ...]).validate_python(
await self.database.query_raw(
"""SELECT DISTINCT CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_type'
"""SELECT DISTINCT CASE WHEN source='performance' THEN record->'performance'->>'scope_type'
ELSE 'deployment' END AS scope_type,
CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_id' END AS scope_id
CASE WHEN source='performance' THEN record->'performance'->>'scope_id' END AS scope_id
FROM "LiteLLM_IssueSignal" WHERE issue_id=$1 ORDER BY scope_type,scope_id LIMIT 11""",
issue_id,
)
)
async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None:
async def add_signal_batch(self, signals: tuple[AlertSignal, ...]) -> None:
if not signals:
return
if len(signals) > 500:
@ -323,14 +323,14 @@ class IssuesRepository:
) UPDATE "LiteLLM_Issue" i SET
updated_at=GREATEST(CURRENT_TIMESTAMP,i.updated_at+interval '1 millisecond'),
status=CASE WHEN i.kind='performance' AND i.active AND i.status<>'open' AND EXISTS (
SELECT 1 FROM added a WHERE a.issue_id=i.id AND a.source='usage_alert'
AND (a.record->'usage_alert'->>'scope_type'<>i.scope_type
OR a.record->'usage_alert'->>'scope_id' IS DISTINCT FROM i.scope_id)
SELECT 1 FROM added a WHERE a.issue_id=i.id AND a.source='performance'
AND (a.record->'performance'->>'scope_type'<>i.scope_type
OR a.record->'performance'->>'scope_id' IS DISTINCT FROM i.scope_id)
AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" prior
WHERE prior.issue_id=i.id AND prior.source='usage_alert'
AND prior.record->'usage_alert'->>'scope_type'=a.record->'usage_alert'->>'scope_type'
AND prior.record->'usage_alert'->>'scope_id'
IS NOT DISTINCT FROM a.record->'usage_alert'->>'scope_id')
WHERE prior.issue_id=i.id AND prior.source='performance'
AND prior.record->'performance'->>'scope_type'=a.record->'performance'->>'scope_type'
AND prior.record->'performance'->>'scope_id'
IS NOT DISTINCT FROM a.record->'performance'->>'scope_id')
) THEN 'open' ELSE i.status END
WHERE i.id IN(SELECT issue_id FROM added) RETURNING i.id""",
json.dumps(tuple(signal.model_dump(mode="json") for signal in signals)),
@ -341,11 +341,11 @@ class IssuesRepository:
issue_id: str,
limit: int = 50,
offset: int = 0,
visible_scope: IssueCohort | None = None,
) -> IssueSignalList:
where: Final = """issue_id=$1 AND ($2::text IS NULL OR (source='usage_alert'
AND record->'usage_alert'->>'scope_type'=$2
AND record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM $3::text))"""
visible_scope: AlertCohort | None = None,
) -> AlertSignalList:
where: Final = """issue_id=$1 AND ($2::text IS NULL OR (source='performance'
AND record->'performance'->>'scope_type'=$2
AND record->'performance'->>'scope_id' IS NOT DISTINCT FROM $3::text))"""
args: Final = (
issue_id,
visible_scope.scope_type if visible_scope else None,
@ -365,7 +365,7 @@ class IssuesRepository:
offset,
)
)
return IssueSignalList(
return AlertSignalList(
data=tuple(row.record for row in rows), total=counts[0].count, limit=limit, offset=offset
)
@ -374,18 +374,18 @@ class IssuesRepository:
day: date,
limit: int = 500,
cursor: str | None = None,
scope_type: IssueScope | None = None,
scope_type: AlertScope | None = None,
metric: UsageAlertMetric | None = None,
scope_ids: tuple[str, ...] | None = None,
completed_only: bool = False,
) -> tuple[IssueProjectionSignal, ...]:
) -> tuple[AlertProjectionSignal, ...]:
if not 1 <= limit <= 500:
raise ValueError("Read between one and 500 source signals")
if scope_ids == ():
return ()
if scope_ids is not None and len(scope_ids) > 500:
raise ValueError("Filter at most 500 source scopes")
return TypeAdapter(tuple[IssueProjectionSignal, ...]).validate_python(
return TypeAdapter(tuple[AlertProjectionSignal, ...]).validate_python(
await self.database.query_raw(
"""SELECT a.record AS alert,r.record AS rule,a.evaluation_day
FROM "LiteLLM_UsageAlertIncident" a JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id
@ -470,14 +470,14 @@ class IssuesRepository:
FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id
LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id
LEFT JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id
WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='usage_alert'
WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='performance'
AND ({selection})
AND s.record->'usage_alert'->>'scope_type'=i.scope_type
AND s.record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM i.scope_id
AND ((i.family='spend' AND s.record->'usage_alert'->>'metric'='spend')
OR (i.family='latency' AND s.record->'usage_alert'->>'metric'='latency')
OR (i.family='traffic' AND s.record->'usage_alert'->>'metric'='requests')
OR (i.family='reliability' AND s.record->'usage_alert'->>'metric' IN ('errors','error_rate')))
AND s.record->'performance'->>'scope_type'=i.scope_type
AND s.record->'performance'->>'scope_id' IS NOT DISTINCT FROM i.scope_id
AND ((i.family='spend' AND s.record->'performance'->>'metric'='spend')
OR (i.family='latency' AND s.record->'performance'->>'metric'='latency')
OR (i.family='traffic' AND s.record->'performance'->>'metric'='requests')
OR (i.family='reliability' AND s.record->'performance'->>'metric' IN ('errors','error_rate')))
), observations AS (
SELECT *,COALESCE(source_id IS NOT NULL AND NOT breached AND evaluation_day<=$1
AND evaluation_day>=latest_observation_day
@ -498,7 +498,7 @@ class IssuesRepository:
)
return rows[0].count
async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]:
async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Alert, ...]:
if not 1 <= limit <= 500:
raise ValueError("Read between one and 500 provider issues")
rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
@ -560,19 +560,19 @@ class IssuesRepository:
)
return rows[0].count
async def family_settings(self) -> "IssueFamilyList":
async def family_settings(self) -> "AlertFamilyList":
from litellm.repositories.issue_families import family_settings
return await family_settings(self.database)
async def update_family(
self, family: "PerformanceIssueFamily", data: "IssueFamilyUpdate"
) -> "IssueFamilySetting | None":
self, family: "PerformanceIssueFamily", data: "AlertFamilyUpdate"
) -> "AlertFamilySetting | None":
from litellm.repositories.issue_families import update_family
return await update_family(self.database, family, data)
async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None:
async def queue_deliveries(self, bindings: tuple[AlertDeliveryBinding, ...]) -> None:
if not bindings:
return
if len(bindings) > 500:
@ -597,17 +597,17 @@ class IssuesRepository:
json.dumps(tuple(binding.model_dump(mode="json") for binding in bindings)),
)
async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]:
async def pending_deliveries(self, limit: int = 100) -> tuple[AlertPendingDelivery, ...]:
return await self._pending_deliveries(limit, None)
async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None:
async def pending_delivery(self, delivery_id: str) -> AlertPendingDelivery | None:
rows: Final = await self._pending_deliveries(1, delivery_id)
return rows[0] if rows else None
async def _pending_deliveries(self, limit: int, delivery_id: str | None) -> tuple[IssuePendingDelivery, ...]:
async def _pending_deliveries(self, limit: int, delivery_id: str | None) -> tuple[AlertPendingDelivery, ...]:
if not 1 <= limit <= 500:
raise ValueError("Read between one and 500 deliveries")
return TypeAdapter(tuple[IssuePendingDelivery, ...]).validate_python(
return TypeAdapter(tuple[AlertPendingDelivery, ...]).validate_python(
await self.database.query_raw(
f"""SELECT d.record || jsonb_build_object('next_attempt',d.next_attempt AT TIME ZONE 'UTC') AS delivery,
{_RECORD} AS issue,bound.rule,bound.alert
@ -635,7 +635,7 @@ class IssuesRepository:
AND custom.record->>'direction'=r.record->>'direction'
AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id)))
AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id
AND s.source='usage_alert' AND s.source_id=a.id)
AND s.source='performance' AND s.source_id=a.id)
ORDER BY b.rule_id,b.source_id LIMIT 1
) bound ON true
WHERE NOT d.sent AND d.next_attempt<=CURRENT_TIMESTAMP AND i.active AND ({_STATUS})='open'
@ -682,10 +682,10 @@ class IssuesRepository:
if not 1 <= limit <= 500 or retention_days < 1:
raise ValueError("Retention and cleanup batch must be bounded positive values")
await self.database.query_raw(
"""WITH expired AS MATERIALIZED (
SELECT id FROM "LiteLLM_Issue" WHERE status<>'open'
AND updated_at<CURRENT_TIMESTAMP-make_interval(days=>$1::int)
ORDER BY updated_at,id LIMIT $2
f"""WITH expired AS MATERIALIZED (
SELECT i.id {_JOIN} WHERE i.status<>'open' AND ({_STATUS})<>'open'
AND i.updated_at<CURRENT_TIMESTAMP-make_interval(days=>$1::int)
ORDER BY i.updated_at,i.id LIMIT $2
), signals AS (
DELETE FROM "LiteLLM_IssueSignal" WHERE (issue_id,source,source_id) IN (
SELECT issue_id,source,source_id FROM "LiteLLM_IssueSignal"

34
litellm/repositories/provider_latency.py
View file

@ -9,8 +9,8 @@ from pydantic import BaseModel, Field, TypeAdapter
from litellm.constants import PTU_SENTINEL_API_KEY
from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket
from litellm.repositories.usage_alert_metrics import MetricsDatabase
from litellm.types.proxy.issues import IssueCohort, IssueComparison, IssueImpact, IssueModelPair
from litellm.types.proxy.usage_alerts import UsageAlertMetricPoint, UsageAlertRuleInput, UsageAlertScope
from litellm.types.proxy.issues import AlertCohort, AlertComparison, AlertImpact, AlertModelPair
from litellm.types.proxy.usage_alerts import AlertRuleInput, UsageAlertMetricPoint, UsageAlertScope
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
@ -18,15 +18,15 @@ if TYPE_CHECKING:
@dataclass(frozen=True, slots=True)
class ProviderLatencyEvidence:
cohort: IssueCohort
comparison: IssueComparison
impact: IssueImpact
cohort: AlertCohort
comparison: AlertComparison
impact: AlertImpact
comparable_timed_requests: int
total_timed_requests: int
class _CohortRow(BaseModel, frozen=True):
pairs: tuple[IssueModelPair, ...]
pairs: tuple[AlertModelPair, ...]
observed: float
expected: float
sample_days: int | None
@ -162,9 +162,9 @@ SELECT (SELECT COUNT(*) FROM covered)=cardinality($4::text[]) AND NOT EXISTS (
def _arguments(
pairs: tuple[IssueModelPair, ...],
pairs: tuple[AlertModelPair, ...],
day: date,
rule: UsageAlertRuleInput,
rule: AlertRuleInput,
source_ids: tuple[str, ...] | None = None,
) -> tuple[object, ...]:
if not 1 <= len(pairs) <= 100 or len(frozenset(pairs)) != len(pairs):
@ -195,8 +195,8 @@ class ProviderLatencyRepository:
def from_prisma(cls, prisma: "PrismaClient") -> "ProviderLatencyRepository":
return cls(cast(MetricsDatabase, prisma.writer_db)) # cast-ok: generated Prisma query_raw interface
async def provider_pairs(self, provider: str, day: date) -> tuple[IssueModelPair, ...]:
return TypeAdapter(tuple[IssueModelPair, ...]).validate_python(
async def provider_pairs(self, provider: str, day: date) -> tuple[AlertModelPair, ...]:
return TypeAdapter(tuple[AlertModelPair, ...]).validate_python(
await self._database.query_raw(
"""SELECT DISTINCT custom_llm_provider AS provider,model FROM "LiteLLM_DailyUserSpend"
WHERE date=$1 AND custom_llm_provider=$2 AND model IS NOT NULL AND model<>'' AND timed_requests>0
@ -209,7 +209,7 @@ class ProviderLatencyRepository:
)
async def latency_cohort(
self, pairs: tuple[IssueModelPair, ...], day: date, rule: UsageAlertRuleInput
self, pairs: tuple[AlertModelPair, ...], day: date, rule: AlertRuleInput
) -> ProviderLatencyEvidence | None:
arguments: Final = _arguments(pairs, day, rule)
if len(pairs) < 2:
@ -219,8 +219,8 @@ class ProviderLatencyRepository:
return None
row: Final = rows[0]
return ProviderLatencyEvidence(
cohort=IssueCohort(scope_type="provider", scope_id=pairs[0].provider, model_pairs=row.pairs),
comparison=IssueComparison(
cohort=AlertCohort(scope_type="provider", scope_id=pairs[0].provider, model_pairs=row.pairs),
comparison=AlertComparison(
metric="latency",
direction="increase",
observed_value=row.observed,
@ -231,7 +231,7 @@ class ProviderLatencyRepository:
baseline_sample_days=row.sample_days,
history=row.history,
),
impact=IssueImpact(
impact=AlertImpact(
keys=row.keys,
users=row.users,
teams=row.teams,
@ -246,9 +246,9 @@ class ProviderLatencyRepository:
async def source_members(
self,
pairs: tuple[IssueModelPair, ...],
pairs: tuple[AlertModelPair, ...],
day: date,
rule: UsageAlertRuleInput,
rule: AlertRuleInput,
scope_type: UsageAlertScope,
source_ids: tuple[str, ...],
) -> frozenset[str]:
@ -273,7 +273,7 @@ class ProviderLatencyRepository:
rows: Final = _MEMBERS.validate_python(await self._database.query_raw(query, *arguments))
return frozenset(api_key_bucket(row.entity_id) if scope_type == "key" else row.entity_id for row in rows)
async def cohort_recovered(self, pairs: tuple[IssueModelPair, ...], day: date, rule: UsageAlertRuleInput) -> bool:
async def cohort_recovered(self, pairs: tuple[AlertModelPair, ...], day: date, rule: AlertRuleInput) -> bool:
arguments: Final = _arguments(pairs, day, rule)
rows: Final = TypeAdapter(tuple[_Recovery, ...]).validate_python(
await self._database.query_raw(_RECOVERY_QUERY, *arguments)

209
litellm/repositories/usage_alerts.py
View file

@ -19,26 +19,21 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS
from litellm.types.proxy.usage_alerts import (
UsageAlert,
AlertModelList,
AlertModelOption,
AlertRule,
AlertRuleInput,
AlertRuleList,
PerformanceSignal,
PerformanceSignalList,
UsageAlertDefaultSummary,
UsageAlertEvaluationStatus,
UsageAlertGroup,
UsageAlertGroupList,
UsageAlertGroupMetric,
UsageAlertGroupUpdate,
UsageAlertGroupUpdateResult,
UsageAlertList,
UsageAlertMetric,
UsageAlertModelList,
UsageAlertModelOption,
UsageAlertObservation,
UsageAlertProgress,
UsageAlertProjectionCursor,
UsageAlertProjectionWork,
UsageAlertRule,
UsageAlertRuleInput,
UsageAlertRuleKind,
UsageAlertRuleList,
UsageAlertScope,
UsageAlertStatus,
UsageAlertWorkCursor,
@ -78,19 +73,6 @@ class _RuleCounts(_Count, frozen=True):
default_summary: UsageAlertDefaultSummary
class _GroupRow(BaseModel, frozen=True):
scope_type: UsageAlertScope
scope_id: str | None
day: date
incident_count: int
metrics: tuple[UsageAlertGroupMetric, ...]
class _GroupPage(BaseModel, frozen=True):
total: int
data: tuple[_GroupRow, ...]
class _ResourceLabel(BaseModel, frozen=True):
scope_type: str
scope_id: str | None
@ -113,8 +95,8 @@ class _Ownership(BaseModel, frozen=True):
_DEFAULT_RULE_IDS: Final = DEFAULT_USAGE_RULE_IDS
def _rule(record: Mapping[str, object]) -> UsageAlertRule:
return UsageAlertRule.model_validate({**record, "is_default": str(record.get("id")) in _DEFAULT_RULE_IDS})
def _rule(record: Mapping[str, object]) -> AlertRule:
return AlertRule.model_validate({**record, "is_default": str(record.get("id")) in _DEFAULT_RULE_IDS})
def activity_path(
@ -137,14 +119,14 @@ def activity_path(
"tab": "keys",
"from": (day - timedelta(days=baseline_days)).isoformat(),
"to": day.isoformat(),
"alert": alert_id,
"signal": alert_id,
}
)
)
query: Final = {
"activity_from": (day - timedelta(days=baseline_days)).isoformat(),
"activity_to": day.isoformat(),
"alert": alert_id,
"signal": alert_id,
}
if parameter and entity_id and not entity_id.startswith("sk-"):
query[parameter] = entity_id
@ -152,7 +134,7 @@ def activity_path(
return route + "?" + urlencode(query)
def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now: datetime) -> Mapping[str, object]:
def _observation_payload(rule: AlertRule, item: UsageAlertObservation, now: datetime) -> Mapping[str, object]:
alert_id: Final = str(
uuid5(
NAMESPACE_URL,
@ -170,7 +152,7 @@ def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now:
),
)
)
incident: Final = UsageAlert(
incident: Final = PerformanceSignal(
id=alert_id,
rule_id=rule.id,
rule_name=rule.name,
@ -208,7 +190,7 @@ class UsageAlertsRepository:
cast(AlertDatabase, prisma.writer_db) # cast-ok: PrismaWrapper forwards query_raw to its generated client
)
async def get_rule(self, rule_id: str) -> UsageAlertRule | None:
async def get_rule(self, rule_id: str) -> AlertRule | None:
rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
await self.database.query_raw(
'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND NOT deleted', rule_id
@ -218,14 +200,14 @@ class UsageAlertsRepository:
async def save_rule(
self,
data: UsageAlertRuleInput,
data: AlertRuleInput,
created_by: str | None,
existing: UsageAlertRule | None = None,
existing: AlertRule | None = None,
*,
overrides_defaults: bool = False,
) -> UsageAlertRule | None:
) -> AlertRule | None:
now: Final = datetime.now(timezone.utc)
rule: Final = UsageAlertRule.model_validate(
rule: Final = AlertRule.model_validate(
{
**data.model_dump(),
"id": existing.id if existing else str(uuid4()),
@ -255,7 +237,7 @@ class UsageAlertsRepository:
saved: Final = TypeAdapter(tuple[_Record, ...]).validate_python(rows)
return _rule(saved[0].record) if saved else None
async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool:
async def delete_rule(self, rule_id: str, *, existing: AlertRule | None = None) -> bool:
return bool(
await self.database.query_raw(
"""UPDATE "LiteLLM_UsageAlertRule" SET deleted=true,enabled=false
@ -276,7 +258,7 @@ class UsageAlertsRepository:
limit: int = 50,
offset: int = 0,
kind: UsageAlertRuleKind = "all",
) -> UsageAlertRuleList:
) -> AlertRuleList:
args: Final = (scope_type, scope_id, json.dumps(tuple(sorted(_DEFAULT_RULE_IDS))), kind)
where: Final = "NOT deleted AND ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2)"
builtin: Final = "id IN (SELECT jsonb_array_elements_text($3::jsonb))"
@ -305,7 +287,7 @@ class UsageAlertsRepository:
offset,
)
)
return UsageAlertRuleList(
return AlertRuleList(
data=tuple(_rule(row.record) for row in rows),
total=count[0].count,
limit=limit,
@ -315,7 +297,7 @@ class UsageAlertsRepository:
async def enabled_rule_page(
self, scope: UsageAlertScope, wildcard: bool, cursor: str | None, limit: int = 500
) -> tuple[UsageAlertRule, ...]:
) -> tuple[AlertRule, ...]:
if not 1 <= limit <= 500:
raise ValueError("Read at most 500 rules")
rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
@ -327,7 +309,7 @@ class UsageAlertsRepository:
limit,
)
)
return tuple(UsageAlertRule.model_validate(row.record) for row in rows)
return tuple(AlertRule.model_validate(row.record) for row in rows)
async def work_cursor(self) -> UsageAlertWorkCursor:
rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
@ -344,7 +326,7 @@ class UsageAlertsRepository:
cursor.model_dump_json(),
)
async def save_progress(self, updates: tuple[tuple[UsageAlertRule, UsageAlertProgress], ...]) -> None:
async def save_progress(self, updates: tuple[tuple[AlertRule, UsageAlertProgress], ...]) -> None:
if not updates:
return
if len(updates) > 500:
@ -369,7 +351,7 @@ class UsageAlertsRepository:
)
async def suppressed_entities(
self, rules: tuple[UsageAlertRule, ...], entities: tuple[str | None, ...]
self, rules: tuple[AlertRule, ...], entities: tuple[str | None, ...]
) -> Mapping[str, frozenset[str | None]]:
defaults: Final = tuple(rule for rule in rules if rule.id in _DEFAULT_RULE_IDS)
if not defaults or not entities:
@ -446,11 +428,11 @@ class UsageAlertsRepository:
work.version,
)
async def get_alert(self, alert_id: str) -> UsageAlert | None:
async def get_alert(self, alert_id: str) -> PerformanceSignal | None:
rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
await self.database.query_raw('SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE id=$1', alert_id)
)
return UsageAlert.model_validate(rows[0].record) if rows else None
return PerformanceSignal.model_validate(rows[0].record) if rows else None
async def list_alerts(
self,
@ -462,7 +444,7 @@ class UsageAlertsRepository:
end_date: date | None = None,
limit: int = 50,
offset: int = 0,
) -> UsageAlertList:
) -> PerformanceSignalList:
args: Final = (
scope_type,
scope_id,
@ -485,14 +467,14 @@ class UsageAlertsRepository:
offset,
)
)
return UsageAlertList(
data=tuple(UsageAlert.model_validate(row.record) for row in rows),
return PerformanceSignalList(
data=tuple(PerformanceSignal.model_validate(row.record) for row in rows),
total=count[0].count,
limit=limit,
offset=offset,
)
async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None:
async def set_status(self, alert_id: str, status: UsageAlertStatus) -> PerformanceSignal | None:
rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
await self.database.query_raw(
"""UPDATE "LiteLLM_UsageAlertIncident" SET status=$2, record=jsonb_set(record,'{status}',to_jsonb($2::text)),updated_at=CURRENT_TIMESTAMP WHERE id=$1 RETURNING record""",
@ -500,92 +482,7 @@ class UsageAlertsRepository:
status,
)
)
return UsageAlert.model_validate(rows[0].record) if rows else None
async def list_groups(
self,
scope_type: UsageAlertScope | None = None,
scope_id: str | None = None,
status: UsageAlertStatus | None = None,
start_date: date | None = None,
end_date: date | None = None,
limit: int = 50,
offset: int = 0,
) -> UsageAlertGroupList:
rows: Final = TypeAdapter(tuple[_GroupPage, ...]).validate_python(
await self.database.query_raw(
"""WITH groups AS MATERIALIZED (
SELECT scope_type,scope_id,day,count(*)::int AS incident_count FROM "LiteLLM_UsageAlertIncident"
WHERE ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2)
AND ($3::text IS NULL OR status=$3) AND ($4::text IS NULL OR day >= $4)
AND ($5::text IS NULL OR day <= $5)
GROUP BY scope_type,scope_id,day
), page AS MATERIALIZED (
SELECT * FROM groups ORDER BY day DESC,scope_type,scope_id NULLS FIRST LIMIT $6 OFFSET $7
), selected AS (
SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i
ON i.scope_type=p.scope_type AND i.scope_id=p.scope_id AND i.day=p.day
WHERE ($3::text IS NULL OR i.status=$3)
UNION ALL
SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i
ON i.scope_type=p.scope_type AND i.scope_id IS NULL AND i.day=p.day
WHERE p.scope_id IS NULL AND ($3::text IS NULL OR i.status=$3)
), ranked AS (
SELECT scope_type,scope_id,day,metric,record,
count(*) OVER (PARTITION BY scope_type,scope_id,day,metric)::int AS incident_count,
row_number() OVER (PARTITION BY scope_type,scope_id,day,metric
ORDER BY record->>'first_seen' DESC,id) AS rank
FROM selected
), packed AS (
SELECT p.scope_type,p.scope_id,p.day,p.incident_count,
jsonb_agg(jsonb_build_object('metric',r.metric,'incident_count',r.incident_count,
'representative',r.record) ORDER BY CASE r.metric
WHEN 'requests' THEN 0 WHEN 'spend' THEN 1 WHEN 'error_rate' THEN 2
WHEN 'errors' THEN 3 ELSE 4 END) AS metrics
FROM page p JOIN ranked r ON p.scope_type=r.scope_type
AND p.scope_id IS NOT DISTINCT FROM r.scope_id AND p.day=r.day AND r.rank=1
GROUP BY p.scope_type,p.scope_id,p.day,p.incident_count
) SELECT (SELECT count(*)::int FROM groups) AS total,
COALESCE((SELECT jsonb_agg(to_jsonb(packed) ORDER BY day DESC,scope_type,scope_id NULLS FIRST)
FROM packed),'[]'::jsonb) AS data""",
scope_type,
scope_id,
status,
start_date.isoformat() if start_date else None,
end_date.isoformat() if end_date else None,
limit,
offset,
)
)
page: Final = rows[0]
labels: Final[Mapping[tuple[str, str | None], str | None]] = (
await self.resource_labels(page.data) if page.data else {}
)
return UsageAlertGroupList(
data=tuple(
UsageAlertGroup(
id=str(
uuid5(
NAMESPACE_URL,
json.dumps(
["litellm:performance-group", row.scope_type, row.scope_id, row.day.isoformat()]
),
)
),
scope_type=row.scope_type,
scope_id=row.scope_id,
scope_label=labels.get((row.scope_type, row.scope_id)),
day=row.day,
incident_count=row.incident_count,
metrics=row.metrics,
activity_path=row.metrics[0].representative.activity_path,
)
for row in page.data
),
total=page.total,
limit=limit,
offset=offset,
)
return PerformanceSignal.model_validate(rows[0].record) if rows else None
async def resource_labels(
self, groups: tuple[ResourceLabelScope, ...]
@ -626,25 +523,7 @@ class UsageAlertsRepository:
)
return MappingProxyType({(row.scope_type, row.scope_id): row.label for row in rows})
async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult:
rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
await self.database.query_raw(
"""WITH changed AS (
UPDATE "LiteLLM_UsageAlertIncident" SET status=$5,
record=jsonb_set(record,'{status}',to_jsonb($5::text)),updated_at=CURRENT_TIMESTAMP
WHERE scope_type=$1 AND scope_id IS NOT DISTINCT FROM $2::text AND day=$3 AND status=$4
RETURNING id
) SELECT count(*)::int AS count FROM changed""",
data.scope_type,
data.scope_id,
data.day.isoformat(),
data.expected_status,
data.status,
)
)
return UsageAlertGroupUpdateResult(updated_count=rows[0].count)
async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList:
async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> AlertModelList:
today: Final = datetime.now(timezone.utc).date()
rows: Final = TypeAdapter(tuple[_ModelIdentity, ...]).validate_python(
await self.database.query_raw(
@ -661,8 +540,8 @@ class UsageAlertsRepository:
)
)
selected: Final = rows[:limit]
return UsageAlertModelList(
data=tuple(UsageAlertModelOption(scope_id=row.model, label=row.model) for row in selected),
return AlertModelList(
data=tuple(AlertModelOption(scope_id=row.model, label=row.model) for row in selected),
limit=limit,
next_cursor=selected[-1].model if len(rows) > limit else None,
start_date=today - timedelta(days=29),
@ -699,10 +578,10 @@ class UsageAlertsRepository:
)
@staticmethod
def _checkpoint(rule: UsageAlertRule, day: date) -> str:
def _checkpoint(rule: AlertRule, day: date) -> str:
return f"{rule.id}:{rule.updated_at.isoformat()}:{day.isoformat()}"
async def completed_rule_ids(self, rules: tuple[UsageAlertRule, ...], day: date) -> frozenset[str]:
async def completed_rule_ids(self, rules: tuple[AlertRule, ...], day: date) -> frozenset[str]:
if len(rules) > 500:
raise ValueError("A checkpoint batch must contain at most 500 rules")
rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python(
@ -717,7 +596,7 @@ class UsageAlertsRepository:
async def record_evaluations(
self,
evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...],
evaluations: tuple[tuple[AlertRule, UsageAlertEvaluationStatus], ...],
day: date,
*,
completed: bool,
@ -764,16 +643,16 @@ class UsageAlertsRepository:
str(uuid4()),
)
async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None:
async def ensure_default_rules(self, rules: tuple[AlertRuleInput, ...]) -> None:
for rule in rules:
await self._ensure_default_rule(rule)
async def _ensure_default_rule(self, rule: UsageAlertRuleInput) -> None:
async def _ensure_default_rule(self, rule: AlertRuleInput) -> None:
rule_id: Final = str(
uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}")
)
now: Final = datetime.now(timezone.utc)
record: Final = UsageAlertRule.model_validate(
record: Final = AlertRule.model_validate(
{**rule.model_dump(), "id": rule_id, "created_at": now, "updated_at": now, "created_by": None}
)
await self.database.query_raw(
@ -786,8 +665,8 @@ class UsageAlertsRepository:
)
async def record_observations(
self, rule: UsageAlertRule, observations: tuple[UsageAlertObservation, ...]
) -> tuple[UsageAlert, ...]:
self, rule: AlertRule, observations: tuple[UsageAlertObservation, ...]
) -> tuple[PerformanceSignal, ...]:
if not observations:
return ()
if len(observations) > 500 or len({item.entity_id for item in observations}) != len(observations):
@ -838,7 +717,7 @@ class UsageAlertsRepository:
str(uuid4()),
)
)
return tuple(UsageAlert.model_validate(row.record) for row in rows)
return tuple(PerformanceSignal.model_validate(row.record) for row in rows)
async def prune(self, retention_days: int = 90, limit: int = 500) -> None:
await self.database.query_raw(

122
litellm/types/proxy/issues.py
View file

@ -10,40 +10,40 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import (
from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo
from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily
from litellm.types.proxy.usage_alerts import (
UsageAlert,
AlertRule,
PerformanceSignal,
UsageAlertChannel,
UsageAlertDirection,
UsageAlertEvaluationCounts,
UsageAlertMetric,
UsageAlertMetricPoint,
UsageAlertRule,
UsageAlertStatus,
)
IssueActivityResponse: TypeAlias = BoundedDailyActivityResponse
AlertActivityResponse: TypeAlias = BoundedDailyActivityResponse
IssueKind: TypeAlias = Literal["performance", "model_deprecation"]
IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"]
IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"]
IssueGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"]
IssueSignalGroupingReason: TypeAlias = Literal[
AlertKind: TypeAlias = Literal["performance", "model_deprecation"]
AlertFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"]
AlertScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"]
AlertGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"]
AlertSignalGroupingReason: TypeAlias = Literal[
"same_resource", "same_key_traffic", "dominant_key", "provider_cohort", "volume_explains_spend", "model_deprecation"
]
class IssueModelPair(BaseModel, frozen=True):
class AlertModelPair(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
provider: str = Field(min_length=1, max_length=256)
model: str = Field(min_length=1, max_length=1024)
class IssueCohort(BaseModel, frozen=True):
class AlertCohort(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
scope_type: IssueScope
scope_type: AlertScope
scope_id: str | None = Field(default=None, min_length=1, max_length=1024)
model_pairs: tuple[IssueModelPair, ...] = Field(default=(), max_length=100)
model_pairs: tuple[AlertModelPair, ...] = Field(default=(), max_length=100)
@model_validator(mode="after")
def validate_scope(self) -> Self:
@ -56,7 +56,7 @@ class IssueCohort(BaseModel, frozen=True):
return self
class IssueComparison(BaseModel, frozen=True):
class AlertComparison(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False)
metric: UsageAlertMetric
@ -78,7 +78,7 @@ class IssueComparison(BaseModel, frozen=True):
return self
class IssueImpact(BaseModel, frozen=True):
class AlertImpact(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
keys: int | None = Field(default=None, ge=0)
@ -90,7 +90,7 @@ class IssueImpact(BaseModel, frozen=True):
measured_models: int | None = Field(default=None, ge=0)
class IssueDeprecation(BaseModel, frozen=True):
class AlertDeprecation(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
model: ModelDeprecationInfo
@ -102,24 +102,24 @@ class IssueDeprecation(BaseModel, frozen=True):
daily_requests: tuple[UsageAlertMetricPoint, ...] = Field(default=(), max_length=30)
class IssueInput(BaseModel, frozen=True):
class AlertInput(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
id: str = Field(min_length=1, max_length=200)
cohort_key: str = Field(min_length=1, max_length=200)
kind: IssueKind
family: IssueFamily
kind: AlertKind
family: AlertFamily
title: str = Field(min_length=1, max_length=2048)
cohort: IssueCohort
cohort: AlertCohort
observed_on: date
observed_at: datetime
range_start: date
range_end: date
grouping_reason: IssueGroupingReason
grouping_reason: AlertGroupingReason
explanation: str = Field(default="", max_length=2000)
comparisons: tuple[IssueComparison, ...] = Field(default=(), max_length=10)
impact: IssueImpact = Field(default_factory=IssueImpact)
deprecation: IssueDeprecation | None = None
comparisons: tuple[AlertComparison, ...] = Field(default=(), max_length=10)
impact: AlertImpact = Field(default_factory=AlertImpact)
deprecation: AlertDeprecation | None = None
@model_validator(mode="after")
def validate_issue(self) -> Self:
@ -134,7 +134,7 @@ class IssueInput(BaseModel, frozen=True):
return self
class Issue(IssueInput, frozen=True):
class Alert(AlertInput, frozen=True):
status: UsageAlertStatus
first_seen: datetime
last_seen: datetime
@ -149,58 +149,60 @@ class Issue(IssueInput, frozen=True):
)
class IssueList(BaseModel, frozen=True):
data: tuple[Issue, ...]
class AlertList(BaseModel, frozen=True):
data: tuple[Alert, ...]
total: int
limit: int = Field(description="Effective page size, reduced for large records. Continue at offset + limit.")
offset: int
class IssueUpdate(BaseModel, frozen=True):
class AlertUpdate(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
status: UsageAlertStatus
expected_status: UsageAlertStatus
class IssueSignalInput(BaseModel, frozen=True):
class AlertSignalInput(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
source: Literal["usage_alert", "model_deprecation"]
source: Literal["performance", "model_deprecation"]
source_id: str = Field(min_length=1, max_length=200)
usage_alert: UsageAlert | None = None
deprecation: IssueDeprecation | None = None
grouping_reason: IssueSignalGroupingReason = "same_resource"
performance: PerformanceSignal | None = None
deprecation: AlertDeprecation | None = None
grouping_reason: AlertSignalGroupingReason = "same_resource"
explanation: str = Field(default="", max_length=2000)
@model_validator(mode="after")
def validate_signal(self) -> Self:
if self.source == "usage_alert":
if self.usage_alert is None or self.deprecation is not None or self.usage_alert.id != self.source_id:
raise ValueError("Usage signals require their matching source alert")
elif self.deprecation is None or self.usage_alert is not None:
if self.source == "performance":
if self.performance is None or self.deprecation is not None or self.performance.id != self.source_id:
raise ValueError("Performance signals require their matching evidence")
elif self.deprecation is None or self.performance is not None:
raise ValueError("Deprecation signals require deprecation details")
return self
class IssueSignal(IssueSignalInput, frozen=True):
issue_id: str
class AlertSignal(AlertSignalInput, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True)
issue_id: str = Field(alias="alert_id")
class IssueSignalList(BaseModel, frozen=True):
data: tuple[IssueSignal, ...]
class AlertSignalList(BaseModel, frozen=True):
data: tuple[AlertSignal, ...]
total: int
limit: int
offset: int
class IssueProjectionSignal(BaseModel, frozen=True):
alert: UsageAlert
rule: UsageAlertRule
class AlertProjectionSignal(BaseModel, frozen=True):
alert: PerformanceSignal
rule: AlertRule
evaluation_day: date
class IssueDeliveryInput(BaseModel, frozen=True):
class AlertDeliveryInput(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
id: str = Field(min_length=1, max_length=200)
@ -211,27 +213,27 @@ class IssueDeliveryInput(BaseModel, frozen=True):
parent_delivery_id: str | None = Field(default=None, min_length=1, max_length=200)
class IssueDeliveryBinding(BaseModel, frozen=True):
delivery: IssueDeliveryInput
class AlertDeliveryBinding(BaseModel, frozen=True):
delivery: AlertDeliveryInput
rule_id: str
rule_revision: str
source_id: str
class IssueDelivery(IssueDeliveryInput, frozen=True):
class AlertDelivery(AlertDeliveryInput, frozen=True):
attempts: int = 0
next_attempt: datetime | None = None
last_error: Literal["delivery_failed", "transport_not_configured"] | None = None
class IssuePendingDelivery(BaseModel, frozen=True):
delivery: IssueDelivery
issue: Issue
rule: UsageAlertRule
alert: UsageAlert
class AlertPendingDelivery(BaseModel, frozen=True):
delivery: AlertDelivery
issue: Alert
rule: AlertRule
alert: PerformanceSignal
class IssueFamilyPolicy(BaseModel, frozen=True):
class AlertFamilyPolicy(BaseModel, frozen=True):
direction: UsageAlertDirection
enabled: bool | None
multiplier: float | None
@ -244,21 +246,21 @@ class IssueFamilyPolicy(BaseModel, frozen=True):
enabled_rules: int
class IssueFamilySetting(BaseModel, frozen=True):
class AlertFamilySetting(BaseModel, frozen=True):
family: PerformanceIssueFamily
enabled: bool | None
total_rules: int
enabled_rules: int
policies: tuple[IssueFamilyPolicy, ...]
policies: tuple[AlertFamilyPolicy, ...]
evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts)
class IssueFamilyList(BaseModel, frozen=True):
data: tuple[IssueFamilySetting, ...]
class AlertFamilyList(BaseModel, frozen=True):
data: tuple[AlertFamilySetting, ...]
projection_pending: bool = False
class IssueFamilyPolicyUpdate(BaseModel, frozen=True):
class AlertFamilyPolicyUpdate(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
direction: UsageAlertDirection
@ -270,11 +272,11 @@ class IssueFamilyPolicyUpdate(BaseModel, frozen=True):
min_history_days: int | None = Field(default=None, ge=1, le=30)
class IssueFamilyUpdate(BaseModel, frozen=True):
class AlertFamilyUpdate(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
enabled: bool | None = None
policies: tuple[IssueFamilyPolicyUpdate, ...] = Field(default=(), max_length=2)
policies: tuple[AlertFamilyPolicyUpdate, ...] = Field(default=(), max_length=2)
@model_validator(mode="after")
def validate_update(self) -> Self:

10
litellm/types/proxy/usage_alert_defaults.py
View file

@ -5,11 +5,11 @@ from typing import Final, Literal, TypeAlias
from uuid import NAMESPACE_URL, uuid5
from litellm.types.proxy.usage_alerts import (
AlertRule,
AlertRuleInput,
UsageAlertChannel,
UsageAlertDirection,
UsageAlertMetric,
UsageAlertRule,
UsageAlertRuleInput,
UsageAlertScope,
)
@ -50,15 +50,15 @@ _DEFAULT_USAGE_IDENTITIES: Final = MappingProxyType(
DEFAULT_USAGE_RULE_IDS: Final = frozenset(_DEFAULT_USAGE_IDENTITIES)
def is_default_usage_rule(rule: UsageAlertRule) -> bool:
def is_default_usage_rule(rule: AlertRule) -> bool:
return _DEFAULT_USAGE_IDENTITIES.get(rule.id) == (rule.scope_type, rule.metric, rule.direction)
def default_usage_rules(
channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False
) -> tuple[UsageAlertRuleInput, ...]:
) -> tuple[AlertRuleInput, ...]:
return tuple(
UsageAlertRuleInput(
AlertRuleInput(
name=f"Daily {scope} {policy.metric.replace('_', ' ')} {policy.direction}",
scope_type=scope,
metric=policy.metric,

70
litellm/types/proxy/usage_alerts.py
View file

@ -24,7 +24,7 @@ class UsageAlertMetricPoint(BaseModel, frozen=True):
value: float | None = Field(default=None, ge=0)
class UsageAlertRuleInput(BaseModel):
class AlertRuleInput(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False)
name: str = Field(min_length=1, max_length=200, pattern=r"^[^\x00-\x1f\x7f]+$")
@ -102,7 +102,7 @@ class UsageAlertProjectionCursor(BaseModel, frozen=True):
observed_at: datetime | None = None
class UsageAlertRule(UsageAlertRuleInput):
class AlertRule(AlertRuleInput):
overrides_defaults: bool = Field(default=False, exclude=True)
id: str
created_at: datetime
@ -114,7 +114,7 @@ class UsageAlertRule(UsageAlertRuleInput):
evaluation_progress: UsageAlertProgress | None = Field(default=None, exclude=True)
class UsageAlert(BaseModel):
class PerformanceSignal(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False)
id: str
@ -142,11 +142,6 @@ class UsageAlert(BaseModel):
notification_error: Literal["delivery_failed", "transport_not_configured"] | None = None
class UsageAlertUpdate(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
status: UsageAlertStatus
class UsageAlertEvaluationCounts(BaseModel, frozen=True):
delayed: int = 0
pending: int = 0
@ -161,73 +156,28 @@ class UsageAlertDefaultSummary(BaseModel, frozen=True):
evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts)
class UsageAlertRuleList(BaseModel, frozen=True):
data: tuple[UsageAlertRule, ...]
class AlertRuleList(BaseModel, frozen=True):
data: tuple[AlertRule, ...]
total: int
limit: int
offset: int
default_summary: UsageAlertDefaultSummary = Field(default_factory=UsageAlertDefaultSummary)
class UsageAlertList(BaseModel, frozen=True):
data: tuple[UsageAlert, ...]
class PerformanceSignalList(BaseModel, frozen=True):
data: tuple[PerformanceSignal, ...]
total: int
limit: int
offset: int
class UsageAlertGroupMetric(BaseModel, frozen=True):
metric: UsageAlertMetric
incident_count: int
representative: UsageAlert
class UsageAlertGroup(BaseModel, frozen=True):
id: str
scope_type: UsageAlertScope
scope_id: str | None
scope_label: str | None
day: date
incident_count: int
metrics: tuple[UsageAlertGroupMetric, ...] = Field(max_length=5)
activity_path: str
class UsageAlertGroupList(BaseModel, frozen=True):
data: tuple[UsageAlertGroup, ...]
total: int
limit: int
offset: int
class UsageAlertGroupUpdate(BaseModel, frozen=True):
model_config = ConfigDict(extra="forbid")
scope_type: UsageAlertScope
scope_id: str | None = Field(default=None, min_length=1, max_length=1024)
day: date
expected_status: UsageAlertStatus
status: UsageAlertStatus
@model_validator(mode="after")
def validate_group(self) -> Self:
if (self.scope_type == "deployment") != (self.scope_id is None):
raise ValueError("Specify a resource ID except for deployment alerts")
if self.expected_status == self.status:
raise ValueError("The new status must differ from the expected status")
return self
class UsageAlertGroupUpdateResult(BaseModel, frozen=True):
updated_count: int
class UsageAlertModelOption(BaseModel, frozen=True):
class AlertModelOption(BaseModel, frozen=True):
scope_id: str
label: str
class UsageAlertModelList(BaseModel, frozen=True):
data: tuple[UsageAlertModelOption, ...]
class AlertModelList(BaseModel, frozen=True):
data: tuple[AlertModelOption, ...]
limit: int
next_cursor: str | None
start_date: date

4
tests/integration/database/test_deprecation_issue_evaluation.py
View file

@ -118,10 +118,10 @@ async def test_retirement_issue_http_state_survives_replay_and_reopens_only_for_
await record(now.date(), 100)
projector = DeprecationIssueProjector.from_prisma(db.prisma, source)
await projector.run(now)
page = gateway.get("/issues", {"scope_type": "model", "scope_id": name})
page = gateway.get("/alerts", {"scope_type": "model", "scope_id": name})
assert page["total"] == 1
original = page["data"][0]
path = f"/issues/{original['id']}"
path = f"/alerts/{original['id']}"
assert original["status"] == "open"
assert original["impact"]["requests"] == 100
assert gateway.get(path + "/signals")["total"] == 1

273
tests/integration/database/test_issues.py
View file

@ -26,28 +26,30 @@ from litellm.repositories.issue_activity import IssueActivityRepository
from litellm.repositories.issues import IssuesRepository
from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository
from litellm.types.proxy.issues import (
IssueActivityResponse,
IssueCohort,
IssueComparison,
IssueDeliveryBinding,
IssueDeliveryInput,
IssueFamilyPolicyUpdate,
IssueFamilyUpdate,
IssueInput,
IssueList,
IssueModelPair,
IssueSignal,
IssueSignalInput,
IssueSignalList,
AlertActivityResponse,
AlertCohort,
AlertComparison,
AlertDeliveryBinding,
AlertDeliveryInput,
AlertDeprecation,
AlertFamilyPolicyUpdate,
AlertFamilyUpdate,
AlertInput,
AlertList,
AlertModelPair,
AlertSignal,
AlertSignalInput,
AlertSignalList,
)
from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo
from litellm.types.proxy.usage_alert_defaults import default_usage_rule_id, default_usage_rules
from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertObservation, UsageAlertRule, UsageAlertRuleInput
from litellm.types.proxy.usage_alerts import AlertRule, AlertRuleInput, PerformanceSignal, UsageAlertObservation
def _issue(cohort: IssueCohort) -> IssueInput:
def _issue(cohort: AlertCohort) -> AlertInput:
identity: Final = str(uuid4())
day: Final = date.today() - timedelta(days=3)
return IssueInput(
return AlertInput(
id=identity,
cohort_key=identity,
kind="performance",
@ -61,7 +63,7 @@ def _issue(cohort: IssueCohort) -> IssueInput:
grouping_reason="same_resource",
explanation="Requests increased for this resource.",
comparisons=(
IssueComparison(
AlertComparison(
metric="requests",
direction="increase",
observed_value=420,
@ -75,12 +77,12 @@ def _issue(cohort: IssueCohort) -> IssueInput:
)
def _signals(issue_id: str, inputs: tuple[IssueSignalInput, ...]) -> tuple[IssueSignal, ...]:
return tuple(IssueSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in inputs)
def _signals(issue_id: str, inputs: tuple[AlertSignalInput, ...]) -> tuple[AlertSignal, ...]:
return tuple(AlertSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in inputs)
def _delivery(data: IssueDeliveryInput, rule: UsageAlertRule, source: UsageAlert) -> IssueDeliveryBinding:
return IssueDeliveryBinding(
def _delivery(data: AlertDeliveryInput, rule: AlertRule, source: PerformanceSignal) -> AlertDeliveryBinding:
return AlertDeliveryBinding(
delivery=data, rule_id=rule.id, rule_revision=rule.updated_at.isoformat(), source_id=source.id
)
@ -106,9 +108,9 @@ async def test_issue_pages_bound_payload_bytes_without_losing_details_or_paginat
store: Final = IssuesRepository.from_prisma(db.prisma)
owner: Final = "page-budget-" + uuid4().hex
pairs: Final = tuple(
IssueModelPair(provider=owner, model=str(index).zfill(4) + "\U0001f680" * 1020) for index in range(100)
AlertModelPair(provider=owner, model=str(index).zfill(4) + "\U0001f680" * 1020) for index in range(100)
)
cohort: Final = IssueCohort(scope_type="provider", scope_id=owner, model_pairs=pairs)
cohort: Final = AlertCohort(scope_type="provider", scope_id=owner, model_pairs=pairs)
inputs: Final = tuple(_issue(cohort) for _ in range(6))
identifiers: Final = tuple(item.id for item in inputs)
try:
@ -149,11 +151,11 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa
viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only")
inference_key: Final = scenario.key(user_id=owner, key_type="llm_api", key_alias="Example checkout")
reporting_id: Final = sha256(inference_key.encode()).hexdigest()
first: Final = _issue(IssueCohort(scope_type="key", scope_id=reporting_id))
first: Final = _issue(AlertCohort(scope_type="key", scope_id=reporting_id))
second: Final = _issue(first.cohort)
global_issue: Final = _issue(IssueCohort(scope_type="provider", scope_id="example-provider"))
global_issue: Final = _issue(AlertCohort(scope_type="provider", scope_id="example-provider"))
rule: Final = await alerts.save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Visibility fixture",
scope_type="key",
scope_id=reporting_id,
@ -193,18 +195,18 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa
_signals(
first.id,
(
IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),
IssueSignalInput(
source="usage_alert",
AlertSignalInput(source="performance", source_id=source.id, performance=source),
AlertSignalInput(
source="performance",
source_id=broader.id,
usage_alert=broader,
performance=broader,
grouping_reason="dominant_key",
explanation="Parent contribution evidence.",
),
),
)
)
path: Final = f"/issues/{first.id}"
path: Final = f"/alerts/{first.id}"
own: Final = gateway.request("GET", path, key=owner_key)
assert own.status_code == 200 and own.json()["scope_label"] == "Example checkout"
assert own.json()["can_update_status"] is False, "Broader source scopes require their own write permission"
@ -221,23 +223,23 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa
pages: Final = tuple(
gateway.request(
"GET",
"/issues",
"/alerts",
key=owner_key,
params={"scope_type": "key", "scope_id": reporting_id, "limit": "1", "offset": str(offset)},
)
for offset in (0, 1, 2)
)
assert all(page.status_code == 200 for page in pages)
for offset, parsed in enumerate(IssueList.model_validate(page.json()) for page in pages):
for offset, parsed in enumerate(AlertList.model_validate(page.json()) for page in pages):
assert parsed.total == 2 and len(parsed.data) == (1 if offset < 2 else 0)
assert gateway.request("GET", "/issues", key=owner_key).status_code == 403
assert gateway.request("GET", "/alerts", key=owner_key).status_code == 403
for suffix in ("", "/signals", "/activity", "/keys", "/users", "/impact"):
assert gateway.request("GET", f"/issues/{global_issue.id}{suffix}", key=owner_key).status_code == 403
assert gateway.request("GET", f"/alerts/{global_issue.id}{suffix}", key=owner_key).status_code == 403
assert gateway.request("GET", path + suffix, key=outsider_key).status_code == 403
assert gateway.request("GET", path + suffix, key=inference_key).status_code == 403
public_signals: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1"})
assert public_signals.status_code == 200
visible: Final = IssueSignalList.model_validate(public_signals.json())
visible: Final = AlertSignalList.model_validate(public_signals.json())
assert visible.total == 1 and tuple(row.source_id for row in visible.data) == (source.id,)
assert "99999" not in public_signals.text
end: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1", "offset": "1"})
@ -258,11 +260,11 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa
)
assert gateway.request("PATCH", path, {"status": "dismissed", "expected_status": "open"}).status_code == 200
assert gateway.request("PATCH", path, {"status": "resolved", "expected_status": "open"}).status_code == 409
assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["status"] == "open"
assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["can_update_status"] is True
assert gateway.request("GET", f"/alerts/{second.id}", key=owner_key).json()["status"] == "open"
assert gateway.request("GET", f"/alerts/{second.id}", key=owner_key).json()["can_update_status"] is True
assert (
gateway.request(
"PATCH", f"/issues/{second.id}", {"status": "dismissed", "expected_status": "open"}, key=owner_key
"PATCH", f"/alerts/{second.id}", {"status": "dismissed", "expected_status": "open"}, key=owner_key
).status_code
== 200
)
@ -293,11 +295,11 @@ async def test_issue_detail_reports_team_status_permission(gateway: Gateway, db:
key_type="management",
metadata={"service_account_id": "example-activity-service"},
)
issue: Final = _issue(IssueCohort(scope_type="team", scope_id=team))
permitted_issue: Final = _issue(IssueCohort(scope_type="team", scope_id=permitted_team))
issue: Final = _issue(AlertCohort(scope_type="team", scope_id=team))
permitted_issue: Final = _issue(AlertCohort(scope_type="team", scope_id=permitted_team))
try:
await store.upsert_issues((issue, permitted_issue))
path: Final = f"/issues/{issue.id}"
path: Final = f"/alerts/{issue.id}"
assert gateway.request("GET", path, key=admin_key).json()["can_update_status"] is True
assert gateway.request("GET", path, key=admin_key).json()["can_view_activity"] is True
assert gateway.request("GET", path, key=member_key).json()["can_update_status"] is False
@ -308,10 +310,10 @@ async def test_issue_detail_reports_team_status_permission(gateway: Gateway, db:
for suffix in ("/activity", "/keys", "/impact"):
assert gateway.request("GET", path + suffix, key=admin_key).status_code == 200
assert (
gateway.request("GET", f"/issues/{permitted_issue.id}" + suffix, key=member_key).status_code == 200
gateway.request("GET", f"/alerts/{permitted_issue.id}" + suffix, key=member_key).status_code == 200
)
assert (
gateway.request("GET", f"/issues/{permitted_issue.id}", key=member_key).json()["can_view_activity"]
gateway.request("GET", f"/alerts/{permitted_issue.id}", key=member_key).json()["can_view_activity"]
is True
)
assert (
@ -342,7 +344,7 @@ async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gatew
provider: Final = f"example-provider-{identity}"
day: Final = date.today() - timedelta(days=5)
model_count: Final = 137
cohort: Final = IssueCohort(scope_type="key", scope_id=key)
cohort: Final = AlertCohort(scope_type="key", scope_id=key)
issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day + timedelta(days=2)})
await _query(
db,
@ -365,9 +367,9 @@ async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gatew
await store.upsert_issues((issue,))
scope: Final = issue_activity_scope(cohort, day, day + timedelta(days=2))
original: Final = await get_daily_activity_aggregated(repository.activity, scope, api_key_limit=20)
response: Final = gateway.request("GET", f"/issues/{issue.id}/activity")
response: Final = gateway.request("GET", f"/alerts/{issue.id}/activity")
assert response.status_code == 200
bounded: Final = IssueActivityResponse.model_validate(response.json())
bounded: Final = AlertActivityResponse.model_validate(response.json())
assert bounded.metadata.model_dump(exclude={"api_key_limit"}) == original.metadata.model_dump(
exclude={"api_key_limit"}
)
@ -388,30 +390,30 @@ async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gatew
await repository.database.query_raw(query.sql, *query.params)
)
assert len(raw) == 3 + 1 + 100 + 1
largest: Final = gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "100"})
largest: Final = gateway.request("GET", f"/alerts/{issue.id}/activity", params={"model_limit": "100"})
assert largest.status_code == 200
large_models: Final = IssueActivityResponse.model_validate(largest.json()).model_breakdown
large_models: Final = AlertActivityResponse.model_validate(largest.json()).model_breakdown
assert large_models is not None and len(large_models.models) == 100
assert large_models.other is not None and large_models.other.model_count == model_count - 100
assert gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "101"}).status_code == 422
assert gateway.request("GET", f"/alerts/{issue.id}/activity", params={"model_limit": "101"}).status_code == 422
assert (
gateway.request(
"GET",
f"/issues/{issue.id}/activity",
f"/alerts/{issue.id}/activity",
params={"start_date": day.isoformat(), "end_date": (day + timedelta(days=366)).isoformat()},
).status_code
== 422
)
non_key: Final = await repository.daily_activity(
IssueCohort(scope_type="provider", scope_id=provider), day, day + timedelta(days=2)
AlertCohort(scope_type="provider", scope_id=provider), day, day + timedelta(days=2)
)
assert non_key.model_breakdown is None and non_key.metadata == bounded.metadata
assert len(non_key.results) == 3
async with db.prisma.writer_db.tx(timeout=timedelta(seconds=10)) as blocker:
await blocker.execute_raw('LOCK TABLE "LiteLLM_DailyUserSpend" IN ACCESS EXCLUSIVE MODE')
busy: Final = gateway.request("GET", f"/issues/{issue.id}/activity")
busy: Final = gateway.request("GET", f"/alerts/{issue.id}/activity")
assert busy.status_code == 503 and busy.headers["Retry-After"] == "5"
assert gateway.request("GET", f"/issues/{issue.id}/activity").status_code == 200
assert gateway.request("GET", f"/alerts/{issue.id}/activity").status_code == 200
finally:
await _remove_issues(db, (issue.id,))
await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1 RETURNING id', identity)
@ -459,22 +461,22 @@ async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports(
RETURNING id""",
json.dumps(entries),
)
cohort: Final = IssueCohort(
cohort: Final = AlertCohort(
scope_type="provider",
scope_id=provider,
model_pairs=(IssueModelPair(provider=provider, model="example-model-a"),),
model_pairs=(AlertModelPair(provider=provider, model="example-model-a"),),
)
store: Final = IssuesRepository.from_prisma(db.prisma)
issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day})
try:
await store.upsert_issues((issue,))
activity_http: Final = gateway.request("GET", f"/issues/{issue.id}/activity")
activity_http: Final = gateway.request("GET", f"/alerts/{issue.id}/activity")
assert activity_http.status_code == 200 and activity_http.json()["metadata"]["total_api_requests"] == 100
keys_http: Final = gateway.request("GET", f"/issues/{issue.id}/keys", params={"limit": "1"})
keys_http: Final = gateway.request("GET", f"/alerts/{issue.id}/keys", params={"limit": "1"})
assert keys_http.status_code == 200 and keys_http.json()["api_keys"][0]["metrics"]["api_requests"] == 100
users_http: Final = gateway.request("GET", f"/issues/{issue.id}/users")
users_http: Final = gateway.request("GET", f"/alerts/{issue.id}/users")
assert users_http.status_code == 200 and users_http.json()["total_users"] == 1
impact_http: Final = gateway.request("GET", f"/issues/{issue.id}/impact")
impact_http: Final = gateway.request("GET", f"/alerts/{issue.id}/impact")
assert impact_http.status_code == 200 and impact_http.json()["requests"] == 100
daily: Final = await repository.daily_activity(cohort, day, day)
assert daily.metadata.total_api_requests == 100 and daily.metadata.total_spend == 10
@ -488,15 +490,15 @@ async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports(
assert users.total_users == 1 and users.users[0].api_requests == 100 and users.users[0].timed_requests == 100
impact: Final = await repository.impact(cohort, day, day)
assert (impact.keys, impact.users, impact.models, impact.requests) == (1, 1, 1, 100)
pairs: Final = IssueCohort(
pairs: Final = AlertCohort(
scope_type="deployment",
model_pairs=(
IssueModelPair(provider=provider, model="example-model-a"),
IssueModelPair(provider=other, model="example-model-b"),
AlertModelPair(provider=provider, model="example-model-a"),
AlertModelPair(provider=other, model="example-model-b"),
),
)
assert (await repository.daily_activity(pairs, day, day)).metadata.total_api_requests == 100
injected: Final = IssueCohort(scope_type="provider", scope_id=provider + "' OR true --")
injected: Final = AlertCohort(scope_type="provider", scope_id=provider + "' OR true --")
assert (await repository.impact(injected, day, day)).requests == 0
from litellm.repositories.issue_activity import issue_activity_scope
from litellm.types.repositories.daily_activity import ExportType
@ -523,8 +525,8 @@ async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports(
async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: ManagementDatabase) -> None:
store: Final = IssuesRepository.from_prisma(db.prisma)
alerts: Final = UsageAlertsRepository.from_prisma(db.prisma)
original: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}"))
definition: Final = UsageAlertRuleInput(
original: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}"))
definition: Final = AlertRuleInput(
name="Delivery fixture",
scope_type="user",
scope_id=original.cohort.scope_id,
@ -538,8 +540,8 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma
observation: Final = UsageAlertObservation(
entity_id=rule.scope_id, day=original.observed_on, observed=420, baseline=120, threshold=360, breached=True
)
delivery: Final = IssueDeliveryInput(id=str(uuid4()), issue_id=original.id, channel="slack")
invalid: Final = IssueDeliveryInput(
delivery: Final = AlertDeliveryInput(id=str(uuid4()), issue_id=original.id, channel="slack")
invalid: Final = AlertDeliveryInput(
id=str(uuid4()), issue_id=original.id, channel="email", email_recipients=("outside@example.com",)
)
next_issue_id: Final = str(uuid4())
@ -555,7 +557,7 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma
assert await store.active_signals(original.observed_on, scope_ids=()) == ()
assert await store.active_signals(original.observed_on, scope_ids=("other-example-user",)) == ()
await store.add_signal_batch(
_signals(initial.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),))
_signals(initial.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),))
)
await store.queue_deliveries((_delivery(delivery, rule, source),))
await store.queue_deliveries((_delivery(invalid, rule, source),))
@ -693,7 +695,7 @@ async def test_late_default_override_stops_queued_delivery_and_recovers_on_fresh
await alerts.ensure_default_rules(default_usage_rules(("slack",)))
rule: Final = await alerts.get_rule(default_usage_rule_id("key", "requests", "increase"))
assert rule is not None
issues: Final = tuple(_issue(IssueCohort(scope_type="key", scope_id=key)) for key in ("key-a", "key-b"))
issues: Final = tuple(_issue(AlertCohort(scope_type="key", scope_id=key)) for key in ("key-a", "key-b"))
observations: Final = tuple(
UsageAlertObservation(
entity_id=issue.cohort.scope_id,
@ -708,15 +710,15 @@ async def test_late_default_override_stops_queued_delivery_and_recovers_on_fresh
sources: Final = await alerts.record_observations(rule, observations)
await store.upsert_issues(issues)
for issue in issues:
source: UsageAlert = next(source for source in sources if source.scope_id == issue.cohort.scope_id)
source: PerformanceSignal = next(source for source in sources if source.scope_id == issue.cohort.scope_id)
await store.add_signal_batch(
_signals(issue.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),))
_signals(issue.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),))
)
await store.queue_deliveries(
(_delivery(IssueDeliveryInput(id=issue.id, issue_id=issue.id, channel="slack"), rule, source),)
(_delivery(AlertDeliveryInput(id=issue.id, issue_id=issue.id, channel="slack"), rule, source),)
)
assert await store.pending_delivery(issue.id) is not None
definition: Final = UsageAlertRuleInput(
definition: Final = AlertRuleInput(
name="Project A threshold",
scope_type="key",
scope_id="key-a",
@ -763,7 +765,7 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren
store: Final = IssuesRepository(borrowed)
await alerts.ensure_default_rules(default_usage_rules())
custom: Final = await alerts.save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Custom spend", scope_type="key", scope_id="example-key", metric="spend", direction="increase"
),
"qa-admin",
@ -773,8 +775,8 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren
await alerts.delete_rule(removed_id)
result: Final = await store.update_family(
"spend",
IssueFamilyUpdate(
enabled=False, policies=(IssueFamilyPolicyUpdate(direction="increase", multiplier=5, min_value=25),)
AlertFamilyUpdate(
enabled=False, policies=(AlertFamilyPolicyUpdate(direction="increase", multiplier=5, min_value=25),)
),
)
assert result is not None and result.total_rules == 4 and result.enabled_rules == 0
@ -785,8 +787,8 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren
with pytest.raises(ValidationError, match="min_history_days"):
await store.update_family(
"spend",
IssueFamilyUpdate(
policies=(IssueFamilyPolicyUpdate(direction="increase", baseline_days=1, min_history_days=7),)
AlertFamilyUpdate(
policies=(AlertFamilyPolicyUpdate(direction="increase", baseline_days=1, min_history_days=7),)
),
)
assert await store.family_settings() == baseline
@ -801,7 +803,7 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren
.validate_python(await transaction.query_raw("SELECT pg_backend_pid() AS pid"))[0]
.pid
)
update: Final = tasks.create_task(store.update_family("spend", IssueFamilyUpdate(enabled=True)))
update: Final = tasks.create_task(store.update_family("spend", AlertFamilyUpdate(enabled=True)))
async with asyncio.timeout(10):
while (
not TypeAdapter(tuple[_Blocked, ...])
@ -840,7 +842,7 @@ async def test_family_edit_requires_a_new_evaluation(db: ManagementDatabase) ->
assert rule is not None
await alerts.record_evaluations(((rule, "complete"),), date.today() - timedelta(days=1), completed=True)
result: Final = await store.update_family(
"spend", IssueFamilyUpdate(policies=(IssueFamilyPolicyUpdate(direction="increase", multiplier=5),))
"spend", AlertFamilyUpdate(policies=(AlertFamilyPolicyUpdate(direction="increase", multiplier=5),))
)
saved: Final = await alerts.get_rule(rule.id)
assert result is not None and saved is not None
@ -860,7 +862,7 @@ async def test_deleted_key_owner_retains_historical_issue_access(gateway: Gatewa
owner_key: Final = scenario.key(user_id=owner, key_type="management")
outsider_key: Final = scenario.key(user_id=outsider, key_type="management")
reporting_id: Final = sha256(uuid4().bytes).hexdigest()
issue: Final = _issue(IssueCohort(scope_type="key", scope_id=reporting_id))
issue: Final = _issue(AlertCohort(scope_type="key", scope_id=reporting_id))
await _query(
db,
'INSERT INTO "LiteLLM_DeletedVerificationToken" (id,token,user_id,models) VALUES ($1,$1,$2,ARRAY[]::text[])',
@ -869,7 +871,7 @@ async def test_deleted_key_owner_retains_historical_issue_access(gateway: Gatewa
)
try:
await store.upsert_issues((issue,))
path: Final = f"/issues/{issue.id}"
path: Final = f"/alerts/{issue.id}"
detail: Final = gateway.request("GET", path, key=owner_key)
assert detail.status_code == 200, "Deleted key owner must retain historical issue access"
assert detail.json()["can_update_status"] is True
@ -898,9 +900,9 @@ async def test_deleted_key_owner_retains_historical_issue_access(gateway: Gatewa
async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: ManagementDatabase) -> None:
store: Final = IssuesRepository.from_prisma(db.prisma)
alerts: Final = UsageAlertsRepository.from_prisma(db.prisma)
issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}"))
issue: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}"))
rule: Final = await alerts.save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Scope guard",
scope_type="user",
scope_id=issue.cohort.scope_id,
@ -928,13 +930,13 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana
)[0]
await store.upsert_issues((issue,))
await store.add_signal_batch(
_signals(issue.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),))
_signals(issue.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),))
)
authorized: Final = await store.status_scopes(issue.id)
assert authorized == (issue.cohort,)
assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=authorized) is not None
context: Final = source.model_copy(update={"id": str(uuid4()), "metric": "spend"})
same_scope: Final = (IssueSignalInput(source="usage_alert", source_id=context.id, usage_alert=context),)
same_scope: Final = (AlertSignalInput(source="performance", source_id=context.id, performance=context),)
await store.add_signal_batch(_signals(issue.id, same_scope))
unchanged: Final = await store.get_issue(issue.id)
assert unchanged is not None and unchanged.status == "dismissed"
@ -942,7 +944,7 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana
assert await store.get_issue(issue.id) == unchanged
broader: Final = source.model_copy(update={"id": str(uuid4()), "scope_type": "deployment", "scope_id": None})
await store.add_signal_batch(
_signals(issue.id, (IssueSignalInput(source="usage_alert", source_id=broader.id, usage_alert=broader),))
_signals(issue.id, (AlertSignalInput(source="performance", source_id=broader.id, performance=broader),))
)
assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=authorized) is None
current: Final = await store.get_issue(issue.id)
@ -953,7 +955,7 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana
await store.add_signal_batch(
_signals(
issue.id,
(IssueSignalInput(source="usage_alert", source_id=broader_context.id, usage_alert=broader_context),),
(AlertSignalInput(source="performance", source_id=broader_context.id, performance=broader_context),),
)
)
repeated_scope: Final = await store.get_issue(issue.id)
@ -989,7 +991,7 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana
)
await locked.add_signal_batch(
_signals(
issue.id, (IssueSignalInput(source="usage_alert", source_id=racing.id, usage_alert=racing),)
issue.id, (AlertSignalInput(source="performance", source_id=racing.id, performance=racing),)
)
)
assert update.result() is None
@ -999,10 +1001,10 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana
_signals(
issue.id,
tuple(
IssueSignalInput(
source="usage_alert",
AlertSignalInput(
source="performance",
source_id=f"scope-{index}",
usage_alert=source.model_copy(
performance=source.model_copy(
update={"id": f"scope-{index}", "scope_id": f"other-user-{index}"}
),
)
@ -1024,22 +1026,22 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana
async def test_resource_recovery_ignores_broader_and_traffic_context(db: ManagementDatabase) -> None:
store: Final = IssuesRepository.from_prisma(db.prisma)
alerts: Final = UsageAlertsRepository.from_prisma(db.prisma)
fixture: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}"))
fixture: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}"))
issue: Final = fixture.model_copy(
update={"family": "spend", "comparisons": (fixture.comparisons[0].model_copy(update={"metric": "spend"}),)}
)
definitions: Final = (
UsageAlertRuleInput(
AlertRuleInput(
name="Primary cost", scope_type="user", scope_id=issue.cohort.scope_id, metric="spend", direction="increase"
),
UsageAlertRuleInput(
AlertRuleInput(
name="Traffic context",
scope_type="user",
scope_id=issue.cohort.scope_id,
metric="requests",
direction="increase",
),
UsageAlertRuleInput(name="Broader cost", scope_type="deployment", metric="spend", direction="increase"),
AlertRuleInput(name="Broader cost", scope_type="deployment", metric="spend", direction="increase"),
)
rules: Final = tuple([await alerts.save_rule(definition, "qa-admin") for definition in definitions])
assert all(rule is not None for rule in rules)
@ -1070,7 +1072,7 @@ async def test_resource_recovery_ignores_broader_and_traffic_context(db: Managem
_signals(
issue.id,
tuple(
IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source)
AlertSignalInput(source="performance", source_id=source.id, performance=source)
for source in sources
),
)
@ -1107,7 +1109,7 @@ async def test_resource_recovery_ignores_broader_and_traffic_context(db: Managem
async def test_provider_recovery_pages_saved_cohorts_and_rejects_stale_observations(db: ManagementDatabase) -> None:
store: Final = IssuesRepository.from_prisma(db.prisma)
prefix: Final = f"fixture-{uuid4()}"
initial: Final = _issue(IssueCohort(scope_type="provider", scope_id="example-provider"))
initial: Final = _issue(AlertCohort(scope_type="provider", scope_id="example-provider"))
template: Final = initial.model_copy(update={"observed_at": initial.observed_at.replace(microsecond=123789)})
inputs: Final = tuple(
template.model_copy(update={"id": prefix + suffix, "cohort_key": prefix + suffix})
@ -1146,11 +1148,11 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes(
) -> None:
store: Final = IssuesRepository.from_prisma(db.prisma)
alerts: Final = UsageAlertsRepository.from_prisma(db.prisma)
expired: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}"))
expired: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}"))
open_issue: Final = _issue(expired.cohort)
recent: Final = _issue(expired.cohort)
rule: Final = await alerts.save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Prune fixture",
scope_type="user",
scope_id=expired.cohort.scope_id,
@ -1161,7 +1163,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes(
)
assert rule is not None
deliveries: Final = tuple(
IssueDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") for _ in range(2)
AlertDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") for _ in range(2)
)
try:
await store.upsert_issues((expired, open_issue, recent))
@ -1181,7 +1183,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes(
)
)[0]
await store.add_signal_batch(
_signals(expired.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),))
_signals(expired.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),))
)
for delivery in deliveries:
await store.queue_deliveries((_delivery(delivery, rule, source),))
@ -1226,6 +1228,61 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes(
)
@pytest.mark.asyncio(loop_scope="session")
async def test_prune_preserves_retirement_reopened_by_new_usage(db: ManagementDatabase) -> None:
store: Final = IssuesRepository.from_prisma(db.prisma)
original: Final = _issue(AlertCohort(scope_type="model", scope_id=f"retirement-{uuid4()}"))
name: Final = original.cohort.scope_id or "example-model"
retirement: Final = original.observed_on + timedelta(days=10)
details: Final = AlertDeprecation(
model=ModelDeprecationInfo(
model_name=name, deprecation_date=retirement, days_until_deprecation=10, status="imminent"
),
usage_model=name,
requests=10,
last_used_date=original.observed_on,
last_day_requests=10,
state=DeprecationAlertState(model=name, deprecation_date=retirement),
)
issue: Final = original.model_copy(
update={
"kind": "model_deprecation",
"family": "retirement",
"deprecation": details,
"grouping_reason": "model_deprecation",
"comparisons": (),
}
)
try:
await store.upsert_issues((issue,))
assert await store.set_status(issue.id, "dismissed", "open") is not None
await store.upsert_issues(
(
issue.model_copy(
update={
"deprecation": details.model_copy(update={"last_day_requests": 11, "requests": 11}),
"observed_at": original.observed_at + timedelta(minutes=1),
}
),
)
)
await _query(
db, "UPDATE \"LiteLLM_Issue\" SET updated_at=TIMESTAMP '2000-01-01' WHERE id=$1 RETURNING id", issue.id
)
reopened: Final = await store.get_issue(issue.id)
assert reopened is not None and reopened.status == "open"
await store.prune(retention_days=90, limit=10)
assert await store.get_issue(issue.id) == reopened
finally:
await _query(
db,
'DELETE FROM "LiteLLM_Config" WHERE param_name IN '
'(SELECT ack_key FROM "LiteLLM_Issue" WHERE id=$1) RETURNING param_name',
issue.id,
)
await _remove_issues(db, (issue.id,))
class _MeasuredActivityDatabase:
def __init__(self, database: DailyActivityQueryDatabase) -> None:
self.database = database
@ -1250,7 +1307,7 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history(
day: Final = date.today() - timedelta(days=3)
with gateway.scenario() as scenario:
owner: Final = scenario.user(user_role="internal_user")
issue: Final = _issue(IssueCohort(scope_type="provider", scope_id=identity)).model_copy(
issue: Final = _issue(AlertCohort(scope_type="provider", scope_id=identity)).model_copy(
update={"range_start": day, "range_end": day}
)
try:
@ -1301,7 +1358,7 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history(
totals: Final = await repository.daily_activity(issue.cohort, day, day)
assert totals.metadata.total_api_requests == 30
for endpoint in ("keys", "users"):
response: Final = gateway.request("GET", f"/issues/{issue.id}/{endpoint}", params={"limit": "1"})
response: Final = gateway.request("GET", f"/alerts/{issue.id}/{endpoint}", params={"limit": "1"})
assert response.status_code == 200 and len(response.content) < 2048
assert "padding" not in response.text
await _query(
@ -1324,8 +1381,8 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history(
async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: ManagementDatabase) -> None:
store: Final = IssuesRepository.from_prisma(db.prisma)
alerts: Final = UsageAlertsRepository.from_prisma(db.prisma)
issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}"))
definition: Final = UsageAlertRuleInput(
issue: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}"))
definition: Final = AlertRuleInput(
name="Primary traffic",
scope_type="user",
scope_id=issue.cohort.scope_id,
@ -1350,7 +1407,7 @@ async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: M
_signals(
issue.id,
tuple(
IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source)
AlertSignalInput(source="performance", source_id=source.id, performance=source)
for source in sources
),
)

76
tests/integration/database/test_performance_issues.py
View file

@ -22,7 +22,7 @@ from litellm.repositories.daily_activity_buckets import api_key_bucket
from litellm.repositories.issues import IssuesRepository
from litellm.repositories.usage_alert_metrics import IssueMetricWindow, UsageAlertMetricsRepository
from litellm.repositories.usage_alerts import UsageAlertsRepository
from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRule, UsageAlertRuleInput, UsageAlertScope
from litellm.types.proxy.usage_alerts import UsageAlertObservation, AlertRule, AlertRuleInput, UsageAlertScope
_DAY: Final = date(2026, 11, 4)
_NOW: Final = datetime(2026, 11, 5, 3, microsecond=987654, tzinfo=timezone.utc)
@ -277,7 +277,7 @@ async def test_due_issue_delivery_retries_after_daily_projection_is_complete(db:
await _seed(prisma)
rules = UsageAlertsRepository.from_prisma(prisma)
rule = await rules.save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Key spend notification",
scope_type="key",
scope_id=_KEY_A,
@ -390,7 +390,7 @@ async def test_same_day_rule_revision_requires_a_fresh_observation(db: Managemen
async with _isolated(db) as prisma:
alerts: Final = UsageAlertsRepository.from_prisma(prisma)
issues: Final = IssuesRepository.from_prisma(prisma)
definition: Final = UsageAlertRuleInput(
definition: Final = AlertRuleInput(
name="Project spend",
scope_type="key",
scope_id=_KEY_A,
@ -427,7 +427,7 @@ async def test_late_custom_key_rule_only_overrides_that_keys_default_signal(db:
_KEY_B,
)
job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[]))
# A rule can change after the daily observations, before Issue projection.
# A rule can change after the daily observations, before Alert projection.
await job._store.ensure_default_rules(job._defaults)
await job._store.ensure_projection_work(_DAY)
await job._evaluate(_DAY, asyncio.get_running_loop().time() + 60)
@ -437,7 +437,7 @@ async def test_late_custom_key_rule_only_overrides_that_keys_default_signal(db:
overridden: Final = next(item for item in original if item.alert.scope_id == _KEY_A)
retained: Final = next(item for item in original if item.alert.scope_id == _KEY_B)
custom: Final = await UsageAlertsRepository.from_prisma(prisma).save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Project A high spend threshold",
scope_type="key",
scope_id=_KEY_A,
@ -609,7 +609,7 @@ async def test_entity_page_progress_resumes_old_day_without_early_completion(db:
await _coverage(prisma, _DAY)
job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[]))
rule: Final = await job._store.save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Resume each key",
scope_type="key",
metric="spend",
@ -625,7 +625,7 @@ async def test_entity_page_progress_resumes_old_day_without_early_completion(db:
assert partial.evaluation_progress is not None and partial.evaluation_progress.cursor is not None
assert not await job._store.completed_rule_ids((rule,), _DAY)
assert "evaluation_progress" not in partial.model_dump()
assert "evaluation_progress" not in UsageAlertRule.model_json_schema(mode="serialization")["properties"]
assert "evaluation_progress" not in AlertRule.model_json_schema(mode="serialization")["properties"]
first: Final = await job._store.list_alerts(limit=500)
assert first.total == 10
projector: Final = job._performance_projection
@ -662,7 +662,7 @@ async def test_entity_page_progress_resumes_old_day_without_early_completion(db:
resumed: Final = await job._store.get_rule(rule.id)
assert resumed is not None and resumed.evaluation_progress is None
edited: Final = await job._store.save_rule(
UsageAlertRuleInput.model_validate(rule.model_dump(include=set(UsageAlertRuleInput.model_fields))),
AlertRuleInput.model_validate(rule.model_dump(include=set(AlertRuleInput.model_fields))),
None,
resumed,
)
@ -674,7 +674,7 @@ async def test_disabled_primary_requires_another_eligible_healthy_source_to_reco
async with _isolated(db) as prisma:
await _seed(prisma)
job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[]))
definition: Final = UsageAlertRuleInput(
definition: Final = AlertRuleInput(
name="Extra key policy",
scope_type="key",
scope_id=_KEY_A,
@ -713,7 +713,7 @@ async def test_lone_disabled_source_does_not_claim_recovery(db: ManagementDataba
_KEY_A,
)
job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[]))
definition: Final = UsageAlertRuleInput(
definition: Final = AlertRuleInput(
name="Sensitive key policy",
scope_type="key",
scope_id=_KEY_A,
@ -743,9 +743,7 @@ async def test_rule_edit_discards_old_cursor_and_rejects_stale_progress(db: Mana
async with _isolated(db) as prisma:
store: Final = UsageAlertsRepository.from_prisma(prisma)
definition: Final = UsageAlertRuleInput(
name="Resumable", scope_type="key", metric="spend", direction="increase"
)
definition: Final = AlertRuleInput(name="Resumable", scope_type="key", metric="spend", direction="increase")
rule: Final = await store.save_rule(definition, None)
assert rule is not None
position: Final = UsageAlertProgress(day=_DAY, revision=rule.updated_at, cursor="a" * 64, evaluated=True)
@ -781,6 +779,38 @@ async def test_busy_background_capacity_defers_and_releases_the_worker_lease(db:
) == [{"count": 0}]
@pytest.mark.asyncio(loop_scope="session")
async def test_capacity_failure_before_coverage_does_not_pin_a_drop_rule_to_an_incomplete_day(
db: ManagementDatabase,
) -> None:
async with _isolated(db) as prisma:
await _seed(prisma)
store: Final = UsageAlertsRepository.from_prisma(prisma)
rule: Final = await store.save_rule(
AlertRuleInput(
name="Project traffic drop",
scope_type="key",
scope_id=_KEY_A,
metric="requests",
direction="decrease",
),
"qa-admin",
)
assert rule is not None
await prisma.writer_db.execute_raw(
'DELETE FROM "LiteLLM_UsageAlertFlush" WHERE hour<$1::timestamp', (_DAY + timedelta(days=1)).isoformat()
)
job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[]))
async with prisma.writer_db.tx(timeout=timedelta(seconds=10)) as transaction:
await transaction.execute_raw('LOCK TABLE "LiteLLM_UsageAlertFlush" IN ACCESS EXCLUSIVE MODE')
assert not await job._attempt_scope_page((rule,), _DAY, False)
saved: Final = await store.get_rule(rule.id)
assert saved is not None and saved.evaluation_progress is None
await _next_day(prisma, _DAY + timedelta(days=1), 1, 1)
assert await job._attempt_scope_page((saved,), _DAY + timedelta(days=1), False)
assert await store.completed_rule_ids((saved,), _DAY + timedelta(days=1)) == frozenset((rule.id,))
@pytest.mark.asyncio(loop_scope="session")
async def test_partial_broad_default_waits_for_its_key_family_only(db: ManagementDatabase) -> None:
from litellm.types.proxy.usage_alert_defaults import default_usage_rules
@ -849,7 +879,7 @@ async def test_projection_resume_keeps_spend_and_traffic_in_one_episode(db: Mana
before: Final = await IssuesRepository.from_prisma(prisma).list_issues()
assert len(before.data) == 1 and before.data[0].family == "spend"
added: Final = await job._store.save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Later policy", scope_type="key", scope_id=_KEY_B, metric="latency", direction="increase"
),
None,
@ -873,7 +903,7 @@ async def test_projection_resume_keeps_spend_and_traffic_in_one_episode(db: Mana
after: Final = await IssuesRepository.from_prisma(prisma).list_issues()
assert tuple(item.id for item in after.data) == tuple(item.id for item in before.data)
links: Final = await IssuesRepository.from_prisma(prisma).list_signals(after.data[0].id)
assert any(link.usage_alert is not None and link.usage_alert.metric == "requests" for link in links.data)
assert any(link.performance is not None and link.performance.metric == "requests" for link in links.data)
@pytest.mark.asyncio(loop_scope="session")
@ -891,7 +921,7 @@ async def test_slack_retry_only_repeats_the_failed_destination(db: ManagementDat
rules: Final = UsageAlertsRepository.from_prisma(prisma)
assert (
await rules.save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Project spend",
scope_type="key",
scope_id=_KEY_A,
@ -905,7 +935,7 @@ async def test_slack_retry_only_repeats_the_failed_destination(db: ManagementDat
)
assert (
await rules.save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Another project rule",
scope_type="key",
scope_id=_KEY_A,
@ -938,8 +968,8 @@ async def test_slack_retry_only_repeats_the_failed_destination(db: ManagementDat
retry: Final = (await IssuesRepository.from_prisma(prisma).pending_deliveries())[0]
assert (
await rules.save_rule(
UsageAlertRuleInput.model_validate(
retry.rule.model_dump(include=set(UsageAlertRuleInput.model_fields))
AlertRuleInput.model_validate(
retry.rule.model_dump(include=set(AlertRuleInput.model_fields))
).model_copy(update={"enabled": False}),
"qa-admin",
retry.rule,
@ -1001,7 +1031,7 @@ async def test_cancelled_email_receipt_survives_a_different_delivery(
rules: Final = UsageAlertsRepository.from_prisma(prisma)
assert (
await rules.save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Project spend",
scope_type="key",
scope_id=_KEY_A,
@ -1056,15 +1086,13 @@ async def test_large_custom_rule_scope_does_not_starve_other_scopes_on_resume(db
await _seed(prisma)
store: Final = UsageAlertsRepository.from_prisma(prisma)
key_rule: Final = await store.save_rule(
UsageAlertRuleInput(
AlertRuleInput(
name="Project spend", scope_type="key", scope_id=_KEY_A, metric="spend", direction="increase"
),
"qa-user",
)
team_rule: Final = await store.save_rule(
UsageAlertRuleInput(
name="Team spend", scope_type="team", scope_id="team", metric="spend", direction="increase"
),
AlertRuleInput(name="Team spend", scope_type="team", scope_id="team", metric="spend", direction="increase"),
"qa-admin",
)
assert key_rule is not None and team_rule is not None

10
tests/integration/database/test_provider_latency.py
View file

@ -5,8 +5,8 @@ import pytest
from integration._support.actors import ManagementDatabase
from litellm.repositories.provider_latency import ProviderLatencyRepository
from litellm.types.proxy.issues import IssueModelPair
from litellm.types.proxy.usage_alerts import UsageAlertRuleInput
from litellm.types.proxy.issues import AlertModelPair
from litellm.types.proxy.usage_alerts import AlertRuleInput
@pytest.mark.asyncio(loop_scope="session")
@ -38,7 +38,7 @@ async def test_provider_lookup_skips_oversized_names_and_preserves_valid_models(
tuple(pair[0] for pair in pairs),
tuple(pair[1] for pair in pairs),
)
expected = tuple(IssueModelPair(provider=p, model=m) for p, m in pairs[2:5])
expected = tuple(AlertModelPair(provider=p, model=m) for p, m in pairs[2:5])
first = await repository.provider_pairs(provider, day)
second = await repository.provider_pairs(wide_provider, day)
invalid = await repository.provider_pairs("a" * 257, day)
@ -58,9 +58,9 @@ async def test_provider_lookup_skips_oversized_names_and_preserves_valid_models(
async def test_provider_latency_requires_established_within_model_regressions(db: ManagementDatabase) -> None:
prefix = f"provider-latency-{uuid4()}"
day = date(2031, 1, 12)
pairs = tuple(IssueModelPair(provider=prefix, model=f"{prefix}-{suffix}") for suffix in ("a", "b"))
pairs = tuple(AlertModelPair(provider=prefix, model=f"{prefix}-{suffix}") for suffix in ("a", "b"))
keys = (prefix + "-key-a", prefix + "-key-b")
rule = UsageAlertRuleInput(name="Latency", scope_type="key", metric="latency", direction="increase", min_value=500)
rule = AlertRuleInput(name="Latency", scope_type="key", metric="latency", direction="increase", min_value=500)
repository = ProviderLatencyRepository(db.prisma.writer_db)
try:
await db.prisma.writer_db.query_raw(

4
tests/integration/database/test_usage_alert_evaluation.py
View file

@ -20,13 +20,13 @@ from litellm.repositories.accounting_delivery_repository import (
)
from litellm.repositories.usage_alert_health import UsageAlertFlushRecorder, UsageAlertHealthRepository
from litellm.repositories.usage_alerts import UsageAlertsRepository
from litellm.types.proxy.usage_alerts import UsageAlertRuleInput
from litellm.types.proxy.usage_alerts import AlertRuleInput
@pytest.mark.asyncio(loop_scope="session")
async def test_batch_checkpoints_preserve_rule_revisions_and_incomplete_status(db: ManagementDatabase) -> None:
repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
definition: Final = UsageAlertRuleInput(
definition: Final = AlertRuleInput(
name="Daily evaluation", scope_type="team", scope_id="team-example", metric="spend", direction="increase"
)
rule: Final = await repository.save_rule(definition, "qa-admin")

277
tests/integration/database/test_usage_alerts.py
View file

@ -9,12 +9,11 @@ from integration._support.client import Gateway, string_value
from litellm.repositories.usage_alerts import UsageAlertsRepository
from litellm.types.proxy.usage_alerts import (
UsageAlertGroupList,
AlertModelList,
AlertRule,
AlertRuleInput,
UsageAlertMetric,
UsageAlertModelList,
UsageAlertObservation,
UsageAlertRule,
UsageAlertRuleInput,
)
@ -22,7 +21,7 @@ from litellm.types.proxy.usage_alerts import (
async def test_alert_resource_labels_bound_legacy_display_values(db: ManagementDatabase) -> None:
repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
identity: Final = "label-bound-" + uuid4().hex
scope: Final = UsageAlertRuleInput(
scope: Final = AlertRuleInput(
name="Display label", scope_type="user", scope_id=identity, metric="spend", direction="increase"
)
await repository.database.query_raw(
@ -43,7 +42,7 @@ async def test_alert_resource_labels_bound_legacy_display_values(db: ManagementD
async def test_deleted_key_labels_use_the_latest_alias_and_prefer_live_keys(db: ManagementDatabase) -> None:
repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
identity: Final = sha256(uuid4().bytes).hexdigest()
scope: Final = UsageAlertRuleInput(
scope: Final = AlertRuleInput(
name="Deleted key", scope_type="key", scope_id=identity, metric="spend", direction="increase"
)
try:
@ -63,7 +62,7 @@ async def test_deleted_key_labels_use_the_latest_alias_and_prefer_live_keys(db:
await repository.database.query_raw('DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', identity)
@pytest.mark.parametrize("grant", ["management_routes", "/usage", "/usage/alert-rules"])
@pytest.mark.parametrize("grant", ["management_routes", "/alerts", "/alerts/rules"])
def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gateway: Gateway, grant: str) -> None:
with gateway.scenario() as scenario:
owner: Final = scenario.user(user_role="internal_user")
@ -81,10 +80,10 @@ def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gate
"direction": "increase",
"multiplier": 3,
}
created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key)
created: Final = gateway.request("POST", "/alerts/rules", config, key=owner_key)
assert created.status_code == 201, created.text
rule_id: Final = string_value(created.json()["id"])
path: Final = f"/usage/alert-rules/{rule_id}"
path: Final = f"/alerts/rules/{rule_id}"
try:
assert gateway.request("GET", path, key=owner_key).status_code == 200
assert gateway.request("GET", path, key=outsider_key).status_code == 403
@ -93,14 +92,13 @@ def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gate
assert gateway.request("PUT", path, config, key=viewer_key).status_code == 403
assert gateway.request("PUT", path, config, key=outsider_key).status_code == 403
assert (
gateway.request("POST", "/usage/alert-rules", {**config, "scope_id": None}, key=owner_key).status_code
== 403
gateway.request("POST", "/alerts/rules", {**config, "scope_id": None}, key=owner_key).status_code == 403
)
disabled: Final = gateway.request("PUT", path, {**config, "enabled": False}, key=owner_key)
assert disabled.status_code == 200 and disabled.json()["enabled"] is False
page: Final = gateway.request(
"GET",
"/usage/alert-rules",
"/alerts/rules",
key=owner_key,
params={"scope_type": "key", "scope_id": str(config["scope_id"]), "limit": "1"},
)
@ -122,9 +120,9 @@ def test_admin_edit_preserves_owner_rule_access(gateway: Gateway) -> None:
"metric": "spend",
"direction": "increase",
}
created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key)
created: Final = gateway.request("POST", "/alerts/rules", config, key=owner_key)
assert created.status_code == 201, created.text
path: Final = f"/usage/alert-rules/{created.json()['id']}"
path: Final = f"/alerts/rules/{created.json()['id']}"
try:
edited: Final = gateway.request("PUT", path, {**config, "name": "Renamed by admin"})
assert edited.status_code == 200, edited.text
@ -151,10 +149,10 @@ def test_team_admin_can_manage_team_alerts_but_member_can_only_read(gateway: Gat
"direction": "decrease",
"multiplier": 2,
}
created: Final = gateway.request("POST", "/usage/alert-rules", config, key=admin_key)
created: Final = gateway.request("POST", "/alerts/rules", config, key=admin_key)
assert created.status_code == 201, created.text
rule_id: Final = string_value(created.json()["id"])
path: Final = f"/usage/alert-rules/{rule_id}"
path: Final = f"/alerts/rules/{rule_id}"
try:
assert gateway.request("GET", path, key=member_key).status_code == 200
assert gateway.request("PUT", path, config, key=member_key).status_code == 403
@ -177,21 +175,21 @@ def test_info_only_credentials_cannot_mutate_alerts(gateway: Gateway) -> None:
"metric": "spend",
"direction": "increase",
}
created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key)
created: Final = gateway.request("POST", "/alerts/rules", config, key=owner_key)
assert created.status_code == 201
path: Final = f"/usage/alert-rules/{created.json()['id']}"
path: Final = f"/alerts/rules/{created.json()['id']}"
try:
assert gateway.request("GET", path, key=reader).status_code == 200
assert gateway.request("POST", "/usage/alert-rules", config, key=reader).status_code == 403
assert gateway.request("POST", "/alerts/rules", config, key=reader).status_code == 403
assert gateway.request("PUT", path, config, key=reader).status_code == 403
assert gateway.request("DELETE", path, key=reader).status_code == 403
assert (
gateway.request("PATCH", "/issues/families/spend", {"enabled": False}, key=admin_reader).status_code
gateway.request("PATCH", "/alerts/defaults/spend", {"enabled": False}, key=admin_reader).status_code
== 403
)
assert (
gateway.request(
"PATCH", "/issues/unavailable", {"status": "resolved", "expected_status": "open"}, key=admin_reader
"PATCH", "/alerts/unavailable", {"status": "resolved", "expected_status": "open"}, key=admin_reader
).status_code
== 403
)
@ -215,23 +213,19 @@ def test_only_admins_can_configure_operator_notifications(gateway: Gateway) -> N
{"notification_channels": ["email"], "email_recipients": ["ops@example.com"]},
):
assert (
gateway.request("POST", "/usage/alert-rules", {**config, **notification}, key=owner_key).status_code
== 403
gateway.request("POST", "/alerts/rules", {**config, **notification}, key=owner_key).status_code == 403
)
assert (
gateway.request(
"POST", "/usage/alert-rules", {**config, "overrides_defaults": True}, key=owner_key
).status_code
gateway.request("POST", "/alerts/rules", {**config, "overrides_defaults": True}, key=owner_key).status_code
== 422
)
assert (
gateway.request("POST", "/usage/alert-rules", {**config, "name": "Spend\r\nInjected: header"}).status_code
== 422
gateway.request("POST", "/alerts/rules", {**config, "name": "Spend\r\nInjected: header"}).status_code == 422
)
created: Final = gateway.request("POST", "/usage/alert-rules", {**config, "notification_channels": ["slack"]})
created: Final = gateway.request("POST", "/alerts/rules", {**config, "notification_channels": ["slack"]})
assert created.status_code == 201
assert "overrides_defaults" not in created.json()
path: Final = f"/usage/alert-rules/{created.json()['id']}"
path: Final = f"/alerts/rules/{created.json()['id']}"
try:
assert gateway.request("PUT", path, config, key=owner_key).status_code == 403
assert gateway.request("DELETE", path, key=owner_key).status_code == 403
@ -252,7 +246,7 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew
viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only")
created: Final = gateway.request(
"POST",
"/usage/alert-rules",
"/alerts/rules",
{
"name": "Incident API fixture",
"scope_type": "user",
@ -264,7 +258,7 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew
key=owner_key,
)
assert created.status_code == 201
rule: Final = UsageAlertRule.model_validate(created.json())
rule: Final = AlertRule.model_validate(created.json())
try:
incident: Final = (
await repository.record_observations(
@ -282,9 +276,9 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew
),
)
)[0]
path: Final = f"/usage/alerts/{incident.id}"
path: Final = f"/alerts/signals/{incident.id}"
params: Final = {"scope_type": "user", "scope_id": owner, "limit": "1"}
page: Final = gateway.request("GET", "/usage/alerts", key=owner_key, params=params)
page: Final = gateway.request("GET", "/alerts/signals", key=owner_key, params=params)
assert page.status_code == 200 and page.json()["data"][0]["id"] == incident.id
detail: Final = gateway.request("GET", path, key=owner_key)
assert detail.status_code == 200
@ -293,22 +287,19 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew
assert detail.json()["threshold"] == 360
assert detail.json()["baseline_sample_days"] == 7
assert "detail_tab=activity" in detail.json()["activity_path"]
assert gateway.request("GET", "/usage/alerts", key=outsider_key, params=params).status_code == 403
assert gateway.request("GET", "/alerts/signals", key=outsider_key, params=params).status_code == 403
assert gateway.request("GET", path, key=outsider_key).status_code == 403
assert gateway.request("PATCH", path, {"status": "dismissed"}, key=outsider_key).status_code == 403
assert gateway.request("GET", path, key=viewer_key).status_code == 200
assert gateway.request("PATCH", path, {"status": "dismissed"}, key=viewer_key).status_code == 403
dismissed: Final = gateway.request("PATCH", path, {"status": "dismissed"}, key=owner_key)
assert dismissed.status_code == 200 and dismissed.json()["status"] == "dismissed"
assert gateway.request("GET", path, key=owner_key).json()["status"] == "dismissed"
assert gateway.request("PATCH", path, {"status": "dismissed"}, key=owner_key).status_code == 405
assert gateway.request("GET", path, key=owner_key).json()["status"] == "open"
finally:
assert gateway.request("DELETE", f"/usage/alert-rules/{rule.id}", key=owner_key).status_code == 204
assert gateway.request("DELETE", f"/alerts/rules/{rule.id}", key=owner_key).status_code == 204
@pytest.mark.asyncio(loop_scope="session")
async def test_ongoing_episode_preserves_its_original_evidence_and_activity_link(db: ManagementDatabase) -> None:
repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
definition: Final = UsageAlertRuleInput(
definition: Final = AlertRuleInput(
name="Original episode", scope_type="user", scope_id="episode-example", metric="requests", direction="increase"
)
rule: Final = await repository.save_rule(definition, "qa-admin")
@ -358,7 +349,7 @@ async def test_ongoing_episode_preserves_its_original_evidence_and_activity_link
@pytest.mark.asyncio(loop_scope="session")
async def test_rule_scope_changes_reject_stale_updates_and_deletes(db: ManagementDatabase) -> None:
repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
definition: Final = UsageAlertRuleInput(
definition: Final = AlertRuleInput(
name="Concurrent scope edit", scope_type="user", scope_id="original-owner", metric="spend", direction="increase"
)
original: Final = await repository.save_rule(definition, "qa-admin")
@ -394,7 +385,7 @@ async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency(
db: ManagementDatabase, changed_field: str, new_value: str
) -> None:
repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
definition: Final = UsageAlertRuleInput(
definition: Final = AlertRuleInput(
name="Semantic identity",
scope_type="team",
scope_id="semantic-example",
@ -410,7 +401,7 @@ async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency(
try:
original: Final = (await repository.record_observations(rule, (initial,)))[0]
edited: Final = await repository.save_rule(
UsageAlertRuleInput.model_validate({**definition.model_dump(), changed_field: new_value}), "qa-admin", rule
AlertRuleInput.model_validate({**definition.model_dump(), changed_field: new_value}), "qa-admin", rule
)
assert edited is not None
observation: Final = initial.model_copy(update={"entity_id": edited.scope_id})
@ -437,190 +428,12 @@ async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency(
await repository.delete_rule(rule.id)
@pytest.mark.asyncio(loop_scope="session")
async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_actions(
gateway: Gateway, db: ManagementDatabase
) -> None:
repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
day: Final = date(2026, 11, 4)
with gateway.scenario() as scenario:
owner: Final = scenario.user(user_role="internal_user", user_alias="Checkout operator")
outsider: Final = scenario.user(user_role="internal_user")
viewer: Final = scenario.user(user_role="proxy_admin_viewer")
owner_key: Final = scenario.key(user_id=owner, key_type="management")
outsider_key: Final = scenario.key(user_id=outsider, key_type="management")
viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only")
inference_key: Final = scenario.key(user_id=owner, key_type="llm_api")
metrics: Final[tuple[UsageAlertMetric, ...]] = (
*(("requests",) * 27),
"spend",
"errors",
"latency",
"error_rate",
)
rules: Final = tuple(
[
await repository.save_rule(
UsageAlertRuleInput(
name=f"Group rule {index}",
scope_type="user",
scope_id=owner,
metric=metric,
direction="increase",
baseline_days=7 if index % 2 else 14,
),
owner,
)
for index, metric in enumerate(metrics)
]
)
assert all(rule is not None for rule in rules)
incidents: Final = tuple(
[
(
await repository.record_observations(
rule,
(
UsageAlertObservation(
entity_id=owner,
day=day,
observed=14 if rule.metric == "error_rate" else 420,
baseline=4 if rule.metric == "error_rate" else 0 if index == 0 else 120,
threshold=12 if rule.metric == "error_rate" else 100 if index == 0 else 360,
breached=True,
),
),
)
)[0]
for index, rule in enumerate(rules)
if rule is not None
]
)
old_rule: Final = await repository.save_rule(
UsageAlertRuleInput(
name="Earlier day", scope_type="user", scope_id=owner, metric="requests", direction="increase"
),
owner,
)
assert old_rule is not None
old: Final = (
await repository.record_observations(
old_rule,
(
UsageAlertObservation(
entity_id=owner,
day=day - timedelta(days=1),
observed=420,
baseline=120,
threshold=360,
breached=True,
),
),
)
)[0]
other_rule: Final = await repository.save_rule(
UsageAlertRuleInput(
name="Other resource", scope_type="user", scope_id=outsider, metric="requests", direction="increase"
),
outsider,
)
assert other_rule is not None
other: Final = (
await repository.record_observations(
other_rule,
(
UsageAlertObservation(
entity_id=outsider,
day=day,
observed=420,
baseline=120,
threshold=360,
breached=True,
),
),
)
)[0]
params: Final = {"scope_type": "user", "scope_id": owner, "status": "open", "limit": "1"}
try:
response: Final = gateway.request("GET", "/usage/alert-groups", key=owner_key, params=params)
assert response.status_code == 200, response.text
page: Final = UsageAlertGroupList.model_validate(response.json())
assert page.total == 2 and len(page.data) == 1
group: Final = page.data[0]
assert group.day == day and group.scope_label == "Checkout operator"
assert group.incident_count == 31 and len(group.metrics) == 5
assert {metric.metric: metric.incident_count for metric in group.metrics} == {
"requests": 27,
"spend": 1,
"errors": 1,
"latency": 1,
"error_rate": 1,
}
assert all(metric.representative.id in {incident.id for incident in incidents} for metric in group.metrics)
assert group.activity_path in {incident.activity_path for incident in incidents}
second: Final = UsageAlertGroupList.model_validate(
gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "1"}).json()
)
assert second.total == 2 and second.data[0].day == old.day
empty: Final = UsageAlertGroupList.model_validate(
gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "2"}).json()
)
assert empty.total == 2 and empty.data == ()
history: Final = gateway.request(
"GET",
"/usage/alerts",
key=owner_key,
params={**params, "start_date": day.isoformat(), "end_date": day.isoformat(), "metric": "requests"},
)
assert history.status_code == 200 and history.json()["total"] == 27
await repository.set_status(incidents[0].id, "dismissed")
body: Final = {
"scope_type": "user",
"scope_id": owner,
"day": day.isoformat(),
"expected_status": "open",
"status": "resolved",
}
for forbidden in (outsider_key, inference_key):
assert gateway.request("GET", "/usage/alert-groups", key=forbidden, params=params).status_code == 403
for forbidden in (outsider_key, inference_key, viewer_key):
assert gateway.request("PATCH", "/usage/alert-groups", body, key=forbidden).status_code == 403
assert gateway.request("GET", "/usage/alert-groups", key=viewer_key, params=params).status_code == 200
updated: Final = gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key)
assert updated.status_code == 200 and updated.json()["updated_count"] == 30
assert gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key).json()["updated_count"] == 0
assert await repository.get_alert(old.id) == old
assert await repository.get_alert(other.id) == other
dismissed: Final = await repository.get_alert(incidents[0].id)
assert dismissed is not None and dismissed.status == "dismissed" and dismissed.baseline_value == 0
resolved: Final = await repository.get_alert(incidents[-1].id)
assert resolved == incidents[-1].model_copy(update={"status": "resolved"})
assert (
gateway.request(
"GET", "/usage/alert-groups", params={"scope_type": "user", "scope_id": "x' OR 1=1 --"}
).json()["total"]
== 0
)
assert (
gateway.request(
"GET", "/usage/alert-groups", params={"start_date": "2026-11-05", "end_date": "2026-11-04"}
).status_code
== 422
)
finally:
for rule in (*rules, old_rule, other_rule):
if rule is not None:
await repository.delete_rule(rule.id)
deleted: Final = await repository.list_groups(scope_type="user", scope_id=owner)
assert deleted.total == 2 and all(group.scope_label is None and group.scope_id == owner for group in deleted.data)
@pytest.mark.asyncio(loop_scope="session")
async def test_default_rule_summary_and_kind_filters_do_not_require_loading_every_rule(
gateway: Gateway, db: ManagementDatabase
) -> None:
repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
default: Final = UsageAlertRuleInput(
default: Final = AlertRuleInput(
name="Built-in daily model requests", scope_type="model", metric="requests", direction="increase"
)
await repository.ensure_default_rules((default,))
@ -645,12 +458,12 @@ async def test_default_rule_summary_and_kind_filters_do_not_require_loading_ever
stored: Final = await repository.get_rule(builtins.data[0].id)
assert stored is not None and stored.is_default
filtered: Final = gateway.request(
"GET", "/usage/alert-rules", params={"scope_type": "model", "kind": "custom", "limit": "1"}
"GET", "/alerts/rules", params={"scope_type": "model", "kind": "custom", "limit": "1"}
)
assert filtered.status_code == 200 and filtered.json()["total"] == customs.total
assert all(not rule["is_default"] for rule in filtered.json()["data"])
configuration: Final = stored.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields))
path: Final = f"/usage/alert-rules/{stored.id}"
configuration: Final = stored.model_dump(mode="json", include=frozenset(AlertRuleInput.model_fields))
path: Final = f"/alerts/rules/{stored.id}"
for change in (
{"scope_type": "key"},
{"scope_id": "model-example"},
@ -665,9 +478,9 @@ async def test_default_rule_summary_and_kind_filters_do_not_require_loading_ever
assert updated.json()["min_value"] == stored.min_value + 1
finally:
assert gateway.request("PUT", path, configuration).status_code == 200
custom_input: Final = custom.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields))
custom_input: Final = custom.model_dump(mode="json", include=frozenset(AlertRuleInput.model_fields))
assert (
gateway.request("PUT", f"/usage/alert-rules/{custom.id}", {**custom_input, "metric": "latency"}).status_code
gateway.request("PUT", f"/alerts/rules/{custom.id}", {**custom_input, "metric": "latency"}).status_code
== 200
)
finally:
@ -704,9 +517,9 @@ async def test_reported_model_search_preserves_exact_identity_and_cursor(
prefix,
f"{prefix}/old",
)
response: Final = gateway.request("GET", "/usage/alert-models", params={"search": prefix, "limit": "1"})
response: Final = gateway.request("GET", "/alerts/models", params={"search": prefix, "limit": "1"})
assert response.status_code == 200, response.text
first: Final = UsageAlertModelList.model_validate(response.json())
first: Final = AlertModelList.model_validate(response.json())
assert first.data[0].scope_id == models[0] and first.next_cursor == models[0]
second: Final = await repository.list_reported_models(prefix, first.next_cursor, 1)
third: Final = await repository.list_reported_models(prefix, second.next_cursor, 1)
@ -720,6 +533,6 @@ async def test_reported_model_search_preserves_exact_identity_and_cursor(
with gateway.scenario() as scenario:
user: Final = scenario.user(user_role="internal_user")
key: Final = scenario.key(user_id=user, key_type="management")
assert gateway.request("GET", "/usage/alert-models", key=key).status_code == 403
assert gateway.request("GET", "/alerts/models", key=key).status_code == 403
finally:
await db.prisma.writer_db.execute_raw('DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', prefix)

16
tests/integration/observability/test_usage_alert_email.py
View file

@ -9,11 +9,11 @@ import pytest
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery
from litellm.types.proxy.issues import Issue, IssueCohort, IssueDelivery, IssuePendingDelivery
from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule
from litellm.types.proxy.issues import Alert, AlertCohort, AlertDelivery, AlertPendingDelivery
from litellm.types.proxy.usage_alerts import PerformanceSignal, AlertRule
_NOW: Final = datetime(2026, 10, 20, tzinfo=timezone.utc)
_RULE: Final = UsageAlertRule(
_RULE: Final = AlertRule(
name="Request spike",
id="rule",
scope_type="key",
@ -25,7 +25,7 @@ _RULE: Final = UsageAlertRule(
created_by=None,
notification_channels=("slack",),
)
_ALERT: Final = UsageAlert(
_ALERT: Final = PerformanceSignal(
id="incident",
rule_id="rule",
rule_name="Request spike",
@ -47,13 +47,13 @@ _ALERT: Final = UsageAlert(
)
_ISSUE: Final = Issue(
_ISSUE: Final = Alert(
id="issue-example",
cohort_key="cohort-example",
kind="performance",
family="traffic",
title="Traffic increased",
cohort=IssueCohort(scope_type="key", scope_id="reporting-key"),
cohort=AlertCohort(scope_type="key", scope_id="reporting-key"),
observed_on=_ALERT.day,
observed_at=_NOW,
range_start=_ALERT.baseline_start,
@ -64,11 +64,11 @@ _ISSUE: Final = Issue(
last_seen=_NOW,
updated_at=_NOW,
)
_PENDING: Final = IssuePendingDelivery(
_PENDING: Final = AlertPendingDelivery(
issue=_ISSUE,
rule=_RULE,
alert=_ALERT,
delivery=IssueDelivery(id="delivery-example", issue_id=_ISSUE.id, channel="slack"),
delivery=AlertDelivery(id="delivery-example", issue_id=_ISSUE.id, channel="slack"),
)

18
tests/unit/integrations/SlackAlerting/test_usage_alerts.py
View file

@ -7,11 +7,11 @@ import pytest
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, activity_url, incident_message
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.proxy.issues import Issue, IssueCohort, IssueDelivery, IssuePendingDelivery
from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule
from litellm.types.proxy.issues import Alert, AlertCohort, AlertDelivery, AlertPendingDelivery
from litellm.types.proxy.usage_alerts import PerformanceSignal, AlertRule
_NOW: Final = datetime(2026, 10, 20, tzinfo=timezone.utc)
_RULE: Final = UsageAlertRule(
_RULE: Final = AlertRule(
name="Request spike",
id="rule",
scope_type="key",
@ -23,7 +23,7 @@ _RULE: Final = UsageAlertRule(
created_by=None,
notification_channels=("slack",),
)
_ALERT: Final = UsageAlert(
_ALERT: Final = PerformanceSignal(
id="incident",
rule_id="rule",
rule_name="Request spike",
@ -45,13 +45,13 @@ _ALERT: Final = UsageAlert(
)
_ISSUE: Final = Issue(
_ISSUE: Final = Alert(
id="issue-example",
cohort_key="cohort-example",
kind="performance",
family="traffic",
title="Traffic increased",
cohort=IssueCohort(scope_type="key", scope_id="reporting-key"),
cohort=AlertCohort(scope_type="key", scope_id="reporting-key"),
observed_on=_ALERT.day,
observed_at=_NOW,
range_start=_ALERT.baseline_start,
@ -62,11 +62,11 @@ _ISSUE: Final = Issue(
last_seen=_NOW,
updated_at=_NOW,
)
_PENDING: Final = IssuePendingDelivery(
_PENDING: Final = AlertPendingDelivery(
issue=_ISSUE,
rule=_RULE,
alert=_ALERT,
delivery=IssueDelivery(id="delivery-example", issue_id=_ISSUE.id, channel="slack"),
delivery=AlertDelivery(id="delivery-example", issue_id=_ISSUE.id, channel="slack"),
)
@ -131,7 +131,7 @@ async def test_issue_delivery_links_to_investigation_and_preserves_the_bound_sou
try:
assert await UsageAlertDelivery(slack).send_issue(_PENDING)
assert len(requests) == 1
assert b"https://gateway.example.com/ui/alerts?issue=issue-example" in requests[0].content
assert b"https://gateway.example.com/ui/alerts?alert=issue-example" in requests[0].content
assert (
b"https://gateway.example.com/ui/api-keys?key=reporting-key&amp;detail_tab=activity" in requests[0].content
)

16
tests/unit/proxy/alerting_endpoints/test_usage_alerts.py
View file

@ -21,20 +21,20 @@ class NoDatabase:
def test_info_grants_do_not_authorize_mutations(role: LitellmUserRoles, key_type: str | None) -> None:
user: Final = UserAPIKeyAuth(user_role=role, key_type=key_type, allowed_routes=["info_routes", "openai_routes"])
with pytest.raises(HTTPException) as error:
require_write_access(user, "/usage/alert-rules/example")
require_write_access(user, "/alerts/rules/example")
assert error.value.status_code == 403
@pytest.mark.parametrize("grant", ["management_routes", "/usage", "/usage/alert-rules", "/usage/*"])
@pytest.mark.parametrize("grant", ["management_routes", "/alerts", "/alerts/rules", "/alerts/*"])
def test_explicit_route_grants_authorize_alert_mutations(grant: str) -> None:
assert _allows_write(grant, "/usage/alert-rules/example") is True
assert _allows_write(grant, "/alerts/rules/example") is True
@pytest.mark.parametrize("grant", ["/usag", "/usage/alert-rule", "/usage/alerts"])
@pytest.mark.parametrize("grant", ["/alert", "/alerts/rule", "/alerts/signals"])
def test_route_grants_require_a_complete_path_segment(grant: str) -> None:
user: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, allowed_routes=[grant])
with pytest.raises(HTTPException) as error:
require_write_access(user, "/usage/alert-rules/example")
require_write_access(user, "/alerts/rules/example")
assert error.value.status_code == 403
@ -91,10 +91,10 @@ def test_global_activity_link_uses_the_usage_page_date_contract() -> None:
@pytest.mark.asyncio
async def test_no_database_returns_an_actionable_service_error() -> None:
from litellm.proxy.alerting_endpoints.usage_alerts import list_usage_alerts
from litellm.proxy.alerting_endpoints.usage_alerts import list_performance_signals
user: Final = UserAPIKeyAuth(user_id="qa-admin", user_role=LitellmUserRoles.PROXY_ADMIN)
with pytest.raises(HTTPException) as error:
await list_usage_alerts(user, None)
await list_performance_signals(user, None)
assert error.value.status_code == 503
assert error.value.detail == "Usage alerts require a database connection"
assert error.value.detail == "Alert signals require a database connection"

56
tests/unit/proxy/db/test_db_spend_update_writer.py
View file

@ -3294,8 +3294,9 @@ class _WindowSpendFakeDB:
class _WindowSpendFakePrisma:
def __init__(self, db):
def __init__(self, db, writer_db=None):
self.db = db
self.writer_db = writer_db if writer_db is not None else db
def _window_spend_upserts(db):
@ -5515,6 +5516,59 @@ async def test_cancel_that_lands_while_the_daily_batch_is_committing_waits_for_t
assert _daily_upserts(final_db, table) == [], "the final flush must not write the committed batch again"
class _FreshnessRecordingDB:
def __init__(self) -> None:
self.started = asyncio.Event()
self.release = asyncio.Event()
self.recorded: tuple[object, ...] = ()
@asynccontextmanager
async def tx(self, *, max_wait: timedelta, timeout: timedelta):
yield self
async def query_raw(self, query: str, *args: object):
if "pg_try_advisory_xact_lock" in query:
return [{"acquired": True}]
self.started.set()
await self.release.wait()
self.recorded = args
return [{"source": args[0]}]
@pytest.mark.parametrize("entity_type", ("user", "team"))
@pytest.mark.parametrize("cancel_during", ("commit", "freshness"))
@pytest.mark.asyncio
async def test_cancelled_daily_flush_records_freshness_after_commit_without_requeuing(entity_type, cancel_during):
writer = DBSpendUpdateWriter()
queue = writer.daily_spend_update_queue if entity_type == "user" else writer.daily_team_spend_update_queue
await queue.add_update({"key-a": _daily_entity_txn(entity_type + "_id")})
db = _CommittingDailySpendFakeDB()
freshness = _FreshnessRecordingDB()
tick = asyncio.ensure_future(
writer._flush_daily_spend_queue(
queue=queue,
entity_type=entity_type,
commit=_DAILY_SPEND_COMMITS[entity_type],
n_retry_times=0,
prisma_client=_WindowSpendFakePrisma(db, freshness),
proxy_logging_obj=MagicMock(),
)
)
await asyncio.wait_for(db.committing.wait(), timeout=5)
if cancel_during == "freshness":
db.commit_release.set()
await asyncio.wait_for(freshness.started.wait(), timeout=5)
tick.cancel()
with pytest.raises(asyncio.CancelledError):
await tick
db.commit_release.set()
freshness.release.set()
await queue.settle_interrupted_commits()
assert freshness.recorded and freshness.recorded[0] == entity_type
assert len(_daily_upserts(db, "LiteLLM_Daily" + entity_type.title() + "Spend")) == 1
assert queue.update_queue.empty()
@pytest.mark.asyncio
async def test_tag_batch_drained_from_redis_and_cancelled_while_committing_is_not_restored():
"""Same in-flight COMMIT as the in-memory path, but the drained rows live in Redis. Restoring

38
tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py
View file

@ -19,8 +19,8 @@ from litellm.repositories.usage_alert_health import UsageAlertHealthRepository
from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, UsageAlertMetricsRepository
from litellm.repositories.usage_alerts import UsageAlertsRepository
from litellm.types.proxy.usage_alert_defaults import default_usage_rule_id, default_usage_rules
from litellm.types.proxy.issues import IssuePendingDelivery
from litellm.types.proxy.usage_alerts import UsageAlertRule, UsageAlertRuleInput
from litellm.types.proxy.issues import AlertPendingDelivery
from litellm.types.proxy.usage_alerts import AlertRule, AlertRuleInput
_DAY: Final = date(2026, 10, 20)
@ -93,7 +93,7 @@ class _BudgetDatabase:
class _UnexpectedDelivery:
async def send_issue(self, pending: IssuePendingDelivery) -> bool:
async def send_issue(self, pending: AlertPendingDelivery) -> bool:
raise AssertionError("stalled evaluation must not deliver alerts")
@ -149,7 +149,7 @@ def _row(day: date, requests: int = 200, spend: float = 20, ms: float = 100_000,
"metric,observed,baseline", [("spend", 100, 20), ("requests", 1000, 200), ("latency", 2000, 500)]
)
def test_spikes_use_recorded_reference_days_and_weighted_latency(metric: str, observed: float, baseline: float) -> None:
rule: Final = UsageAlertRuleInput.model_validate(
rule: Final = AlertRuleInput.model_validate(
{"name": "spike", "scope_type": "key", "metric": metric, "direction": "increase"}
)
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 3, 7))
@ -161,13 +161,13 @@ def test_spikes_use_recorded_reference_days_and_weighted_latency(metric: str, ob
def test_missing_target_is_zero_for_counts_but_never_zero_latency() -> None:
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
rule: Final = UsageAlertRuleInput(name="drop", scope_type="key", metric="requests", direction="decrease")
rule: Final = AlertRuleInput(name="drop", scope_type="key", metric="requests", direction="decrease")
assert compare_daily_usage(rule, history, _DAY) == DailyComparison(3, 0, 200, 200 / 3, True)
assert compare_daily_usage(rule.model_copy(update={"metric": "latency"}), history, _DAY) is None
def test_low_volume_and_recorded_zero_spend_do_not_establish_a_multiplier_baseline() -> None:
rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase")
rule: Final = AlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase")
history: Final = tuple(_row(_DAY - timedelta(days=days), requests=10, spend=0) for days in (1, 2, 3))
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) is None
enough_requests: Final = tuple(replace(row, requests=200) for row in history)
@ -178,7 +178,7 @@ def test_low_volume_and_recorded_zero_spend_do_not_establish_a_multiplier_baseli
def test_absolute_floor_suppresses_small_changes() -> None:
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
rule: Final = UsageAlertRuleInput(
rule: Final = AlertRuleInput(
name="spike", scope_type="key", metric="spend", direction="increase", min_value=100
)
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 120, False)
@ -191,7 +191,7 @@ def test_day_is_eligible_only_after_one_hour_utc_grace() -> None:
def test_latency_weights_request_samples_instead_of_averaging_daily_averages() -> None:
rule: Final = UsageAlertRuleInput(
rule: Final = AlertRuleInput(
name="latency", scope_type="key", metric="latency", direction="increase", min_history_days=2
)
rows: Final = (
@ -203,7 +203,7 @@ def test_latency_weights_request_samples_instead_of_averaging_daily_averages() -
def test_errors_are_failed_request_count_and_require_absolute_change_floor() -> None:
rule: Final = UsageAlertRuleInput(
rule: Final = AlertRuleInput(
name="errors", scope_type="key", metric="errors", direction="increase", min_value=10
)
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
@ -213,7 +213,7 @@ def test_errors_are_failed_request_count_and_require_absolute_change_floor() ->
def test_a_value_at_the_configured_multiplier_alerts() -> None:
history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3))
rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase")
rule: Final = AlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase")
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, True)
@ -234,9 +234,9 @@ def test_default_notifications_require_opt_in_and_do_not_duplicate_legacy_user_s
)
def test_sentinel_entities_cannot_create_alerts_with_unusable_activity_links(scope: str, entity: str) -> None:
from litellm.proxy.spend_tracking.usage_alert_job import _observations
from litellm.types.proxy.usage_alerts import UsageAlertRule
from litellm.types.proxy.usage_alerts import AlertRule
rule: Final = UsageAlertRule.model_validate(
rule: Final = AlertRule.model_validate(
{
"id": "rule",
"name": "spike",
@ -292,9 +292,9 @@ def test_healthy_evaluation_completes_and_recovers_only_with_sufficient_evidence
active: bool, spend: float, history_days: int, evaluated: bool, persisted: bool
) -> None:
from litellm.proxy.spend_tracking.usage_alert_job import _observations
from litellm.types.proxy.usage_alerts import UsageAlertRule
from litellm.types.proxy.usage_alerts import AlertRule
rule: Final = UsageAlertRule(
rule: Final = AlertRule(
id="rule",
name="spike",
scope_type="key",
@ -347,7 +347,7 @@ def test_error_rate_default_requires_two_percentage_points_as_well_as_the_multip
def test_no_requests_cannot_report_zero_error_rate_or_resolve_a_reliability_issue() -> None:
rule: Final = UsageAlertRuleInput(name="rate", scope_type="key", metric="error_rate", direction="decrease")
rule: Final = AlertRuleInput(name="rate", scope_type="key", metric="error_rate", direction="decrease")
history: Final = tuple(_row(_DAY - timedelta(days=day), requests=500) for day in (1, 2, 3))
assert compare_daily_usage(rule, history, _DAY) is None
@ -382,7 +382,7 @@ def test_latency_default_requires_a_material_change_and_enough_timed_requests()
@pytest.mark.parametrize("metric", ["spend", "requests", "latency", "errors"])
def test_multiplier_alerts_need_a_positive_established_baseline(metric: str) -> None:
rule: Final = UsageAlertRuleInput.model_validate(
rule: Final = AlertRuleInput.model_validate(
{"name": "baseline", "scope_type": "key", "metric": metric, "direction": "increase", "min_requests": 0}
)
current: Final = replace(_row(_DAY, requests=2000, spend=500), errors=100)
@ -413,7 +413,7 @@ def test_default_traffic_drop_does_not_treat_a_normal_quiet_weekend_as_an_outage
for rule in default_usage_rules()
if (rule.scope_type, rule.metric, rule.direction) == ("key", "requests", "decrease")
)
rule: Final = UsageAlertRule.model_validate(
rule: Final = AlertRule.model_validate(
{
**definition.model_dump(),
"id": default_usage_rule_id("key", "requests", "decrease"),
@ -432,7 +432,7 @@ def test_default_traffic_drop_does_not_treat_a_normal_quiet_weekend_as_an_outage
def test_error_rate_drop_requires_enough_current_requests() -> None:
rule: Final = UsageAlertRuleInput(
rule: Final = AlertRuleInput(
name="Reliability",
scope_type="key",
metric="error_rate",
@ -448,7 +448,7 @@ def test_error_rate_drop_requires_enough_current_requests() -> None:
def test_suppressed_default_can_recover_existing_episode_without_creating_a_new_one() -> None:
from litellm.proxy.spend_tracking.usage_alert_job import _observations
rule: Final = UsageAlertRule(
rule: Final = AlertRule(
id="rule",
name="Default",
scope_type="key",

6
tests/unit/repositories/test_issue_families.py
View file

@ -8,14 +8,14 @@ from litellm.repositories.issue_families import family_settings
from litellm.types.proxy.usage_alerts import (
UsageAlertEvaluationStatus,
UsageAlertMetric,
UsageAlertRule,
AlertRule,
UsageAlertScope,
)
@dataclass
class RuleRows:
rules: tuple[UsageAlertRule, ...]
rules: tuple[AlertRule, ...]
calls: int = 0
async def query_raw(self, query: str, *args: object) -> object:
@ -36,7 +36,7 @@ async def test_family_counts_only_its_enabled_rules_without_an_extra_query(statu
("model", "latency", True),
)
rules: Final = tuple(
UsageAlertRule(
AlertRule(
id=f"test-{scope}-{metric}",
name="Daily monitoring",
scope_type=scope,

24
tests/unit/types/proxy/test_usage_alerts.py
View file

@ -3,7 +3,7 @@ from typing import Final
import pytest
from pydantic import ValidationError
from litellm.types.proxy.usage_alerts import UsageAlertGroupUpdate, UsageAlertRuleInput
from litellm.types.proxy.usage_alerts import AlertRuleInput
RULE: Final = {
"name": "Daily spend",
@ -34,31 +34,13 @@ RULE: Final = {
)
def test_invalid_rules_cannot_be_stored(changes: dict[str, object]) -> None:
with pytest.raises(ValidationError):
UsageAlertRuleInput.model_validate({**RULE, **changes})
AlertRuleInput.model_validate({**RULE, **changes})
def test_zero_request_floor_and_daily_drop_multiplier_are_supported() -> None:
rule: Final = UsageAlertRuleInput.model_validate(
rule: Final = AlertRuleInput.model_validate(
{**RULE, "metric": "requests", "direction": "decrease", "multiplier": 2, "min_requests": 0}
)
assert rule.direction == "decrease"
assert rule.multiplier == 2
assert rule.min_requests == 0
@pytest.mark.parametrize(
"changes",
[{"scope_id": None}, {"scope_type": "deployment"}, {"status": "open"}, {"status": "unknown"}],
)
def test_group_updates_require_exact_resource_and_a_different_valid_status(changes: dict[str, object]) -> None:
with pytest.raises(ValidationError):
UsageAlertGroupUpdate.model_validate(
{
"scope_type": "team",
"scope_id": "team-example",
"day": "2026-11-04",
"expected_status": "open",
"status": "resolved",
**changes,
}
)

4
ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx
View file

@ -11,10 +11,10 @@ import { issueRouteParsers } from "@/components/Issues/issueRoute";
export default function AlertsPage() {
const { t } = useTranslation();
const [params] = useQueryStates(issueRouteParsers);
if (params.issue)
if (params.alert)
return (
<Page className="mx-auto max-w-[1440px] gap-4">
<IssueDetail id={params.issue} />
<IssueDetail id={params.alert} />
</Page>
);
if (params.model)

2
ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx
View file

@ -607,7 +607,7 @@ describe("UsagePage", () => {
});
it.each([
{ context: "deployment alert", alert: "&alert=example-alert", timezoneOffsetMinutes: 0 },
{ context: "deployment alert", alert: "&signal=example-alert", timezoneOffsetMinutes: 0 },
{ context: "ordinary usage", alert: "", timezoneOffsetMinutes: undefined },
])(
"uses the same daily timezone for totals and tag activity in $context",

8
ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx
View file

@ -124,7 +124,7 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations, teamsLoadin
const defaultRange = useMemo(() => defaultUsageDateRange(defaultTimeRange), [defaultTimeRange]);
const [urlParams, setUrlParams] = useQueryStates(usageUrlParsers);
const [alertId] = useQueryState("alert", entityActivityParsers.alert);
const [signalId] = useQueryState("signal", entityActivityParsers.signal);
const { range, from, to } = urlParams;
const dateValue = useMemo<DateRangePickerValue>(
() => dateRangeFromParams({ range, from, to }) ?? defaultRange.value,
@ -172,7 +172,7 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations, teamsLoadin
// For non-admins or "my-usage" view, always pass their own user_id
const effectiveUserId = usageView === "my-usage" || !isAdmin ? userID || null : urlParams.user;
const isDeploymentUsage = isAdmin && usageView === "global" && effectiveUserId === null;
const showUsageAlert = Boolean(alertId && isDeploymentUsage);
const showUsageAlert = Boolean(signalId && isDeploymentUsage);
const clientScopeInput = {
userId: userID,
@ -462,9 +462,9 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations, teamsLoadin
return (
<div ref={pageRef} className="relative w-full px-4 pt-3 pb-8 sm:px-6">
{showUsageAlert && alertId && (
{showUsageAlert && signalId && (
<div className="mb-3">
<UsageAlertContext alertId={alertId} scope="deployment" entityId={null} />
<UsageAlertContext signalId={signalId} scope="deployment" entityId={null} />
</div>
)}
<header className="mb-3 flex w-full flex-wrap items-center justify-between gap-3 border-b pb-3">

8
ui/litellm-dashboard/src/components/Issues/IssueActivity.tsx
View file

@ -5,9 +5,9 @@ import { getProxyBaseUrl } from "@/components/networking";
import { Button } from "@/components/ui/button";
import { boundedActivityMetrics } from "@/components/UsagePage/keyActivityData";
import { backgroundFetchClient } from "@/lib/http/api";
import type { Issue } from "./issueApi";
import type { Alert } from "./issueApi";
export default function IssueActivity({ issue, from, to }: { issue: Issue; from: string; to: string }) {
export default function IssueActivity({ issue, from, to }: { issue: Alert; from: string; to: string }) {
const { userId, userRole, isAuthorized, accessToken } = useAuthorized();
const authorized = Boolean(isAuthorized && accessToken && issue.can_view_activity === true);
const enabled = authorized && from <= to;
@ -15,8 +15,8 @@ export default function IssueActivity({ issue, from, to }: { issue: Issue; from:
queryKey: ["issueActivity", getProxyBaseUrl(), userId, userRole, issue.id, from, to],
enabled,
queryFn: async ({ signal }: { signal: AbortSignal }) => {
const { data } = await backgroundFetchClient.GET("/issues/{issue_id}/activity", {
params: { path: { issue_id: issue.id }, query: { start_date: from, end_date: to } },
const { data } = await backgroundFetchClient.GET("/alerts/{alert_id}/activity", {
params: { path: { alert_id: issue.id }, query: { start_date: from, end_date: to } },
signal,
});
if (!data) throw new Error("Could not load activity");

4
ui/litellm-dashboard/src/components/Issues/IssueComparisonTable.tsx
View file

@ -1,8 +1,8 @@
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import { METRIC_LABELS, formatAlertValue } from "@/components/UsageAlerts/usageAlertApi";
import type { Issue } from "./issueApi";
import type { Alert } from "./issueApi";
export default function IssueComparisonTable({ issue }: { issue: Issue }) {
export default function IssueComparisonTable({ issue }: { issue: Alert }) {
const comparisons = issue.comparisons ?? [];
if (!comparisons.length) return null;
return (

8
ui/litellm-dashboard/src/components/Issues/IssueDetail.tsx
View file

@ -15,7 +15,7 @@ import { uiHref } from "@/utils/uiHref";
import { ApiError } from "@/lib/http/client";
import { useUrlTableState } from "@/components/shared/DataTable";
import {
Issue,
Alert,
issueDate,
issueComparisonText,
issueImpactText,
@ -31,7 +31,7 @@ import IssueSignals from "./IssueSignals";
import IssueStatusActions from "./IssueStatusActions";
import IssueHistoryChart from "./IssueHistoryChart";
function resourceHref(issue: Issue): string | undefined {
function resourceHref(issue: Alert): string | undefined {
const { scope_type: scope, scope_id: id } = issue.cohort;
if (!id || scope === "provider" || scope === "deployment") return undefined;
return entityActivityHref(scope, scope === "key" ? storedKeyId(id) : id, {
@ -40,7 +40,7 @@ function resourceHref(issue: Issue): string | undefined {
});
}
function RetirementContext({ issue }: { issue: Issue }) {
function RetirementContext({ issue }: { issue: Alert }) {
const deprecation = issue.deprecation;
if (!deprecation) return null;
const copy = () =>
@ -67,7 +67,7 @@ function RetirementContext({ issue }: { issue: Issue }) {
);
}
function IssueBody({ issue }: { issue: Issue }) {
function IssueBody({ issue }: { issue: Alert }) {
const [params, setParams] = useQueryStates(entityActivityParsers);
const resourcesTable = useUrlTableState(ISSUE_RESOURCE_TABLE_STATE);
const [resourcesOpen, setResourcesOpen] = useState(false);

12
ui/litellm-dashboard/src/components/Issues/IssueFamilySettings.tsx
View file

@ -9,10 +9,10 @@ import { backgroundFetchClient } from "@/lib/http/api";
import type { components } from "@/lib/http/schema";
import { FAMILY_LABELS } from "./issueApi";
type Family = components["schemas"]["IssueFamilySetting"];
type Policy = components["schemas"]["IssueFamilyPolicy"];
type PolicyUpdate = components["schemas"]["IssueFamilyPolicyUpdate"];
type FamilyUpdate = components["schemas"]["IssueFamilyUpdate"];
type Family = components["schemas"]["AlertFamilySetting"];
type Policy = components["schemas"]["AlertFamilyPolicy"];
type PolicyUpdate = components["schemas"]["AlertFamilyPolicyUpdate"];
type FamilyUpdate = components["schemas"]["AlertFamilyUpdate"];
const SELECT_CLASS = "h-9 rounded-md border border-input bg-background px-2 text-sm";
const FLOOR_UNITS = { spend: "USD", latency: "ms", reliability: "percentage points", traffic: "requests" };
const NUMBER_FIELDS = ["multiplier", "min_value", "min_requests", "baseline_days", "min_history_days"] as const;
@ -162,7 +162,7 @@ export default function IssueFamilySettings({ accessToken }: { accessToken: stri
queryKey: ["issueFamilies", getProxyBaseUrl(), user?.user_id, user?.user_role],
enabled: Boolean(accessToken && user?.user_id),
queryFn: async ({ signal }: { signal: AbortSignal }) => {
const { data } = await backgroundFetchClient.GET("/issues/families", { signal });
const { data } = await backgroundFetchClient.GET("/alerts/defaults", { signal });
if (!data) throw new Error("Could not load monitoring settings");
return data;
},
@ -172,7 +172,7 @@ export default function IssueFamilySettings({ accessToken }: { accessToken: stri
const save = useMutation({
mutationFn: async (body: FamilyUpdate) => {
if (!editing) throw new Error("Choose an alert type");
const { data } = await backgroundFetchClient.PATCH("/issues/families/{family}", {
const { data } = await backgroundFetchClient.PATCH("/alerts/defaults/{family}", {
params: { path: { family: editing.family } },
body,
});

6
ui/litellm-dashboard/src/components/Issues/IssueHistoryChart.tsx
View file

@ -1,13 +1,13 @@
import { Bar, BarChart, CartesianGrid, Cell, ReferenceLine, XAxis, YAxis } from "recharts";
import { ChartContainer, ChartTooltip, ChartTooltipContent } from "@/components/ui/chart";
import { formatAlertValue, METRIC_LABELS } from "@/components/UsageAlerts/usageAlertApi";
import { Issue, issueDate, primaryComparison } from "./issueApi";
import { Alert, issueDate, primaryComparison } from "./issueApi";
export function formatHistoryValue(metric: Parameters<typeof formatAlertValue>[0], value: unknown) {
return typeof value === "number" && Number.isFinite(value) ? formatAlertValue(metric, value) : "Not recorded";
}
export function issueHistory(issue: Issue) {
export function issueHistory(issue: Alert) {
const comparison = primaryComparison(issue);
const history = issue.deprecation?.daily_requests ?? comparison?.history ?? [];
const metric = issue.deprecation ? "requests" : comparison?.metric ?? "requests";
@ -25,7 +25,7 @@ export function issueHistory(issue: Issue) {
return { data, metric, comparison };
}
export default function IssueHistoryChart({ issue, compact = false }: { issue: Issue; compact?: boolean }) {
export default function IssueHistoryChart({ issue, compact = false }: { issue: Alert; compact?: boolean }) {
const { data, metric, comparison } = issueHistory(issue);
const label = issue.deprecation ? "Daily requests" : `Daily ${METRIC_LABELS[metric].toLowerCase()}`;
if (data.length === 0)

10
ui/litellm-dashboard/src/components/Issues/IssueResources.tsx
View file

@ -10,7 +10,7 @@ import { storedKeyId } from "@/components/UsagePage/keyActivityData";
import { backgroundFetchClient } from "@/lib/http/api";
import { entityActivityHref } from "@/utils/entityLinks";
import { formatAlertValue } from "@/components/UsageAlerts/usageAlertApi";
import type { Issue } from "./issueApi";
import type { Alert } from "./issueApi";
interface ResourceRow {
id: string;
@ -58,7 +58,7 @@ export const ISSUE_RESOURCE_TABLE_STATE = {
keyPrefix: "resources",
};
export default function IssueResources({ issue, from, to }: { issue: Issue; from: string; to: string }) {
export default function IssueResources({ issue, from, to }: { issue: Alert; from: string; to: string }) {
const { userId, userRole, isAuthorized, accessToken } = useAuthorized();
const [requestedResource, setResource] = useQueryState(
"resources",
@ -84,11 +84,11 @@ export default function IssueResources({ issue, from, to }: { issue: Issue; from
enabled: Boolean(isAuthorized && accessToken && from <= to),
queryFn: async ({ signal }: { signal: AbortSignal }) => {
const params = {
path: { issue_id: issue.id },
path: { alert_id: issue.id },
query: { start_date: from, end_date: to, limit: table.pagination.pageSize, offset },
};
if (resource === "keys") {
const { data } = await backgroundFetchClient.GET("/issues/{issue_id}/keys", { params, signal });
const { data } = await backgroundFetchClient.GET("/alerts/{alert_id}/keys", { params, signal });
if (!data) throw new Error("Could not load keys");
const rows: ResourceRow[] = data.api_keys.map((key) => ({
id: key.api_key,
@ -101,7 +101,7 @@ export default function IssueResources({ issue, from, to }: { issue: Issue; from
}));
return { rows, total: data.total_api_keys };
}
const { data } = await backgroundFetchClient.GET("/issues/{issue_id}/users", { params, signal });
const { data } = await backgroundFetchClient.GET("/alerts/{alert_id}/users", { params, signal });
if (!data) throw new Error("Could not load users");
const rows: ResourceRow[] = data.users.map((user) => ({
id: user.user_id ?? "unattributed",

10
ui/litellm-dashboard/src/components/Issues/IssueSignals.tsx
View file

@ -5,9 +5,9 @@ import { Button } from "@/components/ui/button";
import { backgroundFetchClient } from "@/lib/http/api";
import { formatAlertValue, METRIC_LABELS, usageAlertHref } from "@/components/UsageAlerts/usageAlertApi";
import { useQueryState, parseAsInteger } from "nuqs";
import { Issue } from "./issueApi";
import { Alert } from "./issueApi";
export default function IssueSignals({ issue }: { issue: Issue }) {
export default function IssueSignals({ issue }: { issue: Alert }) {
const { userId, userRole, isAuthorized, accessToken } = useAuthorized();
const [page, setPage] = useQueryState("signal_page", parseAsInteger.withDefault(1));
const offset = (Math.max(1, page) - 1) * 20;
@ -15,8 +15,8 @@ export default function IssueSignals({ issue }: { issue: Issue }) {
queryKey: ["issueSignals", getProxyBaseUrl(), userId, userRole, issue.id, offset],
enabled: Boolean(isAuthorized && accessToken),
queryFn: async ({ signal }: { signal: AbortSignal }) => {
const { data } = await backgroundFetchClient.GET("/issues/{issue_id}/signals", {
params: { path: { issue_id: issue.id }, query: { limit: 20, offset } },
const { data } = await backgroundFetchClient.GET("/alerts/{alert_id}/signals", {
params: { path: { alert_id: issue.id }, query: { limit: 20, offset } },
signal,
});
if (!data) throw new Error("Could not load source signals");
@ -48,7 +48,7 @@ export default function IssueSignals({ issue }: { issue: Issue }) {
</p>
<ul className="divide-y">
{query.data.data.map((signal) => {
const alert = signal.usage_alert;
const alert = signal.performance;
return (
<li key={`${signal.source}-${signal.source_id}`} className="py-3">
{alert ? (

8
ui/litellm-dashboard/src/components/Issues/IssueStatusActions.tsx
View file

@ -7,9 +7,9 @@ import { Button } from "@/components/ui/button";
import { hasCapability } from "@/utils/capabilities";
import { ApiError } from "@/lib/http/client";
import type { components } from "@/lib/http/schema";
import { Issue, IssueStatus, updateIssueStatus } from "./issueApi";
import { Alert, IssueStatus, updateIssueStatus } from "./issueApi";
export default function IssueStatusActions({ issue, globalAccess = false }: { issue: Issue; globalAccess?: boolean }) {
export default function IssueStatusActions({ issue, globalAccess = false }: { issue: Alert; globalAccess?: boolean }) {
const { isViewOnly, accessToken, userRole } = useAuthorized();
const { data: user } = useCurrentUser(accessToken);
const client = useQueryClient();
@ -17,7 +17,7 @@ export default function IssueStatusActions({ issue, globalAccess = false }: { is
mutationFn: (status: IssueStatus) => updateIssueStatus(issue, status),
onSuccess: (updated) => {
const scope = ["issues", getProxyBaseUrl(), user?.user_id, user?.user_role];
client.setQueryData<Issue>(
client.setQueryData<Alert>(
[...scope, "detail", issue.id],
(previous) =>
previous && {
@ -26,7 +26,7 @@ export default function IssueStatusActions({ issue, globalAccess = false }: { is
can_update_status: updated.can_update_status ?? previous.can_update_status,
},
);
client.setQueriesData<components["schemas"]["IssueList"]>(
client.setQueriesData<components["schemas"]["AlertList"]>(
{ queryKey: [...scope, "list"] },
(page) =>
page && {

69
ui/litellm-dashboard/src/components/Issues/Issues.integration.test.tsx
View file

@ -10,10 +10,11 @@ import {
within,
} from "../../../tests/test-utils";
import IssuesInbox from "./IssuesInbox";
import { NotificationsBell } from "@/components/Navbar/NotificationsBell/NotificationsBell";
import IssueDetail from "./IssueDetail";
import { FAMILY_SETTINGS, RETIREMENT_ISSUE, SPEND_ISSUE } from "./issueFixtures";
import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi";
import type { Issue } from "./issueApi";
import type { Alert } from "./issueApi";
import type { components } from "@/lib/http/schema";
const auth = vi.hoisted(() => ({
@ -25,7 +26,7 @@ const auth = vi.hoisted(() => ({
}));
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => auth }));
const calls: { path: string; query: URLSearchParams; method: string; body: unknown }[] = [];
let detail: Issue;
let detail: Alert;
let refreshBarrier: Promise<void> | undefined;
let holdReads = false;
let failList = false;
@ -154,10 +155,10 @@ describe("unified issue investigations", () => {
calls.push(call);
if (url.pathname === "/management/v1/users/me")
return json({ user_id: auth.userId, user_role: auth.userRole === "Admin" ? "proxy_admin" : "internal_user" });
if (url.pathname === "/issues/foreign-issue") return json({ detail: "Forbidden private-project metadata" }, 403);
if (url.pathname === "/issues") return refreshed(() => listResponse(url.searchParams));
if (url.pathname === "/issues/families") return json(families);
if (url.pathname === "/issues/families/spend") return json(FAMILY_SETTINGS.data[0]);
if (url.pathname === "/alerts/foreign-issue") return json({ detail: "Forbidden private-project metadata" }, 403);
if (url.pathname === "/alerts") return refreshed(() => listResponse(url.searchParams));
if (url.pathname === "/alerts/defaults") return json(families);
if (url.pathname === "/alerts/defaults/spend") return json(FAMILY_SETTINGS.data[0]);
if (url.pathname.endsWith("/activity"))
return failActivity ? json({ detail: "Unavailable" }, 503) : json(activity());
if (url.pathname.endsWith("/keys")) return resourceKeyPage(url.searchParams);
@ -176,10 +177,10 @@ describe("unified issue investigations", () => {
const page = { data: [], total: 0, limit: 20, offset: 0 };
return json(page);
}
if (url.pathname.startsWith("/issues/")) {
if (url.pathname.startsWith("/alerts/")) {
if (deniedDetail) return json({ detail: "Access revoked" }, 403);
if (request.method === "PATCH") {
if (staleStatus) return json({ detail: "Issue changed" }, 409);
if (staleStatus) return json({ detail: "Alert changed" }, 409);
detail = { ...detail, ...body };
holdReads = Boolean(refreshBarrier);
return json({ ...detail, can_update_status: null, can_view_activity: null });
@ -194,19 +195,33 @@ describe("unified issue investigations", () => {
vi.unstubAllGlobals();
});
it("retries a failed notification feed and opens the same alert detail", async () => {
failList = true;
renderWithProviders(<NotificationsBell accessToken="test-session" />);
await userEvent.click(screen.getByRole("button", { name: /^Notifications/ }));
await screen.findByRole("alert");
failList = false;
await userEvent.click(screen.getByRole("button", { name: "Retry" }));
expect(await screen.findByRole("link", { name: SPEND_ISSUE.title })).toHaveAttribute(
"href",
`/ui/alerts?alert=${SPEND_ISSUE.id}`,
);
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
});
it("adapts pagination to the server byte budget without skipping alerts", async () => {
byteLimitedPage = true;
renderWithProviders(<IssuesInbox />);
await screen.findByRole("heading", { name: /Wide key 0/ });
await waitFor(() =>
expect(calls.some((call) => call.path === "/issues" && call.query.get("limit") === "2")).toBe(true),
expect(calls.some((call) => call.path === "/alerts" && call.query.get("limit") === "2")).toBe(true),
);
await userEvent.click(screen.getByRole("button", { name: "Go to next page" }));
await screen.findByRole("heading", { name: /Wide key 2/ });
await userEvent.click(screen.getByRole("button", { name: "Go to next page" }));
await screen.findByRole("heading", { name: /Wide key 4/ });
expect(screen.getByRole("button", { name: "Go to next page" })).toBeDisabled();
expect(calls.some((call) => call.path === "/issues" && call.query.get("offset") === "4")).toBe(true);
expect(calls.some((call) => call.path === "/alerts" && call.query.get("offset") === "4")).toBe(true);
});
it.each([false, true])("preserves the exact offset when later pages shrink (deep link %s)", async (deepLink) => {
@ -220,7 +235,7 @@ describe("unified issue investigations", () => {
await waitFor(() =>
expect(
calls
.filter((call) => call.path === "/issues")
.filter((call) => call.path === "/alerts")
.at(-1)
?.query.get("limit"),
).toBe("1"),
@ -241,19 +256,19 @@ describe("unified issue investigations", () => {
renderWithProviders(<IssuesInbox />);
expect(await screen.findByRole("link", { name: "Spend on Checkout key rose to 20× normal" })).toHaveAttribute(
"href",
"/ui/alerts?issue=issue-checkout-spend",
"/ui/alerts?alert=issue-checkout-spend",
);
expect(screen.getByRole("link", { name: RETIREMENT_ISSUE.title })).toBeInTheDocument();
expect(screen.getByText("$600.00 on Nov 4, 2026 UTC, compared with the usual $30.00.")).toBeInTheDocument();
expect(screen.getByRole("list", { name: "Alerts", exact: true })).toBeInTheDocument();
expect(screen.queryByRole("table")).not.toBeInTheDocument();
expect(calls.filter((call) => call.path === "/issues")).toHaveLength(1);
expect(calls.filter((call) => call.path === "/alerts")).toHaveLength(1);
expect(calls.some((call) => call.path.includes("activity") || call.path.includes("deprecations"))).toBe(false);
await user.click(screen.getByRole("button", { name: "Go to next page", exact: true }));
expect(await screen.findByRole("link", { name: "Spend on Second key rose to 20× normal" })).toBeInTheDocument();
expect(
calls
.filter((call) => call.path === "/issues")
.filter((call) => call.path === "/alerts")
.at(-1)
?.query.get("offset"),
).toBe("25");
@ -261,7 +276,7 @@ describe("unified issue investigations", () => {
expect(await screen.findByRole("link", { name: RETIREMENT_ISSUE.title })).toBeInTheDocument();
expect(
calls
.filter((call) => call.path === "/issues")
.filter((call) => call.path === "/alerts")
.at(-1)
?.query.get("offset"),
).toBe("0");
@ -270,7 +285,7 @@ describe("unified issue investigations", () => {
it("keeps old performance list links usable", async () => {
renderWithProviders(<IssuesInbox />, { searchParams: "type=usage&usage_status=resolved" });
await screen.findByRole("link", { name: "Spend on Checkout key rose to 20× normal" });
const request = calls.find((call) => call.path === "/issues");
const request = calls.find((call) => call.path === "/alerts");
expect(request?.query.get("kind")).toBe("performance");
expect(request?.query.get("status")).toBe("resolved");
});
@ -307,11 +322,11 @@ describe("unified issue investigations", () => {
const feed = screen.getByRole("list", { name: "Alerts", exact: true });
await userEvent.click(within(feed).getAllByRole("button", { name: "Mark resolved" })[0]);
await waitFor(() =>
expect(calls.some((call) => call.method === "PATCH" && call.path === `/issues/${SPEND_ISSUE.id}`)).toBe(true),
expect(calls.some((call) => call.method === "PATCH" && call.path === `/alerts/${SPEND_ISSUE.id}`)).toBe(true),
);
expect(
calls.filter(
(call) => call.method === "GET" && call.path.startsWith("/issues/") && !call.path.endsWith("/families"),
(call) => call.method === "GET" && call.path.startsWith("/alerts/") && !call.path.endsWith("/families"),
),
).toHaveLength(0);
expect(calls.find((call) => call.method === "PATCH")?.body).toEqual({
@ -328,7 +343,7 @@ describe("unified issue investigations", () => {
try {
await userEvent.click(screen.getByRole("button", { name: "Mark resolved" }));
await waitFor(() =>
expect(calls.filter((call) => call.method === "GET" && call.path === `/issues/${detail.id}`)).toHaveLength(2),
expect(calls.filter((call) => call.method === "GET" && call.path === `/alerts/${detail.id}`)).toHaveLength(2),
);
expect(screen.getByRole("link", { name: "View key activity" })).toBeVisible();
expect(screen.getByRole("button", { name: "Reopen" })).toBeDisabled();
@ -344,7 +359,7 @@ describe("unified issue investigations", () => {
await screen.findByRole("link", { name: "Spend on Checkout key rose to 20× normal" });
try {
await userEvent.click(screen.getAllByRole("button", { name: "Mark resolved" })[0]);
await waitFor(() => expect(calls.filter((call) => call.path === "/issues")).toHaveLength(2));
await waitFor(() => expect(calls.filter((call) => call.path === "/alerts")).toHaveLength(2));
expect(screen.getByRole("button", { name: "Reopen" })).toBeDisabled();
} finally {
await act(async () => refresh.resolve());
@ -358,10 +373,10 @@ describe("unified issue investigations", () => {
await screen.findByRole("link", { name: "Spend on Checkout key rose to 20× normal" });
try {
await userEvent.click(screen.getAllByRole("button", { name: "Mark resolved" })[0]);
await waitFor(() => expect(calls.filter((call) => call.path === "/issues")).toHaveLength(2));
await waitFor(() => expect(calls.filter((call) => call.path === "/alerts")).toHaveLength(2));
view.rerender(<IssueDetail id={detail.id} />);
await waitFor(() =>
expect(calls.some((call) => call.method === "GET" && call.path === `/issues/${detail.id}`)).toBe(true),
expect(calls.some((call) => call.method === "GET" && call.path === `/alerts/${detail.id}`)).toBe(true),
);
expect(screen.queryByText("You don’t have access to this alert’s activity.")).not.toBeInTheDocument();
} finally {
@ -516,7 +531,7 @@ describe("unified issue investigations", () => {
});
vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
const request = input instanceof Request ? input : new Request(input, init);
if (request.method === "GET" && new URL(request.url).pathname.startsWith("/issues")) await stalled;
if (request.method === "GET" && new URL(request.url).pathname.startsWith("/alerts")) await stalled;
return fetchBefore(input, init);
});
try {
@ -546,7 +561,7 @@ describe("unified issue investigations", () => {
"href",
"/ui/models-and-endpoints?activity_model=provider%2Fretiring-model&detail_tab=activity&activity_from=2026-10-28&activity_to=2026-11-04",
);
expect(calls.some((call) => call.path === `/issues/${RETIREMENT_ISSUE.id}/activity`)).toBe(false);
expect(calls.some((call) => call.path === `/alerts/${RETIREMENT_ISSUE.id}/activity`)).toBe(false);
});
it("does not show invalid per-success averages when all requests failed", async () => {
@ -623,7 +638,7 @@ describe("unified issue investigations", () => {
"href",
"/ui/api-keys?key=reporting-test-id&detail_tab=activity&activity_from=2026-10-28&activity_to=2026-11-04",
);
expect(calls.some((call) => call.path === "/issues")).toBe(false);
expect(calls.some((call) => call.path === "/alerts")).toBe(false);
expect(calls.some((call) => call.path.includes("/user/daily"))).toBe(false);
await userEvent.click(screen.getByRole("button", { name: "Mark resolved" }));
await waitFor(() =>
@ -642,7 +657,7 @@ describe("unified issue investigations", () => {
expect(screen.getByRole("link", { name: "My usage" })).toHaveAttribute("href", "/ui/usage?view=my-usage");
expect(screen.queryByText(/private-project/)).not.toBeInTheDocument();
expect(screen.queryByRole("heading", { name: "Spend on Checkout key rose to 20× normal" })).not.toBeInTheDocument();
expect(calls.filter((call) => call.path.startsWith("/issues"))).toHaveLength(1);
expect(calls.filter((call) => call.path.startsWith("/alerts"))).toHaveLength(1);
expect(calls.some((call) => call.path.endsWith("/activity") || call.path.endsWith("/keys"))).toBe(false);
});
@ -726,7 +741,7 @@ describe("unified issue investigations", () => {
fireEvent.change(screen.getByLabelText("Increase factor"), { target: { value: "10" } });
await user.click(screen.getByRole("button", { name: "Save changes" }));
await waitFor(() =>
expect(calls.find((call) => call.path === "/issues/families/spend")?.body).toEqual({
expect(calls.find((call) => call.path === "/alerts/defaults/spend")?.body).toEqual({
policies: [{ direction: "increase", multiplier: 10 }],
}),
);

8
ui/litellm-dashboard/src/components/Issues/IssuesInbox.tsx
View file

@ -11,8 +11,8 @@ import {
FAMILY_LABELS,
ISSUE_FAMILIES,
ISSUE_STATUSES,
Issue,
IssueFamily,
Alert,
AlertFamily,
IssueStatus,
issueComparisonText,
issueHeadline,
@ -25,7 +25,7 @@ import IssueHistoryChart from "./IssueHistoryChart";
import IssueStatusActions from "./IssueStatusActions";
const TABLE_STATE = { sortFields: [], defaultSort: { id: "", desc: false }, defaultPageSize: 25, filterColumns: [] };
function AlertEntry({ issue }: { issue: Issue }) {
function AlertEntry({ issue }: { issue: Alert }) {
const impact = issueImpactText(issue);
return (
<li className="grid gap-4 px-4 py-5 sm:px-5 lg:grid-cols-[minmax(0,1fr)_220px] lg:gap-8">
@ -84,7 +84,7 @@ export default function IssuesInbox() {
};
const changeFamily = (next: string) => {
void setParams({
family: next === "all" || next === "performance" ? null : (next as IssueFamily),
family: next === "all" || next === "performance" ? null : (next as AlertFamily),
type: next === "performance" ? "usage" : null,
});
table.onPaginationChange({ ...table.pagination, pageIndex: 0 });

38
ui/litellm-dashboard/src/components/Issues/issueApi.ts
View file

@ -7,11 +7,11 @@ import { hasCapability } from "@/utils/capabilities";
import { uiHref } from "@/utils/uiHref";
import { formatAlertValue } from "@/components/UsageAlerts/usageAlertApi";
export type Issue = components["schemas"]["Issue"];
export type IssuePage = components["schemas"]["IssueList"];
export type IssueFamily = Issue["family"];
export type IssueStatus = Issue["status"];
export type IssueComparison = components["schemas"]["IssueComparison"];
export type Alert = components["schemas"]["Alert"];
export type IssuePage = components["schemas"]["AlertList"];
export type AlertFamily = Alert["family"];
export type IssueStatus = Alert["status"];
export type AlertComparison = components["schemas"]["AlertComparison"];
export const ISSUE_FAMILIES = ["retirement", "latency", "reliability", "spend", "traffic"] as const;
export const ISSUE_STATUSES = ["open", "resolved", "dismissed"] as const;
export const FAMILY_LABELS = {
@ -37,10 +37,10 @@ const PRIMARY_METRICS = {
traffic: "requests",
};
export const issueHref = (id: string) => uiHref(`alerts?issue=${encodeURIComponent(id)}`);
export const issueHref = (id: string) => uiHref(`alerts?alert=${encodeURIComponent(id)}`);
export const issuesHref = () => uiHref("alerts");
export function issueResource(issue: Issue): string {
export function issueResource(issue: Alert): string {
const { scope_type: scope, scope_id: id } = issue.cohort;
const label = SCOPE_LABELS[scope];
if (scope === "deployment" && issue.scope_label === label) return label;
@ -50,7 +50,7 @@ export function issueResource(issue: Issue): string {
return `${label} · ${short}`;
}
export const primaryComparison = (issue: Issue) =>
export const primaryComparison = (issue: Alert) =>
(issue.comparisons ?? []).find((comparison) => comparison.metric === PRIMARY_METRICS[issue.family]) ??
issue.comparisons?.[0];
@ -62,7 +62,7 @@ const READABLE_METRICS = {
error_rate: "Error rate for",
};
export function issueHeadline(issue: Issue): string {
export function issueHeadline(issue: Alert): string {
const comparison = primaryComparison(issue);
if (issue.deprecation || !comparison) return issue.title;
const { metric, observed_value: actual, baseline_value: baseline, direction } = comparison;
@ -79,7 +79,7 @@ export function issueHeadline(issue: Issue): string {
return `${prefix} rose to ${ratio}× normal`;
}
export function issueComparisonText(issue: Issue): string {
export function issueComparisonText(issue: Alert): string {
if (issue.deprecation) {
const { requests, model } = issue.deprecation;
const replacement = model.successor_model ? ` Suggested replacement: ${model.successor_model}.` : "";
@ -92,7 +92,7 @@ export function issueComparisonText(issue: Issue): string {
return `${actual} on ${issueDate(issue.observed_on)} UTC, compared with the usual ${baseline}.`;
}
export function issueImpactText(issue: Issue): string {
export function issueImpactText(issue: Alert): string {
const impact = issue.impact ?? {};
return (["keys", "teams", "models"] as const)
.map((field) => {
@ -109,8 +109,8 @@ export const issueDate = (value: string) => new Date(value).toLocaleDateString(u
interface IssueListOptions {
status?: IssueStatus;
family?: IssueFamily;
kind?: Issue["kind"];
family?: AlertFamily;
kind?: Alert["kind"];
limit?: number;
offset?: number;
}
@ -134,7 +134,7 @@ export function useIssues(accessToken: string | null, listOptions: IssueListOpti
],
enabled,
queryFn: async ({ signal }: { signal: AbortSignal }) => {
const { data } = await backgroundFetchClient.GET("/issues", {
const { data } = await backgroundFetchClient.GET("/alerts", {
params: { query: { status, family, kind, limit, offset } },
signal,
});
@ -156,8 +156,8 @@ export function useIssue(accessToken: string | null, id: string) {
queryKey: ["issues", getProxyBaseUrl(), user?.user_id, user?.user_role, "detail", id],
enabled,
queryFn: async ({ signal }: { signal: AbortSignal }) => {
const { data } = await backgroundFetchClient.GET("/issues/{issue_id}", {
params: { path: { issue_id: id } },
const { data } = await backgroundFetchClient.GET("/alerts/{alert_id}", {
params: { path: { alert_id: id } },
signal,
});
if (!data) throw new Error("Could not load this alert");
@ -170,9 +170,9 @@ export function useIssue(accessToken: string | null, id: string) {
return { ...query, enabled, data: enabled && !query.isError ? query.data : undefined };
}
export async function updateIssueStatus(issue: Issue, status: IssueStatus) {
const { data } = await backgroundFetchClient.PATCH("/issues/{issue_id}", {
params: { path: { issue_id: issue.id } },
export async function updateIssueStatus(issue: Alert, status: IssueStatus) {
const { data } = await backgroundFetchClient.PATCH("/alerts/{alert_id}", {
params: { path: { alert_id: issue.id } },
body: { status, expected_status: issue.status },
});
if (!data) throw new Error("Could not update this alert");

8
ui/litellm-dashboard/src/components/Issues/issueFixtures.ts
View file

@ -1,7 +1,7 @@
import type { components } from "@/lib/http/schema";
import type { Issue } from "./issueApi";
import type { Alert } from "./issueApi";
export const SPEND_ISSUE: Issue = {
export const SPEND_ISSUE: Alert = {
id: "issue-checkout-spend",
cohort_key: "cohort-checkout-spend",
kind: "performance",
@ -57,7 +57,7 @@ export const SPEND_ISSUE: Issue = {
can_view_activity: true,
};
export const RETIREMENT_ISSUE: Issue = {
export const RETIREMENT_ISSUE: Alert = {
...SPEND_ISSUE,
id: "issue-model-retirement",
cohort_key: "cohort-model-retirement",
@ -99,7 +99,7 @@ export const RETIREMENT_ISSUE: Issue = {
},
};
export const FAMILY_SETTINGS: components["schemas"]["IssueFamilyList"] = {
export const FAMILY_SETTINGS: components["schemas"]["AlertFamilyList"] = {
projection_pending: false,
data: (["spend", "latency", "reliability", "traffic"] as const).map((family) => ({
family,

2
ui/litellm-dashboard/src/components/Issues/issueRoute.ts
View file

@ -2,7 +2,7 @@ import { parseAsString, parseAsStringLiteral } from "nuqs";
import { ISSUE_FAMILIES, ISSUE_STATUSES } from "./issueApi";
export const issueRouteParsers = {
issue: parseAsString.withOptions({ history: "push" }),
alert: parseAsString.withOptions({ history: "push" }),
family: parseAsStringLiteral(ISSUE_FAMILIES),
status: parseAsStringLiteral(ISSUE_STATUSES),
type: parseAsStringLiteral(["usage", "model-deprecations"] as const),

4
ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx
View file

@ -53,7 +53,7 @@ describe("model migration impact", () => {
calls.push(call);
if (url.pathname === "/management/v1/users/me")
return apiResponse({ user_id: "user-1", user_role: "proxy_admin" });
if (url.pathname === "/issues") {
if (url.pathname === "/alerts") {
const page = { data: [RETIREMENT_ISSUE], total: 1, limit: 25, offset: 0 };
return apiResponse(page);
}
@ -81,7 +81,7 @@ describe("model migration impact", () => {
expect(screen.getByRole("heading", { name: "Alerts", exact: true })).toBeInTheDocument();
expect(await screen.findByRole("link", { name: RETIREMENT_ISSUE.title })).toHaveAttribute(
"href",
"/ui/alerts?issue=issue-model-retirement",
"/ui/alerts?alert=issue-model-retirement",
);
});
it.each([undefined, "model-deprecations"])("should preserve migration deep links with type %s", async (type) => {

4
ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx
View file

@ -215,7 +215,7 @@ describe("issue notifications", () => {
renderWithProviders(<NotificationsBellView currentVersion={null} latestRelease={null} issues={ISSUES} />);
await user.click(screen.getByRole("button", { name: "Notifications, 2 alerts need attention" }));
const destination = screen.getByRole("link", { name: SPEND_ISSUE.title });
expect(destination).toHaveAttribute("href", "/ui/alerts?issue=issue-checkout-spend");
expect(destination).toHaveAttribute("href", "/ui/alerts?alert=issue-checkout-spend");
expect(screen.getByRole("article", { name: SPEND_ISSUE.title })).toHaveTextContent(
"$600.00 on Nov 4, 2026 UTC, compared with the usual $30.00.",
);
@ -237,7 +237,7 @@ describe("issue notifications", () => {
await user.click(screen.getByRole("button", { name: "Notifications, 2 alerts need attention" }));
expect(screen.getByRole("link", { name: RETIREMENT_ISSUE.title })).toHaveAttribute(
"href",
"/ui/alerts?issue=issue-model-retirement",
"/ui/alerts?alert=issue-model-retirement",
);
expect(screen.getByRole("link", { name: "LiteLLM model cost map" })).toHaveAttribute(
"href",

13
ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx
View file

@ -159,6 +159,8 @@ interface NotificationsBellViewProps {
issues?: IssuePage;
issuesPending?: boolean;
issuesFailed?: boolean;
retryingAlerts?: boolean;
onRetryAlerts?: () => void;
deploymentHealthLink?: React.ReactNode;
currentVersion: string | null | undefined;
latestRelease: LatestReleaseInfo | null | undefined;
@ -171,6 +173,8 @@ export const NotificationsBellView: React.FC<NotificationsBellViewProps> = ({
issues,
issuesPending,
issuesFailed,
retryingAlerts,
onRetryAlerts,
}) => {
const { t } = useTranslation();
const upgrade = upgradeAnnouncement(currentVersion, latestRelease);
@ -230,7 +234,12 @@ export const NotificationsBellView: React.FC<NotificationsBellViewProps> = ({
</p>
)}
{issuesFailed && (
<p className="text-xs text-muted-foreground">{t("shell.notifications.bell.loadError")}</p>
<div className="flex items-center justify-between gap-2 text-xs text-muted-foreground" role="alert">
<p>{t("shell.notifications.bell.loadError")}</p>
<Button variant="link" size="sm" disabled={retryingAlerts} onClick={onRetryAlerts}>
{t("shell.notifications.bell.retry")}
</Button>
</div>
)}
{issues?.data.map((issue) => (
<article key={issue.id} className="space-y-1 border-t pt-3" aria-label={issue.title}>
@ -276,6 +285,8 @@ export const NotificationsBell: React.FC<{ accessToken: string | null; deploymen
issues={issueQuery.data}
issuesPending={issueQuery.enabled && issueQuery.isPending}
issuesFailed={issueQuery.enabled && issueQuery.isError}
retryingAlerts={issueQuery.isFetching}
onRetryAlerts={() => void issueQuery.refetch()}
deploymentHealthLink={deploymentHealthLink}
/>
);

18
ui/litellm-dashboard/src/components/UsageAlerts/UsageAlertContext.tsx
View file

@ -3,24 +3,24 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import { backgroundFetchClient } from "@/lib/http/api";
import { getProxyBaseUrl } from "@/components/networking";
import { Button } from "@/components/ui/button";
import { formatAlertValue, METRIC_LABELS, UsageAlert } from "./usageAlertApi";
import { formatAlertValue, METRIC_LABELS, PerformanceSignal } from "./usageAlertApi";
export default function UsageAlertContext({
alertId,
signalId,
scope,
entityId,
}: {
alertId: string;
scope: UsageAlert["scope_type"];
signalId: string;
scope: PerformanceSignal["scope_type"];
entityId: string | null;
}) {
const { accessToken, userId, userRole, isAuthorized } = useAuthorized();
const options: UseQueryOptions<UsageAlert> = {
queryKey: ["usageAlert", getProxyBaseUrl(), userId, userRole, alertId],
enabled: Boolean(isAuthorized && accessToken && alertId),
const options: UseQueryOptions<PerformanceSignal> = {
queryKey: ["performanceSignal", getProxyBaseUrl(), userId, userRole, signalId],
enabled: Boolean(isAuthorized && accessToken && signalId),
queryFn: async ({ signal }) => {
const { data } = await backgroundFetchClient.GET("/usage/alerts/{alert_id}", {
params: { path: { alert_id: alertId } },
const { data } = await backgroundFetchClient.GET("/alerts/signals/{signal_id}", {
params: { path: { signal_id: signalId } },
signal,
});
if (!data) throw new Error("Alert unavailable");

18
ui/litellm-dashboard/src/components/UsageAlerts/UsageAlerts.integration.test.tsx
View file

@ -97,12 +97,12 @@ describe("usage alerts API integration", () => {
const call = { path: url.pathname, method: request.method, body, query: url.searchParams };
calls.push(call);
if (url.pathname === "/management/v1/users/me") return json({ user_id: "test-admin", user_role: "proxy_admin" });
if (url.pathname === "/usage/alerts/incident-test") return json({ ...observation, status: "resolved" });
if (url.pathname === "/issues/families") return json(FAMILY_SETTINGS);
if (url.pathname === "/usage/alert-rules" && request.method === "POST") {
if (url.pathname === "/alerts/signals/incident-test") return json({ ...observation, status: "resolved" });
if (url.pathname === "/alerts/defaults") return json(FAMILY_SETTINGS);
if (url.pathname === "/alerts/rules" && request.method === "POST") {
return ruleResponse(body);
}
if (url.pathname === "/usage/alert-rules") {
if (url.pathname === "/alerts/rules") {
const page = {
data: customRules,
total: customRules.length,
@ -122,7 +122,7 @@ describe("usage alerts API integration", () => {
};
return json(page);
}
if (url.pathname === "/usage/alert-models") {
if (url.pathname === "/alerts/models") {
return modelsResponse(url.searchParams);
}
return json({ detail: "Unexpected request" }, 404);
@ -150,7 +150,7 @@ describe("usage alerts API integration", () => {
);
it("shows saved evidence on the matching entity without recomputing from a current rule", async () => {
renderWithProviders(<UsageAlertContext alertId="incident-test" scope="key" entityId="reporting-test-id" />);
renderWithProviders(<UsageAlertContext signalId="incident-test" scope="key" entityId="reporting-test-id" />);
expect(await screen.findByText(/Spend was \$120.00 on 2026-11-04 UTC/)).toHaveTextContent(
"baseline was $30.00 across 5 recorded days",
);
@ -159,7 +159,7 @@ describe("usage alerts API integration", () => {
});
it("does not attach an incident explanation to the wrong entity", async () => {
renderWithProviders(<UsageAlertContext alertId="incident-test" scope="team" entityId="team-example" />);
renderWithProviders(<UsageAlertContext signalId="incident-test" scope="team" entityId="team-example" />);
expect(await screen.findByText("This alert belongs to a different entity.")).toBeInTheDocument();
expect(screen.queryByText(/Spend was/)).not.toBeInTheDocument();
});
@ -204,7 +204,7 @@ describe("usage alerts API integration", () => {
renderWithProviders(<UsageRules accessToken="test-access-token" />);
await user.click(await screen.findByRole("button", { name: "Settings" }));
expect(await screen.findByRole("button", { name: "Edit Unexpected spend" })).toBeInTheDocument();
expect(calls.some((call) => call.path === "/usage/alert-rules")).toBe(false);
expect(calls.some((call) => call.path === "/alerts/rules")).toBe(false);
await user.click(screen.getByText("Custom rules", { selector: "summary" }));
await user.click(screen.getByRole("button", { name: "New rule", exact: true }));
expect(screen.getByLabelText("Name")).not.toBeVisible();
@ -236,7 +236,7 @@ describe("usage alerts API integration", () => {
await user.click(screen.getByRole("button", { name: "Save rule" }));
const expected = { scope_type: "model", scope_id: "provider/recorded-model" };
await waitFor(() => expect(calls.find((call) => call.method === "POST")?.body).toMatchObject(expected));
expect(calls.find((call) => call.path === "/usage/alert-models")?.query.get("limit")).toBe("20");
expect(calls.find((call) => call.path === "/alerts/models")?.query.get("limit")).toBe("20");
expect(calls.some((call) => call.path === "/v2/model/info")).toBe(false);
});

2
ui/litellm-dashboard/src/components/UsageAlerts/UsageRuleResourceSelect.tsx
View file

@ -138,7 +138,7 @@ function ModelResource(props: ResourceProps) {
queryKey: ["usageAlertModels", getProxyBaseUrl(), userId, userRole, search],
initialPageParam: "",
queryFn: async ({ pageParam, signal }: { pageParam: string; signal: AbortSignal }) => {
const { data } = await backgroundFetchClient.GET("/usage/alert-models", {
const { data } = await backgroundFetchClient.GET("/alerts/models", {
params: { query: { search: search || undefined, cursor: pageParam || undefined, limit: PAGE_SIZE } },
signal,
});

8
ui/litellm-dashboard/src/components/UsageAlerts/UsageRules.tsx
View file

@ -34,7 +34,7 @@ export default function UsageRules({ accessToken }: { accessToken: string | null
queryKey: ["usageAlertRules", getProxyBaseUrl(), user?.user_id, user?.user_role, offset, "custom"],
enabled: open && customOpen && Boolean(accessToken && user?.user_id),
queryFn: async ({ signal }) => {
const { data } = await backgroundFetchClient.GET("/usage/alert-rules", {
const { data } = await backgroundFetchClient.GET("/alerts/rules", {
params: { query: { limit: 20, offset, kind: "custom" } },
signal,
});
@ -45,8 +45,8 @@ export default function UsageRules({ accessToken }: { accessToken: string | null
const save = useMutation({
mutationFn: async (body: RuleInput) =>
editing
? backgroundFetchClient.PUT("/usage/alert-rules/{rule_id}", { params: { path: { rule_id: editing.id } }, body })
: backgroundFetchClient.POST("/usage/alert-rules", { body }),
? backgroundFetchClient.PUT("/alerts/rules/{rule_id}", { params: { path: { rule_id: editing.id } }, body })
: backgroundFetchClient.POST("/alerts/rules", { body }),
onSuccess: async () => {
setEditing(undefined);
await queryClient.invalidateQueries({ queryKey: ["usageAlertRules"] });
@ -54,7 +54,7 @@ export default function UsageRules({ accessToken }: { accessToken: string | null
});
const remove = useMutation({
mutationFn: (id: string) =>
backgroundFetchClient.DELETE("/usage/alert-rules/{rule_id}", { params: { path: { rule_id: id } } }),
backgroundFetchClient.DELETE("/alerts/rules/{rule_id}", { params: { path: { rule_id: id } } }),
onSuccess: async () => {
setDeleting(null);
if (query.data?.data.length === 1) setOffset(Math.max(0, offset - 20));

18
ui/litellm-dashboard/src/components/UsageAlerts/usageAlertApi.ts
View file

@ -1,12 +1,12 @@
import type { paths } from "@/lib/http/schema";
import { uiHref } from "@/utils/uiHref";
type AlertPage = paths["/usage/alerts"]["get"]["responses"][200]["content"]["application/json"];
export type UsageAlert = AlertPage["data"][number];
export type AlertModelsPage = paths["/usage/alert-models"]["get"]["responses"][200]["content"]["application/json"];
type RulePage = paths["/usage/alert-rules"]["get"]["responses"][200]["content"]["application/json"];
type AlertPage = paths["/alerts/signals"]["get"]["responses"][200]["content"]["application/json"];
export type PerformanceSignal = AlertPage["data"][number];
export type AlertModelsPage = paths["/alerts/models"]["get"]["responses"][200]["content"]["application/json"];
type RulePage = paths["/alerts/rules"]["get"]["responses"][200]["content"]["application/json"];
export type UsageRule = RulePage["data"][number];
export type RuleInput = paths["/usage/alert-rules"]["post"]["requestBody"]["content"]["application/json"];
export type RuleInput = paths["/alerts/rules"]["post"]["requestBody"]["content"]["application/json"];
export const METRIC_LABELS = {
spend: "Spend",
requests: "Requests",
@ -16,9 +16,11 @@ export const METRIC_LABELS = {
};
export const SCOPE_LABELS = { deployment: "Gateway", key: "Key", team: "Team", user: "User", model: "Model" };
export const usageAlertHref = (alert: UsageAlert) => uiHref(alert.activity_path);
export const usageAlertHref = (alert: PerformanceSignal) => uiHref(alert.activity_path);
export function formatAlertChange(alert: Pick<UsageAlert, "baseline_value" | "observed_value" | "direction">): string {
export function formatAlertChange(
alert: Pick<PerformanceSignal, "baseline_value" | "observed_value" | "direction">,
): string {
if (alert.baseline_value === 0) return alert.observed_value > 0 ? "New activity" : "No change";
if (alert.observed_value === 0) return "↓ 100%";
const ratio =
@ -29,7 +31,7 @@ export function formatAlertChange(alert: Pick<UsageAlert, "baseline_value" | "ob
return `${alert.direction === "increase" ? "↑" : "↓"} ${formatted}×`;
}
export function formatAlertValue(metric: UsageAlert["metric"], value: number): string {
export function formatAlertValue(metric: PerformanceSignal["metric"], value: number): string {
if (metric === "spend")
return new Intl.NumberFormat("en-US", {
style: "currency",

6
ui/litellm-dashboard/src/components/UsagePage/EntityActivity.integration.test.tsx
View file

@ -7,7 +7,7 @@ import EntityActivity from "./EntityActivity";
const auth = vi.hoisted(() => ({ userRole: "Admin", userId: "viewer", accessToken: "test-session" }));
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => auth }));
const fetchMock = vi.fn<typeof fetch>();
const dates = "activity_from=2026-10-01&activity_to=2026-10-07&alert=incident-1";
const dates = "activity_from=2026-10-01&activity_to=2026-10-07&signal=incident-1";
const response = (body: unknown, status = 200) =>
new Response(JSON.stringify(body), {
status,
@ -39,7 +39,7 @@ describe("entity activity", () => {
await userEvent.click(screen.getByRole("button", { name: "Apply" }));
await waitFor(() => expect(urls().at(-1)?.searchParams.get("start_date")).toBe("2026-10-03"));
expect(urls().at(-1)?.searchParams.get("api_key")).toBe("key-hash");
expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("alert")).toBe("incident-1");
expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("signal")).toBe("incident-1");
});
it.each([
@ -59,7 +59,7 @@ describe("entity activity", () => {
expect(url.searchParams.get("timezone")).toBe("0");
expect(url.searchParams.get("bounded")).toBe("true");
expect(url.searchParams.get("include_model_breakdown")).toBe(scope === "key" ? "true" : null);
expect(url.searchParams.has("alert")).toBe(false);
expect(url.searchParams.has("signal")).toBe(false);
});
it("keeps a member's key request within their user scope", async () => {

2
ui/litellm-dashboard/src/components/UsagePage/EntityActivity.tsx
View file

@ -99,7 +99,7 @@ export default function EntityActivity({ scope, id, teamId }: Props) {
return (
<div className="space-y-4">
{params.alert && <UsageAlertContext alertId={params.alert} scope={scope} entityId={id} />}
{params.signal && <UsageAlertContext signalId={params.signal} scope={scope} entityId={id} />}
<div className="flex items-center justify-end gap-2">
<span className="text-sm text-muted-foreground">UTC</span>
<AdvancedDatePicker

2
ui/litellm-dashboard/src/components/UsagePage/entityActivityState.ts
View file

@ -13,7 +13,7 @@ export const parseActivityDay = createParser({
export const entityActivityParsers = {
activity_from: parseActivityDay,
activity_to: parseActivityDay,
alert: parseAsString,
signal: parseAsString,
};
export const modelActivityParser = parseAsString.withOptions({ history: "push" });

1
ui/litellm-dashboard/src/i18n/locales/en.json
View file

@ -8221,6 +8221,7 @@
"viewAlerts": "View all",
"loadingAlerts": "Loading alerts…",
"loadError": "Could not load alerts. Open Alerts to retry.",
"retry": "Retry",
"empty": "No open alerts.",
"triggerDeprecations_one": "Notifications, {{count}} model deprecation alert needs attention",
"triggerDeprecations_other": "Notifications, {{count}} model deprecation alerts need attention",

3135
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

File diff suppressed because it is too large Load diff

4
ui/litellm-dashboard/src/utils/entityLinks.test.ts
View file

@ -75,7 +75,7 @@ describe("modelOrAccessGroupHref", () => {
});
describe("entityActivityHref", () => {
const context = { from: "2026-10-01", to: "2026-10-07", alert: "incident-1" };
const context = { from: "2026-10-01", to: "2026-10-07", signal: "incident-1" };
it.each([
["key", "key-hash", "/api-keys", "key"],
["team", "a b/c", "/teams", "team"],
@ -88,7 +88,7 @@ describe("entityActivityHref", () => {
expect(url.searchParams.get("detail_tab")).toBe("activity");
expect(url.searchParams.get("activity_from")).toBe(context.from);
expect(url.searchParams.get("activity_to")).toBe(context.to);
expect(url.searchParams.get("alert")).toBe(context.alert);
expect(url.searchParams.get("signal")).toBe(context.signal);
});
it("does not link credentials, empty entities, or synthetic entities", () => {
expect(entityActivityHref("key", "sk-test-placeholder", context)).toBeUndefined();

4
ui/litellm-dashboard/src/utils/entityLinks.ts
View file

@ -47,7 +47,7 @@ export type ActivityScope = "key" | "team" | "user" | "model";
export interface ActivityLinkContext {
from: string;
to: string;
alert?: string;
signal?: string;
}
export function entityActivityHref(scope: ActivityScope, id: string, context: ActivityLinkContext): string | undefined {
@ -71,6 +71,6 @@ export function entityActivityHref(scope: ActivityScope, id: string, context: Ac
}
if (!href) return undefined;
const params = new URLSearchParams({ detail_tab: "activity", activity_from: context.from, activity_to: context.to });
if (context.alert) params.set("alert", context.alert);
if (context.signal) params.set("signal", context.signal);
return `${href}&${params}`;
}