diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py index 414c205b2b3..869d8bf1ebb 100644 --- a/backend/routes/allowlist.py +++ b/backend/routes/allowlist.py @@ -110,7 +110,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = ( "/v1/rust_control_plane/", # Alerting / email / IP allowlist "/alerting/", - "/issues/", + "/alerts/", "/email/", "/add/allowed_ip", "/delete/allowed_ip", @@ -154,7 +154,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset( { "/", "/routes", - "/issues", + "/alerts", "/lens", "/openapi.json", "/docs", diff --git a/litellm/integrations/SlackAlerting/usage_alerts.py b/litellm/integrations/SlackAlerting/usage_alerts.py index dd10b5aea6d..3419c445bbb 100644 --- a/litellm/integrations/SlackAlerting/usage_alerts.py +++ b/litellm/integrations/SlackAlerting/usage_alerts.py @@ -15,8 +15,8 @@ from pydantic import TypeAdapter from litellm.models.issues import IssueStore from litellm.types.integrations.slack_alerting import AlertType -from litellm.types.proxy.issues import IssueDeliveryBinding, IssueDeliveryInput, IssuePendingDelivery -from litellm.types.proxy.usage_alerts import UsageAlert +from litellm.types.proxy.issues import AlertDeliveryBinding, AlertDeliveryInput, AlertPendingDelivery +from litellm.types.proxy.usage_alerts import PerformanceSignal if TYPE_CHECKING: from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting @@ -28,7 +28,7 @@ def configured_destinations(slack: "SlackAlerting") -> tuple[str, ...]: return _URLS.validate_python(slack.alerting or ()) # pyright: ignore[reportUnknownMemberType] # validate legacy destination config -def activity_url(alert: UsageAlert, base_url: str | None) -> str: +def activity_url(alert: PerformanceSignal, base_url: str | None) -> str: if not base_url: return alert.activity_path parsed: Final = urlsplit(base_url) @@ -37,7 +37,7 @@ def activity_url(alert: UsageAlert, base_url: str | None) -> str: return base_url.rstrip("/") + "/ui" + alert.activity_path -def incident_message(alert: UsageAlert, url: str | None) -> str: +def incident_message(alert: PerformanceSignal, url: str | None) -> str: scope: Final = "Gateway" if alert.scope_type == "deployment" else f"{alert.scope_type}: {alert.scope_id}" unit: Final = ( " ms" @@ -65,14 +65,14 @@ class UsageAlertDelivery: self._slack = slack self._store = store - async def send_issue(self, pending: IssuePendingDelivery) -> bool: + async def send_issue(self, pending: AlertPendingDelivery) -> bool: urls: Final = self._webhooks() if pending.delivery.channel == "slack" else () if len(urls) > 1 and pending.delivery.slack_destination is None: if self._store is None: raise RuntimeError("Multiple alert destinations require a delivery store") bindings: Final = ( - IssueDeliveryBinding( - delivery=IssueDeliveryInput( + AlertDeliveryBinding( + delivery=AlertDeliveryInput( id=str(uuid5(NAMESPACE_URL, f"{pending.delivery.id}:{destination}")), issue_id=pending.issue.id, channel="slack", @@ -88,7 +88,7 @@ class UsageAlertDelivery: while batch := tuple(islice(bindings, 10)): await self._store.queue_deliveries(batch) return True - path: Final = "/alerts?" + urlencode({"issue": pending.issue.id}) + path: Final = "/alerts?" + urlencode({"alert": pending.issue.id}) issue_url: Final = activity_url( pending.alert.model_copy(update={"activity_path": path}), os.getenv("PROXY_BASE_URL") ) @@ -103,7 +103,7 @@ class UsageAlertDelivery: async def _send( self, - pending: IssuePendingDelivery, + pending: AlertPendingDelivery, urls: tuple[str, ...], message: str, subject: str, diff --git a/litellm/models/issues.py b/litellm/models/issues.py index 108ad877e68..3eba32f1613 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -3,23 +3,23 @@ from typing import Literal, Protocol from litellm.models.team import LiteLLM_TeamTable from litellm.types.proxy.issues import ( - Issue, - IssueActivityResponse, - IssueCohort, - IssueDeliveryBinding, - IssueFamily, - IssueFamilyList, - IssueFamilySetting, - IssueFamilyUpdate, - IssueImpact, - IssueInput, - IssueKind, - IssueList, - IssuePendingDelivery, - IssueProjectionSignal, - IssueScope, - IssueSignal, - IssueSignalList, + Alert, + AlertActivityResponse, + AlertCohort, + AlertDeliveryBinding, + AlertFamily, + AlertFamilyList, + AlertFamilySetting, + AlertFamilyUpdate, + AlertImpact, + AlertInput, + AlertKind, + AlertList, + AlertPendingDelivery, + AlertProjectionSignal, + AlertScope, + AlertSignal, + AlertSignalList, ) from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, @@ -30,20 +30,20 @@ from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus class IssueStore(Protocol): - async def get_issue(self, issue_id: str) -> Issue | None: ... + async def get_issue(self, issue_id: str) -> Alert | None: ... async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: ... async def list_issues( self, - kind: IssueKind | None = None, - family: IssueFamily | None = None, + kind: AlertKind | None = None, + family: AlertFamily | None = None, status: UsageAlertStatus | None = None, - scope_type: IssueScope | None = None, + scope_type: AlertScope | None = None, scope_id: str | None = None, limit: int = 50, offset: int = 0, - ) -> IssueList: ... + ) -> AlertList: ... async def set_status( self, @@ -51,42 +51,42 @@ class IssueStore(Protocol): status: UsageAlertStatus, expected_status: UsageAlertStatus, actor: str | None = None, - authorized_scopes: tuple[IssueCohort, ...] | None = None, - ) -> Issue | None: ... + authorized_scopes: tuple[AlertCohort, ...] | None = None, + ) -> Alert | None: ... - async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]: ... + async def status_scopes(self, issue_id: str) -> tuple[AlertCohort, ...]: ... async def upsert_issues( self, - inputs: tuple[IssueInput, ...], + inputs: tuple[AlertInput, ...], reopen: bool = False, status: UsageAlertStatus | None = None, - ) -> tuple[Issue, ...]: ... + ) -> tuple[Alert, ...]: ... - async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: ... + async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Alert, ...]: ... - async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: ... + async def add_signal_batch(self, signals: tuple[AlertSignal, ...]) -> None: ... async def list_signals( - self, issue_id: str, limit: int = 50, offset: int = 0, visible_scope: IssueCohort | None = None - ) -> IssueSignalList: ... + self, issue_id: str, limit: int = 50, offset: int = 0, visible_scope: AlertCohort | None = None + ) -> AlertSignalList: ... async def active_signals( self, day: date, limit: int = 500, cursor: str | None = None, - scope_type: IssueScope | None = None, + scope_type: AlertScope | None = None, metric: UsageAlertMetric | None = None, scope_ids: tuple[str, ...] | None = None, completed_only: bool = False, - ) -> tuple[IssueProjectionSignal, ...]: ... + ) -> tuple[AlertProjectionSignal, ...]: ... async def active_resource_issue_ids(self, cursor: str | None = None) -> tuple[str, ...]: ... async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: ... - async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: ... + async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Alert, ...]: ... async def recover_issue_ids(self, ids: tuple[str, ...], observed_at: datetime) -> int: ... @@ -96,17 +96,17 @@ class IssueStore(Protocol): async def finish_deprecation_projection(self, observed_at: datetime) -> int: ... - async def family_settings(self) -> IssueFamilyList: ... + async def family_settings(self) -> AlertFamilyList: ... async def update_family( - self, family: PerformanceIssueFamily, data: IssueFamilyUpdate - ) -> IssueFamilySetting | None: ... + self, family: PerformanceIssueFamily, data: AlertFamilyUpdate + ) -> AlertFamilySetting | None: ... - async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: ... + async def queue_deliveries(self, bindings: tuple[AlertDeliveryBinding, ...]) -> None: ... - async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: ... + async def pending_deliveries(self, limit: int = 100) -> tuple[AlertPendingDelivery, ...]: ... - async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: ... + async def pending_delivery(self, delivery_id: str) -> AlertPendingDelivery | None: ... async def reserve_delivery(self, delivery_id: str) -> bool: ... @@ -121,15 +121,15 @@ class IssueStore(Protocol): class IssueActivityStore(Protocol): async def daily_activity( - self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 - ) -> IssueActivityResponse: ... + self, cohort: AlertCohort, start: date, end: date, model_limit: int = 20 + ) -> AlertActivityResponse: ... async def keys( - self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityKeyPageResponse: ... async def users( - self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityUserPageResponse: ... - async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: ... + async def impact(self, cohort: AlertCohort, start: date, end: date) -> AlertImpact: ... diff --git a/litellm/models/usage_alerts.py b/litellm/models/usage_alerts.py index 63e2e05b024..1272511563f 100644 --- a/litellm/models/usage_alerts.py +++ b/litellm/models/usage_alerts.py @@ -2,35 +2,32 @@ from datetime import date from typing import Protocol from litellm.types.proxy.usage_alerts import ( - UsageAlert, - UsageAlertGroupList, - UsageAlertGroupUpdate, - UsageAlertGroupUpdateResult, - UsageAlertList, + AlertModelList, + AlertRule, + AlertRuleInput, + AlertRuleList, + PerformanceSignal, + PerformanceSignalList, UsageAlertMetric, - UsageAlertModelList, - UsageAlertRule, - UsageAlertRuleInput, UsageAlertRuleKind, - UsageAlertRuleList, UsageAlertScope, UsageAlertStatus, ) class UsageAlertStore(Protocol): - async def get_rule(self, rule_id: str) -> UsageAlertRule | None: ... + async def get_rule(self, rule_id: str) -> AlertRule | None: ... async def save_rule( self, - data: UsageAlertRuleInput, + data: AlertRuleInput, created_by: str | None, - existing: UsageAlertRule | None = None, + existing: AlertRule | None = None, *, overrides_defaults: bool = False, - ) -> UsageAlertRule | None: ... + ) -> AlertRule | None: ... - async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: ... + async def delete_rule(self, rule_id: str, *, existing: AlertRule | None = None) -> bool: ... async def list_rules( self, @@ -39,9 +36,9 @@ class UsageAlertStore(Protocol): limit: int = 50, offset: int = 0, kind: UsageAlertRuleKind = "all", - ) -> UsageAlertRuleList: ... + ) -> AlertRuleList: ... - async def get_alert(self, alert_id: str) -> UsageAlert | None: ... + async def get_alert(self, alert_id: str) -> PerformanceSignal | None: ... async def list_alerts( self, @@ -53,24 +50,11 @@ class UsageAlertStore(Protocol): end_date: date | None = None, limit: int = 50, offset: int = 0, - ) -> UsageAlertList: ... + ) -> PerformanceSignalList: ... - async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None: ... + async def set_status(self, alert_id: str, status: UsageAlertStatus) -> PerformanceSignal | None: ... - async def list_groups( - self, - scope_type: UsageAlertScope | None = None, - scope_id: str | None = None, - status: UsageAlertStatus | None = None, - start_date: date | None = None, - end_date: date | None = None, - limit: int = 50, - offset: int = 0, - ) -> UsageAlertGroupList: ... - - async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult: ... - - async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList: ... + async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> AlertModelList: ... async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None: ... diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5d42ce4767d..d3879ab8e75 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -595,21 +595,20 @@ class LiteLLMRoutes(enum.Enum): ) llm_api_routes_without_mcp = sorted(set(llm_api_routes) - set(mcp_inference_routes)) info_routes = [ - "/usage/alert-groups", - "/usage/alert-models", - "/usage/alert-rules", - "/usage/alert-rules/{rule_id}", - "/usage/alerts", - "/usage/alerts/{alert_id}", - "/issues", - "/issues/families", - "/issues/families/{family}", - "/issues/{issue_id}", - "/issues/{issue_id}/signals", - "/issues/{issue_id}/activity", - "/issues/{issue_id}/keys", - "/issues/{issue_id}/users", - "/issues/{issue_id}/impact", + "/alerts/models", + "/alerts/rules", + "/alerts/rules/{rule_id}", + "/alerts/signals", + "/alerts/signals/{signal_id}", + "/alerts", + "/alerts/defaults", + "/alerts/defaults/{family}", + "/alerts/{alert_id}", + "/alerts/{alert_id}/signals", + "/alerts/{alert_id}/activity", + "/alerts/{alert_id}/keys", + "/alerts/{alert_id}/users", + "/alerts/{alert_id}/impact", "/key/info", "/key/health", "/team/info", @@ -677,21 +676,20 @@ class LiteLLMRoutes(enum.Enum): management_routes = ( [ - "/usage/alert-groups", - "/usage/alert-models", - "/usage/alert-rules", - "/usage/alert-rules/{rule_id}", - "/usage/alerts", - "/usage/alerts/{alert_id}", - "/issues", - "/issues/families", - "/issues/families/{family}", - "/issues/{issue_id}", - "/issues/{issue_id}/signals", - "/issues/{issue_id}/activity", - "/issues/{issue_id}/keys", - "/issues/{issue_id}/users", - "/issues/{issue_id}/impact", + "/alerts/models", + "/alerts/rules", + "/alerts/rules/{rule_id}", + "/alerts/signals", + "/alerts/signals/{signal_id}", + "/alerts", + "/alerts/defaults", + "/alerts/defaults/{family}", + "/alerts/{alert_id}", + "/alerts/{alert_id}/signals", + "/alerts/{alert_id}/activity", + "/alerts/{alert_id}/keys", + "/alerts/{alert_id}/users", + "/alerts/{alert_id}/impact", # user "/user/new", "/management/v1/users/bulk", @@ -903,21 +901,20 @@ class LiteLLMRoutes(enum.Enum): ) usage_alert_routes = ( - "/usage/alert-groups", - "/usage/alert-models", - "/usage/alert-rules", - "/usage/alert-rules/{rule_id}", - "/usage/alerts", - "/usage/alerts/{alert_id}", - "/issues", - "/issues/families", - "/issues/families/{family}", - "/issues/{issue_id}", - "/issues/{issue_id}/signals", - "/issues/{issue_id}/activity", - "/issues/{issue_id}/keys", - "/issues/{issue_id}/users", - "/issues/{issue_id}/impact", + "/alerts/models", + "/alerts/rules", + "/alerts/rules/{rule_id}", + "/alerts/signals", + "/alerts/signals/{signal_id}", + "/alerts", + "/alerts/defaults", + "/alerts/defaults/{family}", + "/alerts/{alert_id}", + "/alerts/{alert_id}/signals", + "/alerts/{alert_id}/activity", + "/alerts/{alert_id}/keys", + "/alerts/{alert_id}/users", + "/alerts/{alert_id}/impact", ) self_managed_routes = [ diff --git a/litellm/proxy/alerting_endpoints/issues.py b/litellm/proxy/alerting_endpoints/issues.py index 27b663abc18..296503b3856 100644 --- a/litellm/proxy/alerting_endpoints/issues.py +++ b/litellm/proxy/alerting_endpoints/issues.py @@ -1,4 +1,4 @@ -"""Unified issue inbox and investigations backed by existing daily aggregates.""" +"""Alerts and their supporting Activity backed by existing daily aggregates.""" from collections.abc import Generator from contextlib import contextmanager @@ -23,19 +23,19 @@ from litellm.proxy.management_endpoints.common_utils import team_member_has_perm from litellm.repositories.analytics_budget import AnalyticsCapacityError from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.issues import ( - Issue, - IssueActivityResponse, - IssueCohort, - IssueFamily, - IssueFamilyList, - IssueFamilySetting, - IssueFamilyUpdate, - IssueImpact, - IssueKind, - IssueList, - IssueScope, - IssueSignalList, - IssueUpdate, + Alert, + AlertActivityResponse, + AlertCohort, + AlertFamily, + AlertFamilyList, + AlertFamilySetting, + AlertFamilyUpdate, + AlertImpact, + AlertKind, + AlertList, + AlertScope, + AlertSignalList, + AlertUpdate, ) from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, @@ -44,18 +44,18 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertStatus -router: Final = APIRouter(prefix="/issues", tags=["issues"]) +router: Final = APIRouter(prefix="/alerts", tags=["alerts"]) def require_issues(repository: IssueStore | None) -> IssueStore: if repository is None: - raise HTTPException(status_code=503, detail="Issues require a database connection") + raise HTTPException(status_code=503, detail="Alerts require a database connection") return repository def require_activity(repository: IssueActivityStore | None) -> IssueActivityStore: if repository is None: - raise HTTPException(status_code=503, detail="Issue Activity requires a database connection") + raise HTTPException(status_code=503, detail="Alert Activity requires a database connection") return repository @@ -68,16 +68,16 @@ def activity_budget() -> Generator[None]: async def authorized_issue( - issue_id: str, + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, *, write: bool = False, -) -> Issue: - issue: Final = await require_issues(repository).get_issue(issue_id) +) -> Alert: + issue: Final = await require_issues(repository).get_issue(alert_id) if issue is None: - raise HTTPException(status_code=404, detail="Issue not found") + raise HTTPException(status_code=404, detail="Alert not found") await require_scope( user, require_repository(alerts), @@ -88,7 +88,7 @@ async def authorized_issue( return issue -async def can_view_activity(issue: Issue, user: User, repository: IssueStore) -> bool: +async def can_view_activity(issue: Alert, user: User, repository: IssueStore) -> bool: if issue.cohort.scope_type != "team" or user.user_role in ( LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, @@ -102,14 +102,14 @@ async def can_view_activity(issue: Issue, user: User, repository: IssueStore) -> ) -async def require_activity_access(issue: Issue, user: User, repository: IssuesRepositoryDep) -> None: +async def require_activity_access(issue: Alert, user: User, repository: IssuesRepositoryDep) -> None: if not await can_view_activity(issue, user, require_issues(repository)): - raise HTTPException(status_code=403, detail="Full team Activity permission is required for this investigation") + raise HTTPException(status_code=403, detail="Full team Activity permission is required for this alert") async def require_status_scopes( - issue: Issue, user: User, repository: IssueStore, alerts: UsageAlertsRepositoryDep -) -> tuple[IssueCohort, ...] | None: + issue: Alert, user: User, repository: IssueStore, alerts: UsageAlertsRepositoryDep +) -> tuple[AlertCohort, ...] | None: store: Final = require_repository(alerts) await require_scope( user, @@ -130,7 +130,7 @@ async def require_status_scopes( return scopes -def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]: +def activity_range(issue: Alert, start_date: date | None, end_date: date | None) -> tuple[date, date]: start: Final = start_date or issue.range_start end: Final = end_date or issue.range_end if start > end or (end - start).days > 365: @@ -138,44 +138,44 @@ def activity_range(issue: Issue, start_date: date | None, end_date: date | None) return start, end -@router.get("", response_model=IssueList) -async def list_issues( +@router.get("", response_model=AlertList) +async def list_alerts( user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, - kind: IssueKind | None = None, - family: IssueFamily | None = None, + kind: AlertKind | None = None, + family: AlertFamily | None = None, status: UsageAlertStatus | None = None, - scope_type: IssueScope | None = None, + scope_type: AlertScope | None = None, scope_id: ScopeId = None, limit: Limit = 50, offset: Offset = 0, -) -> IssueList: +) -> AlertList: await require_scope( user, require_repository(alerts), "model" if scope_type == "provider" else scope_type, scope_id, write=False ) return await require_issues(repository).list_issues(kind, family, status, scope_type, scope_id, limit, offset) -@router.get("/families", response_model=IssueFamilyList) -async def list_issue_families( +@router.get("/defaults", response_model=AlertFamilyList) +async def list_alert_defaults( user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, -) -> IssueFamilyList: +) -> AlertFamilyList: await require_scope(user, require_repository(alerts), "deployment", None, write=False) return await require_issues(repository).family_settings() -@router.patch("/families/{family}", response_model=IssueFamilySetting) -async def update_issue_family( +@router.patch("/defaults/{family}", response_model=AlertFamilySetting) +async def update_alert_default( family: PerformanceIssueFamily, - data: IssueFamilyUpdate, + data: AlertFamilyUpdate, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, -) -> IssueFamilySetting: - require_write_access(user, f"/issues/families/{family}") +) -> AlertFamilySetting: + require_write_access(user, f"/alerts/defaults/{family}") await require_scope(user, require_repository(alerts), "deployment", None, write=True) try: result: Final = await require_issues(repository).update_family(family, data) @@ -188,17 +188,17 @@ async def update_issue_family( return result -@router.get("/{issue_id}", response_model=Issue) -async def get_issue( - issue_id: str, +@router.get("/{alert_id}", response_model=Alert) +async def get_alert( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, -) -> Issue: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) +) -> Alert: + issue: Final = await authorized_issue(alert_id, user, repository, alerts) activity_allowed: Final = await can_view_activity(issue, user, require_issues(repository)) try: - require_write_access(user, f"/issues/{issue_id}") + require_write_access(user, f"/alerts/{alert_id}") await require_status_scopes(issue, user, require_issues(repository), alerts) except HTTPException as exc: if exc.status_code != 403: @@ -207,37 +207,37 @@ async def get_issue( return issue.model_copy(update={"can_update_status": True, "can_view_activity": activity_allowed}) -@router.patch("/{issue_id}", response_model=Issue) -async def update_issue( - issue_id: str, - data: IssueUpdate, +@router.patch("/{alert_id}", response_model=Alert) +async def update_alert( + alert_id: str, + data: AlertUpdate, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, -) -> Issue: - require_write_access(user, f"/issues/{issue_id}") - issue: Final = await authorized_issue(issue_id, user, repository, alerts) +) -> Alert: + require_write_access(user, f"/alerts/{alert_id}") + issue: Final = await authorized_issue(alert_id, user, repository, alerts) scopes: Final = await require_status_scopes(issue, user, require_issues(repository), alerts) if issue.kind == "model_deprecation" and not issue.active and data.status == "open": - raise HTTPException(status_code=409, detail="This retirement issue reopens when the model receives new traffic") + raise HTTPException(status_code=409, detail="This retirement alert reopens when the model receives new traffic") changed: Final = await require_issues(repository).set_status( - issue_id, data.status, data.expected_status, user.user_id, authorized_scopes=scopes + alert_id, data.status, data.expected_status, user.user_id, authorized_scopes=scopes ) if changed is None: - raise HTTPException(status_code=409, detail="Issue status changed; reload and retry") + raise HTTPException(status_code=409, detail="Alert status changed; reload and retry") return changed -@router.get("/{issue_id}/signals", response_model=IssueSignalList) -async def list_issue_signals( - issue_id: str, +@router.get("/{alert_id}/signals", response_model=AlertSignalList) +async def list_alert_signals( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, limit: Limit = 50, offset: Offset = 0, -) -> IssueSignalList: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) +) -> AlertSignalList: + issue: Final = await authorized_issue(alert_id, user, repository, alerts) visible_scope: Final = ( None if user.user_role @@ -247,12 +247,12 @@ async def list_issue_signals( ) else issue.cohort ) - return await require_issues(repository).list_signals(issue_id, limit, offset, visible_scope) + return await require_issues(repository).list_signals(alert_id, limit, offset, visible_scope) -@router.get("/{issue_id}/activity", response_model=IssueActivityResponse) -async def get_issue_activity( - issue_id: str, +@router.get("/{alert_id}/activity", response_model=AlertActivityResponse) +async def get_alert_activity( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, @@ -260,17 +260,17 @@ async def get_issue_activity( start_date: date | None = None, end_date: date | None = None, model_limit: Limit = 20, -) -> IssueActivityResponse: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) +) -> AlertActivityResponse: + issue: Final = await authorized_issue(alert_id, user, repository, alerts) await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) with activity_budget(): return await require_activity(activity).daily_activity(issue.cohort, start, end, model_limit) -@router.get("/{issue_id}/keys", response_model=DailyActivityKeyPageResponse) -async def get_issue_keys( - issue_id: str, +@router.get("/{alert_id}/keys", response_model=DailyActivityKeyPageResponse) +async def get_alert_keys( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, @@ -280,16 +280,16 @@ async def get_issue_keys( limit: Limit = 20, offset: Offset = 0, ) -> DailyActivityKeyPageResponse: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) + issue: Final = await authorized_issue(alert_id, user, repository, alerts) await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) with activity_budget(): return await require_activity(activity).keys(issue.cohort, start, end, limit, offset) -@router.get("/{issue_id}/users", response_model=DailyActivityUserPageResponse) -async def get_issue_users( - issue_id: str, +@router.get("/{alert_id}/users", response_model=DailyActivityUserPageResponse) +async def get_alert_users( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, @@ -299,7 +299,7 @@ async def get_issue_users( limit: Limit = 20, offset: Offset = 0, ) -> DailyActivityUserPageResponse: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) + issue: Final = await authorized_issue(alert_id, user, repository, alerts) await require_activity_access(issue, user, repository) if issue.cohort.scope_type == "team": raise HTTPException(status_code=422, detail="Historical team aggregates do not retain user membership") @@ -308,17 +308,17 @@ async def get_issue_users( return await require_activity(activity).users(issue.cohort, start, end, limit, offset) -@router.get("/{issue_id}/impact", response_model=IssueImpact) -async def get_issue_impact( - issue_id: str, +@router.get("/{alert_id}/impact", response_model=AlertImpact) +async def get_alert_impact( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, activity: IssueActivityRepositoryDep, start_date: date | None = None, end_date: date | None = None, -) -> IssueImpact: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) +) -> AlertImpact: + issue: Final = await authorized_issue(alert_id, user, repository, alerts) await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) with activity_budget(): diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py index 3950efdc1b4..e3cc688b4d9 100644 --- a/litellm/proxy/alerting_endpoints/usage_alerts.py +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -1,4 +1,4 @@ -"""Public management API for daily usage alert rules and incidents.""" +"""Performance rules and read-only detection evidence.""" from datetime import date from typing import Annotated, Final, TypeAlias @@ -14,28 +14,24 @@ from litellm.proxy.deps.repositories import UsageAlertsRepositoryDep from litellm.repositories.daily_activity_buckets import api_key_from_bucket from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.usage_alerts import ( - UsageAlert, - UsageAlertGroupList, - UsageAlertGroupUpdate, - UsageAlertGroupUpdateResult, - UsageAlertList, + AlertModelList, + AlertRule, + AlertRuleInput, + AlertRuleList, + PerformanceSignal, + PerformanceSignalList, UsageAlertMetric, - UsageAlertModelList, - UsageAlertRule, - UsageAlertRuleInput, UsageAlertRuleKind, - UsageAlertRuleList, UsageAlertScope, UsageAlertStatus, - UsageAlertUpdate, ) -router: Final = APIRouter(prefix="/usage", tags=["usage alerts"]) +router: Final = APIRouter(prefix="/alerts", tags=["alerts"]) def require_repository(repository: UsageAlertStore | None) -> UsageAlertStore: if repository is None: - raise HTTPException(status_code=503, detail="Usage alerts require a database connection") + raise HTTPException(status_code=503, detail="Alert signals require a database connection") return repository @@ -71,9 +67,7 @@ def require_write_access(user: UserAPIKeyAuth, route: str) -> None: raise HTTPException(status_code=403, detail="This credential does not grant alert write access") -def require_notification_access( - user: UserAPIKeyAuth, data: UsageAlertRuleInput, existing: UsageAlertRule | None = None -) -> None: +def require_notification_access(user: UserAPIKeyAuth, data: AlertRuleInput, existing: AlertRule | None = None) -> None: if user.user_role == LitellmUserRoles.PROXY_ADMIN: return if data.notification_channels or data.email_recipients: @@ -97,16 +91,16 @@ async def require_scope( if user.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: if not write: return - raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts") + raise HTTPException(status_code=403, detail="Read-only users cannot manage alerts") if user.user_role not in ( LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, LitellmUserRoles.ORG_ADMIN, LitellmUserRoles.TEAM, ): - raise HTTPException(status_code=403, detail="Usage alerts require a management role") + raise HTTPException(status_code=403, detail="Alert signals require a management role") if write and user.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY: - raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts") + raise HTTPException(status_code=403, detail="Read-only users cannot manage alerts") if scope_id is None: raise HTTPException(status_code=403, detail="Specify a scope you can access") if scope_type == "user" and user.user_id is not None and scope_id == user.user_id: @@ -128,8 +122,8 @@ async def require_scope( raise HTTPException(status_code=403, detail="You cannot access this alert scope") -@router.get("/alert-rules", response_model=UsageAlertRuleList) -async def list_usage_alert_rules( +@router.get("/rules", response_model=AlertRuleList) +async def list_alert_rules( user: User, repository: Repository, scope_type: UsageAlertScope | None = None, @@ -137,59 +131,30 @@ async def list_usage_alert_rules( limit: Limit = 50, offset: Offset = 0, kind: UsageAlertRuleKind = "all", -) -> UsageAlertRuleList: +) -> AlertRuleList: store: Final = require_repository(repository) await require_scope(user, store, scope_type, scope_id, write=False) return await store.list_rules(scope_type, scope_id, limit, offset, kind) -@router.get("/alert-groups", response_model=UsageAlertGroupList) -async def list_usage_alert_groups( - user: User, - repository: Repository, - scope_type: UsageAlertScope | None = None, - scope_id: ScopeId = None, - status: UsageAlertStatus | None = None, - start_date: date | None = None, - end_date: date | None = None, - limit: Limit = 50, - offset: Offset = 0, -) -> UsageAlertGroupList: - store: Final = require_repository(repository) - await require_scope(user, store, scope_type, scope_id, write=False) - if start_date is not None and end_date is not None and start_date > end_date: - raise HTTPException(status_code=422, detail="start_date must not be after end_date") - return await store.list_groups(scope_type, scope_id, status, start_date, end_date, limit, offset) - - -@router.patch("/alert-groups", response_model=UsageAlertGroupUpdateResult) -async def update_usage_alert_group( - data: UsageAlertGroupUpdate, user: User, repository: Repository -) -> UsageAlertGroupUpdateResult: - store: Final = require_repository(repository) - require_write_access(user, "/usage/alert-groups") - await require_scope(user, store, data.scope_type, data.scope_id, write=True) - return await store.set_group_status(data) - - -@router.get("/alert-models", response_model=UsageAlertModelList) -async def list_usage_alert_models( +@router.get("/models", response_model=AlertModelList) +async def list_alert_models( user: User, repository: Repository, search: ScopeId = None, cursor: ScopeId = None, limit: Limit = 20, -) -> UsageAlertModelList: +) -> AlertModelList: store: Final = require_repository(repository) await require_scope(user, store, "model", None, write=False) return await store.list_reported_models(search, cursor, limit) -@router.post("/alert-rules", response_model=UsageAlertRule, status_code=201) -async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, repository: Repository) -> UsageAlertRule: - """Create a daily usage alert rule for a scope the caller can manage.""" +@router.post("/rules", response_model=AlertRule, status_code=201) +async def create_alert_rule(data: AlertRuleInput, user: User, repository: Repository) -> AlertRule: + """Create a daily performance rule for a scope the caller can manage.""" store: Final = require_repository(repository) - require_write_access(user, "/usage/alert-rules") + require_write_access(user, "/alerts/rules") await require_scope(user, store, data.scope_type, data.scope_id, write=True) require_notification_access(user, data) saved: Final = await store.save_rule( @@ -200,8 +165,8 @@ async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, reposit return saved -@router.get("/alert-rules/{rule_id}", response_model=UsageAlertRule) -async def get_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> UsageAlertRule: +@router.get("/rules/{rule_id}", response_model=AlertRule) +async def get_alert_rule(rule_id: str, user: User, repository: Repository) -> AlertRule: store: Final = require_repository(repository) rule: Final = await store.get_rule(rule_id) if rule is None: @@ -210,12 +175,10 @@ async def get_usage_alert_rule(rule_id: str, user: User, repository: Repository) return rule -@router.put("/alert-rules/{rule_id}", response_model=UsageAlertRule) -async def update_usage_alert_rule( - rule_id: str, data: UsageAlertRuleInput, user: User, repository: Repository -) -> UsageAlertRule: +@router.put("/rules/{rule_id}", response_model=AlertRule) +async def update_alert_rule(rule_id: str, data: AlertRuleInput, user: User, repository: Repository) -> AlertRule: store: Final = require_repository(repository) - require_write_access(user, f"/usage/alert-rules/{rule_id}") + require_write_access(user, f"/alerts/rules/{rule_id}") rule: Final = await store.get_rule(rule_id) if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") @@ -237,10 +200,10 @@ async def update_usage_alert_rule( return saved -@router.delete("/alert-rules/{rule_id}", status_code=204) -async def delete_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> Response: +@router.delete("/rules/{rule_id}", status_code=204) +async def delete_alert_rule(rule_id: str, user: User, repository: Repository) -> Response: store: Final = require_repository(repository) - require_write_access(user, f"/usage/alert-rules/{rule_id}") + require_write_access(user, f"/alerts/rules/{rule_id}") rule: Final = await store.get_rule(rule_id) if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") @@ -251,8 +214,8 @@ async def delete_usage_alert_rule(rule_id: str, user: User, repository: Reposito return Response(status_code=204) -@router.get("/alerts", response_model=UsageAlertList) -async def list_usage_alerts( +@router.get("/signals", response_model=PerformanceSignalList) +async def list_performance_signals( user: User, repository: Repository, scope_type: UsageAlertScope | None = None, @@ -263,7 +226,7 @@ async def list_usage_alerts( end_date: date | None = None, limit: Limit = 50, offset: Offset = 0, -) -> UsageAlertList: +) -> PerformanceSignalList: store: Final = require_repository(repository) await require_scope(user, store, scope_type, scope_id, write=False) if start_date is not None and end_date is not None and start_date > end_date: @@ -271,25 +234,11 @@ async def list_usage_alerts( return await store.list_alerts(scope_type, scope_id, metric, status, start_date, end_date, limit, offset) -@router.get("/alerts/{alert_id}", response_model=UsageAlert) -async def get_usage_alert(alert_id: str, user: User, repository: Repository) -> UsageAlert: +@router.get("/signals/{signal_id}", response_model=PerformanceSignal) +async def get_performance_signal(signal_id: str, user: User, repository: Repository) -> PerformanceSignal: store: Final = require_repository(repository) - alert: Final = await store.get_alert(alert_id) + alert: Final = await store.get_alert(signal_id) if alert is None: - raise HTTPException(status_code=404, detail="Usage alert not found") + raise HTTPException(status_code=404, detail="Alert signal not found") await require_scope(user, store, alert.scope_type, alert.scope_id, write=False) return alert - - -@router.patch("/alerts/{alert_id}", response_model=UsageAlert) -async def update_usage_alert(alert_id: str, data: UsageAlertUpdate, user: User, repository: Repository) -> UsageAlert: - store: Final = require_repository(repository) - require_write_access(user, f"/usage/alerts/{alert_id}") - alert: Final = await store.get_alert(alert_id) - if alert is None: - raise HTTPException(status_code=404, detail="Usage alert not found") - await require_scope(user, store, alert.scope_type, alert.scope_id, write=True) - updated: Final = await store.set_status(alert_id, data.status) - if updated is None: - raise HTTPException(status_code=404, detail="Usage alert not found") - return updated diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 5c8ba44d1b5..cc37fb38d83 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1986,19 +1986,21 @@ class DBSpendUpdateWriter: ) -> None: transactions: Final = await queue.flush_and_get_aggregated_daily_spend_update_transactions() commit_started: Final = asyncio.Event() - commit_task: Final = _start_daily_spend_commit( - commit_started, - lambda: commit( + + async def commit_and_record_freshness() -> None: + await commit( n_retry_times=n_retry_times, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj, daily_spend_transactions=cast(dict[str, _DailySpendTransactionT], transactions), - ), - ) - try: - await asyncio.shield(commit_task) + ) + commit_started.set() if entity_type in ("user", "team"): await self.usage_alert_flush_recorder.record(prisma_client, entity_type) + + commit_task: Final = _start_daily_spend_commit(commit_started, commit_and_record_freshness) + try: + await asyncio.shield(commit_task) except asyncio.CancelledError: if commit_started.is_set(): queue.track_interrupted_commit( diff --git a/litellm/proxy/spend_tracking/daily_usage_alerts.py b/litellm/proxy/spend_tracking/daily_usage_alerts.py index 2e449c86d7c..3a0895b0027 100644 --- a/litellm/proxy/spend_tracking/daily_usage_alerts.py +++ b/litellm/proxy/spend_tracking/daily_usage_alerts.py @@ -8,7 +8,7 @@ from typing import Final, Literal, Protocol from litellm.repositories.usage_alert_metrics import DailyAlertMetrics from litellm.types.proxy.usage_alert_defaults import is_default_usage_rule -from litellm.types.proxy.usage_alerts import UsageAlertRule +from litellm.types.proxy.usage_alerts import AlertRule class ComparisonRule(Protocol): @@ -76,7 +76,7 @@ def compare_daily_usage( if ( rule.metric == "requests" and rule.direction == "decrease" - and isinstance(rule, UsageAlertRule) + and isinstance(rule, AlertRule) and is_default_usage_rule(rule) and any( sum(row.requests for row in group) < rule.min_requests diff --git a/litellm/proxy/spend_tracking/deprecation_issues.py b/litellm/proxy/spend_tracking/deprecation_issues.py index 7215bbca1c8..3113cc1e16c 100644 --- a/litellm/proxy/spend_tracking/deprecation_issues.py +++ b/litellm/proxy/spend_tracking/deprecation_issues.py @@ -22,7 +22,7 @@ from litellm.repositories.analytics_budget import ( ) from litellm.repositories.config_repository import BoundedConfigRepository from litellm.repositories.usage_alert_metrics import UsageAlertMetricsRepository -from litellm.types.proxy.issues import IssueCohort, IssueDeprecation, IssueImpact, IssueInput, IssueSignal +from litellm.types.proxy.issues import AlertCohort, AlertDeprecation, AlertImpact, AlertInput, AlertSignal from litellm.types.proxy.model_deprecation import ( DeprecationAlertState, ModelDeprecationInfo, @@ -84,7 +84,7 @@ def deprecation_issue( usage: ModelDeprecationUsage, state: DeprecationAlertState, now: datetime, -) -> IssueInput: +) -> AlertInput: today: Final = now.astimezone(timezone.utc).date() model: Final = candidate.model name: Final = model.model_name or usage.model @@ -93,21 +93,21 @@ def deprecation_issue( if model.deprecation_date < today else f"{name} retires on {model.deprecation_date.isoformat()}" ) - return IssueInput( + return AlertInput( id=f"deprecation:{candidate.state_key.removeprefix('model_deprecation_alert:')}", cohort_key=candidate.state_key, kind="model_deprecation", family="retirement", title=title, - cohort=IssueCohort(scope_type="model", scope_id=usage.model), + cohort=AlertCohort(scope_type="model", scope_id=usage.model), observed_on=usage.last_used_date, observed_at=now, range_start=today - timedelta(days=29), range_end=today, grouping_reason="model_deprecation", explanation="This model has recorded requests in the last 30 UTC dates and is retiring within 30 days or has retired.", - impact=IssueImpact(keys=usage.keys, users=usage.users, teams=usage.teams, models=1, requests=usage.requests), - deprecation=IssueDeprecation( + impact=AlertImpact(keys=usage.keys, users=usage.users, teams=usage.teams, models=1, requests=usage.requests), + deprecation=AlertDeprecation( model=model, usage_model=usage.model, requests=usage.requests, @@ -124,8 +124,8 @@ def used_deprecation_issues( usage: Mapping[str, ModelDeprecationUsage], states: Mapping[str, DeprecationAlertState], now: datetime, -) -> tuple[IssueInput, ...]: - def build(candidate: DeprecationCandidate) -> IssueInput: +) -> tuple[AlertInput, ...]: + def build(candidate: DeprecationCandidate) -> AlertInput: state: Final = states.get(candidate.state_key) or DeprecationAlertState( model=candidate.usage_model, deprecation_date=candidate.model.deprecation_date ) @@ -175,8 +175,8 @@ class DeprecationIssueProjector: saved: Final = await self._issues.upsert_issues(issues) await self._issues.add_signal_batch( tuple( - IssueSignal( - issue_id=issue.id, source="model_deprecation", source_id=issue.id, deprecation=issue.deprecation + AlertSignal( + alert_id=issue.id, source="model_deprecation", source_id=issue.id, deprecation=issue.deprecation ) for issue in saved ) diff --git a/litellm/proxy/spend_tracking/performance_issues.py b/litellm/proxy/spend_tracking/performance_issues.py index b2e055d0edd..d4a9a6e4654 100644 --- a/litellm/proxy/spend_tracking/performance_issues.py +++ b/litellm/proxy/spend_tracking/performance_issues.py @@ -23,17 +23,17 @@ from litellm.repositories.usage_alert_metrics import ( ) from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.proxy.issues import ( - Issue, - IssueCohort, - IssueComparison, - IssueDeliveryBinding, - IssueDeliveryInput, - IssueImpact, - IssueInput, - IssuePendingDelivery, - IssueProjectionSignal, - IssueSignal, - IssueSignalGroupingReason, + Alert, + AlertCohort, + AlertComparison, + AlertDeliveryBinding, + AlertDeliveryInput, + AlertImpact, + AlertInput, + AlertPendingDelivery, + AlertProjectionSignal, + AlertSignal, + AlertSignalGroupingReason, ) from litellm.types.proxy.usage_alert_defaults import ( PerformanceIssueFamily, @@ -41,12 +41,12 @@ from litellm.types.proxy.usage_alert_defaults import ( is_default_usage_rule, ) from litellm.types.proxy.usage_alerts import ( + AlertRule, UsageAlertDirection, UsageAlertMetric, UsageAlertMetricPoint, UsageAlertProjectionCursor, UsageAlertProjectionWork, - UsageAlertRule, UsageAlertScope, ) @@ -54,30 +54,30 @@ _SCOPES: Final[tuple[UsageAlertScope, ...]] = ("key", "user", "team", "model", " _METRICS: Final[tuple[UsageAlertMetric, ...]] = ("spend", "latency", "error_rate", "errors", "requests") -class IssueDelivery(Protocol): - async def send_issue(self, pending: IssuePendingDelivery) -> bool: ... +class AlertDelivery(Protocol): + async def send_issue(self, pending: AlertPendingDelivery) -> bool: ... @dataclass(frozen=True, slots=True) class _Candidate: - source: IssueProjectionSignal + source: AlertProjectionSignal comparison: DailyComparison rows: tuple[DailyAlertMetrics, ...] - impact: IssueImpact + impact: AlertImpact window: IssueMetricWindow @dataclass(frozen=True, slots=True) class _Target: - issue: IssueInput - reason: IssueSignalGroupingReason = "same_resource" + issue: AlertInput + reason: AlertSignalGroupingReason = "same_resource" explanation: str = "" @dataclass(frozen=True, slots=True) class _ProviderGroup: evidence: ProviderLatencyEvidence - rule: UsageAlertRule + rule: AlertRule def _family(metric: UsageAlertMetric) -> PerformanceIssueFamily: @@ -92,7 +92,7 @@ def _cohort_key(family: str, scope: str, entity: str | None, direction: UsageAle return str(uuid5(NAMESPACE_URL, f"litellm:issue:{family}:{scope}:{entity}:{direction}")) -def _window(source: IssueProjectionSignal, day: date) -> IssueMetricWindow: +def _window(source: AlertProjectionSignal, day: date) -> IssueMetricWindow: return IssueMetricWindow( source.alert.id, source.alert.scope_id or "deployment", day - timedelta(days=source.rule.baseline_days), day ) @@ -109,8 +109,8 @@ def _history(candidate: _Candidate, metric: UsageAlertMetric, observed: float) - ) -def _comparison(candidate: _Candidate) -> IssueComparison: - return IssueComparison( +def _comparison(candidate: _Candidate) -> AlertComparison: + return AlertComparison( metric=candidate.source.rule.metric, direction=candidate.source.rule.direction, observed_value=candidate.comparison.observed, @@ -123,11 +123,11 @@ def _comparison(candidate: _Candidate) -> IssueComparison: ) -def _request_context(candidate: _Candidate) -> IssueComparison: +def _request_context(candidate: _Candidate) -> AlertComparison: history: Final = tuple(row for row in candidate.rows if row.day < candidate.window.day) baseline: Final = sum(row.requests for row in history) / len({row.day for row in history}) observed: Final = sum(row.requests for row in candidate.rows if row.day == candidate.window.day) - return IssueComparison( + return AlertComparison( metric="requests", direction="increase" if observed >= baseline else "decrease", observed_value=observed, @@ -139,7 +139,7 @@ def _request_context(candidate: _Candidate) -> IssueComparison: ) -def _resource_issue(candidate: _Candidate, now: datetime) -> IssueInput: +def _resource_issue(candidate: _Candidate, now: datetime) -> AlertInput: rule: Final = candidate.source.rule family: Final = _family(rule.metric) cohort_key: Final = _cohort_key(family, rule.scope_type, candidate.source.alert.scope_id, rule.direction) @@ -150,7 +150,7 @@ def _resource_issue(candidate: _Candidate, now: datetime) -> IssueInput: "errors": "Failed requests", "requests": "Traffic", }[rule.metric] - return IssueInput( + return AlertInput( id=str( uuid5(NAMESPACE_URL, f"{cohort_key}:{candidate.source.alert.id}:{candidate.window.day}:{rule.updated_at}") ), @@ -158,7 +158,7 @@ def _resource_issue(candidate: _Candidate, now: datetime) -> IssueInput: kind="performance", family=family, title=f"{title} {'increased' if rule.direction == 'increase' else 'decreased'}", - cohort=IssueCohort(scope_type=rule.scope_type, scope_id=candidate.source.alert.scope_id), + cohort=AlertCohort(scope_type=rule.scope_type, scope_id=candidate.source.alert.scope_id), observed_on=candidate.window.day, observed_at=now, range_start=candidate.window.start, @@ -186,7 +186,7 @@ def _volume_explains_spend(traffic: _Candidate, spend: _Candidate) -> bool: return 0.8 <= cost_ratio / request_ratio <= 1.2 -def _bindings(candidate: _Candidate, issue_id: str) -> tuple[IssueDeliveryBinding, ...]: +def _bindings(candidate: _Candidate, issue_id: str) -> tuple[AlertDeliveryBinding, ...]: rule: Final = candidate.source.rule destinations: Final = ( tuple(("email", address) for address in rule.email_recipients) if "email" in rule.notification_channels else () @@ -196,8 +196,8 @@ def _bindings(candidate: _Candidate, issue_id: str) -> tuple[IssueDeliveryBindin *((("slack", "configured-slack-route"),) if "slack" in rule.notification_channels else ()), ) return tuple( - IssueDeliveryBinding( - delivery=IssueDeliveryInput( + AlertDeliveryBinding( + delivery=AlertDeliveryInput( id=str(uuid5(NAMESPACE_URL, f"{issue_id}:{channel}:{destination}")), issue_id=issue_id, channel="email" if channel == "email" else "slack", @@ -217,13 +217,13 @@ class PerformanceIssueProjector: store: IssueStore, metrics: UsageAlertMetricsRepository, providers: ProviderLatencyRepository, - delivery: IssueDelivery, + delivery: AlertDelivery, ) -> None: self._store = store self._metrics = metrics self._providers = providers self._delivery = delivery - self._provider_rule: UsageAlertRule | None = None + self._provider_rule: AlertRule | None = None self._provider_cache: OrderedDict[str, _ProviderGroup | None] = OrderedDict() # mutable-ok: max 32 providers async def run_pending( @@ -300,7 +300,7 @@ class PerformanceIssueProjector: ) async def _candidates( - self, sources: tuple[IssueProjectionSignal, ...], scope: UsageAlertScope, day: date + self, sources: tuple[AlertProjectionSignal, ...], scope: UsageAlertScope, day: date ) -> tuple[_Candidate, ...]: eligible: Final = sources windows: Final = tuple(_window(source, day) for source in eligible) @@ -318,7 +318,7 @@ class PerformanceIssueProjector: ) @staticmethod - def _candidate(source: IssueProjectionSignal, days: tuple[IssueMetricDay, ...], day: date) -> _Candidate | None: + def _candidate(source: AlertProjectionSignal, days: tuple[IssueMetricDay, ...], day: date) -> _Candidate | None: rows: Final = tuple(item.metrics for item in days) comparison: Final = compare_daily_usage(source.rule, rows, day) if comparison is None or not comparison.breached: @@ -327,7 +327,7 @@ class PerformanceIssueProjector: source, comparison, rows, - IssueImpact( + AlertImpact( keys=days[0].keys if days else None, users=days[0].users if days else None, teams=1 if source.rule.scope_type == "team" else None, @@ -339,7 +339,7 @@ class PerformanceIssueProjector: async def _project_page( self, - sources: tuple[IssueProjectionSignal, ...], + sources: tuple[AlertProjectionSignal, ...], scope: UsageAlertScope, day: date, now: datetime, @@ -542,7 +542,7 @@ class PerformanceIssueProjector: ) @staticmethod - def _same_policy(left: UsageAlertRule, right: UsageAlertRule) -> bool: + def _same_policy(left: AlertRule, right: AlertRule) -> bool: return (left.baseline_days, left.min_history_days, left.min_requests, left.multiplier, left.min_value) == ( right.baseline_days, right.min_history_days, @@ -579,21 +579,21 @@ class PerformanceIssueProjector: async def _prior_provider_page( self, - existing: tuple[Issue, ...], + existing: tuple[Alert, ...], day: date, - rule: UsageAlertRule, + rule: AlertRule, now: datetime, - ) -> tuple[tuple[Issue, bool, ProviderLatencyEvidence | None], ...]: + ) -> tuple[tuple[Alert, bool, ProviderLatencyEvidence | None], ...]: states: Final = tuple([await self._prior_provider(issue, day, rule) for issue in existing]) await self._store.recover_issue_ids(tuple(issue.id for issue, recovered, _ in states if recovered), now) return states async def _prior_provider( self, - issue: Issue, + issue: Alert, day: date, - rule: UsageAlertRule, - ) -> tuple[Issue, bool, ProviderLatencyEvidence | None]: + rule: AlertRule, + ) -> tuple[Alert, bool, ProviderLatencyEvidence | None]: if await self._providers.cohort_recovered(issue.cohort.model_pairs, day, rule): return issue, True, None evidence: Final = await self._providers.latency_cohort(issue.cohort.model_pairs, day, rule) @@ -665,7 +665,7 @@ class PerformanceIssueProjector: return None key: Final = _cohort_key("latency", "provider", evidence.cohort.scope_id, "increase") return _Target( - IssueInput( + AlertInput( id=str( uuid5( NAMESPACE_URL, f"{key}:{item.source.alert.id}:{item.window.day}:{item.source.rule.updated_at}" @@ -701,11 +701,11 @@ class PerformanceIssueProjector: saved: Final = {item.cohort_key: item for item in await self._store.upsert_issues(inputs)} await self._store.add_signal_batch( tuple( - IssueSignal( - issue_id=saved[targets[candidate.source.alert.id].issue.cohort_key].id, - source="usage_alert", + AlertSignal( + alert_id=saved[targets[candidate.source.alert.id].issue.cohort_key].id, + source="performance", source_id=candidate.source.alert.id, - usage_alert=candidate.source.alert, + performance=candidate.source.alert, grouping_reason=targets[candidate.source.alert.id].reason, explanation=targets[candidate.source.alert.id].explanation, ) @@ -729,7 +729,7 @@ class PerformanceIssueProjector: return await self._notify_one(item, deadline) - async def _notify_one(self, item: IssuePendingDelivery, deadline: float) -> None: + async def _notify_one(self, item: AlertPendingDelivery, deadline: float) -> None: current: Final = await self._store.pending_delivery(item.delivery.id) if current is None: return diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index de9d81a2f58..2e81e8d0c0d 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -31,11 +31,11 @@ from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.integrations.slack_alerting import AlertType from litellm.types.proxy.usage_alert_defaults import USAGE_ALERT_SCOPES, default_usage_rules from litellm.types.proxy.usage_alerts import ( + AlertRule, + AlertRuleInput, UsageAlertEvaluationStatus, UsageAlertObservation, UsageAlertProgress, - UsageAlertRule, - UsageAlertRuleInput, UsageAlertScope, UsageAlertWorkCursor, ) @@ -72,7 +72,7 @@ class _Observations: def _observations( - rule: UsageAlertRule, + rule: AlertRule, rows: Sequence[DailyAlertMetrics], target: date, suppressed: frozenset[str | None] = frozenset(), @@ -117,7 +117,7 @@ class UsageAlertJob: store: UsageAlertsRepository, metrics: UsageAlertMetricsRepository, health: UsageAlertHealthRepository, - defaults: tuple[UsageAlertRuleInput, ...] | None = None, + defaults: tuple[AlertRuleInput, ...] | None = None, *, deprecation_projection: Callable[[datetime], Awaitable[None]] | None = None, performance_projection: PerformanceIssueProjector, @@ -221,7 +221,7 @@ class UsageAlertJob: await self._store.prune() @staticmethod - def _position(rule: UsageAlertRule, target: date) -> UsageAlertProgress: + def _position(rule: AlertRule, target: date) -> UsageAlertProgress: if rule.evaluation_progress is not None and rule.evaluation_progress.revision == rule.updated_at: return rule.evaluation_progress return UsageAlertProgress(day=target, revision=rule.updated_at) @@ -275,15 +275,14 @@ class UsageAlertJob: return cursor.advance(None), advanced, True return cursor.advance(selected[-1].id), advanced, False - async def _attempt_scope_page(self, rules: tuple[UsageAlertRule, ...], target: date, wildcard: bool) -> bool: + async def _attempt_scope_page(self, rules: tuple[AlertRule, ...], target: date, wildcard: bool) -> bool: try: return await self._scope_page(rules, target, wildcard) except AnalyticsCapacityError: - await self._store.save_progress(tuple((rule, self._position(rule, target)) for rule in rules)) verbose_proxy_logger.info("Usage alert page deferred within its database budget") return False - async def _scope_page(self, rules: tuple[UsageAlertRule, ...], target: date, wildcard: bool) -> bool: + async def _scope_page(self, rules: tuple[AlertRule, ...], target: date, wildcard: bool) -> bool: scope: Final = rules[0].scope_type start: Final = datetime.combine(target, time(), tzinfo=timezone.utc) coverage: Final = await self._health.coverage("team" if scope == "team" else "user", start) @@ -354,7 +353,7 @@ class UsageAlertJob: async def _record_evaluations( self, - evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...], + evaluations: tuple[tuple[AlertRule, UsageAlertEvaluationStatus], ...], target: date, *, completed: bool, @@ -364,7 +363,7 @@ class UsageAlertJob: async def _page( self, - rules: Mapping[str | None, tuple[UsageAlertRule, ...]], + rules: Mapping[str | None, tuple[AlertRule, ...]], rows: tuple[DailyAlertMetrics, ...], target: date, suppressed: Mapping[str, frozenset[str | None]], @@ -386,7 +385,7 @@ class UsageAlertJob: async def _rule_chunk( self, - selected: tuple[UsageAlertRule, ...], + selected: tuple[AlertRule, ...], entities: tuple[str | None, ...], rows: tuple[DailyAlertMetrics, ...], by_entity: Mapping[str, tuple[DailyAlertMetrics, ...]], @@ -410,7 +409,7 @@ class UsageAlertJob: async def _rule_page( self, - rule: UsageAlertRule, + rule: AlertRule, rows: tuple[DailyAlertMetrics, ...], target: date, active: frozenset[str | None], diff --git a/litellm/repositories/issue_activity.py b/litellm/repositories/issue_activity.py index 51ec62d69a8..1a9e6a550dc 100644 --- a/litellm/repositories/issue_activity.py +++ b/litellm/repositories/issue_activity.py @@ -1,4 +1,4 @@ -"""Issue investigations reuse the same aggregate filters as entity Activity.""" +"""Alert investigations reuse the same aggregate filters as entity Activity.""" from collections.abc import Mapping from dataclasses import asdict, replace @@ -15,9 +15,9 @@ from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_ from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase, DailyActivityRepository from litellm.repositories.daily_activity_sql import PRISMA_TO_PG_TABLE, build_where_clause from litellm.types.proxy.issues import ( - IssueActivityResponse, - IssueCohort, - IssueImpact, + AlertActivityResponse, + AlertCohort, + AlertImpact, ) from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, @@ -51,7 +51,7 @@ class _UserLabel(BaseModel, frozen=True): user_email: str | None -def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyActivityScope: +def issue_activity_scope(cohort: AlertCohort, start: date, end: date) -> DailyActivityScope: if start > end or (end - start).days > 365: raise ValueError("Choose an ordered date range of at most 366 days") return DailyActivityScope( @@ -83,14 +83,14 @@ class IssueActivityRepository: return cls(daily_activity_repository(prisma, query_database=database), database) async def daily_activity( - self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 - ) -> IssueActivityResponse: + self, cohort: AlertCohort, start: date, end: date, model_limit: int = 20 + ) -> AlertActivityResponse: return await BoundedDailyActivityRepository(self.database).aggregate( issue_activity_scope(cohort, start, end), model_limit=model_limit if cohort.scope_type == "key" else None ) async def keys( - self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityKeyPageResponse: if not 1 <= limit <= 100 or offset < 0: raise ValueError("Read between one and 100 keys with a nonnegative offset") @@ -113,7 +113,7 @@ class IssueActivityRepository: ) async def users( - self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityUserPageResponse: if not 1 <= limit <= 100 or offset < 0: raise ValueError("Read between one and 100 users with a nonnegative offset") @@ -206,7 +206,7 @@ class IssueActivityRepository: ) return rows[0] - async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: + async def impact(self, cohort: AlertCohort, start: date, end: date) -> AlertImpact: scope: Final = issue_activity_scope(cohort, start, end) counts: Final = await self._counts(scope) teams: Final = ( @@ -214,7 +214,7 @@ class IssueActivityRepository: if cohort.scope_type not in ("user", "team") else None ) - return IssueImpact( + return AlertImpact( keys=counts.keys, models=counts.models, requests=counts.requests, diff --git a/litellm/repositories/issue_families.py b/litellm/repositories/issue_families.py index 698a066db8b..4a8e10d1906 100644 --- a/litellm/repositories/issue_families.py +++ b/litellm/repositories/issue_families.py @@ -10,10 +10,10 @@ from pydantic import BaseModel, TypeAdapter from litellm.repositories.usage_alerts import AlertDatabase from litellm.types.proxy.issues import ( - IssueFamilyList, - IssueFamilyPolicy, - IssueFamilySetting, - IssueFamilyUpdate, + AlertFamilyList, + AlertFamilyPolicy, + AlertFamilySetting, + AlertFamilyUpdate, ) from litellm.types.proxy.usage_alert_defaults import ( DEFAULT_USAGE_POLICIES, @@ -22,11 +22,11 @@ from litellm.types.proxy.usage_alert_defaults import ( PerformanceIssueFamily, default_usage_rule_id, ) -from litellm.types.proxy.usage_alerts import UsageAlertEvaluationCounts, UsageAlertRule +from litellm.types.proxy.usage_alerts import AlertRule, UsageAlertEvaluationCounts class _RuleRow(BaseModel, frozen=True): - record: UsageAlertRule + record: AlertRule class _Count(BaseModel, frozen=True): @@ -37,14 +37,14 @@ class _ProjectionState(BaseModel, frozen=True): pending: bool -def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) -> IssueFamilySetting: +def _setting(family: PerformanceIssueFamily, rules: tuple[AlertRule, ...]) -> AlertFamilySetting: policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family) groups: Final = tuple( (policy, tuple(rule for rule in rules if rule.metric == policy.metric and rule.direction == policy.direction)) for policy in policies ) values: Final = tuple( - IssueFamilyPolicy( + AlertFamilyPolicy( direction=policy.direction, enabled=group[0].enabled if group and len({rule.enabled for rule in group}) == 1 else None, multiplier=group[0].multiplier if group and len({rule.multiplier for rule in group}) == 1 else None, @@ -69,7 +69,7 @@ def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) for policy, group in groups ) family_rules: Final = tuple(chain.from_iterable(group for _, group in groups)) - return IssueFamilySetting( + return AlertFamilySetting( family=family, enabled=family_rules[0].enabled if family_rules and len({r.enabled for r in family_rules}) == 1 else None, total_rules=len(family_rules), @@ -81,7 +81,7 @@ def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) ) -async def family_settings(database: AlertDatabase) -> IssueFamilyList: +async def family_settings(database: AlertDatabase) -> AlertFamilyList: rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python( await database.query_raw( 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted', @@ -94,7 +94,7 @@ async def family_settings(database: AlertDatabase) -> IssueFamilyList: WHERE id LIKE 'performance-dirty:%' AND NOT completed LIMIT 1) AS pending""" ) ) - return IssueFamilyList( + return AlertFamilyList( data=tuple( _setting(family, tuple(row.record for row in rows)) for family in ("spend", "latency", "reliability", "traffic") @@ -106,8 +106,8 @@ async def family_settings(database: AlertDatabase) -> IssueFamilyList: async def update_family( database: AlertDatabase, family: PerformanceIssueFamily, - data: IssueFamilyUpdate, -) -> IssueFamilySetting | None: + data: AlertFamilyUpdate, +) -> AlertFamilySetting | None: policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family) if any(update.direction not in {policy.direction for policy in policies} for update in data.policies): raise ValueError("This family does not support that direction") @@ -126,7 +126,7 @@ async def update_family( policy.direction: policy.model_dump(exclude_none=True, exclude={"direction"}) for policy in data.policies } records: Final = tuple( - UsageAlertRule.model_validate( + AlertRule.model_validate( { **row.record.model_dump(), **({"enabled": data.enabled} if data.enabled is not None else {}), diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 62f3b8fa9de..27615caa3d1 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -14,21 +14,21 @@ from litellm.models.team import LiteLLM_TeamTable from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( - Issue, - IssueCohort, - IssueDeliveryBinding, - IssueFamily, - IssueFamilyList, - IssueFamilySetting, - IssueFamilyUpdate, - IssueInput, - IssueKind, - IssueList, - IssuePendingDelivery, - IssueProjectionSignal, - IssueScope, - IssueSignal, - IssueSignalList, + Alert, + AlertCohort, + AlertDeliveryBinding, + AlertFamily, + AlertFamilyList, + AlertFamilySetting, + AlertFamilyUpdate, + AlertInput, + AlertKind, + AlertList, + AlertPendingDelivery, + AlertProjectionSignal, + AlertScope, + AlertSignal, + AlertSignalList, ) from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS, PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus @@ -38,11 +38,11 @@ if TYPE_CHECKING: class _IssueRow(BaseModel, frozen=True): - record: Issue + record: Alert class _SignalRow(BaseModel, frozen=True): - record: IssueSignal + record: AlertSignal class _Count(BaseModel, frozen=True): @@ -82,8 +82,8 @@ def _utc(value: datetime) -> str: return value.astimezone(timezone.utc).replace(tzinfo=None).isoformat() -def _issue_payload(data: IssueInput, status: UsageAlertStatus | None) -> Mapping[str, object]: - initial: Final = Issue.model_validate( +def _issue_payload(data: AlertInput, status: UsageAlertStatus | None) -> Mapping[str, object]: + initial: Final = Alert.model_validate( { **data.model_dump(), "status": status or "open", @@ -110,10 +110,10 @@ class IssuesRepository: async def upsert_issues( self, - inputs: tuple[IssueInput, ...], + inputs: tuple[AlertInput, ...], reopen: bool = False, status: UsageAlertStatus | None = None, - ) -> tuple[Issue, ...]: + ) -> tuple[Alert, ...]: if not inputs: return () if len(inputs) > 500 or len(frozenset(item.id for item in inputs)) != len(inputs): @@ -155,7 +155,7 @@ class IssuesRepository: raise ValueError("An issue ID already belongs to another cohort") return tuple(row.record for row in rows) - async def get_issue(self, issue_id: str) -> Issue | None: + async def get_issue(self, issue_id: str) -> Alert | None: rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( await self.database.query_raw(f"SELECT {_RECORD} AS record {_JOIN} WHERE i.id=$1", issue_id) ) @@ -182,14 +182,14 @@ class IssuesRepository: async def list_issues( self, - kind: IssueKind | None = None, - family: IssueFamily | None = None, + kind: AlertKind | None = None, + family: AlertFamily | None = None, status: UsageAlertStatus | None = None, - scope_type: IssueScope | None = None, + scope_type: AlertScope | None = None, scope_id: str | None = None, limit: int = 50, offset: int = 0, - ) -> IssueList: + ) -> AlertList: where: Final = f"""($1::text IS NULL OR i.kind=$1) AND ($2::text IS NULL OR i.family=$2) AND ($3::text IS NULL OR ({_STATUS})=$3) AND ($4::text IS NULL OR i.scope_type=$4) AND ($5::text IS NULL OR i.scope_id=$5)""" @@ -212,14 +212,14 @@ class IssuesRepository: offset, ) ) - return IssueList( + return AlertList( data=await self._labels(tuple(row.record for row in rows)), total=count[0].count, limit=len(rows) if rows and offset + len(rows) < count[0].count else limit, offset=offset, ) - async def _labels(self, issues: tuple[Issue, ...]) -> tuple[Issue, ...]: + async def _labels(self, issues: tuple[Alert, ...]) -> tuple[Alert, ...]: if not issues: return () labels: Final = await UsageAlertsRepository(self.database).resource_labels( @@ -230,7 +230,7 @@ class IssuesRepository: for issue in issues ) - async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: + async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Alert, ...]: if len(cohort_keys) > 500: raise ValueError("Read at most 500 issue cohorts per batch") rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( @@ -249,8 +249,8 @@ class IssuesRepository: status: UsageAlertStatus, expected_status: UsageAlertStatus, actor: str | None = None, - authorized_scopes: tuple[IssueCohort, ...] | None = None, - ) -> Issue | None: + authorized_scopes: tuple[AlertCohort, ...] | None = None, + ) -> Alert | None: if authorized_scopes is not None and len(authorized_scopes) > 10: raise ValueError("Authorize at most 10 alert source scopes") changed: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( @@ -261,8 +261,8 @@ class IssuesRepository: AND ($5::jsonb IS NULL OR NOT EXISTS ( SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id AND NOT EXISTS ( SELECT 1 FROM jsonb_to_recordset($5::jsonb) AS x(scope_type text,scope_id text) - WHERE s.source='usage_alert' AND x.scope_type=s.record->'usage_alert'->>'scope_type' - AND x.scope_id IS NOT DISTINCT FROM s.record->'usage_alert'->>'scope_id'))) + WHERE s.source='performance' AND x.scope_type=s.record->'performance'->>'scope_type' + AND x.scope_id IS NOT DISTINCT FROM s.record->'performance'->>'scope_id'))) ), ack AS ( INSERT INTO "LiteLLM_Config" AS prior (param_name,param_value) SELECT ack_key,jsonb_build_object('model',record->'deprecation'->>'usage_model', @@ -292,18 +292,18 @@ class IssuesRepository: ) return await self.get_issue(issue_id) if changed else None - async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]: - return TypeAdapter(tuple[IssueCohort, ...]).validate_python( + async def status_scopes(self, issue_id: str) -> tuple[AlertCohort, ...]: + return TypeAdapter(tuple[AlertCohort, ...]).validate_python( await self.database.query_raw( - """SELECT DISTINCT CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_type' + """SELECT DISTINCT CASE WHEN source='performance' THEN record->'performance'->>'scope_type' ELSE 'deployment' END AS scope_type, - CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_id' END AS scope_id + CASE WHEN source='performance' THEN record->'performance'->>'scope_id' END AS scope_id FROM "LiteLLM_IssueSignal" WHERE issue_id=$1 ORDER BY scope_type,scope_id LIMIT 11""", issue_id, ) ) - async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: + async def add_signal_batch(self, signals: tuple[AlertSignal, ...]) -> None: if not signals: return if len(signals) > 500: @@ -323,14 +323,14 @@ class IssuesRepository: ) UPDATE "LiteLLM_Issue" i SET updated_at=GREATEST(CURRENT_TIMESTAMP,i.updated_at+interval '1 millisecond'), status=CASE WHEN i.kind='performance' AND i.active AND i.status<>'open' AND EXISTS ( - SELECT 1 FROM added a WHERE a.issue_id=i.id AND a.source='usage_alert' - AND (a.record->'usage_alert'->>'scope_type'<>i.scope_type - OR a.record->'usage_alert'->>'scope_id' IS DISTINCT FROM i.scope_id) + SELECT 1 FROM added a WHERE a.issue_id=i.id AND a.source='performance' + AND (a.record->'performance'->>'scope_type'<>i.scope_type + OR a.record->'performance'->>'scope_id' IS DISTINCT FROM i.scope_id) AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" prior - WHERE prior.issue_id=i.id AND prior.source='usage_alert' - AND prior.record->'usage_alert'->>'scope_type'=a.record->'usage_alert'->>'scope_type' - AND prior.record->'usage_alert'->>'scope_id' - IS NOT DISTINCT FROM a.record->'usage_alert'->>'scope_id') + WHERE prior.issue_id=i.id AND prior.source='performance' + AND prior.record->'performance'->>'scope_type'=a.record->'performance'->>'scope_type' + AND prior.record->'performance'->>'scope_id' + IS NOT DISTINCT FROM a.record->'performance'->>'scope_id') ) THEN 'open' ELSE i.status END WHERE i.id IN(SELECT issue_id FROM added) RETURNING i.id""", json.dumps(tuple(signal.model_dump(mode="json") for signal in signals)), @@ -341,11 +341,11 @@ class IssuesRepository: issue_id: str, limit: int = 50, offset: int = 0, - visible_scope: IssueCohort | None = None, - ) -> IssueSignalList: - where: Final = """issue_id=$1 AND ($2::text IS NULL OR (source='usage_alert' - AND record->'usage_alert'->>'scope_type'=$2 - AND record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM $3::text))""" + visible_scope: AlertCohort | None = None, + ) -> AlertSignalList: + where: Final = """issue_id=$1 AND ($2::text IS NULL OR (source='performance' + AND record->'performance'->>'scope_type'=$2 + AND record->'performance'->>'scope_id' IS NOT DISTINCT FROM $3::text))""" args: Final = ( issue_id, visible_scope.scope_type if visible_scope else None, @@ -365,7 +365,7 @@ class IssuesRepository: offset, ) ) - return IssueSignalList( + return AlertSignalList( data=tuple(row.record for row in rows), total=counts[0].count, limit=limit, offset=offset ) @@ -374,18 +374,18 @@ class IssuesRepository: day: date, limit: int = 500, cursor: str | None = None, - scope_type: IssueScope | None = None, + scope_type: AlertScope | None = None, metric: UsageAlertMetric | None = None, scope_ids: tuple[str, ...] | None = None, completed_only: bool = False, - ) -> tuple[IssueProjectionSignal, ...]: + ) -> tuple[AlertProjectionSignal, ...]: if not 1 <= limit <= 500: raise ValueError("Read between one and 500 source signals") if scope_ids == (): return () if scope_ids is not None and len(scope_ids) > 500: raise ValueError("Filter at most 500 source scopes") - return TypeAdapter(tuple[IssueProjectionSignal, ...]).validate_python( + return TypeAdapter(tuple[AlertProjectionSignal, ...]).validate_python( await self.database.query_raw( """SELECT a.record AS alert,r.record AS rule,a.evaluation_day FROM "LiteLLM_UsageAlertIncident" a JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id @@ -470,14 +470,14 @@ class IssuesRepository: FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id LEFT JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id - WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='usage_alert' + WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='performance' AND ({selection}) - AND s.record->'usage_alert'->>'scope_type'=i.scope_type - AND s.record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM i.scope_id - AND ((i.family='spend' AND s.record->'usage_alert'->>'metric'='spend') - OR (i.family='latency' AND s.record->'usage_alert'->>'metric'='latency') - OR (i.family='traffic' AND s.record->'usage_alert'->>'metric'='requests') - OR (i.family='reliability' AND s.record->'usage_alert'->>'metric' IN ('errors','error_rate'))) + AND s.record->'performance'->>'scope_type'=i.scope_type + AND s.record->'performance'->>'scope_id' IS NOT DISTINCT FROM i.scope_id + AND ((i.family='spend' AND s.record->'performance'->>'metric'='spend') + OR (i.family='latency' AND s.record->'performance'->>'metric'='latency') + OR (i.family='traffic' AND s.record->'performance'->>'metric'='requests') + OR (i.family='reliability' AND s.record->'performance'->>'metric' IN ('errors','error_rate'))) ), observations AS ( SELECT *,COALESCE(source_id IS NOT NULL AND NOT breached AND evaluation_day<=$1 AND evaluation_day>=latest_observation_day @@ -498,7 +498,7 @@ class IssuesRepository: ) return rows[0].count - async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: + async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Alert, ...]: if not 1 <= limit <= 500: raise ValueError("Read between one and 500 provider issues") rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( @@ -560,19 +560,19 @@ class IssuesRepository: ) return rows[0].count - async def family_settings(self) -> "IssueFamilyList": + async def family_settings(self) -> "AlertFamilyList": from litellm.repositories.issue_families import family_settings return await family_settings(self.database) async def update_family( - self, family: "PerformanceIssueFamily", data: "IssueFamilyUpdate" - ) -> "IssueFamilySetting | None": + self, family: "PerformanceIssueFamily", data: "AlertFamilyUpdate" + ) -> "AlertFamilySetting | None": from litellm.repositories.issue_families import update_family return await update_family(self.database, family, data) - async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: + async def queue_deliveries(self, bindings: tuple[AlertDeliveryBinding, ...]) -> None: if not bindings: return if len(bindings) > 500: @@ -597,17 +597,17 @@ class IssuesRepository: json.dumps(tuple(binding.model_dump(mode="json") for binding in bindings)), ) - async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: + async def pending_deliveries(self, limit: int = 100) -> tuple[AlertPendingDelivery, ...]: return await self._pending_deliveries(limit, None) - async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: + async def pending_delivery(self, delivery_id: str) -> AlertPendingDelivery | None: rows: Final = await self._pending_deliveries(1, delivery_id) return rows[0] if rows else None - async def _pending_deliveries(self, limit: int, delivery_id: str | None) -> tuple[IssuePendingDelivery, ...]: + async def _pending_deliveries(self, limit: int, delivery_id: str | None) -> tuple[AlertPendingDelivery, ...]: if not 1 <= limit <= 500: raise ValueError("Read between one and 500 deliveries") - return TypeAdapter(tuple[IssuePendingDelivery, ...]).validate_python( + return TypeAdapter(tuple[AlertPendingDelivery, ...]).validate_python( await self.database.query_raw( f"""SELECT d.record || jsonb_build_object('next_attempt',d.next_attempt AT TIME ZONE 'UTC') AS delivery, {_RECORD} AS issue,bound.rule,bound.alert @@ -635,7 +635,7 @@ class IssuesRepository: AND custom.record->>'direction'=r.record->>'direction' AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id))) AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id - AND s.source='usage_alert' AND s.source_id=a.id) + AND s.source='performance' AND s.source_id=a.id) ORDER BY b.rule_id,b.source_id LIMIT 1 ) bound ON true WHERE NOT d.sent AND d.next_attempt<=CURRENT_TIMESTAMP AND i.active AND ({_STATUS})='open' @@ -682,10 +682,10 @@ class IssuesRepository: if not 1 <= limit <= 500 or retention_days < 1: raise ValueError("Retention and cleanup batch must be bounded positive values") await self.database.query_raw( - """WITH expired AS MATERIALIZED ( - SELECT id FROM "LiteLLM_Issue" WHERE status<>'open' - AND updated_at$1::int) - ORDER BY updated_at,id LIMIT $2 + f"""WITH expired AS MATERIALIZED ( + SELECT i.id {_JOIN} WHERE i.status<>'open' AND ({_STATUS})<>'open' + AND i.updated_at$1::int) + ORDER BY i.updated_at,i.id LIMIT $2 ), signals AS ( DELETE FROM "LiteLLM_IssueSignal" WHERE (issue_id,source,source_id) IN ( SELECT issue_id,source,source_id FROM "LiteLLM_IssueSignal" diff --git a/litellm/repositories/provider_latency.py b/litellm/repositories/provider_latency.py index 5c2133f2fcd..45744c90dc3 100644 --- a/litellm/repositories/provider_latency.py +++ b/litellm/repositories/provider_latency.py @@ -9,8 +9,8 @@ from pydantic import BaseModel, Field, TypeAdapter from litellm.constants import PTU_SENTINEL_API_KEY from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket from litellm.repositories.usage_alert_metrics import MetricsDatabase -from litellm.types.proxy.issues import IssueCohort, IssueComparison, IssueImpact, IssueModelPair -from litellm.types.proxy.usage_alerts import UsageAlertMetricPoint, UsageAlertRuleInput, UsageAlertScope +from litellm.types.proxy.issues import AlertCohort, AlertComparison, AlertImpact, AlertModelPair +from litellm.types.proxy.usage_alerts import AlertRuleInput, UsageAlertMetricPoint, UsageAlertScope if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -18,15 +18,15 @@ if TYPE_CHECKING: @dataclass(frozen=True, slots=True) class ProviderLatencyEvidence: - cohort: IssueCohort - comparison: IssueComparison - impact: IssueImpact + cohort: AlertCohort + comparison: AlertComparison + impact: AlertImpact comparable_timed_requests: int total_timed_requests: int class _CohortRow(BaseModel, frozen=True): - pairs: tuple[IssueModelPair, ...] + pairs: tuple[AlertModelPair, ...] observed: float expected: float sample_days: int | None @@ -162,9 +162,9 @@ SELECT (SELECT COUNT(*) FROM covered)=cardinality($4::text[]) AND NOT EXISTS ( def _arguments( - pairs: tuple[IssueModelPair, ...], + pairs: tuple[AlertModelPair, ...], day: date, - rule: UsageAlertRuleInput, + rule: AlertRuleInput, source_ids: tuple[str, ...] | None = None, ) -> tuple[object, ...]: if not 1 <= len(pairs) <= 100 or len(frozenset(pairs)) != len(pairs): @@ -195,8 +195,8 @@ class ProviderLatencyRepository: def from_prisma(cls, prisma: "PrismaClient") -> "ProviderLatencyRepository": return cls(cast(MetricsDatabase, prisma.writer_db)) # cast-ok: generated Prisma query_raw interface - async def provider_pairs(self, provider: str, day: date) -> tuple[IssueModelPair, ...]: - return TypeAdapter(tuple[IssueModelPair, ...]).validate_python( + async def provider_pairs(self, provider: str, day: date) -> tuple[AlertModelPair, ...]: + return TypeAdapter(tuple[AlertModelPair, ...]).validate_python( await self._database.query_raw( """SELECT DISTINCT custom_llm_provider AS provider,model FROM "LiteLLM_DailyUserSpend" WHERE date=$1 AND custom_llm_provider=$2 AND model IS NOT NULL AND model<>'' AND timed_requests>0 @@ -209,7 +209,7 @@ class ProviderLatencyRepository: ) async def latency_cohort( - self, pairs: tuple[IssueModelPair, ...], day: date, rule: UsageAlertRuleInput + self, pairs: tuple[AlertModelPair, ...], day: date, rule: AlertRuleInput ) -> ProviderLatencyEvidence | None: arguments: Final = _arguments(pairs, day, rule) if len(pairs) < 2: @@ -219,8 +219,8 @@ class ProviderLatencyRepository: return None row: Final = rows[0] return ProviderLatencyEvidence( - cohort=IssueCohort(scope_type="provider", scope_id=pairs[0].provider, model_pairs=row.pairs), - comparison=IssueComparison( + cohort=AlertCohort(scope_type="provider", scope_id=pairs[0].provider, model_pairs=row.pairs), + comparison=AlertComparison( metric="latency", direction="increase", observed_value=row.observed, @@ -231,7 +231,7 @@ class ProviderLatencyRepository: baseline_sample_days=row.sample_days, history=row.history, ), - impact=IssueImpact( + impact=AlertImpact( keys=row.keys, users=row.users, teams=row.teams, @@ -246,9 +246,9 @@ class ProviderLatencyRepository: async def source_members( self, - pairs: tuple[IssueModelPair, ...], + pairs: tuple[AlertModelPair, ...], day: date, - rule: UsageAlertRuleInput, + rule: AlertRuleInput, scope_type: UsageAlertScope, source_ids: tuple[str, ...], ) -> frozenset[str]: @@ -273,7 +273,7 @@ class ProviderLatencyRepository: rows: Final = _MEMBERS.validate_python(await self._database.query_raw(query, *arguments)) return frozenset(api_key_bucket(row.entity_id) if scope_type == "key" else row.entity_id for row in rows) - async def cohort_recovered(self, pairs: tuple[IssueModelPair, ...], day: date, rule: UsageAlertRuleInput) -> bool: + async def cohort_recovered(self, pairs: tuple[AlertModelPair, ...], day: date, rule: AlertRuleInput) -> bool: arguments: Final = _arguments(pairs, day, rule) rows: Final = TypeAdapter(tuple[_Recovery, ...]).validate_python( await self._database.query_raw(_RECOVERY_QUERY, *arguments) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 3515ec0d110..ee267bdf7ec 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -19,26 +19,21 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter from litellm.repositories.daily_activity_buckets import api_key_from_bucket from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS from litellm.types.proxy.usage_alerts import ( - UsageAlert, + AlertModelList, + AlertModelOption, + AlertRule, + AlertRuleInput, + AlertRuleList, + PerformanceSignal, + PerformanceSignalList, UsageAlertDefaultSummary, UsageAlertEvaluationStatus, - UsageAlertGroup, - UsageAlertGroupList, - UsageAlertGroupMetric, - UsageAlertGroupUpdate, - UsageAlertGroupUpdateResult, - UsageAlertList, UsageAlertMetric, - UsageAlertModelList, - UsageAlertModelOption, UsageAlertObservation, UsageAlertProgress, UsageAlertProjectionCursor, UsageAlertProjectionWork, - UsageAlertRule, - UsageAlertRuleInput, UsageAlertRuleKind, - UsageAlertRuleList, UsageAlertScope, UsageAlertStatus, UsageAlertWorkCursor, @@ -78,19 +73,6 @@ class _RuleCounts(_Count, frozen=True): default_summary: UsageAlertDefaultSummary -class _GroupRow(BaseModel, frozen=True): - scope_type: UsageAlertScope - scope_id: str | None - day: date - incident_count: int - metrics: tuple[UsageAlertGroupMetric, ...] - - -class _GroupPage(BaseModel, frozen=True): - total: int - data: tuple[_GroupRow, ...] - - class _ResourceLabel(BaseModel, frozen=True): scope_type: str scope_id: str | None @@ -113,8 +95,8 @@ class _Ownership(BaseModel, frozen=True): _DEFAULT_RULE_IDS: Final = DEFAULT_USAGE_RULE_IDS -def _rule(record: Mapping[str, object]) -> UsageAlertRule: - return UsageAlertRule.model_validate({**record, "is_default": str(record.get("id")) in _DEFAULT_RULE_IDS}) +def _rule(record: Mapping[str, object]) -> AlertRule: + return AlertRule.model_validate({**record, "is_default": str(record.get("id")) in _DEFAULT_RULE_IDS}) def activity_path( @@ -137,14 +119,14 @@ def activity_path( "tab": "keys", "from": (day - timedelta(days=baseline_days)).isoformat(), "to": day.isoformat(), - "alert": alert_id, + "signal": alert_id, } ) ) query: Final = { "activity_from": (day - timedelta(days=baseline_days)).isoformat(), "activity_to": day.isoformat(), - "alert": alert_id, + "signal": alert_id, } if parameter and entity_id and not entity_id.startswith("sk-"): query[parameter] = entity_id @@ -152,7 +134,7 @@ def activity_path( return route + "?" + urlencode(query) -def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now: datetime) -> Mapping[str, object]: +def _observation_payload(rule: AlertRule, item: UsageAlertObservation, now: datetime) -> Mapping[str, object]: alert_id: Final = str( uuid5( NAMESPACE_URL, @@ -170,7 +152,7 @@ def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now: ), ) ) - incident: Final = UsageAlert( + incident: Final = PerformanceSignal( id=alert_id, rule_id=rule.id, rule_name=rule.name, @@ -208,7 +190,7 @@ class UsageAlertsRepository: cast(AlertDatabase, prisma.writer_db) # cast-ok: PrismaWrapper forwards query_raw to its generated client ) - async def get_rule(self, rule_id: str) -> UsageAlertRule | None: + async def get_rule(self, rule_id: str) -> AlertRule | None: rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( await self.database.query_raw( 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND NOT deleted', rule_id @@ -218,14 +200,14 @@ class UsageAlertsRepository: async def save_rule( self, - data: UsageAlertRuleInput, + data: AlertRuleInput, created_by: str | None, - existing: UsageAlertRule | None = None, + existing: AlertRule | None = None, *, overrides_defaults: bool = False, - ) -> UsageAlertRule | None: + ) -> AlertRule | None: now: Final = datetime.now(timezone.utc) - rule: Final = UsageAlertRule.model_validate( + rule: Final = AlertRule.model_validate( { **data.model_dump(), "id": existing.id if existing else str(uuid4()), @@ -255,7 +237,7 @@ class UsageAlertsRepository: saved: Final = TypeAdapter(tuple[_Record, ...]).validate_python(rows) return _rule(saved[0].record) if saved else None - async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: + async def delete_rule(self, rule_id: str, *, existing: AlertRule | None = None) -> bool: return bool( await self.database.query_raw( """UPDATE "LiteLLM_UsageAlertRule" SET deleted=true,enabled=false @@ -276,7 +258,7 @@ class UsageAlertsRepository: limit: int = 50, offset: int = 0, kind: UsageAlertRuleKind = "all", - ) -> UsageAlertRuleList: + ) -> AlertRuleList: args: Final = (scope_type, scope_id, json.dumps(tuple(sorted(_DEFAULT_RULE_IDS))), kind) where: Final = "NOT deleted AND ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2)" builtin: Final = "id IN (SELECT jsonb_array_elements_text($3::jsonb))" @@ -305,7 +287,7 @@ class UsageAlertsRepository: offset, ) ) - return UsageAlertRuleList( + return AlertRuleList( data=tuple(_rule(row.record) for row in rows), total=count[0].count, limit=limit, @@ -315,7 +297,7 @@ class UsageAlertsRepository: async def enabled_rule_page( self, scope: UsageAlertScope, wildcard: bool, cursor: str | None, limit: int = 500 - ) -> tuple[UsageAlertRule, ...]: + ) -> tuple[AlertRule, ...]: if not 1 <= limit <= 500: raise ValueError("Read at most 500 rules") rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( @@ -327,7 +309,7 @@ class UsageAlertsRepository: limit, ) ) - return tuple(UsageAlertRule.model_validate(row.record) for row in rows) + return tuple(AlertRule.model_validate(row.record) for row in rows) async def work_cursor(self) -> UsageAlertWorkCursor: rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( @@ -344,7 +326,7 @@ class UsageAlertsRepository: cursor.model_dump_json(), ) - async def save_progress(self, updates: tuple[tuple[UsageAlertRule, UsageAlertProgress], ...]) -> None: + async def save_progress(self, updates: tuple[tuple[AlertRule, UsageAlertProgress], ...]) -> None: if not updates: return if len(updates) > 500: @@ -369,7 +351,7 @@ class UsageAlertsRepository: ) async def suppressed_entities( - self, rules: tuple[UsageAlertRule, ...], entities: tuple[str | None, ...] + self, rules: tuple[AlertRule, ...], entities: tuple[str | None, ...] ) -> Mapping[str, frozenset[str | None]]: defaults: Final = tuple(rule for rule in rules if rule.id in _DEFAULT_RULE_IDS) if not defaults or not entities: @@ -446,11 +428,11 @@ class UsageAlertsRepository: work.version, ) - async def get_alert(self, alert_id: str) -> UsageAlert | None: + async def get_alert(self, alert_id: str) -> PerformanceSignal | None: rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( await self.database.query_raw('SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE id=$1', alert_id) ) - return UsageAlert.model_validate(rows[0].record) if rows else None + return PerformanceSignal.model_validate(rows[0].record) if rows else None async def list_alerts( self, @@ -462,7 +444,7 @@ class UsageAlertsRepository: end_date: date | None = None, limit: int = 50, offset: int = 0, - ) -> UsageAlertList: + ) -> PerformanceSignalList: args: Final = ( scope_type, scope_id, @@ -485,14 +467,14 @@ class UsageAlertsRepository: offset, ) ) - return UsageAlertList( - data=tuple(UsageAlert.model_validate(row.record) for row in rows), + return PerformanceSignalList( + data=tuple(PerformanceSignal.model_validate(row.record) for row in rows), total=count[0].count, limit=limit, offset=offset, ) - async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None: + async def set_status(self, alert_id: str, status: UsageAlertStatus) -> PerformanceSignal | None: rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( await self.database.query_raw( """UPDATE "LiteLLM_UsageAlertIncident" SET status=$2, record=jsonb_set(record,'{status}',to_jsonb($2::text)),updated_at=CURRENT_TIMESTAMP WHERE id=$1 RETURNING record""", @@ -500,92 +482,7 @@ class UsageAlertsRepository: status, ) ) - return UsageAlert.model_validate(rows[0].record) if rows else None - - async def list_groups( - self, - scope_type: UsageAlertScope | None = None, - scope_id: str | None = None, - status: UsageAlertStatus | None = None, - start_date: date | None = None, - end_date: date | None = None, - limit: int = 50, - offset: int = 0, - ) -> UsageAlertGroupList: - rows: Final = TypeAdapter(tuple[_GroupPage, ...]).validate_python( - await self.database.query_raw( - """WITH groups AS MATERIALIZED ( - SELECT scope_type,scope_id,day,count(*)::int AS incident_count FROM "LiteLLM_UsageAlertIncident" - WHERE ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2) - AND ($3::text IS NULL OR status=$3) AND ($4::text IS NULL OR day >= $4) - AND ($5::text IS NULL OR day <= $5) - GROUP BY scope_type,scope_id,day - ), page AS MATERIALIZED ( - SELECT * FROM groups ORDER BY day DESC,scope_type,scope_id NULLS FIRST LIMIT $6 OFFSET $7 - ), selected AS ( - SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i - ON i.scope_type=p.scope_type AND i.scope_id=p.scope_id AND i.day=p.day - WHERE ($3::text IS NULL OR i.status=$3) - UNION ALL - SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i - ON i.scope_type=p.scope_type AND i.scope_id IS NULL AND i.day=p.day - WHERE p.scope_id IS NULL AND ($3::text IS NULL OR i.status=$3) - ), ranked AS ( - SELECT scope_type,scope_id,day,metric,record, - count(*) OVER (PARTITION BY scope_type,scope_id,day,metric)::int AS incident_count, - row_number() OVER (PARTITION BY scope_type,scope_id,day,metric - ORDER BY record->>'first_seen' DESC,id) AS rank - FROM selected - ), packed AS ( - SELECT p.scope_type,p.scope_id,p.day,p.incident_count, - jsonb_agg(jsonb_build_object('metric',r.metric,'incident_count',r.incident_count, - 'representative',r.record) ORDER BY CASE r.metric - WHEN 'requests' THEN 0 WHEN 'spend' THEN 1 WHEN 'error_rate' THEN 2 - WHEN 'errors' THEN 3 ELSE 4 END) AS metrics - FROM page p JOIN ranked r ON p.scope_type=r.scope_type - AND p.scope_id IS NOT DISTINCT FROM r.scope_id AND p.day=r.day AND r.rank=1 - GROUP BY p.scope_type,p.scope_id,p.day,p.incident_count - ) SELECT (SELECT count(*)::int FROM groups) AS total, - COALESCE((SELECT jsonb_agg(to_jsonb(packed) ORDER BY day DESC,scope_type,scope_id NULLS FIRST) - FROM packed),'[]'::jsonb) AS data""", - scope_type, - scope_id, - status, - start_date.isoformat() if start_date else None, - end_date.isoformat() if end_date else None, - limit, - offset, - ) - ) - page: Final = rows[0] - labels: Final[Mapping[tuple[str, str | None], str | None]] = ( - await self.resource_labels(page.data) if page.data else {} - ) - return UsageAlertGroupList( - data=tuple( - UsageAlertGroup( - id=str( - uuid5( - NAMESPACE_URL, - json.dumps( - ["litellm:performance-group", row.scope_type, row.scope_id, row.day.isoformat()] - ), - ) - ), - scope_type=row.scope_type, - scope_id=row.scope_id, - scope_label=labels.get((row.scope_type, row.scope_id)), - day=row.day, - incident_count=row.incident_count, - metrics=row.metrics, - activity_path=row.metrics[0].representative.activity_path, - ) - for row in page.data - ), - total=page.total, - limit=limit, - offset=offset, - ) + return PerformanceSignal.model_validate(rows[0].record) if rows else None async def resource_labels( self, groups: tuple[ResourceLabelScope, ...] @@ -626,25 +523,7 @@ class UsageAlertsRepository: ) return MappingProxyType({(row.scope_type, row.scope_id): row.label for row in rows}) - async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult: - rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( - await self.database.query_raw( - """WITH changed AS ( - UPDATE "LiteLLM_UsageAlertIncident" SET status=$5, - record=jsonb_set(record,'{status}',to_jsonb($5::text)),updated_at=CURRENT_TIMESTAMP - WHERE scope_type=$1 AND scope_id IS NOT DISTINCT FROM $2::text AND day=$3 AND status=$4 - RETURNING id - ) SELECT count(*)::int AS count FROM changed""", - data.scope_type, - data.scope_id, - data.day.isoformat(), - data.expected_status, - data.status, - ) - ) - return UsageAlertGroupUpdateResult(updated_count=rows[0].count) - - async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList: + async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> AlertModelList: today: Final = datetime.now(timezone.utc).date() rows: Final = TypeAdapter(tuple[_ModelIdentity, ...]).validate_python( await self.database.query_raw( @@ -661,8 +540,8 @@ class UsageAlertsRepository: ) ) selected: Final = rows[:limit] - return UsageAlertModelList( - data=tuple(UsageAlertModelOption(scope_id=row.model, label=row.model) for row in selected), + return AlertModelList( + data=tuple(AlertModelOption(scope_id=row.model, label=row.model) for row in selected), limit=limit, next_cursor=selected[-1].model if len(rows) > limit else None, start_date=today - timedelta(days=29), @@ -699,10 +578,10 @@ class UsageAlertsRepository: ) @staticmethod - def _checkpoint(rule: UsageAlertRule, day: date) -> str: + def _checkpoint(rule: AlertRule, day: date) -> str: return f"{rule.id}:{rule.updated_at.isoformat()}:{day.isoformat()}" - async def completed_rule_ids(self, rules: tuple[UsageAlertRule, ...], day: date) -> frozenset[str]: + async def completed_rule_ids(self, rules: tuple[AlertRule, ...], day: date) -> frozenset[str]: if len(rules) > 500: raise ValueError("A checkpoint batch must contain at most 500 rules") rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( @@ -717,7 +596,7 @@ class UsageAlertsRepository: async def record_evaluations( self, - evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...], + evaluations: tuple[tuple[AlertRule, UsageAlertEvaluationStatus], ...], day: date, *, completed: bool, @@ -764,16 +643,16 @@ class UsageAlertsRepository: str(uuid4()), ) - async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None: + async def ensure_default_rules(self, rules: tuple[AlertRuleInput, ...]) -> None: for rule in rules: await self._ensure_default_rule(rule) - async def _ensure_default_rule(self, rule: UsageAlertRuleInput) -> None: + async def _ensure_default_rule(self, rule: AlertRuleInput) -> None: rule_id: Final = str( uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}") ) now: Final = datetime.now(timezone.utc) - record: Final = UsageAlertRule.model_validate( + record: Final = AlertRule.model_validate( {**rule.model_dump(), "id": rule_id, "created_at": now, "updated_at": now, "created_by": None} ) await self.database.query_raw( @@ -786,8 +665,8 @@ class UsageAlertsRepository: ) async def record_observations( - self, rule: UsageAlertRule, observations: tuple[UsageAlertObservation, ...] - ) -> tuple[UsageAlert, ...]: + self, rule: AlertRule, observations: tuple[UsageAlertObservation, ...] + ) -> tuple[PerformanceSignal, ...]: if not observations: return () if len(observations) > 500 or len({item.entity_id for item in observations}) != len(observations): @@ -838,7 +717,7 @@ class UsageAlertsRepository: str(uuid4()), ) ) - return tuple(UsageAlert.model_validate(row.record) for row in rows) + return tuple(PerformanceSignal.model_validate(row.record) for row in rows) async def prune(self, retention_days: int = 90, limit: int = 500) -> None: await self.database.query_raw( diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py index 3341ee8dc7f..609e0d142f1 100644 --- a/litellm/types/proxy/issues.py +++ b/litellm/types/proxy/issues.py @@ -10,40 +10,40 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import ( - UsageAlert, + AlertRule, + PerformanceSignal, UsageAlertChannel, UsageAlertDirection, UsageAlertEvaluationCounts, UsageAlertMetric, UsageAlertMetricPoint, - UsageAlertRule, UsageAlertStatus, ) -IssueActivityResponse: TypeAlias = BoundedDailyActivityResponse +AlertActivityResponse: TypeAlias = BoundedDailyActivityResponse -IssueKind: TypeAlias = Literal["performance", "model_deprecation"] -IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] -IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] -IssueGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"] -IssueSignalGroupingReason: TypeAlias = Literal[ +AlertKind: TypeAlias = Literal["performance", "model_deprecation"] +AlertFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] +AlertScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] +AlertGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"] +AlertSignalGroupingReason: TypeAlias = Literal[ "same_resource", "same_key_traffic", "dominant_key", "provider_cohort", "volume_explains_spend", "model_deprecation" ] -class IssueModelPair(BaseModel, frozen=True): +class AlertModelPair(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") provider: str = Field(min_length=1, max_length=256) model: str = Field(min_length=1, max_length=1024) -class IssueCohort(BaseModel, frozen=True): +class AlertCohort(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") - scope_type: IssueScope + scope_type: AlertScope scope_id: str | None = Field(default=None, min_length=1, max_length=1024) - model_pairs: tuple[IssueModelPair, ...] = Field(default=(), max_length=100) + model_pairs: tuple[AlertModelPair, ...] = Field(default=(), max_length=100) @model_validator(mode="after") def validate_scope(self) -> Self: @@ -56,7 +56,7 @@ class IssueCohort(BaseModel, frozen=True): return self -class IssueComparison(BaseModel, frozen=True): +class AlertComparison(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) metric: UsageAlertMetric @@ -78,7 +78,7 @@ class IssueComparison(BaseModel, frozen=True): return self -class IssueImpact(BaseModel, frozen=True): +class AlertImpact(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") keys: int | None = Field(default=None, ge=0) @@ -90,7 +90,7 @@ class IssueImpact(BaseModel, frozen=True): measured_models: int | None = Field(default=None, ge=0) -class IssueDeprecation(BaseModel, frozen=True): +class AlertDeprecation(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") model: ModelDeprecationInfo @@ -102,24 +102,24 @@ class IssueDeprecation(BaseModel, frozen=True): daily_requests: tuple[UsageAlertMetricPoint, ...] = Field(default=(), max_length=30) -class IssueInput(BaseModel, frozen=True): +class AlertInput(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") id: str = Field(min_length=1, max_length=200) cohort_key: str = Field(min_length=1, max_length=200) - kind: IssueKind - family: IssueFamily + kind: AlertKind + family: AlertFamily title: str = Field(min_length=1, max_length=2048) - cohort: IssueCohort + cohort: AlertCohort observed_on: date observed_at: datetime range_start: date range_end: date - grouping_reason: IssueGroupingReason + grouping_reason: AlertGroupingReason explanation: str = Field(default="", max_length=2000) - comparisons: tuple[IssueComparison, ...] = Field(default=(), max_length=10) - impact: IssueImpact = Field(default_factory=IssueImpact) - deprecation: IssueDeprecation | None = None + comparisons: tuple[AlertComparison, ...] = Field(default=(), max_length=10) + impact: AlertImpact = Field(default_factory=AlertImpact) + deprecation: AlertDeprecation | None = None @model_validator(mode="after") def validate_issue(self) -> Self: @@ -134,7 +134,7 @@ class IssueInput(BaseModel, frozen=True): return self -class Issue(IssueInput, frozen=True): +class Alert(AlertInput, frozen=True): status: UsageAlertStatus first_seen: datetime last_seen: datetime @@ -149,58 +149,60 @@ class Issue(IssueInput, frozen=True): ) -class IssueList(BaseModel, frozen=True): - data: tuple[Issue, ...] +class AlertList(BaseModel, frozen=True): + data: tuple[Alert, ...] total: int limit: int = Field(description="Effective page size, reduced for large records. Continue at offset + limit.") offset: int -class IssueUpdate(BaseModel, frozen=True): +class AlertUpdate(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") status: UsageAlertStatus expected_status: UsageAlertStatus -class IssueSignalInput(BaseModel, frozen=True): +class AlertSignalInput(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") - source: Literal["usage_alert", "model_deprecation"] + source: Literal["performance", "model_deprecation"] source_id: str = Field(min_length=1, max_length=200) - usage_alert: UsageAlert | None = None - deprecation: IssueDeprecation | None = None - grouping_reason: IssueSignalGroupingReason = "same_resource" + performance: PerformanceSignal | None = None + deprecation: AlertDeprecation | None = None + grouping_reason: AlertSignalGroupingReason = "same_resource" explanation: str = Field(default="", max_length=2000) @model_validator(mode="after") def validate_signal(self) -> Self: - if self.source == "usage_alert": - if self.usage_alert is None or self.deprecation is not None or self.usage_alert.id != self.source_id: - raise ValueError("Usage signals require their matching source alert") - elif self.deprecation is None or self.usage_alert is not None: + if self.source == "performance": + if self.performance is None or self.deprecation is not None or self.performance.id != self.source_id: + raise ValueError("Performance signals require their matching evidence") + elif self.deprecation is None or self.performance is not None: raise ValueError("Deprecation signals require deprecation details") return self -class IssueSignal(IssueSignalInput, frozen=True): - issue_id: str +class AlertSignal(AlertSignalInput, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) + + issue_id: str = Field(alias="alert_id") -class IssueSignalList(BaseModel, frozen=True): - data: tuple[IssueSignal, ...] +class AlertSignalList(BaseModel, frozen=True): + data: tuple[AlertSignal, ...] total: int limit: int offset: int -class IssueProjectionSignal(BaseModel, frozen=True): - alert: UsageAlert - rule: UsageAlertRule +class AlertProjectionSignal(BaseModel, frozen=True): + alert: PerformanceSignal + rule: AlertRule evaluation_day: date -class IssueDeliveryInput(BaseModel, frozen=True): +class AlertDeliveryInput(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") id: str = Field(min_length=1, max_length=200) @@ -211,27 +213,27 @@ class IssueDeliveryInput(BaseModel, frozen=True): parent_delivery_id: str | None = Field(default=None, min_length=1, max_length=200) -class IssueDeliveryBinding(BaseModel, frozen=True): - delivery: IssueDeliveryInput +class AlertDeliveryBinding(BaseModel, frozen=True): + delivery: AlertDeliveryInput rule_id: str rule_revision: str source_id: str -class IssueDelivery(IssueDeliveryInput, frozen=True): +class AlertDelivery(AlertDeliveryInput, frozen=True): attempts: int = 0 next_attempt: datetime | None = None last_error: Literal["delivery_failed", "transport_not_configured"] | None = None -class IssuePendingDelivery(BaseModel, frozen=True): - delivery: IssueDelivery - issue: Issue - rule: UsageAlertRule - alert: UsageAlert +class AlertPendingDelivery(BaseModel, frozen=True): + delivery: AlertDelivery + issue: Alert + rule: AlertRule + alert: PerformanceSignal -class IssueFamilyPolicy(BaseModel, frozen=True): +class AlertFamilyPolicy(BaseModel, frozen=True): direction: UsageAlertDirection enabled: bool | None multiplier: float | None @@ -244,21 +246,21 @@ class IssueFamilyPolicy(BaseModel, frozen=True): enabled_rules: int -class IssueFamilySetting(BaseModel, frozen=True): +class AlertFamilySetting(BaseModel, frozen=True): family: PerformanceIssueFamily enabled: bool | None total_rules: int enabled_rules: int - policies: tuple[IssueFamilyPolicy, ...] + policies: tuple[AlertFamilyPolicy, ...] evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts) -class IssueFamilyList(BaseModel, frozen=True): - data: tuple[IssueFamilySetting, ...] +class AlertFamilyList(BaseModel, frozen=True): + data: tuple[AlertFamilySetting, ...] projection_pending: bool = False -class IssueFamilyPolicyUpdate(BaseModel, frozen=True): +class AlertFamilyPolicyUpdate(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") direction: UsageAlertDirection @@ -270,11 +272,11 @@ class IssueFamilyPolicyUpdate(BaseModel, frozen=True): min_history_days: int | None = Field(default=None, ge=1, le=30) -class IssueFamilyUpdate(BaseModel, frozen=True): +class AlertFamilyUpdate(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") enabled: bool | None = None - policies: tuple[IssueFamilyPolicyUpdate, ...] = Field(default=(), max_length=2) + policies: tuple[AlertFamilyPolicyUpdate, ...] = Field(default=(), max_length=2) @model_validator(mode="after") def validate_update(self) -> Self: diff --git a/litellm/types/proxy/usage_alert_defaults.py b/litellm/types/proxy/usage_alert_defaults.py index d6642ac9c81..3aa73566c0c 100644 --- a/litellm/types/proxy/usage_alert_defaults.py +++ b/litellm/types/proxy/usage_alert_defaults.py @@ -5,11 +5,11 @@ from typing import Final, Literal, TypeAlias from uuid import NAMESPACE_URL, uuid5 from litellm.types.proxy.usage_alerts import ( + AlertRule, + AlertRuleInput, UsageAlertChannel, UsageAlertDirection, UsageAlertMetric, - UsageAlertRule, - UsageAlertRuleInput, UsageAlertScope, ) @@ -50,15 +50,15 @@ _DEFAULT_USAGE_IDENTITIES: Final = MappingProxyType( DEFAULT_USAGE_RULE_IDS: Final = frozenset(_DEFAULT_USAGE_IDENTITIES) -def is_default_usage_rule(rule: UsageAlertRule) -> bool: +def is_default_usage_rule(rule: AlertRule) -> bool: return _DEFAULT_USAGE_IDENTITIES.get(rule.id) == (rule.scope_type, rule.metric, rule.direction) def default_usage_rules( channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False -) -> tuple[UsageAlertRuleInput, ...]: +) -> tuple[AlertRuleInput, ...]: return tuple( - UsageAlertRuleInput( + AlertRuleInput( name=f"Daily {scope} {policy.metric.replace('_', ' ')} {policy.direction}", scope_type=scope, metric=policy.metric, diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index b5b3956fdc1..265150d5b06 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -24,7 +24,7 @@ class UsageAlertMetricPoint(BaseModel, frozen=True): value: float | None = Field(default=None, ge=0) -class UsageAlertRuleInput(BaseModel): +class AlertRuleInput(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) name: str = Field(min_length=1, max_length=200, pattern=r"^[^\x00-\x1f\x7f]+$") @@ -102,7 +102,7 @@ class UsageAlertProjectionCursor(BaseModel, frozen=True): observed_at: datetime | None = None -class UsageAlertRule(UsageAlertRuleInput): +class AlertRule(AlertRuleInput): overrides_defaults: bool = Field(default=False, exclude=True) id: str created_at: datetime @@ -114,7 +114,7 @@ class UsageAlertRule(UsageAlertRuleInput): evaluation_progress: UsageAlertProgress | None = Field(default=None, exclude=True) -class UsageAlert(BaseModel): +class PerformanceSignal(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) id: str @@ -142,11 +142,6 @@ class UsageAlert(BaseModel): notification_error: Literal["delivery_failed", "transport_not_configured"] | None = None -class UsageAlertUpdate(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - status: UsageAlertStatus - - class UsageAlertEvaluationCounts(BaseModel, frozen=True): delayed: int = 0 pending: int = 0 @@ -161,73 +156,28 @@ class UsageAlertDefaultSummary(BaseModel, frozen=True): evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts) -class UsageAlertRuleList(BaseModel, frozen=True): - data: tuple[UsageAlertRule, ...] +class AlertRuleList(BaseModel, frozen=True): + data: tuple[AlertRule, ...] total: int limit: int offset: int default_summary: UsageAlertDefaultSummary = Field(default_factory=UsageAlertDefaultSummary) -class UsageAlertList(BaseModel, frozen=True): - data: tuple[UsageAlert, ...] +class PerformanceSignalList(BaseModel, frozen=True): + data: tuple[PerformanceSignal, ...] total: int limit: int offset: int -class UsageAlertGroupMetric(BaseModel, frozen=True): - metric: UsageAlertMetric - incident_count: int - representative: UsageAlert - - -class UsageAlertGroup(BaseModel, frozen=True): - id: str - scope_type: UsageAlertScope - scope_id: str | None - scope_label: str | None - day: date - incident_count: int - metrics: tuple[UsageAlertGroupMetric, ...] = Field(max_length=5) - activity_path: str - - -class UsageAlertGroupList(BaseModel, frozen=True): - data: tuple[UsageAlertGroup, ...] - total: int - limit: int - offset: int - - -class UsageAlertGroupUpdate(BaseModel, frozen=True): - model_config = ConfigDict(extra="forbid") - scope_type: UsageAlertScope - scope_id: str | None = Field(default=None, min_length=1, max_length=1024) - day: date - expected_status: UsageAlertStatus - status: UsageAlertStatus - - @model_validator(mode="after") - def validate_group(self) -> Self: - if (self.scope_type == "deployment") != (self.scope_id is None): - raise ValueError("Specify a resource ID except for deployment alerts") - if self.expected_status == self.status: - raise ValueError("The new status must differ from the expected status") - return self - - -class UsageAlertGroupUpdateResult(BaseModel, frozen=True): - updated_count: int - - -class UsageAlertModelOption(BaseModel, frozen=True): +class AlertModelOption(BaseModel, frozen=True): scope_id: str label: str -class UsageAlertModelList(BaseModel, frozen=True): - data: tuple[UsageAlertModelOption, ...] +class AlertModelList(BaseModel, frozen=True): + data: tuple[AlertModelOption, ...] limit: int next_cursor: str | None start_date: date diff --git a/tests/integration/database/test_deprecation_issue_evaluation.py b/tests/integration/database/test_deprecation_issue_evaluation.py index 3417ca92ebe..501cb057533 100644 --- a/tests/integration/database/test_deprecation_issue_evaluation.py +++ b/tests/integration/database/test_deprecation_issue_evaluation.py @@ -118,10 +118,10 @@ async def test_retirement_issue_http_state_survives_replay_and_reopens_only_for_ await record(now.date(), 100) projector = DeprecationIssueProjector.from_prisma(db.prisma, source) await projector.run(now) - page = gateway.get("/issues", {"scope_type": "model", "scope_id": name}) + page = gateway.get("/alerts", {"scope_type": "model", "scope_id": name}) assert page["total"] == 1 original = page["data"][0] - path = f"/issues/{original['id']}" + path = f"/alerts/{original['id']}" assert original["status"] == "open" assert original["impact"]["requests"] == 100 assert gateway.get(path + "/signals")["total"] == 1 diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index 670396bd0ed..d14a84a0ce0 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -26,28 +26,30 @@ from litellm.repositories.issue_activity import IssueActivityRepository from litellm.repositories.issues import IssuesRepository from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( - IssueActivityResponse, - IssueCohort, - IssueComparison, - IssueDeliveryBinding, - IssueDeliveryInput, - IssueFamilyPolicyUpdate, - IssueFamilyUpdate, - IssueInput, - IssueList, - IssueModelPair, - IssueSignal, - IssueSignalInput, - IssueSignalList, + AlertActivityResponse, + AlertCohort, + AlertComparison, + AlertDeliveryBinding, + AlertDeliveryInput, + AlertDeprecation, + AlertFamilyPolicyUpdate, + AlertFamilyUpdate, + AlertInput, + AlertList, + AlertModelPair, + AlertSignal, + AlertSignalInput, + AlertSignalList, ) +from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo from litellm.types.proxy.usage_alert_defaults import default_usage_rule_id, default_usage_rules -from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertObservation, UsageAlertRule, UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import AlertRule, AlertRuleInput, PerformanceSignal, UsageAlertObservation -def _issue(cohort: IssueCohort) -> IssueInput: +def _issue(cohort: AlertCohort) -> AlertInput: identity: Final = str(uuid4()) day: Final = date.today() - timedelta(days=3) - return IssueInput( + return AlertInput( id=identity, cohort_key=identity, kind="performance", @@ -61,7 +63,7 @@ def _issue(cohort: IssueCohort) -> IssueInput: grouping_reason="same_resource", explanation="Requests increased for this resource.", comparisons=( - IssueComparison( + AlertComparison( metric="requests", direction="increase", observed_value=420, @@ -75,12 +77,12 @@ def _issue(cohort: IssueCohort) -> IssueInput: ) -def _signals(issue_id: str, inputs: tuple[IssueSignalInput, ...]) -> tuple[IssueSignal, ...]: - return tuple(IssueSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in inputs) +def _signals(issue_id: str, inputs: tuple[AlertSignalInput, ...]) -> tuple[AlertSignal, ...]: + return tuple(AlertSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in inputs) -def _delivery(data: IssueDeliveryInput, rule: UsageAlertRule, source: UsageAlert) -> IssueDeliveryBinding: - return IssueDeliveryBinding( +def _delivery(data: AlertDeliveryInput, rule: AlertRule, source: PerformanceSignal) -> AlertDeliveryBinding: + return AlertDeliveryBinding( delivery=data, rule_id=rule.id, rule_revision=rule.updated_at.isoformat(), source_id=source.id ) @@ -106,9 +108,9 @@ async def test_issue_pages_bound_payload_bytes_without_losing_details_or_paginat store: Final = IssuesRepository.from_prisma(db.prisma) owner: Final = "page-budget-" + uuid4().hex pairs: Final = tuple( - IssueModelPair(provider=owner, model=str(index).zfill(4) + "\U0001f680" * 1020) for index in range(100) + AlertModelPair(provider=owner, model=str(index).zfill(4) + "\U0001f680" * 1020) for index in range(100) ) - cohort: Final = IssueCohort(scope_type="provider", scope_id=owner, model_pairs=pairs) + cohort: Final = AlertCohort(scope_type="provider", scope_id=owner, model_pairs=pairs) inputs: Final = tuple(_issue(cohort) for _ in range(6)) identifiers: Final = tuple(item.id for item in inputs) try: @@ -149,11 +151,11 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") inference_key: Final = scenario.key(user_id=owner, key_type="llm_api", key_alias="Example checkout") reporting_id: Final = sha256(inference_key.encode()).hexdigest() - first: Final = _issue(IssueCohort(scope_type="key", scope_id=reporting_id)) + first: Final = _issue(AlertCohort(scope_type="key", scope_id=reporting_id)) second: Final = _issue(first.cohort) - global_issue: Final = _issue(IssueCohort(scope_type="provider", scope_id="example-provider")) + global_issue: Final = _issue(AlertCohort(scope_type="provider", scope_id="example-provider")) rule: Final = await alerts.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Visibility fixture", scope_type="key", scope_id=reporting_id, @@ -193,18 +195,18 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa _signals( first.id, ( - IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source), - IssueSignalInput( - source="usage_alert", + AlertSignalInput(source="performance", source_id=source.id, performance=source), + AlertSignalInput( + source="performance", source_id=broader.id, - usage_alert=broader, + performance=broader, grouping_reason="dominant_key", explanation="Parent contribution evidence.", ), ), ) ) - path: Final = f"/issues/{first.id}" + path: Final = f"/alerts/{first.id}" own: Final = gateway.request("GET", path, key=owner_key) assert own.status_code == 200 and own.json()["scope_label"] == "Example checkout" assert own.json()["can_update_status"] is False, "Broader source scopes require their own write permission" @@ -221,23 +223,23 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa pages: Final = tuple( gateway.request( "GET", - "/issues", + "/alerts", key=owner_key, params={"scope_type": "key", "scope_id": reporting_id, "limit": "1", "offset": str(offset)}, ) for offset in (0, 1, 2) ) assert all(page.status_code == 200 for page in pages) - for offset, parsed in enumerate(IssueList.model_validate(page.json()) for page in pages): + for offset, parsed in enumerate(AlertList.model_validate(page.json()) for page in pages): assert parsed.total == 2 and len(parsed.data) == (1 if offset < 2 else 0) - assert gateway.request("GET", "/issues", key=owner_key).status_code == 403 + assert gateway.request("GET", "/alerts", key=owner_key).status_code == 403 for suffix in ("", "/signals", "/activity", "/keys", "/users", "/impact"): - assert gateway.request("GET", f"/issues/{global_issue.id}{suffix}", key=owner_key).status_code == 403 + assert gateway.request("GET", f"/alerts/{global_issue.id}{suffix}", key=owner_key).status_code == 403 assert gateway.request("GET", path + suffix, key=outsider_key).status_code == 403 assert gateway.request("GET", path + suffix, key=inference_key).status_code == 403 public_signals: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1"}) assert public_signals.status_code == 200 - visible: Final = IssueSignalList.model_validate(public_signals.json()) + visible: Final = AlertSignalList.model_validate(public_signals.json()) assert visible.total == 1 and tuple(row.source_id for row in visible.data) == (source.id,) assert "99999" not in public_signals.text end: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1", "offset": "1"}) @@ -258,11 +260,11 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa ) assert gateway.request("PATCH", path, {"status": "dismissed", "expected_status": "open"}).status_code == 200 assert gateway.request("PATCH", path, {"status": "resolved", "expected_status": "open"}).status_code == 409 - assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["status"] == "open" - assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["can_update_status"] is True + assert gateway.request("GET", f"/alerts/{second.id}", key=owner_key).json()["status"] == "open" + assert gateway.request("GET", f"/alerts/{second.id}", key=owner_key).json()["can_update_status"] is True assert ( gateway.request( - "PATCH", f"/issues/{second.id}", {"status": "dismissed", "expected_status": "open"}, key=owner_key + "PATCH", f"/alerts/{second.id}", {"status": "dismissed", "expected_status": "open"}, key=owner_key ).status_code == 200 ) @@ -293,11 +295,11 @@ async def test_issue_detail_reports_team_status_permission(gateway: Gateway, db: key_type="management", metadata={"service_account_id": "example-activity-service"}, ) - issue: Final = _issue(IssueCohort(scope_type="team", scope_id=team)) - permitted_issue: Final = _issue(IssueCohort(scope_type="team", scope_id=permitted_team)) + issue: Final = _issue(AlertCohort(scope_type="team", scope_id=team)) + permitted_issue: Final = _issue(AlertCohort(scope_type="team", scope_id=permitted_team)) try: await store.upsert_issues((issue, permitted_issue)) - path: Final = f"/issues/{issue.id}" + path: Final = f"/alerts/{issue.id}" assert gateway.request("GET", path, key=admin_key).json()["can_update_status"] is True assert gateway.request("GET", path, key=admin_key).json()["can_view_activity"] is True assert gateway.request("GET", path, key=member_key).json()["can_update_status"] is False @@ -308,10 +310,10 @@ async def test_issue_detail_reports_team_status_permission(gateway: Gateway, db: for suffix in ("/activity", "/keys", "/impact"): assert gateway.request("GET", path + suffix, key=admin_key).status_code == 200 assert ( - gateway.request("GET", f"/issues/{permitted_issue.id}" + suffix, key=member_key).status_code == 200 + gateway.request("GET", f"/alerts/{permitted_issue.id}" + suffix, key=member_key).status_code == 200 ) assert ( - gateway.request("GET", f"/issues/{permitted_issue.id}", key=member_key).json()["can_view_activity"] + gateway.request("GET", f"/alerts/{permitted_issue.id}", key=member_key).json()["can_view_activity"] is True ) assert ( @@ -342,7 +344,7 @@ async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gatew provider: Final = f"example-provider-{identity}" day: Final = date.today() - timedelta(days=5) model_count: Final = 137 - cohort: Final = IssueCohort(scope_type="key", scope_id=key) + cohort: Final = AlertCohort(scope_type="key", scope_id=key) issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day + timedelta(days=2)}) await _query( db, @@ -365,9 +367,9 @@ async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gatew await store.upsert_issues((issue,)) scope: Final = issue_activity_scope(cohort, day, day + timedelta(days=2)) original: Final = await get_daily_activity_aggregated(repository.activity, scope, api_key_limit=20) - response: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + response: Final = gateway.request("GET", f"/alerts/{issue.id}/activity") assert response.status_code == 200 - bounded: Final = IssueActivityResponse.model_validate(response.json()) + bounded: Final = AlertActivityResponse.model_validate(response.json()) assert bounded.metadata.model_dump(exclude={"api_key_limit"}) == original.metadata.model_dump( exclude={"api_key_limit"} ) @@ -388,30 +390,30 @@ async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gatew await repository.database.query_raw(query.sql, *query.params) ) assert len(raw) == 3 + 1 + 100 + 1 - largest: Final = gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "100"}) + largest: Final = gateway.request("GET", f"/alerts/{issue.id}/activity", params={"model_limit": "100"}) assert largest.status_code == 200 - large_models: Final = IssueActivityResponse.model_validate(largest.json()).model_breakdown + large_models: Final = AlertActivityResponse.model_validate(largest.json()).model_breakdown assert large_models is not None and len(large_models.models) == 100 assert large_models.other is not None and large_models.other.model_count == model_count - 100 - assert gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "101"}).status_code == 422 + assert gateway.request("GET", f"/alerts/{issue.id}/activity", params={"model_limit": "101"}).status_code == 422 assert ( gateway.request( "GET", - f"/issues/{issue.id}/activity", + f"/alerts/{issue.id}/activity", params={"start_date": day.isoformat(), "end_date": (day + timedelta(days=366)).isoformat()}, ).status_code == 422 ) non_key: Final = await repository.daily_activity( - IssueCohort(scope_type="provider", scope_id=provider), day, day + timedelta(days=2) + AlertCohort(scope_type="provider", scope_id=provider), day, day + timedelta(days=2) ) assert non_key.model_breakdown is None and non_key.metadata == bounded.metadata assert len(non_key.results) == 3 async with db.prisma.writer_db.tx(timeout=timedelta(seconds=10)) as blocker: await blocker.execute_raw('LOCK TABLE "LiteLLM_DailyUserSpend" IN ACCESS EXCLUSIVE MODE') - busy: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + busy: Final = gateway.request("GET", f"/alerts/{issue.id}/activity") assert busy.status_code == 503 and busy.headers["Retry-After"] == "5" - assert gateway.request("GET", f"/issues/{issue.id}/activity").status_code == 200 + assert gateway.request("GET", f"/alerts/{issue.id}/activity").status_code == 200 finally: await _remove_issues(db, (issue.id,)) await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1 RETURNING id', identity) @@ -459,22 +461,22 @@ async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( RETURNING id""", json.dumps(entries), ) - cohort: Final = IssueCohort( + cohort: Final = AlertCohort( scope_type="provider", scope_id=provider, - model_pairs=(IssueModelPair(provider=provider, model="example-model-a"),), + model_pairs=(AlertModelPair(provider=provider, model="example-model-a"),), ) store: Final = IssuesRepository.from_prisma(db.prisma) issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day}) try: await store.upsert_issues((issue,)) - activity_http: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + activity_http: Final = gateway.request("GET", f"/alerts/{issue.id}/activity") assert activity_http.status_code == 200 and activity_http.json()["metadata"]["total_api_requests"] == 100 - keys_http: Final = gateway.request("GET", f"/issues/{issue.id}/keys", params={"limit": "1"}) + keys_http: Final = gateway.request("GET", f"/alerts/{issue.id}/keys", params={"limit": "1"}) assert keys_http.status_code == 200 and keys_http.json()["api_keys"][0]["metrics"]["api_requests"] == 100 - users_http: Final = gateway.request("GET", f"/issues/{issue.id}/users") + users_http: Final = gateway.request("GET", f"/alerts/{issue.id}/users") assert users_http.status_code == 200 and users_http.json()["total_users"] == 1 - impact_http: Final = gateway.request("GET", f"/issues/{issue.id}/impact") + impact_http: Final = gateway.request("GET", f"/alerts/{issue.id}/impact") assert impact_http.status_code == 200 and impact_http.json()["requests"] == 100 daily: Final = await repository.daily_activity(cohort, day, day) assert daily.metadata.total_api_requests == 100 and daily.metadata.total_spend == 10 @@ -488,15 +490,15 @@ async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( assert users.total_users == 1 and users.users[0].api_requests == 100 and users.users[0].timed_requests == 100 impact: Final = await repository.impact(cohort, day, day) assert (impact.keys, impact.users, impact.models, impact.requests) == (1, 1, 1, 100) - pairs: Final = IssueCohort( + pairs: Final = AlertCohort( scope_type="deployment", model_pairs=( - IssueModelPair(provider=provider, model="example-model-a"), - IssueModelPair(provider=other, model="example-model-b"), + AlertModelPair(provider=provider, model="example-model-a"), + AlertModelPair(provider=other, model="example-model-b"), ), ) assert (await repository.daily_activity(pairs, day, day)).metadata.total_api_requests == 100 - injected: Final = IssueCohort(scope_type="provider", scope_id=provider + "' OR true --") + injected: Final = AlertCohort(scope_type="provider", scope_id=provider + "' OR true --") assert (await repository.impact(injected, day, day)).requests == 0 from litellm.repositories.issue_activity import issue_activity_scope from litellm.types.repositories.daily_activity import ExportType @@ -523,8 +525,8 @@ async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: ManagementDatabase) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - original: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) - definition: Final = UsageAlertRuleInput( + original: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + definition: Final = AlertRuleInput( name="Delivery fixture", scope_type="user", scope_id=original.cohort.scope_id, @@ -538,8 +540,8 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma observation: Final = UsageAlertObservation( entity_id=rule.scope_id, day=original.observed_on, observed=420, baseline=120, threshold=360, breached=True ) - delivery: Final = IssueDeliveryInput(id=str(uuid4()), issue_id=original.id, channel="slack") - invalid: Final = IssueDeliveryInput( + delivery: Final = AlertDeliveryInput(id=str(uuid4()), issue_id=original.id, channel="slack") + invalid: Final = AlertDeliveryInput( id=str(uuid4()), issue_id=original.id, channel="email", email_recipients=("outside@example.com",) ) next_issue_id: Final = str(uuid4()) @@ -555,7 +557,7 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma assert await store.active_signals(original.observed_on, scope_ids=()) == () assert await store.active_signals(original.observed_on, scope_ids=("other-example-user",)) == () await store.add_signal_batch( - _signals(initial.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),)) + _signals(initial.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),)) ) await store.queue_deliveries((_delivery(delivery, rule, source),)) await store.queue_deliveries((_delivery(invalid, rule, source),)) @@ -693,7 +695,7 @@ async def test_late_default_override_stops_queued_delivery_and_recovers_on_fresh await alerts.ensure_default_rules(default_usage_rules(("slack",))) rule: Final = await alerts.get_rule(default_usage_rule_id("key", "requests", "increase")) assert rule is not None - issues: Final = tuple(_issue(IssueCohort(scope_type="key", scope_id=key)) for key in ("key-a", "key-b")) + issues: Final = tuple(_issue(AlertCohort(scope_type="key", scope_id=key)) for key in ("key-a", "key-b")) observations: Final = tuple( UsageAlertObservation( entity_id=issue.cohort.scope_id, @@ -708,15 +710,15 @@ async def test_late_default_override_stops_queued_delivery_and_recovers_on_fresh sources: Final = await alerts.record_observations(rule, observations) await store.upsert_issues(issues) for issue in issues: - source: UsageAlert = next(source for source in sources if source.scope_id == issue.cohort.scope_id) + source: PerformanceSignal = next(source for source in sources if source.scope_id == issue.cohort.scope_id) await store.add_signal_batch( - _signals(issue.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),)) + _signals(issue.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),)) ) await store.queue_deliveries( - (_delivery(IssueDeliveryInput(id=issue.id, issue_id=issue.id, channel="slack"), rule, source),) + (_delivery(AlertDeliveryInput(id=issue.id, issue_id=issue.id, channel="slack"), rule, source),) ) assert await store.pending_delivery(issue.id) is not None - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Project A threshold", scope_type="key", scope_id="key-a", @@ -763,7 +765,7 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren store: Final = IssuesRepository(borrowed) await alerts.ensure_default_rules(default_usage_rules()) custom: Final = await alerts.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Custom spend", scope_type="key", scope_id="example-key", metric="spend", direction="increase" ), "qa-admin", @@ -773,8 +775,8 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren await alerts.delete_rule(removed_id) result: Final = await store.update_family( "spend", - IssueFamilyUpdate( - enabled=False, policies=(IssueFamilyPolicyUpdate(direction="increase", multiplier=5, min_value=25),) + AlertFamilyUpdate( + enabled=False, policies=(AlertFamilyPolicyUpdate(direction="increase", multiplier=5, min_value=25),) ), ) assert result is not None and result.total_rules == 4 and result.enabled_rules == 0 @@ -785,8 +787,8 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren with pytest.raises(ValidationError, match="min_history_days"): await store.update_family( "spend", - IssueFamilyUpdate( - policies=(IssueFamilyPolicyUpdate(direction="increase", baseline_days=1, min_history_days=7),) + AlertFamilyUpdate( + policies=(AlertFamilyPolicyUpdate(direction="increase", baseline_days=1, min_history_days=7),) ), ) assert await store.family_settings() == baseline @@ -801,7 +803,7 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren .validate_python(await transaction.query_raw("SELECT pg_backend_pid() AS pid"))[0] .pid ) - update: Final = tasks.create_task(store.update_family("spend", IssueFamilyUpdate(enabled=True))) + update: Final = tasks.create_task(store.update_family("spend", AlertFamilyUpdate(enabled=True))) async with asyncio.timeout(10): while ( not TypeAdapter(tuple[_Blocked, ...]) @@ -840,7 +842,7 @@ async def test_family_edit_requires_a_new_evaluation(db: ManagementDatabase) -> assert rule is not None await alerts.record_evaluations(((rule, "complete"),), date.today() - timedelta(days=1), completed=True) result: Final = await store.update_family( - "spend", IssueFamilyUpdate(policies=(IssueFamilyPolicyUpdate(direction="increase", multiplier=5),)) + "spend", AlertFamilyUpdate(policies=(AlertFamilyPolicyUpdate(direction="increase", multiplier=5),)) ) saved: Final = await alerts.get_rule(rule.id) assert result is not None and saved is not None @@ -860,7 +862,7 @@ async def test_deleted_key_owner_retains_historical_issue_access(gateway: Gatewa owner_key: Final = scenario.key(user_id=owner, key_type="management") outsider_key: Final = scenario.key(user_id=outsider, key_type="management") reporting_id: Final = sha256(uuid4().bytes).hexdigest() - issue: Final = _issue(IssueCohort(scope_type="key", scope_id=reporting_id)) + issue: Final = _issue(AlertCohort(scope_type="key", scope_id=reporting_id)) await _query( db, 'INSERT INTO "LiteLLM_DeletedVerificationToken" (id,token,user_id,models) VALUES ($1,$1,$2,ARRAY[]::text[])', @@ -869,7 +871,7 @@ async def test_deleted_key_owner_retains_historical_issue_access(gateway: Gatewa ) try: await store.upsert_issues((issue,)) - path: Final = f"/issues/{issue.id}" + path: Final = f"/alerts/{issue.id}" detail: Final = gateway.request("GET", path, key=owner_key) assert detail.status_code == 200, "Deleted key owner must retain historical issue access" assert detail.json()["can_update_status"] is True @@ -898,9 +900,9 @@ async def test_deleted_key_owner_retains_historical_issue_access(gateway: Gatewa async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: ManagementDatabase) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + issue: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) rule: Final = await alerts.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Scope guard", scope_type="user", scope_id=issue.cohort.scope_id, @@ -928,13 +930,13 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana )[0] await store.upsert_issues((issue,)) await store.add_signal_batch( - _signals(issue.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),)) + _signals(issue.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),)) ) authorized: Final = await store.status_scopes(issue.id) assert authorized == (issue.cohort,) assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=authorized) is not None context: Final = source.model_copy(update={"id": str(uuid4()), "metric": "spend"}) - same_scope: Final = (IssueSignalInput(source="usage_alert", source_id=context.id, usage_alert=context),) + same_scope: Final = (AlertSignalInput(source="performance", source_id=context.id, performance=context),) await store.add_signal_batch(_signals(issue.id, same_scope)) unchanged: Final = await store.get_issue(issue.id) assert unchanged is not None and unchanged.status == "dismissed" @@ -942,7 +944,7 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana assert await store.get_issue(issue.id) == unchanged broader: Final = source.model_copy(update={"id": str(uuid4()), "scope_type": "deployment", "scope_id": None}) await store.add_signal_batch( - _signals(issue.id, (IssueSignalInput(source="usage_alert", source_id=broader.id, usage_alert=broader),)) + _signals(issue.id, (AlertSignalInput(source="performance", source_id=broader.id, performance=broader),)) ) assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=authorized) is None current: Final = await store.get_issue(issue.id) @@ -953,7 +955,7 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana await store.add_signal_batch( _signals( issue.id, - (IssueSignalInput(source="usage_alert", source_id=broader_context.id, usage_alert=broader_context),), + (AlertSignalInput(source="performance", source_id=broader_context.id, performance=broader_context),), ) ) repeated_scope: Final = await store.get_issue(issue.id) @@ -989,7 +991,7 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana ) await locked.add_signal_batch( _signals( - issue.id, (IssueSignalInput(source="usage_alert", source_id=racing.id, usage_alert=racing),) + issue.id, (AlertSignalInput(source="performance", source_id=racing.id, performance=racing),) ) ) assert update.result() is None @@ -999,10 +1001,10 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana _signals( issue.id, tuple( - IssueSignalInput( - source="usage_alert", + AlertSignalInput( + source="performance", source_id=f"scope-{index}", - usage_alert=source.model_copy( + performance=source.model_copy( update={"id": f"scope-{index}", "scope_id": f"other-user-{index}"} ), ) @@ -1024,22 +1026,22 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana async def test_resource_recovery_ignores_broader_and_traffic_context(db: ManagementDatabase) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - fixture: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + fixture: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) issue: Final = fixture.model_copy( update={"family": "spend", "comparisons": (fixture.comparisons[0].model_copy(update={"metric": "spend"}),)} ) definitions: Final = ( - UsageAlertRuleInput( + AlertRuleInput( name="Primary cost", scope_type="user", scope_id=issue.cohort.scope_id, metric="spend", direction="increase" ), - UsageAlertRuleInput( + AlertRuleInput( name="Traffic context", scope_type="user", scope_id=issue.cohort.scope_id, metric="requests", direction="increase", ), - UsageAlertRuleInput(name="Broader cost", scope_type="deployment", metric="spend", direction="increase"), + AlertRuleInput(name="Broader cost", scope_type="deployment", metric="spend", direction="increase"), ) rules: Final = tuple([await alerts.save_rule(definition, "qa-admin") for definition in definitions]) assert all(rule is not None for rule in rules) @@ -1070,7 +1072,7 @@ async def test_resource_recovery_ignores_broader_and_traffic_context(db: Managem _signals( issue.id, tuple( - IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source) + AlertSignalInput(source="performance", source_id=source.id, performance=source) for source in sources ), ) @@ -1107,7 +1109,7 @@ async def test_resource_recovery_ignores_broader_and_traffic_context(db: Managem async def test_provider_recovery_pages_saved_cohorts_and_rejects_stale_observations(db: ManagementDatabase) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) prefix: Final = f"fixture-{uuid4()}" - initial: Final = _issue(IssueCohort(scope_type="provider", scope_id="example-provider")) + initial: Final = _issue(AlertCohort(scope_type="provider", scope_id="example-provider")) template: Final = initial.model_copy(update={"observed_at": initial.observed_at.replace(microsecond=123789)}) inputs: Final = tuple( template.model_copy(update={"id": prefix + suffix, "cohort_key": prefix + suffix}) @@ -1146,11 +1148,11 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - expired: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + expired: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) open_issue: Final = _issue(expired.cohort) recent: Final = _issue(expired.cohort) rule: Final = await alerts.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Prune fixture", scope_type="user", scope_id=expired.cohort.scope_id, @@ -1161,7 +1163,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) assert rule is not None deliveries: Final = tuple( - IssueDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") for _ in range(2) + AlertDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") for _ in range(2) ) try: await store.upsert_issues((expired, open_issue, recent)) @@ -1181,7 +1183,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) )[0] await store.add_signal_batch( - _signals(expired.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),)) + _signals(expired.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),)) ) for delivery in deliveries: await store.queue_deliveries((_delivery(delivery, rule, source),)) @@ -1226,6 +1228,61 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) +@pytest.mark.asyncio(loop_scope="session") +async def test_prune_preserves_retirement_reopened_by_new_usage(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + original: Final = _issue(AlertCohort(scope_type="model", scope_id=f"retirement-{uuid4()}")) + name: Final = original.cohort.scope_id or "example-model" + retirement: Final = original.observed_on + timedelta(days=10) + details: Final = AlertDeprecation( + model=ModelDeprecationInfo( + model_name=name, deprecation_date=retirement, days_until_deprecation=10, status="imminent" + ), + usage_model=name, + requests=10, + last_used_date=original.observed_on, + last_day_requests=10, + state=DeprecationAlertState(model=name, deprecation_date=retirement), + ) + issue: Final = original.model_copy( + update={ + "kind": "model_deprecation", + "family": "retirement", + "deprecation": details, + "grouping_reason": "model_deprecation", + "comparisons": (), + } + ) + try: + await store.upsert_issues((issue,)) + assert await store.set_status(issue.id, "dismissed", "open") is not None + await store.upsert_issues( + ( + issue.model_copy( + update={ + "deprecation": details.model_copy(update={"last_day_requests": 11, "requests": 11}), + "observed_at": original.observed_at + timedelta(minutes=1), + } + ), + ) + ) + await _query( + db, "UPDATE \"LiteLLM_Issue\" SET updated_at=TIMESTAMP '2000-01-01' WHERE id=$1 RETURNING id", issue.id + ) + reopened: Final = await store.get_issue(issue.id) + assert reopened is not None and reopened.status == "open" + await store.prune(retention_days=90, limit=10) + assert await store.get_issue(issue.id) == reopened + finally: + await _query( + db, + 'DELETE FROM "LiteLLM_Config" WHERE param_name IN ' + '(SELECT ack_key FROM "LiteLLM_Issue" WHERE id=$1) RETURNING param_name', + issue.id, + ) + await _remove_issues(db, (issue.id,)) + + class _MeasuredActivityDatabase: def __init__(self, database: DailyActivityQueryDatabase) -> None: self.database = database @@ -1250,7 +1307,7 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( day: Final = date.today() - timedelta(days=3) with gateway.scenario() as scenario: owner: Final = scenario.user(user_role="internal_user") - issue: Final = _issue(IssueCohort(scope_type="provider", scope_id=identity)).model_copy( + issue: Final = _issue(AlertCohort(scope_type="provider", scope_id=identity)).model_copy( update={"range_start": day, "range_end": day} ) try: @@ -1301,7 +1358,7 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( totals: Final = await repository.daily_activity(issue.cohort, day, day) assert totals.metadata.total_api_requests == 30 for endpoint in ("keys", "users"): - response: Final = gateway.request("GET", f"/issues/{issue.id}/{endpoint}", params={"limit": "1"}) + response: Final = gateway.request("GET", f"/alerts/{issue.id}/{endpoint}", params={"limit": "1"}) assert response.status_code == 200 and len(response.content) < 2048 assert "padding" not in response.text await _query( @@ -1324,8 +1381,8 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: ManagementDatabase) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) - definition: Final = UsageAlertRuleInput( + issue: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + definition: Final = AlertRuleInput( name="Primary traffic", scope_type="user", scope_id=issue.cohort.scope_id, @@ -1350,7 +1407,7 @@ async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: M _signals( issue.id, tuple( - IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source) + AlertSignalInput(source="performance", source_id=source.id, performance=source) for source in sources ), ) diff --git a/tests/integration/database/test_performance_issues.py b/tests/integration/database/test_performance_issues.py index addfafd2617..990eca901a5 100644 --- a/tests/integration/database/test_performance_issues.py +++ b/tests/integration/database/test_performance_issues.py @@ -22,7 +22,7 @@ from litellm.repositories.daily_activity_buckets import api_key_bucket from litellm.repositories.issues import IssuesRepository from litellm.repositories.usage_alert_metrics import IssueMetricWindow, UsageAlertMetricsRepository from litellm.repositories.usage_alerts import UsageAlertsRepository -from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRule, UsageAlertRuleInput, UsageAlertScope +from litellm.types.proxy.usage_alerts import UsageAlertObservation, AlertRule, AlertRuleInput, UsageAlertScope _DAY: Final = date(2026, 11, 4) _NOW: Final = datetime(2026, 11, 5, 3, microsecond=987654, tzinfo=timezone.utc) @@ -277,7 +277,7 @@ async def test_due_issue_delivery_retries_after_daily_projection_is_complete(db: await _seed(prisma) rules = UsageAlertsRepository.from_prisma(prisma) rule = await rules.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Key spend notification", scope_type="key", scope_id=_KEY_A, @@ -390,7 +390,7 @@ async def test_same_day_rule_revision_requires_a_fresh_observation(db: Managemen async with _isolated(db) as prisma: alerts: Final = UsageAlertsRepository.from_prisma(prisma) issues: Final = IssuesRepository.from_prisma(prisma) - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Project spend", scope_type="key", scope_id=_KEY_A, @@ -427,7 +427,7 @@ async def test_late_custom_key_rule_only_overrides_that_keys_default_signal(db: _KEY_B, ) job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) - # A rule can change after the daily observations, before Issue projection. + # A rule can change after the daily observations, before Alert projection. await job._store.ensure_default_rules(job._defaults) await job._store.ensure_projection_work(_DAY) await job._evaluate(_DAY, asyncio.get_running_loop().time() + 60) @@ -437,7 +437,7 @@ async def test_late_custom_key_rule_only_overrides_that_keys_default_signal(db: overridden: Final = next(item for item in original if item.alert.scope_id == _KEY_A) retained: Final = next(item for item in original if item.alert.scope_id == _KEY_B) custom: Final = await UsageAlertsRepository.from_prisma(prisma).save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Project A high spend threshold", scope_type="key", scope_id=_KEY_A, @@ -609,7 +609,7 @@ async def test_entity_page_progress_resumes_old_day_without_early_completion(db: await _coverage(prisma, _DAY) job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) rule: Final = await job._store.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Resume each key", scope_type="key", metric="spend", @@ -625,7 +625,7 @@ async def test_entity_page_progress_resumes_old_day_without_early_completion(db: assert partial.evaluation_progress is not None and partial.evaluation_progress.cursor is not None assert not await job._store.completed_rule_ids((rule,), _DAY) assert "evaluation_progress" not in partial.model_dump() - assert "evaluation_progress" not in UsageAlertRule.model_json_schema(mode="serialization")["properties"] + assert "evaluation_progress" not in AlertRule.model_json_schema(mode="serialization")["properties"] first: Final = await job._store.list_alerts(limit=500) assert first.total == 10 projector: Final = job._performance_projection @@ -662,7 +662,7 @@ async def test_entity_page_progress_resumes_old_day_without_early_completion(db: resumed: Final = await job._store.get_rule(rule.id) assert resumed is not None and resumed.evaluation_progress is None edited: Final = await job._store.save_rule( - UsageAlertRuleInput.model_validate(rule.model_dump(include=set(UsageAlertRuleInput.model_fields))), + AlertRuleInput.model_validate(rule.model_dump(include=set(AlertRuleInput.model_fields))), None, resumed, ) @@ -674,7 +674,7 @@ async def test_disabled_primary_requires_another_eligible_healthy_source_to_reco async with _isolated(db) as prisma: await _seed(prisma) job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Extra key policy", scope_type="key", scope_id=_KEY_A, @@ -713,7 +713,7 @@ async def test_lone_disabled_source_does_not_claim_recovery(db: ManagementDataba _KEY_A, ) job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Sensitive key policy", scope_type="key", scope_id=_KEY_A, @@ -743,9 +743,7 @@ async def test_rule_edit_discards_old_cursor_and_rejects_stale_progress(db: Mana async with _isolated(db) as prisma: store: Final = UsageAlertsRepository.from_prisma(prisma) - definition: Final = UsageAlertRuleInput( - name="Resumable", scope_type="key", metric="spend", direction="increase" - ) + definition: Final = AlertRuleInput(name="Resumable", scope_type="key", metric="spend", direction="increase") rule: Final = await store.save_rule(definition, None) assert rule is not None position: Final = UsageAlertProgress(day=_DAY, revision=rule.updated_at, cursor="a" * 64, evaluated=True) @@ -781,6 +779,38 @@ async def test_busy_background_capacity_defers_and_releases_the_worker_lease(db: ) == [{"count": 0}] +@pytest.mark.asyncio(loop_scope="session") +async def test_capacity_failure_before_coverage_does_not_pin_a_drop_rule_to_an_incomplete_day( + db: ManagementDatabase, +) -> None: + async with _isolated(db) as prisma: + await _seed(prisma) + store: Final = UsageAlertsRepository.from_prisma(prisma) + rule: Final = await store.save_rule( + AlertRuleInput( + name="Project traffic drop", + scope_type="key", + scope_id=_KEY_A, + metric="requests", + direction="decrease", + ), + "qa-admin", + ) + assert rule is not None + await prisma.writer_db.execute_raw( + 'DELETE FROM "LiteLLM_UsageAlertFlush" WHERE hour<$1::timestamp', (_DAY + timedelta(days=1)).isoformat() + ) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + async with prisma.writer_db.tx(timeout=timedelta(seconds=10)) as transaction: + await transaction.execute_raw('LOCK TABLE "LiteLLM_UsageAlertFlush" IN ACCESS EXCLUSIVE MODE') + assert not await job._attempt_scope_page((rule,), _DAY, False) + saved: Final = await store.get_rule(rule.id) + assert saved is not None and saved.evaluation_progress is None + await _next_day(prisma, _DAY + timedelta(days=1), 1, 1) + assert await job._attempt_scope_page((saved,), _DAY + timedelta(days=1), False) + assert await store.completed_rule_ids((saved,), _DAY + timedelta(days=1)) == frozenset((rule.id,)) + + @pytest.mark.asyncio(loop_scope="session") async def test_partial_broad_default_waits_for_its_key_family_only(db: ManagementDatabase) -> None: from litellm.types.proxy.usage_alert_defaults import default_usage_rules @@ -849,7 +879,7 @@ async def test_projection_resume_keeps_spend_and_traffic_in_one_episode(db: Mana before: Final = await IssuesRepository.from_prisma(prisma).list_issues() assert len(before.data) == 1 and before.data[0].family == "spend" added: Final = await job._store.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Later policy", scope_type="key", scope_id=_KEY_B, metric="latency", direction="increase" ), None, @@ -873,7 +903,7 @@ async def test_projection_resume_keeps_spend_and_traffic_in_one_episode(db: Mana after: Final = await IssuesRepository.from_prisma(prisma).list_issues() assert tuple(item.id for item in after.data) == tuple(item.id for item in before.data) links: Final = await IssuesRepository.from_prisma(prisma).list_signals(after.data[0].id) - assert any(link.usage_alert is not None and link.usage_alert.metric == "requests" for link in links.data) + assert any(link.performance is not None and link.performance.metric == "requests" for link in links.data) @pytest.mark.asyncio(loop_scope="session") @@ -891,7 +921,7 @@ async def test_slack_retry_only_repeats_the_failed_destination(db: ManagementDat rules: Final = UsageAlertsRepository.from_prisma(prisma) assert ( await rules.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Project spend", scope_type="key", scope_id=_KEY_A, @@ -905,7 +935,7 @@ async def test_slack_retry_only_repeats_the_failed_destination(db: ManagementDat ) assert ( await rules.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Another project rule", scope_type="key", scope_id=_KEY_A, @@ -938,8 +968,8 @@ async def test_slack_retry_only_repeats_the_failed_destination(db: ManagementDat retry: Final = (await IssuesRepository.from_prisma(prisma).pending_deliveries())[0] assert ( await rules.save_rule( - UsageAlertRuleInput.model_validate( - retry.rule.model_dump(include=set(UsageAlertRuleInput.model_fields)) + AlertRuleInput.model_validate( + retry.rule.model_dump(include=set(AlertRuleInput.model_fields)) ).model_copy(update={"enabled": False}), "qa-admin", retry.rule, @@ -1001,7 +1031,7 @@ async def test_cancelled_email_receipt_survives_a_different_delivery( rules: Final = UsageAlertsRepository.from_prisma(prisma) assert ( await rules.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Project spend", scope_type="key", scope_id=_KEY_A, @@ -1056,15 +1086,13 @@ async def test_large_custom_rule_scope_does_not_starve_other_scopes_on_resume(db await _seed(prisma) store: Final = UsageAlertsRepository.from_prisma(prisma) key_rule: Final = await store.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Project spend", scope_type="key", scope_id=_KEY_A, metric="spend", direction="increase" ), "qa-user", ) team_rule: Final = await store.save_rule( - UsageAlertRuleInput( - name="Team spend", scope_type="team", scope_id="team", metric="spend", direction="increase" - ), + AlertRuleInput(name="Team spend", scope_type="team", scope_id="team", metric="spend", direction="increase"), "qa-admin", ) assert key_rule is not None and team_rule is not None diff --git a/tests/integration/database/test_provider_latency.py b/tests/integration/database/test_provider_latency.py index e8413460ed3..387f40858da 100644 --- a/tests/integration/database/test_provider_latency.py +++ b/tests/integration/database/test_provider_latency.py @@ -5,8 +5,8 @@ import pytest from integration._support.actors import ManagementDatabase from litellm.repositories.provider_latency import ProviderLatencyRepository -from litellm.types.proxy.issues import IssueModelPair -from litellm.types.proxy.usage_alerts import UsageAlertRuleInput +from litellm.types.proxy.issues import AlertModelPair +from litellm.types.proxy.usage_alerts import AlertRuleInput @pytest.mark.asyncio(loop_scope="session") @@ -38,7 +38,7 @@ async def test_provider_lookup_skips_oversized_names_and_preserves_valid_models( tuple(pair[0] for pair in pairs), tuple(pair[1] for pair in pairs), ) - expected = tuple(IssueModelPair(provider=p, model=m) for p, m in pairs[2:5]) + expected = tuple(AlertModelPair(provider=p, model=m) for p, m in pairs[2:5]) first = await repository.provider_pairs(provider, day) second = await repository.provider_pairs(wide_provider, day) invalid = await repository.provider_pairs("a" * 257, day) @@ -58,9 +58,9 @@ async def test_provider_lookup_skips_oversized_names_and_preserves_valid_models( async def test_provider_latency_requires_established_within_model_regressions(db: ManagementDatabase) -> None: prefix = f"provider-latency-{uuid4()}" day = date(2031, 1, 12) - pairs = tuple(IssueModelPair(provider=prefix, model=f"{prefix}-{suffix}") for suffix in ("a", "b")) + pairs = tuple(AlertModelPair(provider=prefix, model=f"{prefix}-{suffix}") for suffix in ("a", "b")) keys = (prefix + "-key-a", prefix + "-key-b") - rule = UsageAlertRuleInput(name="Latency", scope_type="key", metric="latency", direction="increase", min_value=500) + rule = AlertRuleInput(name="Latency", scope_type="key", metric="latency", direction="increase", min_value=500) repository = ProviderLatencyRepository(db.prisma.writer_db) try: await db.prisma.writer_db.query_raw( diff --git a/tests/integration/database/test_usage_alert_evaluation.py b/tests/integration/database/test_usage_alert_evaluation.py index 799c36df03f..fdc12679230 100644 --- a/tests/integration/database/test_usage_alert_evaluation.py +++ b/tests/integration/database/test_usage_alert_evaluation.py @@ -20,13 +20,13 @@ from litellm.repositories.accounting_delivery_repository import ( ) from litellm.repositories.usage_alert_health import UsageAlertFlushRecorder, UsageAlertHealthRepository from litellm.repositories.usage_alerts import UsageAlertsRepository -from litellm.types.proxy.usage_alerts import UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import AlertRuleInput @pytest.mark.asyncio(loop_scope="session") async def test_batch_checkpoints_preserve_rule_revisions_and_incomplete_status(db: ManagementDatabase) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Daily evaluation", scope_type="team", scope_id="team-example", metric="spend", direction="increase" ) rule: Final = await repository.save_rule(definition, "qa-admin") diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index 5ebe5808014..b477f07d447 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -9,12 +9,11 @@ from integration._support.client import Gateway, string_value from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.proxy.usage_alerts import ( - UsageAlertGroupList, + AlertModelList, + AlertRule, + AlertRuleInput, UsageAlertMetric, - UsageAlertModelList, UsageAlertObservation, - UsageAlertRule, - UsageAlertRuleInput, ) @@ -22,7 +21,7 @@ from litellm.types.proxy.usage_alerts import ( async def test_alert_resource_labels_bound_legacy_display_values(db: ManagementDatabase) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) identity: Final = "label-bound-" + uuid4().hex - scope: Final = UsageAlertRuleInput( + scope: Final = AlertRuleInput( name="Display label", scope_type="user", scope_id=identity, metric="spend", direction="increase" ) await repository.database.query_raw( @@ -43,7 +42,7 @@ async def test_alert_resource_labels_bound_legacy_display_values(db: ManagementD async def test_deleted_key_labels_use_the_latest_alias_and_prefer_live_keys(db: ManagementDatabase) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) identity: Final = sha256(uuid4().bytes).hexdigest() - scope: Final = UsageAlertRuleInput( + scope: Final = AlertRuleInput( name="Deleted key", scope_type="key", scope_id=identity, metric="spend", direction="increase" ) try: @@ -63,7 +62,7 @@ async def test_deleted_key_labels_use_the_latest_alias_and_prefer_live_keys(db: await repository.database.query_raw('DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', identity) -@pytest.mark.parametrize("grant", ["management_routes", "/usage", "/usage/alert-rules"]) +@pytest.mark.parametrize("grant", ["management_routes", "/alerts", "/alerts/rules"]) def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gateway: Gateway, grant: str) -> None: with gateway.scenario() as scenario: owner: Final = scenario.user(user_role="internal_user") @@ -81,10 +80,10 @@ def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gate "direction": "increase", "multiplier": 3, } - created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + created: Final = gateway.request("POST", "/alerts/rules", config, key=owner_key) assert created.status_code == 201, created.text rule_id: Final = string_value(created.json()["id"]) - path: Final = f"/usage/alert-rules/{rule_id}" + path: Final = f"/alerts/rules/{rule_id}" try: assert gateway.request("GET", path, key=owner_key).status_code == 200 assert gateway.request("GET", path, key=outsider_key).status_code == 403 @@ -93,14 +92,13 @@ def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gate assert gateway.request("PUT", path, config, key=viewer_key).status_code == 403 assert gateway.request("PUT", path, config, key=outsider_key).status_code == 403 assert ( - gateway.request("POST", "/usage/alert-rules", {**config, "scope_id": None}, key=owner_key).status_code - == 403 + gateway.request("POST", "/alerts/rules", {**config, "scope_id": None}, key=owner_key).status_code == 403 ) disabled: Final = gateway.request("PUT", path, {**config, "enabled": False}, key=owner_key) assert disabled.status_code == 200 and disabled.json()["enabled"] is False page: Final = gateway.request( "GET", - "/usage/alert-rules", + "/alerts/rules", key=owner_key, params={"scope_type": "key", "scope_id": str(config["scope_id"]), "limit": "1"}, ) @@ -122,9 +120,9 @@ def test_admin_edit_preserves_owner_rule_access(gateway: Gateway) -> None: "metric": "spend", "direction": "increase", } - created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + created: Final = gateway.request("POST", "/alerts/rules", config, key=owner_key) assert created.status_code == 201, created.text - path: Final = f"/usage/alert-rules/{created.json()['id']}" + path: Final = f"/alerts/rules/{created.json()['id']}" try: edited: Final = gateway.request("PUT", path, {**config, "name": "Renamed by admin"}) assert edited.status_code == 200, edited.text @@ -151,10 +149,10 @@ def test_team_admin_can_manage_team_alerts_but_member_can_only_read(gateway: Gat "direction": "decrease", "multiplier": 2, } - created: Final = gateway.request("POST", "/usage/alert-rules", config, key=admin_key) + created: Final = gateway.request("POST", "/alerts/rules", config, key=admin_key) assert created.status_code == 201, created.text rule_id: Final = string_value(created.json()["id"]) - path: Final = f"/usage/alert-rules/{rule_id}" + path: Final = f"/alerts/rules/{rule_id}" try: assert gateway.request("GET", path, key=member_key).status_code == 200 assert gateway.request("PUT", path, config, key=member_key).status_code == 403 @@ -177,21 +175,21 @@ def test_info_only_credentials_cannot_mutate_alerts(gateway: Gateway) -> None: "metric": "spend", "direction": "increase", } - created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + created: Final = gateway.request("POST", "/alerts/rules", config, key=owner_key) assert created.status_code == 201 - path: Final = f"/usage/alert-rules/{created.json()['id']}" + path: Final = f"/alerts/rules/{created.json()['id']}" try: assert gateway.request("GET", path, key=reader).status_code == 200 - assert gateway.request("POST", "/usage/alert-rules", config, key=reader).status_code == 403 + assert gateway.request("POST", "/alerts/rules", config, key=reader).status_code == 403 assert gateway.request("PUT", path, config, key=reader).status_code == 403 assert gateway.request("DELETE", path, key=reader).status_code == 403 assert ( - gateway.request("PATCH", "/issues/families/spend", {"enabled": False}, key=admin_reader).status_code + gateway.request("PATCH", "/alerts/defaults/spend", {"enabled": False}, key=admin_reader).status_code == 403 ) assert ( gateway.request( - "PATCH", "/issues/unavailable", {"status": "resolved", "expected_status": "open"}, key=admin_reader + "PATCH", "/alerts/unavailable", {"status": "resolved", "expected_status": "open"}, key=admin_reader ).status_code == 403 ) @@ -215,23 +213,19 @@ def test_only_admins_can_configure_operator_notifications(gateway: Gateway) -> N {"notification_channels": ["email"], "email_recipients": ["ops@example.com"]}, ): assert ( - gateway.request("POST", "/usage/alert-rules", {**config, **notification}, key=owner_key).status_code - == 403 + gateway.request("POST", "/alerts/rules", {**config, **notification}, key=owner_key).status_code == 403 ) assert ( - gateway.request( - "POST", "/usage/alert-rules", {**config, "overrides_defaults": True}, key=owner_key - ).status_code + gateway.request("POST", "/alerts/rules", {**config, "overrides_defaults": True}, key=owner_key).status_code == 422 ) assert ( - gateway.request("POST", "/usage/alert-rules", {**config, "name": "Spend\r\nInjected: header"}).status_code - == 422 + gateway.request("POST", "/alerts/rules", {**config, "name": "Spend\r\nInjected: header"}).status_code == 422 ) - created: Final = gateway.request("POST", "/usage/alert-rules", {**config, "notification_channels": ["slack"]}) + created: Final = gateway.request("POST", "/alerts/rules", {**config, "notification_channels": ["slack"]}) assert created.status_code == 201 assert "overrides_defaults" not in created.json() - path: Final = f"/usage/alert-rules/{created.json()['id']}" + path: Final = f"/alerts/rules/{created.json()['id']}" try: assert gateway.request("PUT", path, config, key=owner_key).status_code == 403 assert gateway.request("DELETE", path, key=owner_key).status_code == 403 @@ -252,7 +246,7 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") created: Final = gateway.request( "POST", - "/usage/alert-rules", + "/alerts/rules", { "name": "Incident API fixture", "scope_type": "user", @@ -264,7 +258,7 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew key=owner_key, ) assert created.status_code == 201 - rule: Final = UsageAlertRule.model_validate(created.json()) + rule: Final = AlertRule.model_validate(created.json()) try: incident: Final = ( await repository.record_observations( @@ -282,9 +276,9 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew ), ) )[0] - path: Final = f"/usage/alerts/{incident.id}" + path: Final = f"/alerts/signals/{incident.id}" params: Final = {"scope_type": "user", "scope_id": owner, "limit": "1"} - page: Final = gateway.request("GET", "/usage/alerts", key=owner_key, params=params) + page: Final = gateway.request("GET", "/alerts/signals", key=owner_key, params=params) assert page.status_code == 200 and page.json()["data"][0]["id"] == incident.id detail: Final = gateway.request("GET", path, key=owner_key) assert detail.status_code == 200 @@ -293,22 +287,19 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew assert detail.json()["threshold"] == 360 assert detail.json()["baseline_sample_days"] == 7 assert "detail_tab=activity" in detail.json()["activity_path"] - assert gateway.request("GET", "/usage/alerts", key=outsider_key, params=params).status_code == 403 + assert gateway.request("GET", "/alerts/signals", key=outsider_key, params=params).status_code == 403 assert gateway.request("GET", path, key=outsider_key).status_code == 403 - assert gateway.request("PATCH", path, {"status": "dismissed"}, key=outsider_key).status_code == 403 assert gateway.request("GET", path, key=viewer_key).status_code == 200 - assert gateway.request("PATCH", path, {"status": "dismissed"}, key=viewer_key).status_code == 403 - dismissed: Final = gateway.request("PATCH", path, {"status": "dismissed"}, key=owner_key) - assert dismissed.status_code == 200 and dismissed.json()["status"] == "dismissed" - assert gateway.request("GET", path, key=owner_key).json()["status"] == "dismissed" + assert gateway.request("PATCH", path, {"status": "dismissed"}, key=owner_key).status_code == 405 + assert gateway.request("GET", path, key=owner_key).json()["status"] == "open" finally: - assert gateway.request("DELETE", f"/usage/alert-rules/{rule.id}", key=owner_key).status_code == 204 + assert gateway.request("DELETE", f"/alerts/rules/{rule.id}", key=owner_key).status_code == 204 @pytest.mark.asyncio(loop_scope="session") async def test_ongoing_episode_preserves_its_original_evidence_and_activity_link(db: ManagementDatabase) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Original episode", scope_type="user", scope_id="episode-example", metric="requests", direction="increase" ) rule: Final = await repository.save_rule(definition, "qa-admin") @@ -358,7 +349,7 @@ async def test_ongoing_episode_preserves_its_original_evidence_and_activity_link @pytest.mark.asyncio(loop_scope="session") async def test_rule_scope_changes_reject_stale_updates_and_deletes(db: ManagementDatabase) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Concurrent scope edit", scope_type="user", scope_id="original-owner", metric="spend", direction="increase" ) original: Final = await repository.save_rule(definition, "qa-admin") @@ -394,7 +385,7 @@ async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency( db: ManagementDatabase, changed_field: str, new_value: str ) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Semantic identity", scope_type="team", scope_id="semantic-example", @@ -410,7 +401,7 @@ async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency( try: original: Final = (await repository.record_observations(rule, (initial,)))[0] edited: Final = await repository.save_rule( - UsageAlertRuleInput.model_validate({**definition.model_dump(), changed_field: new_value}), "qa-admin", rule + AlertRuleInput.model_validate({**definition.model_dump(), changed_field: new_value}), "qa-admin", rule ) assert edited is not None observation: Final = initial.model_copy(update={"entity_id": edited.scope_id}) @@ -437,190 +428,12 @@ async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency( await repository.delete_rule(rule.id) -@pytest.mark.asyncio(loop_scope="session") -async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_actions( - gateway: Gateway, db: ManagementDatabase -) -> None: - repository: Final = UsageAlertsRepository.from_prisma(db.prisma) - day: Final = date(2026, 11, 4) - with gateway.scenario() as scenario: - owner: Final = scenario.user(user_role="internal_user", user_alias="Checkout operator") - outsider: Final = scenario.user(user_role="internal_user") - viewer: Final = scenario.user(user_role="proxy_admin_viewer") - owner_key: Final = scenario.key(user_id=owner, key_type="management") - outsider_key: Final = scenario.key(user_id=outsider, key_type="management") - viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") - inference_key: Final = scenario.key(user_id=owner, key_type="llm_api") - metrics: Final[tuple[UsageAlertMetric, ...]] = ( - *(("requests",) * 27), - "spend", - "errors", - "latency", - "error_rate", - ) - rules: Final = tuple( - [ - await repository.save_rule( - UsageAlertRuleInput( - name=f"Group rule {index}", - scope_type="user", - scope_id=owner, - metric=metric, - direction="increase", - baseline_days=7 if index % 2 else 14, - ), - owner, - ) - for index, metric in enumerate(metrics) - ] - ) - assert all(rule is not None for rule in rules) - incidents: Final = tuple( - [ - ( - await repository.record_observations( - rule, - ( - UsageAlertObservation( - entity_id=owner, - day=day, - observed=14 if rule.metric == "error_rate" else 420, - baseline=4 if rule.metric == "error_rate" else 0 if index == 0 else 120, - threshold=12 if rule.metric == "error_rate" else 100 if index == 0 else 360, - breached=True, - ), - ), - ) - )[0] - for index, rule in enumerate(rules) - if rule is not None - ] - ) - old_rule: Final = await repository.save_rule( - UsageAlertRuleInput( - name="Earlier day", scope_type="user", scope_id=owner, metric="requests", direction="increase" - ), - owner, - ) - assert old_rule is not None - old: Final = ( - await repository.record_observations( - old_rule, - ( - UsageAlertObservation( - entity_id=owner, - day=day - timedelta(days=1), - observed=420, - baseline=120, - threshold=360, - breached=True, - ), - ), - ) - )[0] - other_rule: Final = await repository.save_rule( - UsageAlertRuleInput( - name="Other resource", scope_type="user", scope_id=outsider, metric="requests", direction="increase" - ), - outsider, - ) - assert other_rule is not None - other: Final = ( - await repository.record_observations( - other_rule, - ( - UsageAlertObservation( - entity_id=outsider, - day=day, - observed=420, - baseline=120, - threshold=360, - breached=True, - ), - ), - ) - )[0] - params: Final = {"scope_type": "user", "scope_id": owner, "status": "open", "limit": "1"} - try: - response: Final = gateway.request("GET", "/usage/alert-groups", key=owner_key, params=params) - assert response.status_code == 200, response.text - page: Final = UsageAlertGroupList.model_validate(response.json()) - assert page.total == 2 and len(page.data) == 1 - group: Final = page.data[0] - assert group.day == day and group.scope_label == "Checkout operator" - assert group.incident_count == 31 and len(group.metrics) == 5 - assert {metric.metric: metric.incident_count for metric in group.metrics} == { - "requests": 27, - "spend": 1, - "errors": 1, - "latency": 1, - "error_rate": 1, - } - assert all(metric.representative.id in {incident.id for incident in incidents} for metric in group.metrics) - assert group.activity_path in {incident.activity_path for incident in incidents} - second: Final = UsageAlertGroupList.model_validate( - gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "1"}).json() - ) - assert second.total == 2 and second.data[0].day == old.day - empty: Final = UsageAlertGroupList.model_validate( - gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "2"}).json() - ) - assert empty.total == 2 and empty.data == () - history: Final = gateway.request( - "GET", - "/usage/alerts", - key=owner_key, - params={**params, "start_date": day.isoformat(), "end_date": day.isoformat(), "metric": "requests"}, - ) - assert history.status_code == 200 and history.json()["total"] == 27 - await repository.set_status(incidents[0].id, "dismissed") - body: Final = { - "scope_type": "user", - "scope_id": owner, - "day": day.isoformat(), - "expected_status": "open", - "status": "resolved", - } - for forbidden in (outsider_key, inference_key): - assert gateway.request("GET", "/usage/alert-groups", key=forbidden, params=params).status_code == 403 - for forbidden in (outsider_key, inference_key, viewer_key): - assert gateway.request("PATCH", "/usage/alert-groups", body, key=forbidden).status_code == 403 - assert gateway.request("GET", "/usage/alert-groups", key=viewer_key, params=params).status_code == 200 - updated: Final = gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key) - assert updated.status_code == 200 and updated.json()["updated_count"] == 30 - assert gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key).json()["updated_count"] == 0 - assert await repository.get_alert(old.id) == old - assert await repository.get_alert(other.id) == other - dismissed: Final = await repository.get_alert(incidents[0].id) - assert dismissed is not None and dismissed.status == "dismissed" and dismissed.baseline_value == 0 - resolved: Final = await repository.get_alert(incidents[-1].id) - assert resolved == incidents[-1].model_copy(update={"status": "resolved"}) - assert ( - gateway.request( - "GET", "/usage/alert-groups", params={"scope_type": "user", "scope_id": "x' OR 1=1 --"} - ).json()["total"] - == 0 - ) - assert ( - gateway.request( - "GET", "/usage/alert-groups", params={"start_date": "2026-11-05", "end_date": "2026-11-04"} - ).status_code - == 422 - ) - finally: - for rule in (*rules, old_rule, other_rule): - if rule is not None: - await repository.delete_rule(rule.id) - deleted: Final = await repository.list_groups(scope_type="user", scope_id=owner) - assert deleted.total == 2 and all(group.scope_label is None and group.scope_id == owner for group in deleted.data) - - @pytest.mark.asyncio(loop_scope="session") async def test_default_rule_summary_and_kind_filters_do_not_require_loading_every_rule( gateway: Gateway, db: ManagementDatabase ) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) - default: Final = UsageAlertRuleInput( + default: Final = AlertRuleInput( name="Built-in daily model requests", scope_type="model", metric="requests", direction="increase" ) await repository.ensure_default_rules((default,)) @@ -645,12 +458,12 @@ async def test_default_rule_summary_and_kind_filters_do_not_require_loading_ever stored: Final = await repository.get_rule(builtins.data[0].id) assert stored is not None and stored.is_default filtered: Final = gateway.request( - "GET", "/usage/alert-rules", params={"scope_type": "model", "kind": "custom", "limit": "1"} + "GET", "/alerts/rules", params={"scope_type": "model", "kind": "custom", "limit": "1"} ) assert filtered.status_code == 200 and filtered.json()["total"] == customs.total assert all(not rule["is_default"] for rule in filtered.json()["data"]) - configuration: Final = stored.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields)) - path: Final = f"/usage/alert-rules/{stored.id}" + configuration: Final = stored.model_dump(mode="json", include=frozenset(AlertRuleInput.model_fields)) + path: Final = f"/alerts/rules/{stored.id}" for change in ( {"scope_type": "key"}, {"scope_id": "model-example"}, @@ -665,9 +478,9 @@ async def test_default_rule_summary_and_kind_filters_do_not_require_loading_ever assert updated.json()["min_value"] == stored.min_value + 1 finally: assert gateway.request("PUT", path, configuration).status_code == 200 - custom_input: Final = custom.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields)) + custom_input: Final = custom.model_dump(mode="json", include=frozenset(AlertRuleInput.model_fields)) assert ( - gateway.request("PUT", f"/usage/alert-rules/{custom.id}", {**custom_input, "metric": "latency"}).status_code + gateway.request("PUT", f"/alerts/rules/{custom.id}", {**custom_input, "metric": "latency"}).status_code == 200 ) finally: @@ -704,9 +517,9 @@ async def test_reported_model_search_preserves_exact_identity_and_cursor( prefix, f"{prefix}/old", ) - response: Final = gateway.request("GET", "/usage/alert-models", params={"search": prefix, "limit": "1"}) + response: Final = gateway.request("GET", "/alerts/models", params={"search": prefix, "limit": "1"}) assert response.status_code == 200, response.text - first: Final = UsageAlertModelList.model_validate(response.json()) + first: Final = AlertModelList.model_validate(response.json()) assert first.data[0].scope_id == models[0] and first.next_cursor == models[0] second: Final = await repository.list_reported_models(prefix, first.next_cursor, 1) third: Final = await repository.list_reported_models(prefix, second.next_cursor, 1) @@ -720,6 +533,6 @@ async def test_reported_model_search_preserves_exact_identity_and_cursor( with gateway.scenario() as scenario: user: Final = scenario.user(user_role="internal_user") key: Final = scenario.key(user_id=user, key_type="management") - assert gateway.request("GET", "/usage/alert-models", key=key).status_code == 403 + assert gateway.request("GET", "/alerts/models", key=key).status_code == 403 finally: await db.prisma.writer_db.execute_raw('DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', prefix) diff --git a/tests/integration/observability/test_usage_alert_email.py b/tests/integration/observability/test_usage_alert_email.py index 961f304f547..efb65919550 100644 --- a/tests/integration/observability/test_usage_alert_email.py +++ b/tests/integration/observability/test_usage_alert_email.py @@ -9,11 +9,11 @@ import pytest from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery -from litellm.types.proxy.issues import Issue, IssueCohort, IssueDelivery, IssuePendingDelivery -from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule +from litellm.types.proxy.issues import Alert, AlertCohort, AlertDelivery, AlertPendingDelivery +from litellm.types.proxy.usage_alerts import PerformanceSignal, AlertRule _NOW: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) -_RULE: Final = UsageAlertRule( +_RULE: Final = AlertRule( name="Request spike", id="rule", scope_type="key", @@ -25,7 +25,7 @@ _RULE: Final = UsageAlertRule( created_by=None, notification_channels=("slack",), ) -_ALERT: Final = UsageAlert( +_ALERT: Final = PerformanceSignal( id="incident", rule_id="rule", rule_name="Request spike", @@ -47,13 +47,13 @@ _ALERT: Final = UsageAlert( ) -_ISSUE: Final = Issue( +_ISSUE: Final = Alert( id="issue-example", cohort_key="cohort-example", kind="performance", family="traffic", title="Traffic increased", - cohort=IssueCohort(scope_type="key", scope_id="reporting-key"), + cohort=AlertCohort(scope_type="key", scope_id="reporting-key"), observed_on=_ALERT.day, observed_at=_NOW, range_start=_ALERT.baseline_start, @@ -64,11 +64,11 @@ _ISSUE: Final = Issue( last_seen=_NOW, updated_at=_NOW, ) -_PENDING: Final = IssuePendingDelivery( +_PENDING: Final = AlertPendingDelivery( issue=_ISSUE, rule=_RULE, alert=_ALERT, - delivery=IssueDelivery(id="delivery-example", issue_id=_ISSUE.id, channel="slack"), + delivery=AlertDelivery(id="delivery-example", issue_id=_ISSUE.id, channel="slack"), ) diff --git a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py index 985fbff0328..1a23434cd10 100644 --- a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py +++ b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py @@ -7,11 +7,11 @@ import pytest from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, activity_url, incident_message from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.types.proxy.issues import Issue, IssueCohort, IssueDelivery, IssuePendingDelivery -from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule +from litellm.types.proxy.issues import Alert, AlertCohort, AlertDelivery, AlertPendingDelivery +from litellm.types.proxy.usage_alerts import PerformanceSignal, AlertRule _NOW: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) -_RULE: Final = UsageAlertRule( +_RULE: Final = AlertRule( name="Request spike", id="rule", scope_type="key", @@ -23,7 +23,7 @@ _RULE: Final = UsageAlertRule( created_by=None, notification_channels=("slack",), ) -_ALERT: Final = UsageAlert( +_ALERT: Final = PerformanceSignal( id="incident", rule_id="rule", rule_name="Request spike", @@ -45,13 +45,13 @@ _ALERT: Final = UsageAlert( ) -_ISSUE: Final = Issue( +_ISSUE: Final = Alert( id="issue-example", cohort_key="cohort-example", kind="performance", family="traffic", title="Traffic increased", - cohort=IssueCohort(scope_type="key", scope_id="reporting-key"), + cohort=AlertCohort(scope_type="key", scope_id="reporting-key"), observed_on=_ALERT.day, observed_at=_NOW, range_start=_ALERT.baseline_start, @@ -62,11 +62,11 @@ _ISSUE: Final = Issue( last_seen=_NOW, updated_at=_NOW, ) -_PENDING: Final = IssuePendingDelivery( +_PENDING: Final = AlertPendingDelivery( issue=_ISSUE, rule=_RULE, alert=_ALERT, - delivery=IssueDelivery(id="delivery-example", issue_id=_ISSUE.id, channel="slack"), + delivery=AlertDelivery(id="delivery-example", issue_id=_ISSUE.id, channel="slack"), ) @@ -131,7 +131,7 @@ async def test_issue_delivery_links_to_investigation_and_preserves_the_bound_sou try: assert await UsageAlertDelivery(slack).send_issue(_PENDING) assert len(requests) == 1 - assert b"https://gateway.example.com/ui/alerts?issue=issue-example" in requests[0].content + assert b"https://gateway.example.com/ui/alerts?alert=issue-example" in requests[0].content assert ( b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content ) diff --git a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py index 239151f9ce1..5107e55743a 100644 --- a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py +++ b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py @@ -21,20 +21,20 @@ class NoDatabase: def test_info_grants_do_not_authorize_mutations(role: LitellmUserRoles, key_type: str | None) -> None: user: Final = UserAPIKeyAuth(user_role=role, key_type=key_type, allowed_routes=["info_routes", "openai_routes"]) with pytest.raises(HTTPException) as error: - require_write_access(user, "/usage/alert-rules/example") + require_write_access(user, "/alerts/rules/example") assert error.value.status_code == 403 -@pytest.mark.parametrize("grant", ["management_routes", "/usage", "/usage/alert-rules", "/usage/*"]) +@pytest.mark.parametrize("grant", ["management_routes", "/alerts", "/alerts/rules", "/alerts/*"]) def test_explicit_route_grants_authorize_alert_mutations(grant: str) -> None: - assert _allows_write(grant, "/usage/alert-rules/example") is True + assert _allows_write(grant, "/alerts/rules/example") is True -@pytest.mark.parametrize("grant", ["/usag", "/usage/alert-rule", "/usage/alerts"]) +@pytest.mark.parametrize("grant", ["/alert", "/alerts/rule", "/alerts/signals"]) def test_route_grants_require_a_complete_path_segment(grant: str) -> None: user: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, allowed_routes=[grant]) with pytest.raises(HTTPException) as error: - require_write_access(user, "/usage/alert-rules/example") + require_write_access(user, "/alerts/rules/example") assert error.value.status_code == 403 @@ -91,10 +91,10 @@ def test_global_activity_link_uses_the_usage_page_date_contract() -> None: @pytest.mark.asyncio async def test_no_database_returns_an_actionable_service_error() -> None: - from litellm.proxy.alerting_endpoints.usage_alerts import list_usage_alerts + from litellm.proxy.alerting_endpoints.usage_alerts import list_performance_signals user: Final = UserAPIKeyAuth(user_id="qa-admin", user_role=LitellmUserRoles.PROXY_ADMIN) with pytest.raises(HTTPException) as error: - await list_usage_alerts(user, None) + await list_performance_signals(user, None) assert error.value.status_code == 503 - assert error.value.detail == "Usage alerts require a database connection" + assert error.value.detail == "Alert signals require a database connection" diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index e2debe05c03..9d588c8556c 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -3294,8 +3294,9 @@ class _WindowSpendFakeDB: class _WindowSpendFakePrisma: - def __init__(self, db): + def __init__(self, db, writer_db=None): self.db = db + self.writer_db = writer_db if writer_db is not None else db def _window_spend_upserts(db): @@ -5515,6 +5516,59 @@ async def test_cancel_that_lands_while_the_daily_batch_is_committing_waits_for_t assert _daily_upserts(final_db, table) == [], "the final flush must not write the committed batch again" +class _FreshnessRecordingDB: + def __init__(self) -> None: + self.started = asyncio.Event() + self.release = asyncio.Event() + self.recorded: tuple[object, ...] = () + + @asynccontextmanager + async def tx(self, *, max_wait: timedelta, timeout: timedelta): + yield self + + async def query_raw(self, query: str, *args: object): + if "pg_try_advisory_xact_lock" in query: + return [{"acquired": True}] + self.started.set() + await self.release.wait() + self.recorded = args + return [{"source": args[0]}] + + +@pytest.mark.parametrize("entity_type", ("user", "team")) +@pytest.mark.parametrize("cancel_during", ("commit", "freshness")) +@pytest.mark.asyncio +async def test_cancelled_daily_flush_records_freshness_after_commit_without_requeuing(entity_type, cancel_during): + writer = DBSpendUpdateWriter() + queue = writer.daily_spend_update_queue if entity_type == "user" else writer.daily_team_spend_update_queue + await queue.add_update({"key-a": _daily_entity_txn(entity_type + "_id")}) + db = _CommittingDailySpendFakeDB() + freshness = _FreshnessRecordingDB() + tick = asyncio.ensure_future( + writer._flush_daily_spend_queue( + queue=queue, + entity_type=entity_type, + commit=_DAILY_SPEND_COMMITS[entity_type], + n_retry_times=0, + prisma_client=_WindowSpendFakePrisma(db, freshness), + proxy_logging_obj=MagicMock(), + ) + ) + await asyncio.wait_for(db.committing.wait(), timeout=5) + if cancel_during == "freshness": + db.commit_release.set() + await asyncio.wait_for(freshness.started.wait(), timeout=5) + tick.cancel() + with pytest.raises(asyncio.CancelledError): + await tick + db.commit_release.set() + freshness.release.set() + await queue.settle_interrupted_commits() + assert freshness.recorded and freshness.recorded[0] == entity_type + assert len(_daily_upserts(db, "LiteLLM_Daily" + entity_type.title() + "Spend")) == 1 + assert queue.update_queue.empty() + + @pytest.mark.asyncio async def test_tag_batch_drained_from_redis_and_cancelled_while_committing_is_not_restored(): """Same in-flight COMMIT as the in-memory path, but the drained rows live in Redis. Restoring diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py index a7f2ef5c2e3..5dd3f96c57d 100644 --- a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -19,8 +19,8 @@ from litellm.repositories.usage_alert_health import UsageAlertHealthRepository from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, UsageAlertMetricsRepository from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.proxy.usage_alert_defaults import default_usage_rule_id, default_usage_rules -from litellm.types.proxy.issues import IssuePendingDelivery -from litellm.types.proxy.usage_alerts import UsageAlertRule, UsageAlertRuleInput +from litellm.types.proxy.issues import AlertPendingDelivery +from litellm.types.proxy.usage_alerts import AlertRule, AlertRuleInput _DAY: Final = date(2026, 10, 20) @@ -93,7 +93,7 @@ class _BudgetDatabase: class _UnexpectedDelivery: - async def send_issue(self, pending: IssuePendingDelivery) -> bool: + async def send_issue(self, pending: AlertPendingDelivery) -> bool: raise AssertionError("stalled evaluation must not deliver alerts") @@ -149,7 +149,7 @@ def _row(day: date, requests: int = 200, spend: float = 20, ms: float = 100_000, "metric,observed,baseline", [("spend", 100, 20), ("requests", 1000, 200), ("latency", 2000, 500)] ) def test_spikes_use_recorded_reference_days_and_weighted_latency(metric: str, observed: float, baseline: float) -> None: - rule: Final = UsageAlertRuleInput.model_validate( + rule: Final = AlertRuleInput.model_validate( {"name": "spike", "scope_type": "key", "metric": metric, "direction": "increase"} ) history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 3, 7)) @@ -161,13 +161,13 @@ def test_spikes_use_recorded_reference_days_and_weighted_latency(metric: str, ob def test_missing_target_is_zero_for_counts_but_never_zero_latency() -> None: history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) - rule: Final = UsageAlertRuleInput(name="drop", scope_type="key", metric="requests", direction="decrease") + rule: Final = AlertRuleInput(name="drop", scope_type="key", metric="requests", direction="decrease") assert compare_daily_usage(rule, history, _DAY) == DailyComparison(3, 0, 200, 200 / 3, True) assert compare_daily_usage(rule.model_copy(update={"metric": "latency"}), history, _DAY) is None def test_low_volume_and_recorded_zero_spend_do_not_establish_a_multiplier_baseline() -> None: - rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase") + rule: Final = AlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase") history: Final = tuple(_row(_DAY - timedelta(days=days), requests=10, spend=0) for days in (1, 2, 3)) assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) is None enough_requests: Final = tuple(replace(row, requests=200) for row in history) @@ -178,7 +178,7 @@ def test_low_volume_and_recorded_zero_spend_do_not_establish_a_multiplier_baseli def test_absolute_floor_suppresses_small_changes() -> None: history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) - rule: Final = UsageAlertRuleInput( + rule: Final = AlertRuleInput( name="spike", scope_type="key", metric="spend", direction="increase", min_value=100 ) assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 120, False) @@ -191,7 +191,7 @@ def test_day_is_eligible_only_after_one_hour_utc_grace() -> None: def test_latency_weights_request_samples_instead_of_averaging_daily_averages() -> None: - rule: Final = UsageAlertRuleInput( + rule: Final = AlertRuleInput( name="latency", scope_type="key", metric="latency", direction="increase", min_history_days=2 ) rows: Final = ( @@ -203,7 +203,7 @@ def test_latency_weights_request_samples_instead_of_averaging_daily_averages() - def test_errors_are_failed_request_count_and_require_absolute_change_floor() -> None: - rule: Final = UsageAlertRuleInput( + rule: Final = AlertRuleInput( name="errors", scope_type="key", metric="errors", direction="increase", min_value=10 ) history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) @@ -213,7 +213,7 @@ def test_errors_are_failed_request_count_and_require_absolute_change_floor() -> def test_a_value_at_the_configured_multiplier_alerts() -> None: history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) - rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase") + rule: Final = AlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase") assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, True) @@ -234,9 +234,9 @@ def test_default_notifications_require_opt_in_and_do_not_duplicate_legacy_user_s ) def test_sentinel_entities_cannot_create_alerts_with_unusable_activity_links(scope: str, entity: str) -> None: from litellm.proxy.spend_tracking.usage_alert_job import _observations - from litellm.types.proxy.usage_alerts import UsageAlertRule + from litellm.types.proxy.usage_alerts import AlertRule - rule: Final = UsageAlertRule.model_validate( + rule: Final = AlertRule.model_validate( { "id": "rule", "name": "spike", @@ -292,9 +292,9 @@ def test_healthy_evaluation_completes_and_recovers_only_with_sufficient_evidence active: bool, spend: float, history_days: int, evaluated: bool, persisted: bool ) -> None: from litellm.proxy.spend_tracking.usage_alert_job import _observations - from litellm.types.proxy.usage_alerts import UsageAlertRule + from litellm.types.proxy.usage_alerts import AlertRule - rule: Final = UsageAlertRule( + rule: Final = AlertRule( id="rule", name="spike", scope_type="key", @@ -347,7 +347,7 @@ def test_error_rate_default_requires_two_percentage_points_as_well_as_the_multip def test_no_requests_cannot_report_zero_error_rate_or_resolve_a_reliability_issue() -> None: - rule: Final = UsageAlertRuleInput(name="rate", scope_type="key", metric="error_rate", direction="decrease") + rule: Final = AlertRuleInput(name="rate", scope_type="key", metric="error_rate", direction="decrease") history: Final = tuple(_row(_DAY - timedelta(days=day), requests=500) for day in (1, 2, 3)) assert compare_daily_usage(rule, history, _DAY) is None @@ -382,7 +382,7 @@ def test_latency_default_requires_a_material_change_and_enough_timed_requests() @pytest.mark.parametrize("metric", ["spend", "requests", "latency", "errors"]) def test_multiplier_alerts_need_a_positive_established_baseline(metric: str) -> None: - rule: Final = UsageAlertRuleInput.model_validate( + rule: Final = AlertRuleInput.model_validate( {"name": "baseline", "scope_type": "key", "metric": metric, "direction": "increase", "min_requests": 0} ) current: Final = replace(_row(_DAY, requests=2000, spend=500), errors=100) @@ -413,7 +413,7 @@ def test_default_traffic_drop_does_not_treat_a_normal_quiet_weekend_as_an_outage for rule in default_usage_rules() if (rule.scope_type, rule.metric, rule.direction) == ("key", "requests", "decrease") ) - rule: Final = UsageAlertRule.model_validate( + rule: Final = AlertRule.model_validate( { **definition.model_dump(), "id": default_usage_rule_id("key", "requests", "decrease"), @@ -432,7 +432,7 @@ def test_default_traffic_drop_does_not_treat_a_normal_quiet_weekend_as_an_outage def test_error_rate_drop_requires_enough_current_requests() -> None: - rule: Final = UsageAlertRuleInput( + rule: Final = AlertRuleInput( name="Reliability", scope_type="key", metric="error_rate", @@ -448,7 +448,7 @@ def test_error_rate_drop_requires_enough_current_requests() -> None: def test_suppressed_default_can_recover_existing_episode_without_creating_a_new_one() -> None: from litellm.proxy.spend_tracking.usage_alert_job import _observations - rule: Final = UsageAlertRule( + rule: Final = AlertRule( id="rule", name="Default", scope_type="key", diff --git a/tests/unit/repositories/test_issue_families.py b/tests/unit/repositories/test_issue_families.py index a22159e9b5a..200fe0c3fee 100644 --- a/tests/unit/repositories/test_issue_families.py +++ b/tests/unit/repositories/test_issue_families.py @@ -8,14 +8,14 @@ from litellm.repositories.issue_families import family_settings from litellm.types.proxy.usage_alerts import ( UsageAlertEvaluationStatus, UsageAlertMetric, - UsageAlertRule, + AlertRule, UsageAlertScope, ) @dataclass class RuleRows: - rules: tuple[UsageAlertRule, ...] + rules: tuple[AlertRule, ...] calls: int = 0 async def query_raw(self, query: str, *args: object) -> object: @@ -36,7 +36,7 @@ async def test_family_counts_only_its_enabled_rules_without_an_extra_query(statu ("model", "latency", True), ) rules: Final = tuple( - UsageAlertRule( + AlertRule( id=f"test-{scope}-{metric}", name="Daily monitoring", scope_type=scope, diff --git a/tests/unit/types/proxy/test_usage_alerts.py b/tests/unit/types/proxy/test_usage_alerts.py index 3ee4618d619..6ca22300fbb 100644 --- a/tests/unit/types/proxy/test_usage_alerts.py +++ b/tests/unit/types/proxy/test_usage_alerts.py @@ -3,7 +3,7 @@ from typing import Final import pytest from pydantic import ValidationError -from litellm.types.proxy.usage_alerts import UsageAlertGroupUpdate, UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import AlertRuleInput RULE: Final = { "name": "Daily spend", @@ -34,31 +34,13 @@ RULE: Final = { ) def test_invalid_rules_cannot_be_stored(changes: dict[str, object]) -> None: with pytest.raises(ValidationError): - UsageAlertRuleInput.model_validate({**RULE, **changes}) + AlertRuleInput.model_validate({**RULE, **changes}) def test_zero_request_floor_and_daily_drop_multiplier_are_supported() -> None: - rule: Final = UsageAlertRuleInput.model_validate( + rule: Final = AlertRuleInput.model_validate( {**RULE, "metric": "requests", "direction": "decrease", "multiplier": 2, "min_requests": 0} ) assert rule.direction == "decrease" assert rule.multiplier == 2 assert rule.min_requests == 0 - - -@pytest.mark.parametrize( - "changes", - [{"scope_id": None}, {"scope_type": "deployment"}, {"status": "open"}, {"status": "unknown"}], -) -def test_group_updates_require_exact_resource_and_a_different_valid_status(changes: dict[str, object]) -> None: - with pytest.raises(ValidationError): - UsageAlertGroupUpdate.model_validate( - { - "scope_type": "team", - "scope_id": "team-example", - "day": "2026-11-04", - "expected_status": "open", - "status": "resolved", - **changes, - } - ) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx index 66d6fbbee24..c05a1a69310 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx @@ -11,10 +11,10 @@ import { issueRouteParsers } from "@/components/Issues/issueRoute"; export default function AlertsPage() { const { t } = useTranslation(); const [params] = useQueryStates(issueRouteParsers); - if (params.issue) + if (params.alert) return ( - + ); if (params.model) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx index 5fd74aca99d..d4cdd57dbb3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx @@ -607,7 +607,7 @@ describe("UsagePage", () => { }); it.each([ - { context: "deployment alert", alert: "&alert=example-alert", timezoneOffsetMinutes: 0 }, + { context: "deployment alert", alert: "&signal=example-alert", timezoneOffsetMinutes: 0 }, { context: "ordinary usage", alert: "", timezoneOffsetMinutes: undefined }, ])( "uses the same daily timezone for totals and tag activity in $context", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index afbc1ba8391..4ba7a55d336 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -124,7 +124,7 @@ const UsagePage: React.FC = ({ teams, organizations, teamsLoadin const defaultRange = useMemo(() => defaultUsageDateRange(defaultTimeRange), [defaultTimeRange]); const [urlParams, setUrlParams] = useQueryStates(usageUrlParsers); - const [alertId] = useQueryState("alert", entityActivityParsers.alert); + const [signalId] = useQueryState("signal", entityActivityParsers.signal); const { range, from, to } = urlParams; const dateValue = useMemo( () => dateRangeFromParams({ range, from, to }) ?? defaultRange.value, @@ -172,7 +172,7 @@ const UsagePage: React.FC = ({ teams, organizations, teamsLoadin // For non-admins or "my-usage" view, always pass their own user_id const effectiveUserId = usageView === "my-usage" || !isAdmin ? userID || null : urlParams.user; const isDeploymentUsage = isAdmin && usageView === "global" && effectiveUserId === null; - const showUsageAlert = Boolean(alertId && isDeploymentUsage); + const showUsageAlert = Boolean(signalId && isDeploymentUsage); const clientScopeInput = { userId: userID, @@ -462,9 +462,9 @@ const UsagePage: React.FC = ({ teams, organizations, teamsLoadin return (
- {showUsageAlert && alertId && ( + {showUsageAlert && signalId && (
- +
)}
diff --git a/ui/litellm-dashboard/src/components/Issues/IssueActivity.tsx b/ui/litellm-dashboard/src/components/Issues/IssueActivity.tsx index cb60f45e249..6accfa4341b 100644 --- a/ui/litellm-dashboard/src/components/Issues/IssueActivity.tsx +++ b/ui/litellm-dashboard/src/components/Issues/IssueActivity.tsx @@ -5,9 +5,9 @@ import { getProxyBaseUrl } from "@/components/networking"; import { Button } from "@/components/ui/button"; import { boundedActivityMetrics } from "@/components/UsagePage/keyActivityData"; import { backgroundFetchClient } from "@/lib/http/api"; -import type { Issue } from "./issueApi"; +import type { Alert } from "./issueApi"; -export default function IssueActivity({ issue, from, to }: { issue: Issue; from: string; to: string }) { +export default function IssueActivity({ issue, from, to }: { issue: Alert; from: string; to: string }) { const { userId, userRole, isAuthorized, accessToken } = useAuthorized(); const authorized = Boolean(isAuthorized && accessToken && issue.can_view_activity === true); const enabled = authorized && from <= to; @@ -15,8 +15,8 @@ export default function IssueActivity({ issue, from, to }: { issue: Issue; from: queryKey: ["issueActivity", getProxyBaseUrl(), userId, userRole, issue.id, from, to], enabled, queryFn: async ({ signal }: { signal: AbortSignal }) => { - const { data } = await backgroundFetchClient.GET("/issues/{issue_id}/activity", { - params: { path: { issue_id: issue.id }, query: { start_date: from, end_date: to } }, + const { data } = await backgroundFetchClient.GET("/alerts/{alert_id}/activity", { + params: { path: { alert_id: issue.id }, query: { start_date: from, end_date: to } }, signal, }); if (!data) throw new Error("Could not load activity"); diff --git a/ui/litellm-dashboard/src/components/Issues/IssueComparisonTable.tsx b/ui/litellm-dashboard/src/components/Issues/IssueComparisonTable.tsx index a195d947fc2..2602bada1f6 100644 --- a/ui/litellm-dashboard/src/components/Issues/IssueComparisonTable.tsx +++ b/ui/litellm-dashboard/src/components/Issues/IssueComparisonTable.tsx @@ -1,8 +1,8 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { METRIC_LABELS, formatAlertValue } from "@/components/UsageAlerts/usageAlertApi"; -import type { Issue } from "./issueApi"; +import type { Alert } from "./issueApi"; -export default function IssueComparisonTable({ issue }: { issue: Issue }) { +export default function IssueComparisonTable({ issue }: { issue: Alert }) { const comparisons = issue.comparisons ?? []; if (!comparisons.length) return null; return ( diff --git a/ui/litellm-dashboard/src/components/Issues/IssueDetail.tsx b/ui/litellm-dashboard/src/components/Issues/IssueDetail.tsx index a4ef2e63746..97a0a81d2f2 100644 --- a/ui/litellm-dashboard/src/components/Issues/IssueDetail.tsx +++ b/ui/litellm-dashboard/src/components/Issues/IssueDetail.tsx @@ -15,7 +15,7 @@ import { uiHref } from "@/utils/uiHref"; import { ApiError } from "@/lib/http/client"; import { useUrlTableState } from "@/components/shared/DataTable"; import { - Issue, + Alert, issueDate, issueComparisonText, issueImpactText, @@ -31,7 +31,7 @@ import IssueSignals from "./IssueSignals"; import IssueStatusActions from "./IssueStatusActions"; import IssueHistoryChart from "./IssueHistoryChart"; -function resourceHref(issue: Issue): string | undefined { +function resourceHref(issue: Alert): string | undefined { const { scope_type: scope, scope_id: id } = issue.cohort; if (!id || scope === "provider" || scope === "deployment") return undefined; return entityActivityHref(scope, scope === "key" ? storedKeyId(id) : id, { @@ -40,7 +40,7 @@ function resourceHref(issue: Issue): string | undefined { }); } -function RetirementContext({ issue }: { issue: Issue }) { +function RetirementContext({ issue }: { issue: Alert }) { const deprecation = issue.deprecation; if (!deprecation) return null; const copy = () => @@ -67,7 +67,7 @@ function RetirementContext({ issue }: { issue: Issue }) { ); } -function IssueBody({ issue }: { issue: Issue }) { +function IssueBody({ issue }: { issue: Alert }) { const [params, setParams] = useQueryStates(entityActivityParsers); const resourcesTable = useUrlTableState(ISSUE_RESOURCE_TABLE_STATE); const [resourcesOpen, setResourcesOpen] = useState(false); diff --git a/ui/litellm-dashboard/src/components/Issues/IssueFamilySettings.tsx b/ui/litellm-dashboard/src/components/Issues/IssueFamilySettings.tsx index 25953f6be35..619257fc517 100644 --- a/ui/litellm-dashboard/src/components/Issues/IssueFamilySettings.tsx +++ b/ui/litellm-dashboard/src/components/Issues/IssueFamilySettings.tsx @@ -9,10 +9,10 @@ import { backgroundFetchClient } from "@/lib/http/api"; import type { components } from "@/lib/http/schema"; import { FAMILY_LABELS } from "./issueApi"; -type Family = components["schemas"]["IssueFamilySetting"]; -type Policy = components["schemas"]["IssueFamilyPolicy"]; -type PolicyUpdate = components["schemas"]["IssueFamilyPolicyUpdate"]; -type FamilyUpdate = components["schemas"]["IssueFamilyUpdate"]; +type Family = components["schemas"]["AlertFamilySetting"]; +type Policy = components["schemas"]["AlertFamilyPolicy"]; +type PolicyUpdate = components["schemas"]["AlertFamilyPolicyUpdate"]; +type FamilyUpdate = components["schemas"]["AlertFamilyUpdate"]; const SELECT_CLASS = "h-9 rounded-md border border-input bg-background px-2 text-sm"; const FLOOR_UNITS = { spend: "USD", latency: "ms", reliability: "percentage points", traffic: "requests" }; const NUMBER_FIELDS = ["multiplier", "min_value", "min_requests", "baseline_days", "min_history_days"] as const; @@ -162,7 +162,7 @@ export default function IssueFamilySettings({ accessToken }: { accessToken: stri queryKey: ["issueFamilies", getProxyBaseUrl(), user?.user_id, user?.user_role], enabled: Boolean(accessToken && user?.user_id), queryFn: async ({ signal }: { signal: AbortSignal }) => { - const { data } = await backgroundFetchClient.GET("/issues/families", { signal }); + const { data } = await backgroundFetchClient.GET("/alerts/defaults", { signal }); if (!data) throw new Error("Could not load monitoring settings"); return data; }, @@ -172,7 +172,7 @@ export default function IssueFamilySettings({ accessToken }: { accessToken: stri const save = useMutation({ mutationFn: async (body: FamilyUpdate) => { if (!editing) throw new Error("Choose an alert type"); - const { data } = await backgroundFetchClient.PATCH("/issues/families/{family}", { + const { data } = await backgroundFetchClient.PATCH("/alerts/defaults/{family}", { params: { path: { family: editing.family } }, body, }); diff --git a/ui/litellm-dashboard/src/components/Issues/IssueHistoryChart.tsx b/ui/litellm-dashboard/src/components/Issues/IssueHistoryChart.tsx index dba76ae5878..4f3d3b1141d 100644 --- a/ui/litellm-dashboard/src/components/Issues/IssueHistoryChart.tsx +++ b/ui/litellm-dashboard/src/components/Issues/IssueHistoryChart.tsx @@ -1,13 +1,13 @@ import { Bar, BarChart, CartesianGrid, Cell, ReferenceLine, XAxis, YAxis } from "recharts"; import { ChartContainer, ChartTooltip, ChartTooltipContent } from "@/components/ui/chart"; import { formatAlertValue, METRIC_LABELS } from "@/components/UsageAlerts/usageAlertApi"; -import { Issue, issueDate, primaryComparison } from "./issueApi"; +import { Alert, issueDate, primaryComparison } from "./issueApi"; export function formatHistoryValue(metric: Parameters[0], value: unknown) { return typeof value === "number" && Number.isFinite(value) ? formatAlertValue(metric, value) : "Not recorded"; } -export function issueHistory(issue: Issue) { +export function issueHistory(issue: Alert) { const comparison = primaryComparison(issue); const history = issue.deprecation?.daily_requests ?? comparison?.history ?? []; const metric = issue.deprecation ? "requests" : comparison?.metric ?? "requests"; @@ -25,7 +25,7 @@ export function issueHistory(issue: Issue) { return { data, metric, comparison }; } -export default function IssueHistoryChart({ issue, compact = false }: { issue: Issue; compact?: boolean }) { +export default function IssueHistoryChart({ issue, compact = false }: { issue: Alert; compact?: boolean }) { const { data, metric, comparison } = issueHistory(issue); const label = issue.deprecation ? "Daily requests" : `Daily ${METRIC_LABELS[metric].toLowerCase()}`; if (data.length === 0) diff --git a/ui/litellm-dashboard/src/components/Issues/IssueResources.tsx b/ui/litellm-dashboard/src/components/Issues/IssueResources.tsx index bbf1e5653ed..dfd9167eeb3 100644 --- a/ui/litellm-dashboard/src/components/Issues/IssueResources.tsx +++ b/ui/litellm-dashboard/src/components/Issues/IssueResources.tsx @@ -10,7 +10,7 @@ import { storedKeyId } from "@/components/UsagePage/keyActivityData"; import { backgroundFetchClient } from "@/lib/http/api"; import { entityActivityHref } from "@/utils/entityLinks"; import { formatAlertValue } from "@/components/UsageAlerts/usageAlertApi"; -import type { Issue } from "./issueApi"; +import type { Alert } from "./issueApi"; interface ResourceRow { id: string; @@ -58,7 +58,7 @@ export const ISSUE_RESOURCE_TABLE_STATE = { keyPrefix: "resources", }; -export default function IssueResources({ issue, from, to }: { issue: Issue; from: string; to: string }) { +export default function IssueResources({ issue, from, to }: { issue: Alert; from: string; to: string }) { const { userId, userRole, isAuthorized, accessToken } = useAuthorized(); const [requestedResource, setResource] = useQueryState( "resources", @@ -84,11 +84,11 @@ export default function IssueResources({ issue, from, to }: { issue: Issue; from enabled: Boolean(isAuthorized && accessToken && from <= to), queryFn: async ({ signal }: { signal: AbortSignal }) => { const params = { - path: { issue_id: issue.id }, + path: { alert_id: issue.id }, query: { start_date: from, end_date: to, limit: table.pagination.pageSize, offset }, }; if (resource === "keys") { - const { data } = await backgroundFetchClient.GET("/issues/{issue_id}/keys", { params, signal }); + const { data } = await backgroundFetchClient.GET("/alerts/{alert_id}/keys", { params, signal }); if (!data) throw new Error("Could not load keys"); const rows: ResourceRow[] = data.api_keys.map((key) => ({ id: key.api_key, @@ -101,7 +101,7 @@ export default function IssueResources({ issue, from, to }: { issue: Issue; from })); return { rows, total: data.total_api_keys }; } - const { data } = await backgroundFetchClient.GET("/issues/{issue_id}/users", { params, signal }); + const { data } = await backgroundFetchClient.GET("/alerts/{alert_id}/users", { params, signal }); if (!data) throw new Error("Could not load users"); const rows: ResourceRow[] = data.users.map((user) => ({ id: user.user_id ?? "unattributed", diff --git a/ui/litellm-dashboard/src/components/Issues/IssueSignals.tsx b/ui/litellm-dashboard/src/components/Issues/IssueSignals.tsx index d7e9a985f53..c20cf46ddda 100644 --- a/ui/litellm-dashboard/src/components/Issues/IssueSignals.tsx +++ b/ui/litellm-dashboard/src/components/Issues/IssueSignals.tsx @@ -5,9 +5,9 @@ import { Button } from "@/components/ui/button"; import { backgroundFetchClient } from "@/lib/http/api"; import { formatAlertValue, METRIC_LABELS, usageAlertHref } from "@/components/UsageAlerts/usageAlertApi"; import { useQueryState, parseAsInteger } from "nuqs"; -import { Issue } from "./issueApi"; +import { Alert } from "./issueApi"; -export default function IssueSignals({ issue }: { issue: Issue }) { +export default function IssueSignals({ issue }: { issue: Alert }) { const { userId, userRole, isAuthorized, accessToken } = useAuthorized(); const [page, setPage] = useQueryState("signal_page", parseAsInteger.withDefault(1)); const offset = (Math.max(1, page) - 1) * 20; @@ -15,8 +15,8 @@ export default function IssueSignals({ issue }: { issue: Issue }) { queryKey: ["issueSignals", getProxyBaseUrl(), userId, userRole, issue.id, offset], enabled: Boolean(isAuthorized && accessToken), queryFn: async ({ signal }: { signal: AbortSignal }) => { - const { data } = await backgroundFetchClient.GET("/issues/{issue_id}/signals", { - params: { path: { issue_id: issue.id }, query: { limit: 20, offset } }, + const { data } = await backgroundFetchClient.GET("/alerts/{alert_id}/signals", { + params: { path: { alert_id: issue.id }, query: { limit: 20, offset } }, signal, }); if (!data) throw new Error("Could not load source signals"); @@ -48,7 +48,7 @@ export default function IssueSignals({ issue }: { issue: Issue }) {

    {query.data.data.map((signal) => { - const alert = signal.usage_alert; + const alert = signal.performance; return (
  • {alert ? ( diff --git a/ui/litellm-dashboard/src/components/Issues/IssueStatusActions.tsx b/ui/litellm-dashboard/src/components/Issues/IssueStatusActions.tsx index d25d39a9a5a..1f6423a06ba 100644 --- a/ui/litellm-dashboard/src/components/Issues/IssueStatusActions.tsx +++ b/ui/litellm-dashboard/src/components/Issues/IssueStatusActions.tsx @@ -7,9 +7,9 @@ import { Button } from "@/components/ui/button"; import { hasCapability } from "@/utils/capabilities"; import { ApiError } from "@/lib/http/client"; import type { components } from "@/lib/http/schema"; -import { Issue, IssueStatus, updateIssueStatus } from "./issueApi"; +import { Alert, IssueStatus, updateIssueStatus } from "./issueApi"; -export default function IssueStatusActions({ issue, globalAccess = false }: { issue: Issue; globalAccess?: boolean }) { +export default function IssueStatusActions({ issue, globalAccess = false }: { issue: Alert; globalAccess?: boolean }) { const { isViewOnly, accessToken, userRole } = useAuthorized(); const { data: user } = useCurrentUser(accessToken); const client = useQueryClient(); @@ -17,7 +17,7 @@ export default function IssueStatusActions({ issue, globalAccess = false }: { is mutationFn: (status: IssueStatus) => updateIssueStatus(issue, status), onSuccess: (updated) => { const scope = ["issues", getProxyBaseUrl(), user?.user_id, user?.user_role]; - client.setQueryData( + client.setQueryData( [...scope, "detail", issue.id], (previous) => previous && { @@ -26,7 +26,7 @@ export default function IssueStatusActions({ issue, globalAccess = false }: { is can_update_status: updated.can_update_status ?? previous.can_update_status, }, ); - client.setQueriesData( + client.setQueriesData( { queryKey: [...scope, "list"] }, (page) => page && { diff --git a/ui/litellm-dashboard/src/components/Issues/Issues.integration.test.tsx b/ui/litellm-dashboard/src/components/Issues/Issues.integration.test.tsx index 61a1fb5778e..8d3f1f54c18 100644 --- a/ui/litellm-dashboard/src/components/Issues/Issues.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/Issues/Issues.integration.test.tsx @@ -10,10 +10,11 @@ import { within, } from "../../../tests/test-utils"; import IssuesInbox from "./IssuesInbox"; +import { NotificationsBell } from "@/components/Navbar/NotificationsBell/NotificationsBell"; import IssueDetail from "./IssueDetail"; import { FAMILY_SETTINGS, RETIREMENT_ISSUE, SPEND_ISSUE } from "./issueFixtures"; import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; -import type { Issue } from "./issueApi"; +import type { Alert } from "./issueApi"; import type { components } from "@/lib/http/schema"; const auth = vi.hoisted(() => ({ @@ -25,7 +26,7 @@ const auth = vi.hoisted(() => ({ })); vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => auth })); const calls: { path: string; query: URLSearchParams; method: string; body: unknown }[] = []; -let detail: Issue; +let detail: Alert; let refreshBarrier: Promise | undefined; let holdReads = false; let failList = false; @@ -154,10 +155,10 @@ describe("unified issue investigations", () => { calls.push(call); if (url.pathname === "/management/v1/users/me") return json({ user_id: auth.userId, user_role: auth.userRole === "Admin" ? "proxy_admin" : "internal_user" }); - if (url.pathname === "/issues/foreign-issue") return json({ detail: "Forbidden private-project metadata" }, 403); - if (url.pathname === "/issues") return refreshed(() => listResponse(url.searchParams)); - if (url.pathname === "/issues/families") return json(families); - if (url.pathname === "/issues/families/spend") return json(FAMILY_SETTINGS.data[0]); + if (url.pathname === "/alerts/foreign-issue") return json({ detail: "Forbidden private-project metadata" }, 403); + if (url.pathname === "/alerts") return refreshed(() => listResponse(url.searchParams)); + if (url.pathname === "/alerts/defaults") return json(families); + if (url.pathname === "/alerts/defaults/spend") return json(FAMILY_SETTINGS.data[0]); if (url.pathname.endsWith("/activity")) return failActivity ? json({ detail: "Unavailable" }, 503) : json(activity()); if (url.pathname.endsWith("/keys")) return resourceKeyPage(url.searchParams); @@ -176,10 +177,10 @@ describe("unified issue investigations", () => { const page = { data: [], total: 0, limit: 20, offset: 0 }; return json(page); } - if (url.pathname.startsWith("/issues/")) { + if (url.pathname.startsWith("/alerts/")) { if (deniedDetail) return json({ detail: "Access revoked" }, 403); if (request.method === "PATCH") { - if (staleStatus) return json({ detail: "Issue changed" }, 409); + if (staleStatus) return json({ detail: "Alert changed" }, 409); detail = { ...detail, ...body }; holdReads = Boolean(refreshBarrier); return json({ ...detail, can_update_status: null, can_view_activity: null }); @@ -194,19 +195,33 @@ describe("unified issue investigations", () => { vi.unstubAllGlobals(); }); + it("retries a failed notification feed and opens the same alert detail", async () => { + failList = true; + renderWithProviders(); + await userEvent.click(screen.getByRole("button", { name: /^Notifications/ })); + await screen.findByRole("alert"); + failList = false; + await userEvent.click(screen.getByRole("button", { name: "Retry" })); + expect(await screen.findByRole("link", { name: SPEND_ISSUE.title })).toHaveAttribute( + "href", + `/ui/alerts?alert=${SPEND_ISSUE.id}`, + ); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + }); + it("adapts pagination to the server byte budget without skipping alerts", async () => { byteLimitedPage = true; renderWithProviders(); await screen.findByRole("heading", { name: /Wide key 0/ }); await waitFor(() => - expect(calls.some((call) => call.path === "/issues" && call.query.get("limit") === "2")).toBe(true), + expect(calls.some((call) => call.path === "/alerts" && call.query.get("limit") === "2")).toBe(true), ); await userEvent.click(screen.getByRole("button", { name: "Go to next page" })); await screen.findByRole("heading", { name: /Wide key 2/ }); await userEvent.click(screen.getByRole("button", { name: "Go to next page" })); await screen.findByRole("heading", { name: /Wide key 4/ }); expect(screen.getByRole("button", { name: "Go to next page" })).toBeDisabled(); - expect(calls.some((call) => call.path === "/issues" && call.query.get("offset") === "4")).toBe(true); + expect(calls.some((call) => call.path === "/alerts" && call.query.get("offset") === "4")).toBe(true); }); it.each([false, true])("preserves the exact offset when later pages shrink (deep link %s)", async (deepLink) => { @@ -220,7 +235,7 @@ describe("unified issue investigations", () => { await waitFor(() => expect( calls - .filter((call) => call.path === "/issues") + .filter((call) => call.path === "/alerts") .at(-1) ?.query.get("limit"), ).toBe("1"), @@ -241,19 +256,19 @@ describe("unified issue investigations", () => { renderWithProviders(); expect(await screen.findByRole("link", { name: "Spend on Checkout key rose to 20× normal" })).toHaveAttribute( "href", - "/ui/alerts?issue=issue-checkout-spend", + "/ui/alerts?alert=issue-checkout-spend", ); expect(screen.getByRole("link", { name: RETIREMENT_ISSUE.title })).toBeInTheDocument(); expect(screen.getByText("$600.00 on Nov 4, 2026 UTC, compared with the usual $30.00.")).toBeInTheDocument(); expect(screen.getByRole("list", { name: "Alerts", exact: true })).toBeInTheDocument(); expect(screen.queryByRole("table")).not.toBeInTheDocument(); - expect(calls.filter((call) => call.path === "/issues")).toHaveLength(1); + expect(calls.filter((call) => call.path === "/alerts")).toHaveLength(1); expect(calls.some((call) => call.path.includes("activity") || call.path.includes("deprecations"))).toBe(false); await user.click(screen.getByRole("button", { name: "Go to next page", exact: true })); expect(await screen.findByRole("link", { name: "Spend on Second key rose to 20× normal" })).toBeInTheDocument(); expect( calls - .filter((call) => call.path === "/issues") + .filter((call) => call.path === "/alerts") .at(-1) ?.query.get("offset"), ).toBe("25"); @@ -261,7 +276,7 @@ describe("unified issue investigations", () => { expect(await screen.findByRole("link", { name: RETIREMENT_ISSUE.title })).toBeInTheDocument(); expect( calls - .filter((call) => call.path === "/issues") + .filter((call) => call.path === "/alerts") .at(-1) ?.query.get("offset"), ).toBe("0"); @@ -270,7 +285,7 @@ describe("unified issue investigations", () => { it("keeps old performance list links usable", async () => { renderWithProviders(, { searchParams: "type=usage&usage_status=resolved" }); await screen.findByRole("link", { name: "Spend on Checkout key rose to 20× normal" }); - const request = calls.find((call) => call.path === "/issues"); + const request = calls.find((call) => call.path === "/alerts"); expect(request?.query.get("kind")).toBe("performance"); expect(request?.query.get("status")).toBe("resolved"); }); @@ -307,11 +322,11 @@ describe("unified issue investigations", () => { const feed = screen.getByRole("list", { name: "Alerts", exact: true }); await userEvent.click(within(feed).getAllByRole("button", { name: "Mark resolved" })[0]); await waitFor(() => - expect(calls.some((call) => call.method === "PATCH" && call.path === `/issues/${SPEND_ISSUE.id}`)).toBe(true), + expect(calls.some((call) => call.method === "PATCH" && call.path === `/alerts/${SPEND_ISSUE.id}`)).toBe(true), ); expect( calls.filter( - (call) => call.method === "GET" && call.path.startsWith("/issues/") && !call.path.endsWith("/families"), + (call) => call.method === "GET" && call.path.startsWith("/alerts/") && !call.path.endsWith("/families"), ), ).toHaveLength(0); expect(calls.find((call) => call.method === "PATCH")?.body).toEqual({ @@ -328,7 +343,7 @@ describe("unified issue investigations", () => { try { await userEvent.click(screen.getByRole("button", { name: "Mark resolved" })); await waitFor(() => - expect(calls.filter((call) => call.method === "GET" && call.path === `/issues/${detail.id}`)).toHaveLength(2), + expect(calls.filter((call) => call.method === "GET" && call.path === `/alerts/${detail.id}`)).toHaveLength(2), ); expect(screen.getByRole("link", { name: "View key activity" })).toBeVisible(); expect(screen.getByRole("button", { name: "Reopen" })).toBeDisabled(); @@ -344,7 +359,7 @@ describe("unified issue investigations", () => { await screen.findByRole("link", { name: "Spend on Checkout key rose to 20× normal" }); try { await userEvent.click(screen.getAllByRole("button", { name: "Mark resolved" })[0]); - await waitFor(() => expect(calls.filter((call) => call.path === "/issues")).toHaveLength(2)); + await waitFor(() => expect(calls.filter((call) => call.path === "/alerts")).toHaveLength(2)); expect(screen.getByRole("button", { name: "Reopen" })).toBeDisabled(); } finally { await act(async () => refresh.resolve()); @@ -358,10 +373,10 @@ describe("unified issue investigations", () => { await screen.findByRole("link", { name: "Spend on Checkout key rose to 20× normal" }); try { await userEvent.click(screen.getAllByRole("button", { name: "Mark resolved" })[0]); - await waitFor(() => expect(calls.filter((call) => call.path === "/issues")).toHaveLength(2)); + await waitFor(() => expect(calls.filter((call) => call.path === "/alerts")).toHaveLength(2)); view.rerender(); await waitFor(() => - expect(calls.some((call) => call.method === "GET" && call.path === `/issues/${detail.id}`)).toBe(true), + expect(calls.some((call) => call.method === "GET" && call.path === `/alerts/${detail.id}`)).toBe(true), ); expect(screen.queryByText("You don’t have access to this alert’s activity.")).not.toBeInTheDocument(); } finally { @@ -516,7 +531,7 @@ describe("unified issue investigations", () => { }); vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); - if (request.method === "GET" && new URL(request.url).pathname.startsWith("/issues")) await stalled; + if (request.method === "GET" && new URL(request.url).pathname.startsWith("/alerts")) await stalled; return fetchBefore(input, init); }); try { @@ -546,7 +561,7 @@ describe("unified issue investigations", () => { "href", "/ui/models-and-endpoints?activity_model=provider%2Fretiring-model&detail_tab=activity&activity_from=2026-10-28&activity_to=2026-11-04", ); - expect(calls.some((call) => call.path === `/issues/${RETIREMENT_ISSUE.id}/activity`)).toBe(false); + expect(calls.some((call) => call.path === `/alerts/${RETIREMENT_ISSUE.id}/activity`)).toBe(false); }); it("does not show invalid per-success averages when all requests failed", async () => { @@ -623,7 +638,7 @@ describe("unified issue investigations", () => { "href", "/ui/api-keys?key=reporting-test-id&detail_tab=activity&activity_from=2026-10-28&activity_to=2026-11-04", ); - expect(calls.some((call) => call.path === "/issues")).toBe(false); + expect(calls.some((call) => call.path === "/alerts")).toBe(false); expect(calls.some((call) => call.path.includes("/user/daily"))).toBe(false); await userEvent.click(screen.getByRole("button", { name: "Mark resolved" })); await waitFor(() => @@ -642,7 +657,7 @@ describe("unified issue investigations", () => { expect(screen.getByRole("link", { name: "My usage" })).toHaveAttribute("href", "/ui/usage?view=my-usage"); expect(screen.queryByText(/private-project/)).not.toBeInTheDocument(); expect(screen.queryByRole("heading", { name: "Spend on Checkout key rose to 20× normal" })).not.toBeInTheDocument(); - expect(calls.filter((call) => call.path.startsWith("/issues"))).toHaveLength(1); + expect(calls.filter((call) => call.path.startsWith("/alerts"))).toHaveLength(1); expect(calls.some((call) => call.path.endsWith("/activity") || call.path.endsWith("/keys"))).toBe(false); }); @@ -726,7 +741,7 @@ describe("unified issue investigations", () => { fireEvent.change(screen.getByLabelText("Increase factor"), { target: { value: "10" } }); await user.click(screen.getByRole("button", { name: "Save changes" })); await waitFor(() => - expect(calls.find((call) => call.path === "/issues/families/spend")?.body).toEqual({ + expect(calls.find((call) => call.path === "/alerts/defaults/spend")?.body).toEqual({ policies: [{ direction: "increase", multiplier: 10 }], }), ); diff --git a/ui/litellm-dashboard/src/components/Issues/IssuesInbox.tsx b/ui/litellm-dashboard/src/components/Issues/IssuesInbox.tsx index cbd8834352b..a7a39e0ffb3 100644 --- a/ui/litellm-dashboard/src/components/Issues/IssuesInbox.tsx +++ b/ui/litellm-dashboard/src/components/Issues/IssuesInbox.tsx @@ -11,8 +11,8 @@ import { FAMILY_LABELS, ISSUE_FAMILIES, ISSUE_STATUSES, - Issue, - IssueFamily, + Alert, + AlertFamily, IssueStatus, issueComparisonText, issueHeadline, @@ -25,7 +25,7 @@ import IssueHistoryChart from "./IssueHistoryChart"; import IssueStatusActions from "./IssueStatusActions"; const TABLE_STATE = { sortFields: [], defaultSort: { id: "", desc: false }, defaultPageSize: 25, filterColumns: [] }; -function AlertEntry({ issue }: { issue: Issue }) { +function AlertEntry({ issue }: { issue: Alert }) { const impact = issueImpactText(issue); return (
  • @@ -84,7 +84,7 @@ export default function IssuesInbox() { }; const changeFamily = (next: string) => { void setParams({ - family: next === "all" || next === "performance" ? null : (next as IssueFamily), + family: next === "all" || next === "performance" ? null : (next as AlertFamily), type: next === "performance" ? "usage" : null, }); table.onPaginationChange({ ...table.pagination, pageIndex: 0 }); diff --git a/ui/litellm-dashboard/src/components/Issues/issueApi.ts b/ui/litellm-dashboard/src/components/Issues/issueApi.ts index b8d90638697..030625f31e2 100644 --- a/ui/litellm-dashboard/src/components/Issues/issueApi.ts +++ b/ui/litellm-dashboard/src/components/Issues/issueApi.ts @@ -7,11 +7,11 @@ import { hasCapability } from "@/utils/capabilities"; import { uiHref } from "@/utils/uiHref"; import { formatAlertValue } from "@/components/UsageAlerts/usageAlertApi"; -export type Issue = components["schemas"]["Issue"]; -export type IssuePage = components["schemas"]["IssueList"]; -export type IssueFamily = Issue["family"]; -export type IssueStatus = Issue["status"]; -export type IssueComparison = components["schemas"]["IssueComparison"]; +export type Alert = components["schemas"]["Alert"]; +export type IssuePage = components["schemas"]["AlertList"]; +export type AlertFamily = Alert["family"]; +export type IssueStatus = Alert["status"]; +export type AlertComparison = components["schemas"]["AlertComparison"]; export const ISSUE_FAMILIES = ["retirement", "latency", "reliability", "spend", "traffic"] as const; export const ISSUE_STATUSES = ["open", "resolved", "dismissed"] as const; export const FAMILY_LABELS = { @@ -37,10 +37,10 @@ const PRIMARY_METRICS = { traffic: "requests", }; -export const issueHref = (id: string) => uiHref(`alerts?issue=${encodeURIComponent(id)}`); +export const issueHref = (id: string) => uiHref(`alerts?alert=${encodeURIComponent(id)}`); export const issuesHref = () => uiHref("alerts"); -export function issueResource(issue: Issue): string { +export function issueResource(issue: Alert): string { const { scope_type: scope, scope_id: id } = issue.cohort; const label = SCOPE_LABELS[scope]; if (scope === "deployment" && issue.scope_label === label) return label; @@ -50,7 +50,7 @@ export function issueResource(issue: Issue): string { return `${label} · ${short}`; } -export const primaryComparison = (issue: Issue) => +export const primaryComparison = (issue: Alert) => (issue.comparisons ?? []).find((comparison) => comparison.metric === PRIMARY_METRICS[issue.family]) ?? issue.comparisons?.[0]; @@ -62,7 +62,7 @@ const READABLE_METRICS = { error_rate: "Error rate for", }; -export function issueHeadline(issue: Issue): string { +export function issueHeadline(issue: Alert): string { const comparison = primaryComparison(issue); if (issue.deprecation || !comparison) return issue.title; const { metric, observed_value: actual, baseline_value: baseline, direction } = comparison; @@ -79,7 +79,7 @@ export function issueHeadline(issue: Issue): string { return `${prefix} rose to ${ratio}× normal`; } -export function issueComparisonText(issue: Issue): string { +export function issueComparisonText(issue: Alert): string { if (issue.deprecation) { const { requests, model } = issue.deprecation; const replacement = model.successor_model ? ` Suggested replacement: ${model.successor_model}.` : ""; @@ -92,7 +92,7 @@ export function issueComparisonText(issue: Issue): string { return `${actual} on ${issueDate(issue.observed_on)} UTC, compared with the usual ${baseline}.`; } -export function issueImpactText(issue: Issue): string { +export function issueImpactText(issue: Alert): string { const impact = issue.impact ?? {}; return (["keys", "teams", "models"] as const) .map((field) => { @@ -109,8 +109,8 @@ export const issueDate = (value: string) => new Date(value).toLocaleDateString(u interface IssueListOptions { status?: IssueStatus; - family?: IssueFamily; - kind?: Issue["kind"]; + family?: AlertFamily; + kind?: Alert["kind"]; limit?: number; offset?: number; } @@ -134,7 +134,7 @@ export function useIssues(accessToken: string | null, listOptions: IssueListOpti ], enabled, queryFn: async ({ signal }: { signal: AbortSignal }) => { - const { data } = await backgroundFetchClient.GET("/issues", { + const { data } = await backgroundFetchClient.GET("/alerts", { params: { query: { status, family, kind, limit, offset } }, signal, }); @@ -156,8 +156,8 @@ export function useIssue(accessToken: string | null, id: string) { queryKey: ["issues", getProxyBaseUrl(), user?.user_id, user?.user_role, "detail", id], enabled, queryFn: async ({ signal }: { signal: AbortSignal }) => { - const { data } = await backgroundFetchClient.GET("/issues/{issue_id}", { - params: { path: { issue_id: id } }, + const { data } = await backgroundFetchClient.GET("/alerts/{alert_id}", { + params: { path: { alert_id: id } }, signal, }); if (!data) throw new Error("Could not load this alert"); @@ -170,9 +170,9 @@ export function useIssue(accessToken: string | null, id: string) { return { ...query, enabled, data: enabled && !query.isError ? query.data : undefined }; } -export async function updateIssueStatus(issue: Issue, status: IssueStatus) { - const { data } = await backgroundFetchClient.PATCH("/issues/{issue_id}", { - params: { path: { issue_id: issue.id } }, +export async function updateIssueStatus(issue: Alert, status: IssueStatus) { + const { data } = await backgroundFetchClient.PATCH("/alerts/{alert_id}", { + params: { path: { alert_id: issue.id } }, body: { status, expected_status: issue.status }, }); if (!data) throw new Error("Could not update this alert"); diff --git a/ui/litellm-dashboard/src/components/Issues/issueFixtures.ts b/ui/litellm-dashboard/src/components/Issues/issueFixtures.ts index 22b46b451a0..1864b990ab7 100644 --- a/ui/litellm-dashboard/src/components/Issues/issueFixtures.ts +++ b/ui/litellm-dashboard/src/components/Issues/issueFixtures.ts @@ -1,7 +1,7 @@ import type { components } from "@/lib/http/schema"; -import type { Issue } from "./issueApi"; +import type { Alert } from "./issueApi"; -export const SPEND_ISSUE: Issue = { +export const SPEND_ISSUE: Alert = { id: "issue-checkout-spend", cohort_key: "cohort-checkout-spend", kind: "performance", @@ -57,7 +57,7 @@ export const SPEND_ISSUE: Issue = { can_view_activity: true, }; -export const RETIREMENT_ISSUE: Issue = { +export const RETIREMENT_ISSUE: Alert = { ...SPEND_ISSUE, id: "issue-model-retirement", cohort_key: "cohort-model-retirement", @@ -99,7 +99,7 @@ export const RETIREMENT_ISSUE: Issue = { }, }; -export const FAMILY_SETTINGS: components["schemas"]["IssueFamilyList"] = { +export const FAMILY_SETTINGS: components["schemas"]["AlertFamilyList"] = { projection_pending: false, data: (["spend", "latency", "reliability", "traffic"] as const).map((family) => ({ family, diff --git a/ui/litellm-dashboard/src/components/Issues/issueRoute.ts b/ui/litellm-dashboard/src/components/Issues/issueRoute.ts index 4f900482be4..7211ec4ca10 100644 --- a/ui/litellm-dashboard/src/components/Issues/issueRoute.ts +++ b/ui/litellm-dashboard/src/components/Issues/issueRoute.ts @@ -2,7 +2,7 @@ import { parseAsString, parseAsStringLiteral } from "nuqs"; import { ISSUE_FAMILIES, ISSUE_STATUSES } from "./issueApi"; export const issueRouteParsers = { - issue: parseAsString.withOptions({ history: "push" }), + alert: parseAsString.withOptions({ history: "push" }), family: parseAsStringLiteral(ISSUE_FAMILIES), status: parseAsStringLiteral(ISSUE_STATUSES), type: parseAsStringLiteral(["usage", "model-deprecations"] as const), diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx index 178ba77b9eb..cd7f644d355 100644 --- a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx @@ -53,7 +53,7 @@ describe("model migration impact", () => { calls.push(call); if (url.pathname === "/management/v1/users/me") return apiResponse({ user_id: "user-1", user_role: "proxy_admin" }); - if (url.pathname === "/issues") { + if (url.pathname === "/alerts") { const page = { data: [RETIREMENT_ISSUE], total: 1, limit: 25, offset: 0 }; return apiResponse(page); } @@ -81,7 +81,7 @@ describe("model migration impact", () => { expect(screen.getByRole("heading", { name: "Alerts", exact: true })).toBeInTheDocument(); expect(await screen.findByRole("link", { name: RETIREMENT_ISSUE.title })).toHaveAttribute( "href", - "/ui/alerts?issue=issue-model-retirement", + "/ui/alerts?alert=issue-model-retirement", ); }); it.each([undefined, "model-deprecations"])("should preserve migration deep links with type %s", async (type) => { diff --git a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx index e013368417d..b032a317765 100644 --- a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx @@ -215,7 +215,7 @@ describe("issue notifications", () => { renderWithProviders(); await user.click(screen.getByRole("button", { name: "Notifications, 2 alerts need attention" })); const destination = screen.getByRole("link", { name: SPEND_ISSUE.title }); - expect(destination).toHaveAttribute("href", "/ui/alerts?issue=issue-checkout-spend"); + expect(destination).toHaveAttribute("href", "/ui/alerts?alert=issue-checkout-spend"); expect(screen.getByRole("article", { name: SPEND_ISSUE.title })).toHaveTextContent( "$600.00 on Nov 4, 2026 UTC, compared with the usual $30.00.", ); @@ -237,7 +237,7 @@ describe("issue notifications", () => { await user.click(screen.getByRole("button", { name: "Notifications, 2 alerts need attention" })); expect(screen.getByRole("link", { name: RETIREMENT_ISSUE.title })).toHaveAttribute( "href", - "/ui/alerts?issue=issue-model-retirement", + "/ui/alerts?alert=issue-model-retirement", ); expect(screen.getByRole("link", { name: "LiteLLM model cost map" })).toHaveAttribute( "href", diff --git a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx index 2acc0e9f145..1ad397a1ce7 100644 --- a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx @@ -159,6 +159,8 @@ interface NotificationsBellViewProps { issues?: IssuePage; issuesPending?: boolean; issuesFailed?: boolean; + retryingAlerts?: boolean; + onRetryAlerts?: () => void; deploymentHealthLink?: React.ReactNode; currentVersion: string | null | undefined; latestRelease: LatestReleaseInfo | null | undefined; @@ -171,6 +173,8 @@ export const NotificationsBellView: React.FC = ({ issues, issuesPending, issuesFailed, + retryingAlerts, + onRetryAlerts, }) => { const { t } = useTranslation(); const upgrade = upgradeAnnouncement(currentVersion, latestRelease); @@ -230,7 +234,12 @@ export const NotificationsBellView: React.FC = ({

    )} {issuesFailed && ( -

    {t("shell.notifications.bell.loadError")}

    +
    +

    {t("shell.notifications.bell.loadError")}

    + +
    )} {issues?.data.map((issue) => (
    @@ -276,6 +285,8 @@ export const NotificationsBell: React.FC<{ accessToken: string | null; deploymen issues={issueQuery.data} issuesPending={issueQuery.enabled && issueQuery.isPending} issuesFailed={issueQuery.enabled && issueQuery.isError} + retryingAlerts={issueQuery.isFetching} + onRetryAlerts={() => void issueQuery.refetch()} deploymentHealthLink={deploymentHealthLink} /> ); diff --git a/ui/litellm-dashboard/src/components/UsageAlerts/UsageAlertContext.tsx b/ui/litellm-dashboard/src/components/UsageAlerts/UsageAlertContext.tsx index 3aa771389f1..4c9bb37dc59 100644 --- a/ui/litellm-dashboard/src/components/UsageAlerts/UsageAlertContext.tsx +++ b/ui/litellm-dashboard/src/components/UsageAlerts/UsageAlertContext.tsx @@ -3,24 +3,24 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { backgroundFetchClient } from "@/lib/http/api"; import { getProxyBaseUrl } from "@/components/networking"; import { Button } from "@/components/ui/button"; -import { formatAlertValue, METRIC_LABELS, UsageAlert } from "./usageAlertApi"; +import { formatAlertValue, METRIC_LABELS, PerformanceSignal } from "./usageAlertApi"; export default function UsageAlertContext({ - alertId, + signalId, scope, entityId, }: { - alertId: string; - scope: UsageAlert["scope_type"]; + signalId: string; + scope: PerformanceSignal["scope_type"]; entityId: string | null; }) { const { accessToken, userId, userRole, isAuthorized } = useAuthorized(); - const options: UseQueryOptions = { - queryKey: ["usageAlert", getProxyBaseUrl(), userId, userRole, alertId], - enabled: Boolean(isAuthorized && accessToken && alertId), + const options: UseQueryOptions = { + queryKey: ["performanceSignal", getProxyBaseUrl(), userId, userRole, signalId], + enabled: Boolean(isAuthorized && accessToken && signalId), queryFn: async ({ signal }) => { - const { data } = await backgroundFetchClient.GET("/usage/alerts/{alert_id}", { - params: { path: { alert_id: alertId } }, + const { data } = await backgroundFetchClient.GET("/alerts/signals/{signal_id}", { + params: { path: { signal_id: signalId } }, signal, }); if (!data) throw new Error("Alert unavailable"); diff --git a/ui/litellm-dashboard/src/components/UsageAlerts/UsageAlerts.integration.test.tsx b/ui/litellm-dashboard/src/components/UsageAlerts/UsageAlerts.integration.test.tsx index d17c6366feb..14b3fcbcff3 100644 --- a/ui/litellm-dashboard/src/components/UsageAlerts/UsageAlerts.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/UsageAlerts/UsageAlerts.integration.test.tsx @@ -97,12 +97,12 @@ describe("usage alerts API integration", () => { const call = { path: url.pathname, method: request.method, body, query: url.searchParams }; calls.push(call); if (url.pathname === "/management/v1/users/me") return json({ user_id: "test-admin", user_role: "proxy_admin" }); - if (url.pathname === "/usage/alerts/incident-test") return json({ ...observation, status: "resolved" }); - if (url.pathname === "/issues/families") return json(FAMILY_SETTINGS); - if (url.pathname === "/usage/alert-rules" && request.method === "POST") { + if (url.pathname === "/alerts/signals/incident-test") return json({ ...observation, status: "resolved" }); + if (url.pathname === "/alerts/defaults") return json(FAMILY_SETTINGS); + if (url.pathname === "/alerts/rules" && request.method === "POST") { return ruleResponse(body); } - if (url.pathname === "/usage/alert-rules") { + if (url.pathname === "/alerts/rules") { const page = { data: customRules, total: customRules.length, @@ -122,7 +122,7 @@ describe("usage alerts API integration", () => { }; return json(page); } - if (url.pathname === "/usage/alert-models") { + if (url.pathname === "/alerts/models") { return modelsResponse(url.searchParams); } return json({ detail: "Unexpected request" }, 404); @@ -150,7 +150,7 @@ describe("usage alerts API integration", () => { ); it("shows saved evidence on the matching entity without recomputing from a current rule", async () => { - renderWithProviders(); + renderWithProviders(); expect(await screen.findByText(/Spend was \$120.00 on 2026-11-04 UTC/)).toHaveTextContent( "baseline was $30.00 across 5 recorded days", ); @@ -159,7 +159,7 @@ describe("usage alerts API integration", () => { }); it("does not attach an incident explanation to the wrong entity", async () => { - renderWithProviders(); + renderWithProviders(); expect(await screen.findByText("This alert belongs to a different entity.")).toBeInTheDocument(); expect(screen.queryByText(/Spend was/)).not.toBeInTheDocument(); }); @@ -204,7 +204,7 @@ describe("usage alerts API integration", () => { renderWithProviders(); await user.click(await screen.findByRole("button", { name: "Settings" })); expect(await screen.findByRole("button", { name: "Edit Unexpected spend" })).toBeInTheDocument(); - expect(calls.some((call) => call.path === "/usage/alert-rules")).toBe(false); + expect(calls.some((call) => call.path === "/alerts/rules")).toBe(false); await user.click(screen.getByText("Custom rules", { selector: "summary" })); await user.click(screen.getByRole("button", { name: "New rule", exact: true })); expect(screen.getByLabelText("Name")).not.toBeVisible(); @@ -236,7 +236,7 @@ describe("usage alerts API integration", () => { await user.click(screen.getByRole("button", { name: "Save rule" })); const expected = { scope_type: "model", scope_id: "provider/recorded-model" }; await waitFor(() => expect(calls.find((call) => call.method === "POST")?.body).toMatchObject(expected)); - expect(calls.find((call) => call.path === "/usage/alert-models")?.query.get("limit")).toBe("20"); + expect(calls.find((call) => call.path === "/alerts/models")?.query.get("limit")).toBe("20"); expect(calls.some((call) => call.path === "/v2/model/info")).toBe(false); }); diff --git a/ui/litellm-dashboard/src/components/UsageAlerts/UsageRuleResourceSelect.tsx b/ui/litellm-dashboard/src/components/UsageAlerts/UsageRuleResourceSelect.tsx index e234325d05c..d492d92710e 100644 --- a/ui/litellm-dashboard/src/components/UsageAlerts/UsageRuleResourceSelect.tsx +++ b/ui/litellm-dashboard/src/components/UsageAlerts/UsageRuleResourceSelect.tsx @@ -138,7 +138,7 @@ function ModelResource(props: ResourceProps) { queryKey: ["usageAlertModels", getProxyBaseUrl(), userId, userRole, search], initialPageParam: "", queryFn: async ({ pageParam, signal }: { pageParam: string; signal: AbortSignal }) => { - const { data } = await backgroundFetchClient.GET("/usage/alert-models", { + const { data } = await backgroundFetchClient.GET("/alerts/models", { params: { query: { search: search || undefined, cursor: pageParam || undefined, limit: PAGE_SIZE } }, signal, }); diff --git a/ui/litellm-dashboard/src/components/UsageAlerts/UsageRules.tsx b/ui/litellm-dashboard/src/components/UsageAlerts/UsageRules.tsx index 3aacd146664..026d8c0d44c 100644 --- a/ui/litellm-dashboard/src/components/UsageAlerts/UsageRules.tsx +++ b/ui/litellm-dashboard/src/components/UsageAlerts/UsageRules.tsx @@ -34,7 +34,7 @@ export default function UsageRules({ accessToken }: { accessToken: string | null queryKey: ["usageAlertRules", getProxyBaseUrl(), user?.user_id, user?.user_role, offset, "custom"], enabled: open && customOpen && Boolean(accessToken && user?.user_id), queryFn: async ({ signal }) => { - const { data } = await backgroundFetchClient.GET("/usage/alert-rules", { + const { data } = await backgroundFetchClient.GET("/alerts/rules", { params: { query: { limit: 20, offset, kind: "custom" } }, signal, }); @@ -45,8 +45,8 @@ export default function UsageRules({ accessToken }: { accessToken: string | null const save = useMutation({ mutationFn: async (body: RuleInput) => editing - ? backgroundFetchClient.PUT("/usage/alert-rules/{rule_id}", { params: { path: { rule_id: editing.id } }, body }) - : backgroundFetchClient.POST("/usage/alert-rules", { body }), + ? backgroundFetchClient.PUT("/alerts/rules/{rule_id}", { params: { path: { rule_id: editing.id } }, body }) + : backgroundFetchClient.POST("/alerts/rules", { body }), onSuccess: async () => { setEditing(undefined); await queryClient.invalidateQueries({ queryKey: ["usageAlertRules"] }); @@ -54,7 +54,7 @@ export default function UsageRules({ accessToken }: { accessToken: string | null }); const remove = useMutation({ mutationFn: (id: string) => - backgroundFetchClient.DELETE("/usage/alert-rules/{rule_id}", { params: { path: { rule_id: id } } }), + backgroundFetchClient.DELETE("/alerts/rules/{rule_id}", { params: { path: { rule_id: id } } }), onSuccess: async () => { setDeleting(null); if (query.data?.data.length === 1) setOffset(Math.max(0, offset - 20)); diff --git a/ui/litellm-dashboard/src/components/UsageAlerts/usageAlertApi.ts b/ui/litellm-dashboard/src/components/UsageAlerts/usageAlertApi.ts index a9d0ba7bdd4..004ca95000b 100644 --- a/ui/litellm-dashboard/src/components/UsageAlerts/usageAlertApi.ts +++ b/ui/litellm-dashboard/src/components/UsageAlerts/usageAlertApi.ts @@ -1,12 +1,12 @@ import type { paths } from "@/lib/http/schema"; import { uiHref } from "@/utils/uiHref"; -type AlertPage = paths["/usage/alerts"]["get"]["responses"][200]["content"]["application/json"]; -export type UsageAlert = AlertPage["data"][number]; -export type AlertModelsPage = paths["/usage/alert-models"]["get"]["responses"][200]["content"]["application/json"]; -type RulePage = paths["/usage/alert-rules"]["get"]["responses"][200]["content"]["application/json"]; +type AlertPage = paths["/alerts/signals"]["get"]["responses"][200]["content"]["application/json"]; +export type PerformanceSignal = AlertPage["data"][number]; +export type AlertModelsPage = paths["/alerts/models"]["get"]["responses"][200]["content"]["application/json"]; +type RulePage = paths["/alerts/rules"]["get"]["responses"][200]["content"]["application/json"]; export type UsageRule = RulePage["data"][number]; -export type RuleInput = paths["/usage/alert-rules"]["post"]["requestBody"]["content"]["application/json"]; +export type RuleInput = paths["/alerts/rules"]["post"]["requestBody"]["content"]["application/json"]; export const METRIC_LABELS = { spend: "Spend", requests: "Requests", @@ -16,9 +16,11 @@ export const METRIC_LABELS = { }; export const SCOPE_LABELS = { deployment: "Gateway", key: "Key", team: "Team", user: "User", model: "Model" }; -export const usageAlertHref = (alert: UsageAlert) => uiHref(alert.activity_path); +export const usageAlertHref = (alert: PerformanceSignal) => uiHref(alert.activity_path); -export function formatAlertChange(alert: Pick): string { +export function formatAlertChange( + alert: Pick, +): string { if (alert.baseline_value === 0) return alert.observed_value > 0 ? "New activity" : "No change"; if (alert.observed_value === 0) return "↓ 100%"; const ratio = @@ -29,7 +31,7 @@ export function formatAlertChange(alert: Pick ({ userRole: "Admin", userId: "viewer", accessToken: "test-session" })); vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => auth })); const fetchMock = vi.fn(); -const dates = "activity_from=2026-10-01&activity_to=2026-10-07&alert=incident-1"; +const dates = "activity_from=2026-10-01&activity_to=2026-10-07&signal=incident-1"; const response = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status, @@ -39,7 +39,7 @@ describe("entity activity", () => { await userEvent.click(screen.getByRole("button", { name: "Apply" })); await waitFor(() => expect(urls().at(-1)?.searchParams.get("start_date")).toBe("2026-10-03")); expect(urls().at(-1)?.searchParams.get("api_key")).toBe("key-hash"); - expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("alert")).toBe("incident-1"); + expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("signal")).toBe("incident-1"); }); it.each([ @@ -59,7 +59,7 @@ describe("entity activity", () => { expect(url.searchParams.get("timezone")).toBe("0"); expect(url.searchParams.get("bounded")).toBe("true"); expect(url.searchParams.get("include_model_breakdown")).toBe(scope === "key" ? "true" : null); - expect(url.searchParams.has("alert")).toBe(false); + expect(url.searchParams.has("signal")).toBe(false); }); it("keeps a member's key request within their user scope", async () => { diff --git a/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.tsx b/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.tsx index 1ffbe521954..5e2e4c1fc32 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.tsx @@ -99,7 +99,7 @@ export default function EntityActivity({ scope, id, teamId }: Props) { return (
    - {params.alert && } + {params.signal && }
    UTC { }); describe("entityActivityHref", () => { - const context = { from: "2026-10-01", to: "2026-10-07", alert: "incident-1" }; + const context = { from: "2026-10-01", to: "2026-10-07", signal: "incident-1" }; it.each([ ["key", "key-hash", "/api-keys", "key"], ["team", "a b/c", "/teams", "team"], @@ -88,7 +88,7 @@ describe("entityActivityHref", () => { expect(url.searchParams.get("detail_tab")).toBe("activity"); expect(url.searchParams.get("activity_from")).toBe(context.from); expect(url.searchParams.get("activity_to")).toBe(context.to); - expect(url.searchParams.get("alert")).toBe(context.alert); + expect(url.searchParams.get("signal")).toBe(context.signal); }); it("does not link credentials, empty entities, or synthetic entities", () => { expect(entityActivityHref("key", "sk-test-placeholder", context)).toBeUndefined(); diff --git a/ui/litellm-dashboard/src/utils/entityLinks.ts b/ui/litellm-dashboard/src/utils/entityLinks.ts index 19c02a5f910..e6272e667c8 100644 --- a/ui/litellm-dashboard/src/utils/entityLinks.ts +++ b/ui/litellm-dashboard/src/utils/entityLinks.ts @@ -47,7 +47,7 @@ export type ActivityScope = "key" | "team" | "user" | "model"; export interface ActivityLinkContext { from: string; to: string; - alert?: string; + signal?: string; } export function entityActivityHref(scope: ActivityScope, id: string, context: ActivityLinkContext): string | undefined { @@ -71,6 +71,6 @@ export function entityActivityHref(scope: ActivityScope, id: string, context: Ac } if (!href) return undefined; const params = new URLSearchParams({ detail_tab: "activity", activity_from: context.from, activity_to: context.to }); - if (context.alert) params.set("alert", context.alert); + if (context.signal) params.set("signal", context.signal); return `${href}&${params}`; }