From 7ad3fb000a356966c5fe64c9cfad5ac9c83902d1 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sun, 8 Nov 2026 15:56:17 +0800 Subject: [PATCH] refactor(alerts): unify public alert and signal contracts --- litellm/models/issues.py | 90 +- litellm/models/usage_alerts.py | 48 +- litellm/proxy/_types.py | 87 +- litellm/proxy/alerting_endpoints/issues.py | 162 +- .../proxy/alerting_endpoints/usage_alerts.py | 127 +- litellm/repositories/issue_activity.py | 22 +- litellm/repositories/issue_families.py | 28 +- litellm/repositories/issues.py | 150 +- litellm/repositories/usage_alerts.py | 197 +- litellm/types/proxy/issues.py | 122 +- litellm/types/proxy/usage_alert_defaults.py | 10 +- litellm/types/proxy/usage_alerts.py | 70 +- tests/integration/database/test_issues.py | 269 +- .../integration/database/test_usage_alerts.py | 277 +- .../alerting_endpoints/test_usage_alerts.py | 16 +- .../unit/repositories/test_issue_families.py | 6 +- tests/unit/types/proxy/test_usage_alerts.py | 24 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 3077 ++++++++--------- 18 files changed, 2092 insertions(+), 2690 deletions(-) diff --git a/litellm/models/issues.py b/litellm/models/issues.py index 60962682614..36e7d087b6d 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -3,23 +3,23 @@ from typing import Literal, Protocol from litellm.models.team import LiteLLM_TeamTable from litellm.types.proxy.issues import ( - Issue, - IssueActivityResponse, - IssueCohort, - IssueDeliveryBinding, - IssueFamily, - IssueFamilyList, - IssueFamilySetting, - IssueFamilyUpdate, - IssueImpact, - IssueInput, - IssueKind, - IssueList, - IssuePendingDelivery, - IssueProjectionSignal, - IssueScope, - IssueSignal, - IssueSignalList, + Alert, + AlertActivityResponse, + AlertCohort, + AlertDeliveryBinding, + AlertFamily, + AlertFamilyList, + AlertFamilySetting, + AlertFamilyUpdate, + AlertImpact, + AlertInput, + AlertKind, + AlertList, + AlertPendingDelivery, + AlertProjectionSignal, + AlertScope, + AlertSignal, + AlertSignalList, ) from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, @@ -30,20 +30,20 @@ from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus class IssueStore(Protocol): - async def get_issue(self, issue_id: str) -> Issue | None: ... + async def get_issue(self, issue_id: str) -> Alert | None: ... async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: ... async def list_issues( self, - kind: IssueKind | None = None, - family: IssueFamily | None = None, + kind: AlertKind | None = None, + family: AlertFamily | None = None, status: UsageAlertStatus | None = None, - scope_type: IssueScope | None = None, + scope_type: AlertScope | None = None, scope_id: str | None = None, limit: int = 50, offset: int = 0, - ) -> IssueList: ... + ) -> AlertList: ... async def set_status( self, @@ -51,39 +51,39 @@ class IssueStore(Protocol): status: UsageAlertStatus, expected_status: UsageAlertStatus, actor: str | None = None, - authorized_scopes: tuple[IssueCohort, ...] | None = None, - ) -> Issue | None: ... + authorized_scopes: tuple[AlertCohort, ...] | None = None, + ) -> Alert | None: ... - async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]: ... + async def status_scopes(self, issue_id: str) -> tuple[AlertCohort, ...]: ... async def upsert_issues( self, - inputs: tuple[IssueInput, ...], + inputs: tuple[AlertInput, ...], reopen: bool = False, status: UsageAlertStatus | None = None, - ) -> tuple[Issue, ...]: ... + ) -> tuple[Alert, ...]: ... - async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: ... + async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Alert, ...]: ... - async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: ... + async def add_signal_batch(self, signals: tuple[AlertSignal, ...]) -> None: ... async def list_signals( - self, issue_id: str, limit: int = 50, offset: int = 0, visible_scope: IssueCohort | None = None - ) -> IssueSignalList: ... + self, issue_id: str, limit: int = 50, offset: int = 0, visible_scope: AlertCohort | None = None + ) -> AlertSignalList: ... async def active_signals( self, day: date, limit: int = 500, cursor: str | None = None, - scope_type: IssueScope | None = None, + scope_type: AlertScope | None = None, metric: UsageAlertMetric | None = None, scope_ids: tuple[str, ...] | None = None, - ) -> tuple[IssueProjectionSignal, ...]: ... + ) -> tuple[AlertProjectionSignal, ...]: ... async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: ... - async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: ... + async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Alert, ...]: ... async def recover_issue_ids(self, ids: tuple[str, ...], observed_at: datetime) -> int: ... @@ -93,17 +93,17 @@ class IssueStore(Protocol): async def finish_deprecation_projection(self, observed_at: datetime) -> int: ... - async def family_settings(self) -> IssueFamilyList: ... + async def family_settings(self) -> AlertFamilyList: ... async def update_family( - self, family: PerformanceIssueFamily, data: IssueFamilyUpdate - ) -> IssueFamilySetting | None: ... + self, family: PerformanceIssueFamily, data: AlertFamilyUpdate + ) -> AlertFamilySetting | None: ... - async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: ... + async def queue_deliveries(self, bindings: tuple[AlertDeliveryBinding, ...]) -> None: ... - async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: ... + async def pending_deliveries(self, limit: int = 100) -> tuple[AlertPendingDelivery, ...]: ... - async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: ... + async def pending_delivery(self, delivery_id: str) -> AlertPendingDelivery | None: ... async def mark_delivery_sent(self, delivery_id: str) -> None: ... @@ -116,15 +116,15 @@ class IssueStore(Protocol): class IssueActivityStore(Protocol): async def daily_activity( - self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 - ) -> IssueActivityResponse: ... + self, cohort: AlertCohort, start: date, end: date, model_limit: int = 20 + ) -> AlertActivityResponse: ... async def keys( - self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityKeyPageResponse: ... async def users( - self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityUserPageResponse: ... - async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: ... + async def impact(self, cohort: AlertCohort, start: date, end: date) -> AlertImpact: ... diff --git a/litellm/models/usage_alerts.py b/litellm/models/usage_alerts.py index 63e2e05b024..1272511563f 100644 --- a/litellm/models/usage_alerts.py +++ b/litellm/models/usage_alerts.py @@ -2,35 +2,32 @@ from datetime import date from typing import Protocol from litellm.types.proxy.usage_alerts import ( - UsageAlert, - UsageAlertGroupList, - UsageAlertGroupUpdate, - UsageAlertGroupUpdateResult, - UsageAlertList, + AlertModelList, + AlertRule, + AlertRuleInput, + AlertRuleList, + PerformanceSignal, + PerformanceSignalList, UsageAlertMetric, - UsageAlertModelList, - UsageAlertRule, - UsageAlertRuleInput, UsageAlertRuleKind, - UsageAlertRuleList, UsageAlertScope, UsageAlertStatus, ) class UsageAlertStore(Protocol): - async def get_rule(self, rule_id: str) -> UsageAlertRule | None: ... + async def get_rule(self, rule_id: str) -> AlertRule | None: ... async def save_rule( self, - data: UsageAlertRuleInput, + data: AlertRuleInput, created_by: str | None, - existing: UsageAlertRule | None = None, + existing: AlertRule | None = None, *, overrides_defaults: bool = False, - ) -> UsageAlertRule | None: ... + ) -> AlertRule | None: ... - async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: ... + async def delete_rule(self, rule_id: str, *, existing: AlertRule | None = None) -> bool: ... async def list_rules( self, @@ -39,9 +36,9 @@ class UsageAlertStore(Protocol): limit: int = 50, offset: int = 0, kind: UsageAlertRuleKind = "all", - ) -> UsageAlertRuleList: ... + ) -> AlertRuleList: ... - async def get_alert(self, alert_id: str) -> UsageAlert | None: ... + async def get_alert(self, alert_id: str) -> PerformanceSignal | None: ... async def list_alerts( self, @@ -53,24 +50,11 @@ class UsageAlertStore(Protocol): end_date: date | None = None, limit: int = 50, offset: int = 0, - ) -> UsageAlertList: ... + ) -> PerformanceSignalList: ... - async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None: ... + async def set_status(self, alert_id: str, status: UsageAlertStatus) -> PerformanceSignal | None: ... - async def list_groups( - self, - scope_type: UsageAlertScope | None = None, - scope_id: str | None = None, - status: UsageAlertStatus | None = None, - start_date: date | None = None, - end_date: date | None = None, - limit: int = 50, - offset: int = 0, - ) -> UsageAlertGroupList: ... - - async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult: ... - - async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList: ... + async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> AlertModelList: ... async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None: ... diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5d42ce4767d..d3879ab8e75 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -595,21 +595,20 @@ class LiteLLMRoutes(enum.Enum): ) llm_api_routes_without_mcp = sorted(set(llm_api_routes) - set(mcp_inference_routes)) info_routes = [ - "/usage/alert-groups", - "/usage/alert-models", - "/usage/alert-rules", - "/usage/alert-rules/{rule_id}", - "/usage/alerts", - "/usage/alerts/{alert_id}", - "/issues", - "/issues/families", - "/issues/families/{family}", - "/issues/{issue_id}", - "/issues/{issue_id}/signals", - "/issues/{issue_id}/activity", - "/issues/{issue_id}/keys", - "/issues/{issue_id}/users", - "/issues/{issue_id}/impact", + "/alerts/models", + "/alerts/rules", + "/alerts/rules/{rule_id}", + "/alerts/signals", + "/alerts/signals/{signal_id}", + "/alerts", + "/alerts/defaults", + "/alerts/defaults/{family}", + "/alerts/{alert_id}", + "/alerts/{alert_id}/signals", + "/alerts/{alert_id}/activity", + "/alerts/{alert_id}/keys", + "/alerts/{alert_id}/users", + "/alerts/{alert_id}/impact", "/key/info", "/key/health", "/team/info", @@ -677,21 +676,20 @@ class LiteLLMRoutes(enum.Enum): management_routes = ( [ - "/usage/alert-groups", - "/usage/alert-models", - "/usage/alert-rules", - "/usage/alert-rules/{rule_id}", - "/usage/alerts", - "/usage/alerts/{alert_id}", - "/issues", - "/issues/families", - "/issues/families/{family}", - "/issues/{issue_id}", - "/issues/{issue_id}/signals", - "/issues/{issue_id}/activity", - "/issues/{issue_id}/keys", - "/issues/{issue_id}/users", - "/issues/{issue_id}/impact", + "/alerts/models", + "/alerts/rules", + "/alerts/rules/{rule_id}", + "/alerts/signals", + "/alerts/signals/{signal_id}", + "/alerts", + "/alerts/defaults", + "/alerts/defaults/{family}", + "/alerts/{alert_id}", + "/alerts/{alert_id}/signals", + "/alerts/{alert_id}/activity", + "/alerts/{alert_id}/keys", + "/alerts/{alert_id}/users", + "/alerts/{alert_id}/impact", # user "/user/new", "/management/v1/users/bulk", @@ -903,21 +901,20 @@ class LiteLLMRoutes(enum.Enum): ) usage_alert_routes = ( - "/usage/alert-groups", - "/usage/alert-models", - "/usage/alert-rules", - "/usage/alert-rules/{rule_id}", - "/usage/alerts", - "/usage/alerts/{alert_id}", - "/issues", - "/issues/families", - "/issues/families/{family}", - "/issues/{issue_id}", - "/issues/{issue_id}/signals", - "/issues/{issue_id}/activity", - "/issues/{issue_id}/keys", - "/issues/{issue_id}/users", - "/issues/{issue_id}/impact", + "/alerts/models", + "/alerts/rules", + "/alerts/rules/{rule_id}", + "/alerts/signals", + "/alerts/signals/{signal_id}", + "/alerts", + "/alerts/defaults", + "/alerts/defaults/{family}", + "/alerts/{alert_id}", + "/alerts/{alert_id}/signals", + "/alerts/{alert_id}/activity", + "/alerts/{alert_id}/keys", + "/alerts/{alert_id}/users", + "/alerts/{alert_id}/impact", ) self_managed_routes = [ diff --git a/litellm/proxy/alerting_endpoints/issues.py b/litellm/proxy/alerting_endpoints/issues.py index 27b663abc18..296503b3856 100644 --- a/litellm/proxy/alerting_endpoints/issues.py +++ b/litellm/proxy/alerting_endpoints/issues.py @@ -1,4 +1,4 @@ -"""Unified issue inbox and investigations backed by existing daily aggregates.""" +"""Alerts and their supporting Activity backed by existing daily aggregates.""" from collections.abc import Generator from contextlib import contextmanager @@ -23,19 +23,19 @@ from litellm.proxy.management_endpoints.common_utils import team_member_has_perm from litellm.repositories.analytics_budget import AnalyticsCapacityError from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.issues import ( - Issue, - IssueActivityResponse, - IssueCohort, - IssueFamily, - IssueFamilyList, - IssueFamilySetting, - IssueFamilyUpdate, - IssueImpact, - IssueKind, - IssueList, - IssueScope, - IssueSignalList, - IssueUpdate, + Alert, + AlertActivityResponse, + AlertCohort, + AlertFamily, + AlertFamilyList, + AlertFamilySetting, + AlertFamilyUpdate, + AlertImpact, + AlertKind, + AlertList, + AlertScope, + AlertSignalList, + AlertUpdate, ) from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, @@ -44,18 +44,18 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertStatus -router: Final = APIRouter(prefix="/issues", tags=["issues"]) +router: Final = APIRouter(prefix="/alerts", tags=["alerts"]) def require_issues(repository: IssueStore | None) -> IssueStore: if repository is None: - raise HTTPException(status_code=503, detail="Issues require a database connection") + raise HTTPException(status_code=503, detail="Alerts require a database connection") return repository def require_activity(repository: IssueActivityStore | None) -> IssueActivityStore: if repository is None: - raise HTTPException(status_code=503, detail="Issue Activity requires a database connection") + raise HTTPException(status_code=503, detail="Alert Activity requires a database connection") return repository @@ -68,16 +68,16 @@ def activity_budget() -> Generator[None]: async def authorized_issue( - issue_id: str, + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, *, write: bool = False, -) -> Issue: - issue: Final = await require_issues(repository).get_issue(issue_id) +) -> Alert: + issue: Final = await require_issues(repository).get_issue(alert_id) if issue is None: - raise HTTPException(status_code=404, detail="Issue not found") + raise HTTPException(status_code=404, detail="Alert not found") await require_scope( user, require_repository(alerts), @@ -88,7 +88,7 @@ async def authorized_issue( return issue -async def can_view_activity(issue: Issue, user: User, repository: IssueStore) -> bool: +async def can_view_activity(issue: Alert, user: User, repository: IssueStore) -> bool: if issue.cohort.scope_type != "team" or user.user_role in ( LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, @@ -102,14 +102,14 @@ async def can_view_activity(issue: Issue, user: User, repository: IssueStore) -> ) -async def require_activity_access(issue: Issue, user: User, repository: IssuesRepositoryDep) -> None: +async def require_activity_access(issue: Alert, user: User, repository: IssuesRepositoryDep) -> None: if not await can_view_activity(issue, user, require_issues(repository)): - raise HTTPException(status_code=403, detail="Full team Activity permission is required for this investigation") + raise HTTPException(status_code=403, detail="Full team Activity permission is required for this alert") async def require_status_scopes( - issue: Issue, user: User, repository: IssueStore, alerts: UsageAlertsRepositoryDep -) -> tuple[IssueCohort, ...] | None: + issue: Alert, user: User, repository: IssueStore, alerts: UsageAlertsRepositoryDep +) -> tuple[AlertCohort, ...] | None: store: Final = require_repository(alerts) await require_scope( user, @@ -130,7 +130,7 @@ async def require_status_scopes( return scopes -def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]: +def activity_range(issue: Alert, start_date: date | None, end_date: date | None) -> tuple[date, date]: start: Final = start_date or issue.range_start end: Final = end_date or issue.range_end if start > end or (end - start).days > 365: @@ -138,44 +138,44 @@ def activity_range(issue: Issue, start_date: date | None, end_date: date | None) return start, end -@router.get("", response_model=IssueList) -async def list_issues( +@router.get("", response_model=AlertList) +async def list_alerts( user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, - kind: IssueKind | None = None, - family: IssueFamily | None = None, + kind: AlertKind | None = None, + family: AlertFamily | None = None, status: UsageAlertStatus | None = None, - scope_type: IssueScope | None = None, + scope_type: AlertScope | None = None, scope_id: ScopeId = None, limit: Limit = 50, offset: Offset = 0, -) -> IssueList: +) -> AlertList: await require_scope( user, require_repository(alerts), "model" if scope_type == "provider" else scope_type, scope_id, write=False ) return await require_issues(repository).list_issues(kind, family, status, scope_type, scope_id, limit, offset) -@router.get("/families", response_model=IssueFamilyList) -async def list_issue_families( +@router.get("/defaults", response_model=AlertFamilyList) +async def list_alert_defaults( user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, -) -> IssueFamilyList: +) -> AlertFamilyList: await require_scope(user, require_repository(alerts), "deployment", None, write=False) return await require_issues(repository).family_settings() -@router.patch("/families/{family}", response_model=IssueFamilySetting) -async def update_issue_family( +@router.patch("/defaults/{family}", response_model=AlertFamilySetting) +async def update_alert_default( family: PerformanceIssueFamily, - data: IssueFamilyUpdate, + data: AlertFamilyUpdate, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, -) -> IssueFamilySetting: - require_write_access(user, f"/issues/families/{family}") +) -> AlertFamilySetting: + require_write_access(user, f"/alerts/defaults/{family}") await require_scope(user, require_repository(alerts), "deployment", None, write=True) try: result: Final = await require_issues(repository).update_family(family, data) @@ -188,17 +188,17 @@ async def update_issue_family( return result -@router.get("/{issue_id}", response_model=Issue) -async def get_issue( - issue_id: str, +@router.get("/{alert_id}", response_model=Alert) +async def get_alert( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, -) -> Issue: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) +) -> Alert: + issue: Final = await authorized_issue(alert_id, user, repository, alerts) activity_allowed: Final = await can_view_activity(issue, user, require_issues(repository)) try: - require_write_access(user, f"/issues/{issue_id}") + require_write_access(user, f"/alerts/{alert_id}") await require_status_scopes(issue, user, require_issues(repository), alerts) except HTTPException as exc: if exc.status_code != 403: @@ -207,37 +207,37 @@ async def get_issue( return issue.model_copy(update={"can_update_status": True, "can_view_activity": activity_allowed}) -@router.patch("/{issue_id}", response_model=Issue) -async def update_issue( - issue_id: str, - data: IssueUpdate, +@router.patch("/{alert_id}", response_model=Alert) +async def update_alert( + alert_id: str, + data: AlertUpdate, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, -) -> Issue: - require_write_access(user, f"/issues/{issue_id}") - issue: Final = await authorized_issue(issue_id, user, repository, alerts) +) -> Alert: + require_write_access(user, f"/alerts/{alert_id}") + issue: Final = await authorized_issue(alert_id, user, repository, alerts) scopes: Final = await require_status_scopes(issue, user, require_issues(repository), alerts) if issue.kind == "model_deprecation" and not issue.active and data.status == "open": - raise HTTPException(status_code=409, detail="This retirement issue reopens when the model receives new traffic") + raise HTTPException(status_code=409, detail="This retirement alert reopens when the model receives new traffic") changed: Final = await require_issues(repository).set_status( - issue_id, data.status, data.expected_status, user.user_id, authorized_scopes=scopes + alert_id, data.status, data.expected_status, user.user_id, authorized_scopes=scopes ) if changed is None: - raise HTTPException(status_code=409, detail="Issue status changed; reload and retry") + raise HTTPException(status_code=409, detail="Alert status changed; reload and retry") return changed -@router.get("/{issue_id}/signals", response_model=IssueSignalList) -async def list_issue_signals( - issue_id: str, +@router.get("/{alert_id}/signals", response_model=AlertSignalList) +async def list_alert_signals( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, limit: Limit = 50, offset: Offset = 0, -) -> IssueSignalList: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) +) -> AlertSignalList: + issue: Final = await authorized_issue(alert_id, user, repository, alerts) visible_scope: Final = ( None if user.user_role @@ -247,12 +247,12 @@ async def list_issue_signals( ) else issue.cohort ) - return await require_issues(repository).list_signals(issue_id, limit, offset, visible_scope) + return await require_issues(repository).list_signals(alert_id, limit, offset, visible_scope) -@router.get("/{issue_id}/activity", response_model=IssueActivityResponse) -async def get_issue_activity( - issue_id: str, +@router.get("/{alert_id}/activity", response_model=AlertActivityResponse) +async def get_alert_activity( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, @@ -260,17 +260,17 @@ async def get_issue_activity( start_date: date | None = None, end_date: date | None = None, model_limit: Limit = 20, -) -> IssueActivityResponse: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) +) -> AlertActivityResponse: + issue: Final = await authorized_issue(alert_id, user, repository, alerts) await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) with activity_budget(): return await require_activity(activity).daily_activity(issue.cohort, start, end, model_limit) -@router.get("/{issue_id}/keys", response_model=DailyActivityKeyPageResponse) -async def get_issue_keys( - issue_id: str, +@router.get("/{alert_id}/keys", response_model=DailyActivityKeyPageResponse) +async def get_alert_keys( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, @@ -280,16 +280,16 @@ async def get_issue_keys( limit: Limit = 20, offset: Offset = 0, ) -> DailyActivityKeyPageResponse: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) + issue: Final = await authorized_issue(alert_id, user, repository, alerts) await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) with activity_budget(): return await require_activity(activity).keys(issue.cohort, start, end, limit, offset) -@router.get("/{issue_id}/users", response_model=DailyActivityUserPageResponse) -async def get_issue_users( - issue_id: str, +@router.get("/{alert_id}/users", response_model=DailyActivityUserPageResponse) +async def get_alert_users( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, @@ -299,7 +299,7 @@ async def get_issue_users( limit: Limit = 20, offset: Offset = 0, ) -> DailyActivityUserPageResponse: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) + issue: Final = await authorized_issue(alert_id, user, repository, alerts) await require_activity_access(issue, user, repository) if issue.cohort.scope_type == "team": raise HTTPException(status_code=422, detail="Historical team aggregates do not retain user membership") @@ -308,17 +308,17 @@ async def get_issue_users( return await require_activity(activity).users(issue.cohort, start, end, limit, offset) -@router.get("/{issue_id}/impact", response_model=IssueImpact) -async def get_issue_impact( - issue_id: str, +@router.get("/{alert_id}/impact", response_model=AlertImpact) +async def get_alert_impact( + alert_id: str, user: User, repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, activity: IssueActivityRepositoryDep, start_date: date | None = None, end_date: date | None = None, -) -> IssueImpact: - issue: Final = await authorized_issue(issue_id, user, repository, alerts) +) -> AlertImpact: + issue: Final = await authorized_issue(alert_id, user, repository, alerts) await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) with activity_budget(): diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py index 3950efdc1b4..e3cc688b4d9 100644 --- a/litellm/proxy/alerting_endpoints/usage_alerts.py +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -1,4 +1,4 @@ -"""Public management API for daily usage alert rules and incidents.""" +"""Performance rules and read-only detection evidence.""" from datetime import date from typing import Annotated, Final, TypeAlias @@ -14,28 +14,24 @@ from litellm.proxy.deps.repositories import UsageAlertsRepositoryDep from litellm.repositories.daily_activity_buckets import api_key_from_bucket from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.usage_alerts import ( - UsageAlert, - UsageAlertGroupList, - UsageAlertGroupUpdate, - UsageAlertGroupUpdateResult, - UsageAlertList, + AlertModelList, + AlertRule, + AlertRuleInput, + AlertRuleList, + PerformanceSignal, + PerformanceSignalList, UsageAlertMetric, - UsageAlertModelList, - UsageAlertRule, - UsageAlertRuleInput, UsageAlertRuleKind, - UsageAlertRuleList, UsageAlertScope, UsageAlertStatus, - UsageAlertUpdate, ) -router: Final = APIRouter(prefix="/usage", tags=["usage alerts"]) +router: Final = APIRouter(prefix="/alerts", tags=["alerts"]) def require_repository(repository: UsageAlertStore | None) -> UsageAlertStore: if repository is None: - raise HTTPException(status_code=503, detail="Usage alerts require a database connection") + raise HTTPException(status_code=503, detail="Alert signals require a database connection") return repository @@ -71,9 +67,7 @@ def require_write_access(user: UserAPIKeyAuth, route: str) -> None: raise HTTPException(status_code=403, detail="This credential does not grant alert write access") -def require_notification_access( - user: UserAPIKeyAuth, data: UsageAlertRuleInput, existing: UsageAlertRule | None = None -) -> None: +def require_notification_access(user: UserAPIKeyAuth, data: AlertRuleInput, existing: AlertRule | None = None) -> None: if user.user_role == LitellmUserRoles.PROXY_ADMIN: return if data.notification_channels or data.email_recipients: @@ -97,16 +91,16 @@ async def require_scope( if user.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: if not write: return - raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts") + raise HTTPException(status_code=403, detail="Read-only users cannot manage alerts") if user.user_role not in ( LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, LitellmUserRoles.ORG_ADMIN, LitellmUserRoles.TEAM, ): - raise HTTPException(status_code=403, detail="Usage alerts require a management role") + raise HTTPException(status_code=403, detail="Alert signals require a management role") if write and user.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY: - raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts") + raise HTTPException(status_code=403, detail="Read-only users cannot manage alerts") if scope_id is None: raise HTTPException(status_code=403, detail="Specify a scope you can access") if scope_type == "user" and user.user_id is not None and scope_id == user.user_id: @@ -128,8 +122,8 @@ async def require_scope( raise HTTPException(status_code=403, detail="You cannot access this alert scope") -@router.get("/alert-rules", response_model=UsageAlertRuleList) -async def list_usage_alert_rules( +@router.get("/rules", response_model=AlertRuleList) +async def list_alert_rules( user: User, repository: Repository, scope_type: UsageAlertScope | None = None, @@ -137,59 +131,30 @@ async def list_usage_alert_rules( limit: Limit = 50, offset: Offset = 0, kind: UsageAlertRuleKind = "all", -) -> UsageAlertRuleList: +) -> AlertRuleList: store: Final = require_repository(repository) await require_scope(user, store, scope_type, scope_id, write=False) return await store.list_rules(scope_type, scope_id, limit, offset, kind) -@router.get("/alert-groups", response_model=UsageAlertGroupList) -async def list_usage_alert_groups( - user: User, - repository: Repository, - scope_type: UsageAlertScope | None = None, - scope_id: ScopeId = None, - status: UsageAlertStatus | None = None, - start_date: date | None = None, - end_date: date | None = None, - limit: Limit = 50, - offset: Offset = 0, -) -> UsageAlertGroupList: - store: Final = require_repository(repository) - await require_scope(user, store, scope_type, scope_id, write=False) - if start_date is not None and end_date is not None and start_date > end_date: - raise HTTPException(status_code=422, detail="start_date must not be after end_date") - return await store.list_groups(scope_type, scope_id, status, start_date, end_date, limit, offset) - - -@router.patch("/alert-groups", response_model=UsageAlertGroupUpdateResult) -async def update_usage_alert_group( - data: UsageAlertGroupUpdate, user: User, repository: Repository -) -> UsageAlertGroupUpdateResult: - store: Final = require_repository(repository) - require_write_access(user, "/usage/alert-groups") - await require_scope(user, store, data.scope_type, data.scope_id, write=True) - return await store.set_group_status(data) - - -@router.get("/alert-models", response_model=UsageAlertModelList) -async def list_usage_alert_models( +@router.get("/models", response_model=AlertModelList) +async def list_alert_models( user: User, repository: Repository, search: ScopeId = None, cursor: ScopeId = None, limit: Limit = 20, -) -> UsageAlertModelList: +) -> AlertModelList: store: Final = require_repository(repository) await require_scope(user, store, "model", None, write=False) return await store.list_reported_models(search, cursor, limit) -@router.post("/alert-rules", response_model=UsageAlertRule, status_code=201) -async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, repository: Repository) -> UsageAlertRule: - """Create a daily usage alert rule for a scope the caller can manage.""" +@router.post("/rules", response_model=AlertRule, status_code=201) +async def create_alert_rule(data: AlertRuleInput, user: User, repository: Repository) -> AlertRule: + """Create a daily performance rule for a scope the caller can manage.""" store: Final = require_repository(repository) - require_write_access(user, "/usage/alert-rules") + require_write_access(user, "/alerts/rules") await require_scope(user, store, data.scope_type, data.scope_id, write=True) require_notification_access(user, data) saved: Final = await store.save_rule( @@ -200,8 +165,8 @@ async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, reposit return saved -@router.get("/alert-rules/{rule_id}", response_model=UsageAlertRule) -async def get_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> UsageAlertRule: +@router.get("/rules/{rule_id}", response_model=AlertRule) +async def get_alert_rule(rule_id: str, user: User, repository: Repository) -> AlertRule: store: Final = require_repository(repository) rule: Final = await store.get_rule(rule_id) if rule is None: @@ -210,12 +175,10 @@ async def get_usage_alert_rule(rule_id: str, user: User, repository: Repository) return rule -@router.put("/alert-rules/{rule_id}", response_model=UsageAlertRule) -async def update_usage_alert_rule( - rule_id: str, data: UsageAlertRuleInput, user: User, repository: Repository -) -> UsageAlertRule: +@router.put("/rules/{rule_id}", response_model=AlertRule) +async def update_alert_rule(rule_id: str, data: AlertRuleInput, user: User, repository: Repository) -> AlertRule: store: Final = require_repository(repository) - require_write_access(user, f"/usage/alert-rules/{rule_id}") + require_write_access(user, f"/alerts/rules/{rule_id}") rule: Final = await store.get_rule(rule_id) if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") @@ -237,10 +200,10 @@ async def update_usage_alert_rule( return saved -@router.delete("/alert-rules/{rule_id}", status_code=204) -async def delete_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> Response: +@router.delete("/rules/{rule_id}", status_code=204) +async def delete_alert_rule(rule_id: str, user: User, repository: Repository) -> Response: store: Final = require_repository(repository) - require_write_access(user, f"/usage/alert-rules/{rule_id}") + require_write_access(user, f"/alerts/rules/{rule_id}") rule: Final = await store.get_rule(rule_id) if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") @@ -251,8 +214,8 @@ async def delete_usage_alert_rule(rule_id: str, user: User, repository: Reposito return Response(status_code=204) -@router.get("/alerts", response_model=UsageAlertList) -async def list_usage_alerts( +@router.get("/signals", response_model=PerformanceSignalList) +async def list_performance_signals( user: User, repository: Repository, scope_type: UsageAlertScope | None = None, @@ -263,7 +226,7 @@ async def list_usage_alerts( end_date: date | None = None, limit: Limit = 50, offset: Offset = 0, -) -> UsageAlertList: +) -> PerformanceSignalList: store: Final = require_repository(repository) await require_scope(user, store, scope_type, scope_id, write=False) if start_date is not None and end_date is not None and start_date > end_date: @@ -271,25 +234,11 @@ async def list_usage_alerts( return await store.list_alerts(scope_type, scope_id, metric, status, start_date, end_date, limit, offset) -@router.get("/alerts/{alert_id}", response_model=UsageAlert) -async def get_usage_alert(alert_id: str, user: User, repository: Repository) -> UsageAlert: +@router.get("/signals/{signal_id}", response_model=PerformanceSignal) +async def get_performance_signal(signal_id: str, user: User, repository: Repository) -> PerformanceSignal: store: Final = require_repository(repository) - alert: Final = await store.get_alert(alert_id) + alert: Final = await store.get_alert(signal_id) if alert is None: - raise HTTPException(status_code=404, detail="Usage alert not found") + raise HTTPException(status_code=404, detail="Alert signal not found") await require_scope(user, store, alert.scope_type, alert.scope_id, write=False) return alert - - -@router.patch("/alerts/{alert_id}", response_model=UsageAlert) -async def update_usage_alert(alert_id: str, data: UsageAlertUpdate, user: User, repository: Repository) -> UsageAlert: - store: Final = require_repository(repository) - require_write_access(user, f"/usage/alerts/{alert_id}") - alert: Final = await store.get_alert(alert_id) - if alert is None: - raise HTTPException(status_code=404, detail="Usage alert not found") - await require_scope(user, store, alert.scope_type, alert.scope_id, write=True) - updated: Final = await store.set_status(alert_id, data.status) - if updated is None: - raise HTTPException(status_code=404, detail="Usage alert not found") - return updated diff --git a/litellm/repositories/issue_activity.py b/litellm/repositories/issue_activity.py index 51ec62d69a8..1a9e6a550dc 100644 --- a/litellm/repositories/issue_activity.py +++ b/litellm/repositories/issue_activity.py @@ -1,4 +1,4 @@ -"""Issue investigations reuse the same aggregate filters as entity Activity.""" +"""Alert investigations reuse the same aggregate filters as entity Activity.""" from collections.abc import Mapping from dataclasses import asdict, replace @@ -15,9 +15,9 @@ from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_ from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase, DailyActivityRepository from litellm.repositories.daily_activity_sql import PRISMA_TO_PG_TABLE, build_where_clause from litellm.types.proxy.issues import ( - IssueActivityResponse, - IssueCohort, - IssueImpact, + AlertActivityResponse, + AlertCohort, + AlertImpact, ) from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, @@ -51,7 +51,7 @@ class _UserLabel(BaseModel, frozen=True): user_email: str | None -def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyActivityScope: +def issue_activity_scope(cohort: AlertCohort, start: date, end: date) -> DailyActivityScope: if start > end or (end - start).days > 365: raise ValueError("Choose an ordered date range of at most 366 days") return DailyActivityScope( @@ -83,14 +83,14 @@ class IssueActivityRepository: return cls(daily_activity_repository(prisma, query_database=database), database) async def daily_activity( - self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 - ) -> IssueActivityResponse: + self, cohort: AlertCohort, start: date, end: date, model_limit: int = 20 + ) -> AlertActivityResponse: return await BoundedDailyActivityRepository(self.database).aggregate( issue_activity_scope(cohort, start, end), model_limit=model_limit if cohort.scope_type == "key" else None ) async def keys( - self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityKeyPageResponse: if not 1 <= limit <= 100 or offset < 0: raise ValueError("Read between one and 100 keys with a nonnegative offset") @@ -113,7 +113,7 @@ class IssueActivityRepository: ) async def users( - self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + self, cohort: AlertCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityUserPageResponse: if not 1 <= limit <= 100 or offset < 0: raise ValueError("Read between one and 100 users with a nonnegative offset") @@ -206,7 +206,7 @@ class IssueActivityRepository: ) return rows[0] - async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: + async def impact(self, cohort: AlertCohort, start: date, end: date) -> AlertImpact: scope: Final = issue_activity_scope(cohort, start, end) counts: Final = await self._counts(scope) teams: Final = ( @@ -214,7 +214,7 @@ class IssueActivityRepository: if cohort.scope_type not in ("user", "team") else None ) - return IssueImpact( + return AlertImpact( keys=counts.keys, models=counts.models, requests=counts.requests, diff --git a/litellm/repositories/issue_families.py b/litellm/repositories/issue_families.py index 698a066db8b..4a8e10d1906 100644 --- a/litellm/repositories/issue_families.py +++ b/litellm/repositories/issue_families.py @@ -10,10 +10,10 @@ from pydantic import BaseModel, TypeAdapter from litellm.repositories.usage_alerts import AlertDatabase from litellm.types.proxy.issues import ( - IssueFamilyList, - IssueFamilyPolicy, - IssueFamilySetting, - IssueFamilyUpdate, + AlertFamilyList, + AlertFamilyPolicy, + AlertFamilySetting, + AlertFamilyUpdate, ) from litellm.types.proxy.usage_alert_defaults import ( DEFAULT_USAGE_POLICIES, @@ -22,11 +22,11 @@ from litellm.types.proxy.usage_alert_defaults import ( PerformanceIssueFamily, default_usage_rule_id, ) -from litellm.types.proxy.usage_alerts import UsageAlertEvaluationCounts, UsageAlertRule +from litellm.types.proxy.usage_alerts import AlertRule, UsageAlertEvaluationCounts class _RuleRow(BaseModel, frozen=True): - record: UsageAlertRule + record: AlertRule class _Count(BaseModel, frozen=True): @@ -37,14 +37,14 @@ class _ProjectionState(BaseModel, frozen=True): pending: bool -def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) -> IssueFamilySetting: +def _setting(family: PerformanceIssueFamily, rules: tuple[AlertRule, ...]) -> AlertFamilySetting: policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family) groups: Final = tuple( (policy, tuple(rule for rule in rules if rule.metric == policy.metric and rule.direction == policy.direction)) for policy in policies ) values: Final = tuple( - IssueFamilyPolicy( + AlertFamilyPolicy( direction=policy.direction, enabled=group[0].enabled if group and len({rule.enabled for rule in group}) == 1 else None, multiplier=group[0].multiplier if group and len({rule.multiplier for rule in group}) == 1 else None, @@ -69,7 +69,7 @@ def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) for policy, group in groups ) family_rules: Final = tuple(chain.from_iterable(group for _, group in groups)) - return IssueFamilySetting( + return AlertFamilySetting( family=family, enabled=family_rules[0].enabled if family_rules and len({r.enabled for r in family_rules}) == 1 else None, total_rules=len(family_rules), @@ -81,7 +81,7 @@ def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) ) -async def family_settings(database: AlertDatabase) -> IssueFamilyList: +async def family_settings(database: AlertDatabase) -> AlertFamilyList: rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python( await database.query_raw( 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted', @@ -94,7 +94,7 @@ async def family_settings(database: AlertDatabase) -> IssueFamilyList: WHERE id LIKE 'performance-dirty:%' AND NOT completed LIMIT 1) AS pending""" ) ) - return IssueFamilyList( + return AlertFamilyList( data=tuple( _setting(family, tuple(row.record for row in rows)) for family in ("spend", "latency", "reliability", "traffic") @@ -106,8 +106,8 @@ async def family_settings(database: AlertDatabase) -> IssueFamilyList: async def update_family( database: AlertDatabase, family: PerformanceIssueFamily, - data: IssueFamilyUpdate, -) -> IssueFamilySetting | None: + data: AlertFamilyUpdate, +) -> AlertFamilySetting | None: policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family) if any(update.direction not in {policy.direction for policy in policies} for update in data.policies): raise ValueError("This family does not support that direction") @@ -126,7 +126,7 @@ async def update_family( policy.direction: policy.model_dump(exclude_none=True, exclude={"direction"}) for policy in data.policies } records: Final = tuple( - UsageAlertRule.model_validate( + AlertRule.model_validate( { **row.record.model_dump(), **({"enabled": data.enabled} if data.enabled is not None else {}), diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 0caf70b9cbb..a902ffefbe3 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -14,21 +14,21 @@ from litellm.models.team import LiteLLM_TeamTable from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( - Issue, - IssueCohort, - IssueDeliveryBinding, - IssueFamily, - IssueFamilyList, - IssueFamilySetting, - IssueFamilyUpdate, - IssueInput, - IssueKind, - IssueList, - IssuePendingDelivery, - IssueProjectionSignal, - IssueScope, - IssueSignal, - IssueSignalList, + Alert, + AlertCohort, + AlertDeliveryBinding, + AlertFamily, + AlertFamilyList, + AlertFamilySetting, + AlertFamilyUpdate, + AlertInput, + AlertKind, + AlertList, + AlertPendingDelivery, + AlertProjectionSignal, + AlertScope, + AlertSignal, + AlertSignalList, ) from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS, PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus @@ -38,11 +38,11 @@ if TYPE_CHECKING: class _IssueRow(BaseModel, frozen=True): - record: Issue + record: Alert class _SignalRow(BaseModel, frozen=True): - record: IssueSignal + record: AlertSignal class _Count(BaseModel, frozen=True): @@ -82,8 +82,8 @@ def _utc(value: datetime) -> str: return value.astimezone(timezone.utc).replace(tzinfo=None).isoformat() -def _issue_payload(data: IssueInput, status: UsageAlertStatus | None) -> Mapping[str, object]: - initial: Final = Issue.model_validate( +def _issue_payload(data: AlertInput, status: UsageAlertStatus | None) -> Mapping[str, object]: + initial: Final = Alert.model_validate( { **data.model_dump(), "status": status or "open", @@ -110,10 +110,10 @@ class IssuesRepository: async def upsert_issues( self, - inputs: tuple[IssueInput, ...], + inputs: tuple[AlertInput, ...], reopen: bool = False, status: UsageAlertStatus | None = None, - ) -> tuple[Issue, ...]: + ) -> tuple[Alert, ...]: if not inputs: return () if len(inputs) > 500 or len(frozenset(item.id for item in inputs)) != len(inputs): @@ -152,7 +152,7 @@ class IssuesRepository: raise ValueError("An issue ID already belongs to another cohort") return tuple(row.record for row in rows) - async def get_issue(self, issue_id: str) -> Issue | None: + async def get_issue(self, issue_id: str) -> Alert | None: rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( await self.database.query_raw(f"SELECT {_RECORD} AS record {_JOIN} WHERE i.id=$1", issue_id) ) @@ -179,14 +179,14 @@ class IssuesRepository: async def list_issues( self, - kind: IssueKind | None = None, - family: IssueFamily | None = None, + kind: AlertKind | None = None, + family: AlertFamily | None = None, status: UsageAlertStatus | None = None, - scope_type: IssueScope | None = None, + scope_type: AlertScope | None = None, scope_id: str | None = None, limit: int = 50, offset: int = 0, - ) -> IssueList: + ) -> AlertList: where: Final = f"""($1::text IS NULL OR i.kind=$1) AND ($2::text IS NULL OR i.family=$2) AND ($3::text IS NULL OR ({_STATUS})=$3) AND ($4::text IS NULL OR i.scope_type=$4) AND ($5::text IS NULL OR i.scope_id=$5)""" @@ -202,11 +202,11 @@ class IssuesRepository: offset, ) ) - return IssueList( + return AlertList( data=await self._labels(tuple(row.record for row in rows)), total=count[0].count, limit=limit, offset=offset ) - async def _labels(self, issues: tuple[Issue, ...]) -> tuple[Issue, ...]: + async def _labels(self, issues: tuple[Alert, ...]) -> tuple[Alert, ...]: if not issues: return () labels: Final = await UsageAlertsRepository(self.database).resource_labels( @@ -217,7 +217,7 @@ class IssuesRepository: for issue in issues ) - async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: + async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Alert, ...]: if len(cohort_keys) > 500: raise ValueError("Read at most 500 issue cohorts per batch") rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( @@ -236,8 +236,8 @@ class IssuesRepository: status: UsageAlertStatus, expected_status: UsageAlertStatus, actor: str | None = None, - authorized_scopes: tuple[IssueCohort, ...] | None = None, - ) -> Issue | None: + authorized_scopes: tuple[AlertCohort, ...] | None = None, + ) -> Alert | None: if authorized_scopes is not None and len(authorized_scopes) > 10: raise ValueError("Authorize at most 10 alert source scopes") changed: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( @@ -248,8 +248,8 @@ class IssuesRepository: AND ($5::jsonb IS NULL OR NOT EXISTS ( SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id AND NOT EXISTS ( SELECT 1 FROM jsonb_to_recordset($5::jsonb) AS x(scope_type text,scope_id text) - WHERE s.source='usage_alert' AND x.scope_type=s.record->'usage_alert'->>'scope_type' - AND x.scope_id IS NOT DISTINCT FROM s.record->'usage_alert'->>'scope_id'))) + WHERE s.source='performance' AND x.scope_type=s.record->'performance'->>'scope_type' + AND x.scope_id IS NOT DISTINCT FROM s.record->'performance'->>'scope_id'))) ), ack AS ( INSERT INTO "LiteLLM_Config" AS prior (param_name,param_value) SELECT ack_key,jsonb_build_object('model',record->'deprecation'->>'usage_model', @@ -279,18 +279,18 @@ class IssuesRepository: ) return await self.get_issue(issue_id) if changed else None - async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]: - return TypeAdapter(tuple[IssueCohort, ...]).validate_python( + async def status_scopes(self, issue_id: str) -> tuple[AlertCohort, ...]: + return TypeAdapter(tuple[AlertCohort, ...]).validate_python( await self.database.query_raw( - """SELECT DISTINCT CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_type' + """SELECT DISTINCT CASE WHEN source='performance' THEN record->'performance'->>'scope_type' ELSE 'deployment' END AS scope_type, - CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_id' END AS scope_id + CASE WHEN source='performance' THEN record->'performance'->>'scope_id' END AS scope_id FROM "LiteLLM_IssueSignal" WHERE issue_id=$1 ORDER BY scope_type,scope_id LIMIT 11""", issue_id, ) ) - async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: + async def add_signal_batch(self, signals: tuple[AlertSignal, ...]) -> None: if not signals: return if len(signals) > 500: @@ -310,14 +310,14 @@ class IssuesRepository: ) UPDATE "LiteLLM_Issue" i SET updated_at=GREATEST(CURRENT_TIMESTAMP,i.updated_at+interval '1 millisecond'), status=CASE WHEN i.kind='performance' AND i.active AND i.status<>'open' AND EXISTS ( - SELECT 1 FROM added a WHERE a.issue_id=i.id AND a.source='usage_alert' - AND (a.record->'usage_alert'->>'scope_type'<>i.scope_type - OR a.record->'usage_alert'->>'scope_id' IS DISTINCT FROM i.scope_id) + SELECT 1 FROM added a WHERE a.issue_id=i.id AND a.source='performance' + AND (a.record->'performance'->>'scope_type'<>i.scope_type + OR a.record->'performance'->>'scope_id' IS DISTINCT FROM i.scope_id) AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" prior - WHERE prior.issue_id=i.id AND prior.source='usage_alert' - AND prior.record->'usage_alert'->>'scope_type'=a.record->'usage_alert'->>'scope_type' - AND prior.record->'usage_alert'->>'scope_id' - IS NOT DISTINCT FROM a.record->'usage_alert'->>'scope_id') + WHERE prior.issue_id=i.id AND prior.source='performance' + AND prior.record->'performance'->>'scope_type'=a.record->'performance'->>'scope_type' + AND prior.record->'performance'->>'scope_id' + IS NOT DISTINCT FROM a.record->'performance'->>'scope_id') ) THEN 'open' ELSE i.status END WHERE i.id IN(SELECT issue_id FROM added) RETURNING i.id""", json.dumps(tuple(signal.model_dump(mode="json") for signal in signals)), @@ -328,11 +328,11 @@ class IssuesRepository: issue_id: str, limit: int = 50, offset: int = 0, - visible_scope: IssueCohort | None = None, - ) -> IssueSignalList: - where: Final = """issue_id=$1 AND ($2::text IS NULL OR (source='usage_alert' - AND record->'usage_alert'->>'scope_type'=$2 - AND record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM $3::text))""" + visible_scope: AlertCohort | None = None, + ) -> AlertSignalList: + where: Final = """issue_id=$1 AND ($2::text IS NULL OR (source='performance' + AND record->'performance'->>'scope_type'=$2 + AND record->'performance'->>'scope_id' IS NOT DISTINCT FROM $3::text))""" args: Final = ( issue_id, visible_scope.scope_type if visible_scope else None, @@ -352,7 +352,7 @@ class IssuesRepository: offset, ) ) - return IssueSignalList( + return AlertSignalList( data=tuple(row.record for row in rows), total=counts[0].count, limit=limit, offset=offset ) @@ -361,17 +361,17 @@ class IssuesRepository: day: date, limit: int = 500, cursor: str | None = None, - scope_type: IssueScope | None = None, + scope_type: AlertScope | None = None, metric: UsageAlertMetric | None = None, scope_ids: tuple[str, ...] | None = None, - ) -> tuple[IssueProjectionSignal, ...]: + ) -> tuple[AlertProjectionSignal, ...]: if not 1 <= limit <= 500: raise ValueError("Read between one and 500 source signals") if scope_ids == (): return () if scope_ids is not None and len(scope_ids) > 500: raise ValueError("Filter at most 500 source scopes") - return TypeAdapter(tuple[IssueProjectionSignal, ...]).validate_python( + return TypeAdapter(tuple[AlertProjectionSignal, ...]).validate_python( await self.database.query_raw( """SELECT a.record AS alert,r.record AS rule,a.evaluation_day FROM "LiteLLM_UsageAlertIncident" a JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id @@ -418,14 +418,14 @@ class IssuesRepository: FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id LEFT JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id - WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='usage_alert' + WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='performance' AND ({selection}) - AND s.record->'usage_alert'->>'scope_type'=i.scope_type - AND s.record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM i.scope_id - AND ((i.family='spend' AND s.record->'usage_alert'->>'metric'='spend') - OR (i.family='latency' AND s.record->'usage_alert'->>'metric'='latency') - OR (i.family='traffic' AND s.record->'usage_alert'->>'metric'='requests') - OR (i.family='reliability' AND s.record->'usage_alert'->>'metric' IN ('errors','error_rate'))) + AND s.record->'performance'->>'scope_type'=i.scope_type + AND s.record->'performance'->>'scope_id' IS NOT DISTINCT FROM i.scope_id + AND ((i.family='spend' AND s.record->'performance'->>'metric'='spend') + OR (i.family='latency' AND s.record->'performance'->>'metric'='latency') + OR (i.family='traffic' AND s.record->'performance'->>'metric'='requests') + OR (i.family='reliability' AND s.record->'performance'->>'metric' IN ('errors','error_rate'))) ), observations AS ( SELECT *,COALESCE(source_id IS NOT NULL AND NOT breached AND evaluation_day<=$1 AND evaluation_day>=latest_observation_day @@ -446,7 +446,7 @@ class IssuesRepository: ) return rows[0].count - async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: + async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Alert, ...]: if not 1 <= limit <= 500: raise ValueError("Read between one and 500 provider issues") rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( @@ -508,19 +508,19 @@ class IssuesRepository: ) return rows[0].count - async def family_settings(self) -> "IssueFamilyList": + async def family_settings(self) -> "AlertFamilyList": from litellm.repositories.issue_families import family_settings return await family_settings(self.database) async def update_family( - self, family: "PerformanceIssueFamily", data: "IssueFamilyUpdate" - ) -> "IssueFamilySetting | None": + self, family: "PerformanceIssueFamily", data: "AlertFamilyUpdate" + ) -> "AlertFamilySetting | None": from litellm.repositories.issue_families import update_family return await update_family(self.database, family, data) - async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: + async def queue_deliveries(self, bindings: tuple[AlertDeliveryBinding, ...]) -> None: if not bindings: return if len(bindings) > 500: @@ -545,17 +545,17 @@ class IssuesRepository: json.dumps(tuple(binding.model_dump(mode="json") for binding in bindings)), ) - async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: + async def pending_deliveries(self, limit: int = 100) -> tuple[AlertPendingDelivery, ...]: return await self._pending_deliveries(limit, None) - async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: + async def pending_delivery(self, delivery_id: str) -> AlertPendingDelivery | None: rows: Final = await self._pending_deliveries(1, delivery_id) return rows[0] if rows else None - async def _pending_deliveries(self, limit: int, delivery_id: str | None) -> tuple[IssuePendingDelivery, ...]: + async def _pending_deliveries(self, limit: int, delivery_id: str | None) -> tuple[AlertPendingDelivery, ...]: if not 1 <= limit <= 500: raise ValueError("Read between one and 500 deliveries") - return TypeAdapter(tuple[IssuePendingDelivery, ...]).validate_python( + return TypeAdapter(tuple[AlertPendingDelivery, ...]).validate_python( await self.database.query_raw( f"""SELECT d.record || jsonb_build_object('next_attempt',d.next_attempt AT TIME ZONE 'UTC') AS delivery, {_RECORD} AS issue,bound.rule,bound.alert @@ -583,7 +583,7 @@ class IssuesRepository: AND custom.record->>'direction'=r.record->>'direction' AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id))) AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id - AND s.source='usage_alert' AND s.source_id=a.id) + AND s.source='performance' AND s.source_id=a.id) ORDER BY b.rule_id,b.source_id LIMIT 1 ) bound ON true WHERE NOT d.sent AND d.next_attempt<=CURRENT_TIMESTAMP AND i.active AND ({_STATUS})='open' @@ -622,10 +622,10 @@ class IssuesRepository: if not 1 <= limit <= 500 or retention_days < 1: raise ValueError("Retention and cleanup batch must be bounded positive values") await self.database.query_raw( - """WITH expired AS MATERIALIZED ( - SELECT id FROM "LiteLLM_Issue" WHERE status<>'open' - AND updated_at$1::int) - ORDER BY updated_at,id LIMIT $2 + f"""WITH expired AS MATERIALIZED ( + SELECT i.id {_JOIN} WHERE i.status<>'open' AND ({_STATUS})<>'open' + AND i.updated_at$1::int) + ORDER BY i.updated_at,i.id LIMIT $2 ), signals AS ( DELETE FROM "LiteLLM_IssueSignal" WHERE (issue_id,source,source_id) IN ( SELECT issue_id,source,source_id FROM "LiteLLM_IssueSignal" diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 9380dd86f44..664f0c6466d 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -18,23 +18,18 @@ from pydantic import BaseModel, TypeAdapter from litellm.repositories.daily_activity_buckets import api_key_from_bucket from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS from litellm.types.proxy.usage_alerts import ( - UsageAlert, + AlertModelList, + AlertModelOption, + AlertRule, + AlertRuleInput, + AlertRuleList, + PerformanceSignal, + PerformanceSignalList, UsageAlertDefaultSummary, UsageAlertEvaluationStatus, - UsageAlertGroup, - UsageAlertGroupList, - UsageAlertGroupMetric, - UsageAlertGroupUpdate, - UsageAlertGroupUpdateResult, - UsageAlertList, UsageAlertMetric, - UsageAlertModelList, - UsageAlertModelOption, UsageAlertObservation, - UsageAlertRule, - UsageAlertRuleInput, UsageAlertRuleKind, - UsageAlertRuleList, UsageAlertScope, UsageAlertStatus, ) @@ -67,19 +62,6 @@ class _RuleCounts(_Count, frozen=True): default_summary: UsageAlertDefaultSummary -class _GroupRow(BaseModel, frozen=True): - scope_type: UsageAlertScope - scope_id: str | None - day: date - incident_count: int - metrics: tuple[UsageAlertGroupMetric, ...] - - -class _GroupPage(BaseModel, frozen=True): - total: int - data: tuple[_GroupRow, ...] - - class _ResourceLabel(BaseModel, frozen=True): scope_type: str scope_id: str | None @@ -98,8 +80,8 @@ class _Ownership(BaseModel, frozen=True): _DEFAULT_RULE_IDS: Final = DEFAULT_USAGE_RULE_IDS -def _rule(record: Mapping[str, object]) -> UsageAlertRule: - return UsageAlertRule.model_validate({**record, "is_default": str(record.get("id")) in _DEFAULT_RULE_IDS}) +def _rule(record: Mapping[str, object]) -> AlertRule: + return AlertRule.model_validate({**record, "is_default": str(record.get("id")) in _DEFAULT_RULE_IDS}) def activity_path( @@ -137,7 +119,7 @@ def activity_path( return route + "?" + urlencode(query) -def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now: datetime) -> Mapping[str, object]: +def _observation_payload(rule: AlertRule, item: UsageAlertObservation, now: datetime) -> Mapping[str, object]: alert_id: Final = str( uuid5( NAMESPACE_URL, @@ -155,7 +137,7 @@ def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now: ), ) ) - incident: Final = UsageAlert( + incident: Final = PerformanceSignal( id=alert_id, rule_id=rule.id, rule_name=rule.name, @@ -193,7 +175,7 @@ class UsageAlertsRepository: cast(AlertDatabase, prisma.writer_db) # cast-ok: PrismaWrapper forwards query_raw to its generated client ) - async def get_rule(self, rule_id: str) -> UsageAlertRule | None: + async def get_rule(self, rule_id: str) -> AlertRule | None: rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( await self.database.query_raw( 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND NOT deleted', rule_id @@ -203,14 +185,14 @@ class UsageAlertsRepository: async def save_rule( self, - data: UsageAlertRuleInput, + data: AlertRuleInput, created_by: str | None, - existing: UsageAlertRule | None = None, + existing: AlertRule | None = None, *, overrides_defaults: bool = False, - ) -> UsageAlertRule | None: + ) -> AlertRule | None: now: Final = datetime.now(timezone.utc) - rule: Final = UsageAlertRule.model_validate( + rule: Final = AlertRule.model_validate( { **data.model_dump(), "id": existing.id if existing else str(uuid4()), @@ -240,7 +222,7 @@ class UsageAlertsRepository: saved: Final = TypeAdapter(tuple[_Record, ...]).validate_python(rows) return _rule(saved[0].record) if saved else None - async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: + async def delete_rule(self, rule_id: str, *, existing: AlertRule | None = None) -> bool: return bool( await self.database.query_raw( """UPDATE "LiteLLM_UsageAlertRule" SET deleted=true,enabled=false @@ -261,7 +243,7 @@ class UsageAlertsRepository: limit: int = 50, offset: int = 0, kind: UsageAlertRuleKind = "all", - ) -> UsageAlertRuleList: + ) -> AlertRuleList: args: Final = (scope_type, scope_id, json.dumps(tuple(sorted(_DEFAULT_RULE_IDS))), kind) where: Final = "NOT deleted AND ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2)" builtin: Final = "id IN (SELECT jsonb_array_elements_text($3::jsonb))" @@ -289,7 +271,7 @@ class UsageAlertsRepository: offset, ) ) - return UsageAlertRuleList( + return AlertRuleList( data=tuple(_rule(row.record) for row in rows), total=count[0].count, limit=limit, @@ -297,11 +279,11 @@ class UsageAlertsRepository: default_summary=count[0].default_summary, ) - async def get_alert(self, alert_id: str) -> UsageAlert | None: + async def get_alert(self, alert_id: str) -> PerformanceSignal | None: rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( await self.database.query_raw('SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE id=$1', alert_id) ) - return UsageAlert.model_validate(rows[0].record) if rows else None + return PerformanceSignal.model_validate(rows[0].record) if rows else None async def list_alerts( self, @@ -313,7 +295,7 @@ class UsageAlertsRepository: end_date: date | None = None, limit: int = 50, offset: int = 0, - ) -> UsageAlertList: + ) -> PerformanceSignalList: args: Final = ( scope_type, scope_id, @@ -336,14 +318,14 @@ class UsageAlertsRepository: offset, ) ) - return UsageAlertList( - data=tuple(UsageAlert.model_validate(row.record) for row in rows), + return PerformanceSignalList( + data=tuple(PerformanceSignal.model_validate(row.record) for row in rows), total=count[0].count, limit=limit, offset=offset, ) - async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None: + async def set_status(self, alert_id: str, status: UsageAlertStatus) -> PerformanceSignal | None: rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( await self.database.query_raw( """UPDATE "LiteLLM_UsageAlertIncident" SET status=$2, record=jsonb_set(record,'{status}',to_jsonb($2::text)),updated_at=CURRENT_TIMESTAMP WHERE id=$1 RETURNING record""", @@ -351,92 +333,7 @@ class UsageAlertsRepository: status, ) ) - return UsageAlert.model_validate(rows[0].record) if rows else None - - async def list_groups( - self, - scope_type: UsageAlertScope | None = None, - scope_id: str | None = None, - status: UsageAlertStatus | None = None, - start_date: date | None = None, - end_date: date | None = None, - limit: int = 50, - offset: int = 0, - ) -> UsageAlertGroupList: - rows: Final = TypeAdapter(tuple[_GroupPage, ...]).validate_python( - await self.database.query_raw( - """WITH groups AS MATERIALIZED ( - SELECT scope_type,scope_id,day,count(*)::int AS incident_count FROM "LiteLLM_UsageAlertIncident" - WHERE ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2) - AND ($3::text IS NULL OR status=$3) AND ($4::text IS NULL OR day >= $4) - AND ($5::text IS NULL OR day <= $5) - GROUP BY scope_type,scope_id,day - ), page AS MATERIALIZED ( - SELECT * FROM groups ORDER BY day DESC,scope_type,scope_id NULLS FIRST LIMIT $6 OFFSET $7 - ), selected AS ( - SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i - ON i.scope_type=p.scope_type AND i.scope_id=p.scope_id AND i.day=p.day - WHERE ($3::text IS NULL OR i.status=$3) - UNION ALL - SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i - ON i.scope_type=p.scope_type AND i.scope_id IS NULL AND i.day=p.day - WHERE p.scope_id IS NULL AND ($3::text IS NULL OR i.status=$3) - ), ranked AS ( - SELECT scope_type,scope_id,day,metric,record, - count(*) OVER (PARTITION BY scope_type,scope_id,day,metric)::int AS incident_count, - row_number() OVER (PARTITION BY scope_type,scope_id,day,metric - ORDER BY record->>'first_seen' DESC,id) AS rank - FROM selected - ), packed AS ( - SELECT p.scope_type,p.scope_id,p.day,p.incident_count, - jsonb_agg(jsonb_build_object('metric',r.metric,'incident_count',r.incident_count, - 'representative',r.record) ORDER BY CASE r.metric - WHEN 'requests' THEN 0 WHEN 'spend' THEN 1 WHEN 'error_rate' THEN 2 - WHEN 'errors' THEN 3 ELSE 4 END) AS metrics - FROM page p JOIN ranked r ON p.scope_type=r.scope_type - AND p.scope_id IS NOT DISTINCT FROM r.scope_id AND p.day=r.day AND r.rank=1 - GROUP BY p.scope_type,p.scope_id,p.day,p.incident_count - ) SELECT (SELECT count(*)::int FROM groups) AS total, - COALESCE((SELECT jsonb_agg(to_jsonb(packed) ORDER BY day DESC,scope_type,scope_id NULLS FIRST) - FROM packed),'[]'::jsonb) AS data""", - scope_type, - scope_id, - status, - start_date.isoformat() if start_date else None, - end_date.isoformat() if end_date else None, - limit, - offset, - ) - ) - page: Final = rows[0] - labels: Final[Mapping[tuple[str, str | None], str | None]] = ( - await self.resource_labels(page.data) if page.data else {} - ) - return UsageAlertGroupList( - data=tuple( - UsageAlertGroup( - id=str( - uuid5( - NAMESPACE_URL, - json.dumps( - ["litellm:performance-group", row.scope_type, row.scope_id, row.day.isoformat()] - ), - ) - ), - scope_type=row.scope_type, - scope_id=row.scope_id, - scope_label=labels.get((row.scope_type, row.scope_id)), - day=row.day, - incident_count=row.incident_count, - metrics=row.metrics, - activity_path=row.metrics[0].representative.activity_path, - ) - for row in page.data - ), - total=page.total, - limit=limit, - offset=offset, - ) + return PerformanceSignal.model_validate(rows[0].record) if rows else None async def resource_labels( self, groups: tuple[ResourceLabelScope, ...] @@ -477,25 +374,7 @@ class UsageAlertsRepository: ) return MappingProxyType({(row.scope_type, row.scope_id): row.label for row in rows}) - async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult: - rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( - await self.database.query_raw( - """WITH changed AS ( - UPDATE "LiteLLM_UsageAlertIncident" SET status=$5, - record=jsonb_set(record,'{status}',to_jsonb($5::text)),updated_at=CURRENT_TIMESTAMP - WHERE scope_type=$1 AND scope_id IS NOT DISTINCT FROM $2::text AND day=$3 AND status=$4 - RETURNING id - ) SELECT count(*)::int AS count FROM changed""", - data.scope_type, - data.scope_id, - data.day.isoformat(), - data.expected_status, - data.status, - ) - ) - return UsageAlertGroupUpdateResult(updated_count=rows[0].count) - - async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList: + async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> AlertModelList: today: Final = datetime.now(timezone.utc).date() rows: Final = TypeAdapter(tuple[_ModelIdentity, ...]).validate_python( await self.database.query_raw( @@ -512,8 +391,8 @@ class UsageAlertsRepository: ) ) selected: Final = rows[:limit] - return UsageAlertModelList( - data=tuple(UsageAlertModelOption(scope_id=row.model, label=row.model) for row in selected), + return AlertModelList( + data=tuple(AlertModelOption(scope_id=row.model, label=row.model) for row in selected), limit=limit, next_cursor=selected[-1].model if len(rows) > limit else None, start_date=today - timedelta(days=29), @@ -550,19 +429,19 @@ class UsageAlertsRepository: ) @staticmethod - def _checkpoint(rule: UsageAlertRule, day: date) -> str: + def _checkpoint(rule: AlertRule, day: date) -> str: return f"{rule.id}:{rule.updated_at.isoformat()}:{day.isoformat()}" - async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None: + async def ensure_default_rules(self, rules: tuple[AlertRuleInput, ...]) -> None: for rule in rules: await self._ensure_default_rule(rule) - async def _ensure_default_rule(self, rule: UsageAlertRuleInput) -> None: + async def _ensure_default_rule(self, rule: AlertRuleInput) -> None: rule_id: Final = str( uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}") ) now: Final = datetime.now(timezone.utc) - record: Final = UsageAlertRule.model_validate( + record: Final = AlertRule.model_validate( {**rule.model_dump(), "id": rule_id, "created_at": now, "updated_at": now, "created_by": None} ) await self.database.query_raw( @@ -574,9 +453,7 @@ class UsageAlertsRepository: record.model_dump_json(), ) - async def record_evaluation_status( - self, rule: UsageAlertRule, day: date, status: UsageAlertEvaluationStatus - ) -> None: + async def record_evaluation_status(self, rule: AlertRule, day: date, status: UsageAlertEvaluationStatus) -> None: await self.database.query_raw( """UPDATE "LiteLLM_UsageAlertRule" SET record=record || jsonb_build_object('last_evaluated_day',$2::text,'evaluation_status',$3::text) WHERE id=$1 AND (record->>'updated_at')::timestamptz=$4::timestamptz AND NOT deleted RETURNING id""", rule.id, @@ -586,8 +463,8 @@ class UsageAlertsRepository: ) async def record_observations( - self, rule: UsageAlertRule, observations: tuple[UsageAlertObservation, ...] - ) -> tuple[UsageAlert, ...]: + self, rule: AlertRule, observations: tuple[UsageAlertObservation, ...] + ) -> tuple[PerformanceSignal, ...]: if not observations: return () if len(observations) > 500 or len({item.entity_id for item in observations}) != len(observations): @@ -633,7 +510,7 @@ class UsageAlertsRepository: rule.updated_at.isoformat(), ) ) - return tuple(UsageAlert.model_validate(row.record) for row in rows) + return tuple(PerformanceSignal.model_validate(row.record) for row in rows) async def prune(self, retention_days: int = 90, limit: int = 500) -> None: await self.database.query_raw( diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py index 4ad8f50cf42..405fbfa2d81 100644 --- a/litellm/types/proxy/issues.py +++ b/litellm/types/proxy/issues.py @@ -10,39 +10,39 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import ( - UsageAlert, + AlertRule, + PerformanceSignal, UsageAlertChannel, UsageAlertDirection, UsageAlertEvaluationCounts, UsageAlertMetric, - UsageAlertRule, UsageAlertStatus, ) -IssueActivityResponse: TypeAlias = BoundedDailyActivityResponse +AlertActivityResponse: TypeAlias = BoundedDailyActivityResponse -IssueKind: TypeAlias = Literal["performance", "model_deprecation"] -IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] -IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] -IssueGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"] -IssueSignalGroupingReason: TypeAlias = Literal[ +AlertKind: TypeAlias = Literal["performance", "model_deprecation"] +AlertFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] +AlertScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] +AlertGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"] +AlertSignalGroupingReason: TypeAlias = Literal[ "same_resource", "same_key_traffic", "dominant_key", "provider_cohort", "volume_explains_spend", "model_deprecation" ] -class IssueModelPair(BaseModel, frozen=True): +class AlertModelPair(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") provider: str = Field(min_length=1, max_length=256) model: str = Field(min_length=1, max_length=1024) -class IssueCohort(BaseModel, frozen=True): +class AlertCohort(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") - scope_type: IssueScope + scope_type: AlertScope scope_id: str | None = Field(default=None, min_length=1, max_length=1024) - model_pairs: tuple[IssueModelPair, ...] = Field(default=(), max_length=100) + model_pairs: tuple[AlertModelPair, ...] = Field(default=(), max_length=100) @model_validator(mode="after") def validate_scope(self) -> Self: @@ -55,7 +55,7 @@ class IssueCohort(BaseModel, frozen=True): return self -class IssueComparison(BaseModel, frozen=True): +class AlertComparison(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) metric: UsageAlertMetric @@ -76,7 +76,7 @@ class IssueComparison(BaseModel, frozen=True): return self -class IssueImpact(BaseModel, frozen=True): +class AlertImpact(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") keys: int | None = Field(default=None, ge=0) @@ -88,7 +88,7 @@ class IssueImpact(BaseModel, frozen=True): measured_models: int | None = Field(default=None, ge=0) -class IssueDeprecation(BaseModel, frozen=True): +class AlertDeprecation(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") model: ModelDeprecationInfo @@ -99,24 +99,24 @@ class IssueDeprecation(BaseModel, frozen=True): state: DeprecationAlertState -class IssueInput(BaseModel, frozen=True): +class AlertInput(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") id: str = Field(min_length=1, max_length=200) cohort_key: str = Field(min_length=1, max_length=200) - kind: IssueKind - family: IssueFamily + kind: AlertKind + family: AlertFamily title: str = Field(min_length=1, max_length=2048) - cohort: IssueCohort + cohort: AlertCohort observed_on: date observed_at: datetime range_start: date range_end: date - grouping_reason: IssueGroupingReason + grouping_reason: AlertGroupingReason explanation: str = Field(default="", max_length=2000) - comparisons: tuple[IssueComparison, ...] = Field(default=(), max_length=10) - impact: IssueImpact = Field(default_factory=IssueImpact) - deprecation: IssueDeprecation | None = None + comparisons: tuple[AlertComparison, ...] = Field(default=(), max_length=10) + impact: AlertImpact = Field(default_factory=AlertImpact) + deprecation: AlertDeprecation | None = None @model_validator(mode="after") def validate_issue(self) -> Self: @@ -131,7 +131,7 @@ class IssueInput(BaseModel, frozen=True): return self -class Issue(IssueInput, frozen=True): +class Alert(AlertInput, frozen=True): status: UsageAlertStatus first_seen: datetime last_seen: datetime @@ -146,58 +146,60 @@ class Issue(IssueInput, frozen=True): ) -class IssueList(BaseModel, frozen=True): - data: tuple[Issue, ...] +class AlertList(BaseModel, frozen=True): + data: tuple[Alert, ...] total: int limit: int offset: int -class IssueUpdate(BaseModel, frozen=True): +class AlertUpdate(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") status: UsageAlertStatus expected_status: UsageAlertStatus -class IssueSignalInput(BaseModel, frozen=True): +class AlertSignalInput(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") - source: Literal["usage_alert", "model_deprecation"] + source: Literal["performance", "model_deprecation"] source_id: str = Field(min_length=1, max_length=200) - usage_alert: UsageAlert | None = None - deprecation: IssueDeprecation | None = None - grouping_reason: IssueSignalGroupingReason = "same_resource" + performance: PerformanceSignal | None = None + deprecation: AlertDeprecation | None = None + grouping_reason: AlertSignalGroupingReason = "same_resource" explanation: str = Field(default="", max_length=2000) @model_validator(mode="after") def validate_signal(self) -> Self: - if self.source == "usage_alert": - if self.usage_alert is None or self.deprecation is not None or self.usage_alert.id != self.source_id: - raise ValueError("Usage signals require their matching source alert") - elif self.deprecation is None or self.usage_alert is not None: + if self.source == "performance": + if self.performance is None or self.deprecation is not None or self.performance.id != self.source_id: + raise ValueError("Performance signals require their matching evidence") + elif self.deprecation is None or self.performance is not None: raise ValueError("Deprecation signals require deprecation details") return self -class IssueSignal(IssueSignalInput, frozen=True): - issue_id: str +class AlertSignal(AlertSignalInput, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) + + issue_id: str = Field(alias="alert_id") -class IssueSignalList(BaseModel, frozen=True): - data: tuple[IssueSignal, ...] +class AlertSignalList(BaseModel, frozen=True): + data: tuple[AlertSignal, ...] total: int limit: int offset: int -class IssueProjectionSignal(BaseModel, frozen=True): - alert: UsageAlert - rule: UsageAlertRule +class AlertProjectionSignal(BaseModel, frozen=True): + alert: PerformanceSignal + rule: AlertRule evaluation_day: date -class IssueDeliveryInput(BaseModel, frozen=True): +class AlertDeliveryInput(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") id: str = Field(min_length=1, max_length=200) @@ -206,27 +208,27 @@ class IssueDeliveryInput(BaseModel, frozen=True): email_recipients: tuple[str, ...] = Field(default=(), max_length=20) -class IssueDeliveryBinding(BaseModel, frozen=True): - delivery: IssueDeliveryInput +class AlertDeliveryBinding(BaseModel, frozen=True): + delivery: AlertDeliveryInput rule_id: str rule_revision: str source_id: str -class IssueDelivery(IssueDeliveryInput, frozen=True): +class AlertDelivery(AlertDeliveryInput, frozen=True): attempts: int = 0 next_attempt: datetime | None = None last_error: Literal["delivery_failed", "transport_not_configured"] | None = None -class IssuePendingDelivery(BaseModel, frozen=True): - delivery: IssueDelivery - issue: Issue - rule: UsageAlertRule - alert: UsageAlert +class AlertPendingDelivery(BaseModel, frozen=True): + delivery: AlertDelivery + issue: Alert + rule: AlertRule + alert: PerformanceSignal -class IssueFamilyPolicy(BaseModel, frozen=True): +class AlertFamilyPolicy(BaseModel, frozen=True): direction: UsageAlertDirection enabled: bool | None multiplier: float | None @@ -239,21 +241,21 @@ class IssueFamilyPolicy(BaseModel, frozen=True): enabled_rules: int -class IssueFamilySetting(BaseModel, frozen=True): +class AlertFamilySetting(BaseModel, frozen=True): family: PerformanceIssueFamily enabled: bool | None total_rules: int enabled_rules: int - policies: tuple[IssueFamilyPolicy, ...] + policies: tuple[AlertFamilyPolicy, ...] evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts) -class IssueFamilyList(BaseModel, frozen=True): - data: tuple[IssueFamilySetting, ...] +class AlertFamilyList(BaseModel, frozen=True): + data: tuple[AlertFamilySetting, ...] projection_pending: bool = False -class IssueFamilyPolicyUpdate(BaseModel, frozen=True): +class AlertFamilyPolicyUpdate(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") direction: UsageAlertDirection @@ -265,11 +267,11 @@ class IssueFamilyPolicyUpdate(BaseModel, frozen=True): min_history_days: int | None = Field(default=None, ge=1, le=30) -class IssueFamilyUpdate(BaseModel, frozen=True): +class AlertFamilyUpdate(BaseModel, frozen=True): model_config = ConfigDict(frozen=True, extra="forbid") enabled: bool | None = None - policies: tuple[IssueFamilyPolicyUpdate, ...] = Field(default=(), max_length=2) + policies: tuple[AlertFamilyPolicyUpdate, ...] = Field(default=(), max_length=2) @model_validator(mode="after") def validate_update(self) -> Self: diff --git a/litellm/types/proxy/usage_alert_defaults.py b/litellm/types/proxy/usage_alert_defaults.py index 887932a5194..b3ed48eee14 100644 --- a/litellm/types/proxy/usage_alert_defaults.py +++ b/litellm/types/proxy/usage_alert_defaults.py @@ -4,11 +4,11 @@ from typing import Final, Literal, TypeAlias from uuid import NAMESPACE_URL, uuid5 from litellm.types.proxy.usage_alerts import ( + AlertRule, + AlertRuleInput, UsageAlertChannel, UsageAlertDirection, UsageAlertMetric, - UsageAlertRule, - UsageAlertRuleInput, UsageAlertScope, ) @@ -46,7 +46,7 @@ DEFAULT_USAGE_RULE_IDS: Final = frozenset( ) -def is_default_usage_rule(rule: UsageAlertRule) -> bool: +def is_default_usage_rule(rule: AlertRule) -> bool: return rule.id in DEFAULT_USAGE_RULE_IDS and rule.id == default_usage_rule_id( rule.scope_type, rule.metric, rule.direction ) @@ -54,9 +54,9 @@ def is_default_usage_rule(rule: UsageAlertRule) -> bool: def default_usage_rules( channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False -) -> tuple[UsageAlertRuleInput, ...]: +) -> tuple[AlertRuleInput, ...]: return tuple( - UsageAlertRuleInput( + AlertRuleInput( name=f"Daily {scope} {policy.metric.replace('_', ' ')} {policy.direction}", scope_type=scope, metric=policy.metric, diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index b91707e61d2..78240bda1b1 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -15,7 +15,7 @@ UsageAlertEvaluationStatus: TypeAlias = Literal["pending", "complete", "insuffic UsageAlertRuleKind: TypeAlias = Literal["all", "default", "custom"] -class UsageAlertRuleInput(BaseModel): +class AlertRuleInput(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) name: str = Field(min_length=1, max_length=200, pattern=r"^[^\x00-\x1f\x7f]+$") @@ -55,7 +55,7 @@ class UsageAlertRuleInput(BaseModel): return self -class UsageAlertRule(UsageAlertRuleInput): +class AlertRule(AlertRuleInput): overrides_defaults: bool = Field(default=False, exclude=True) id: str created_at: datetime @@ -66,7 +66,7 @@ class UsageAlertRule(UsageAlertRuleInput): is_default: bool = False -class UsageAlert(BaseModel): +class PerformanceSignal(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) id: str @@ -89,11 +89,6 @@ class UsageAlert(BaseModel): activity_path: str -class UsageAlertUpdate(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - status: UsageAlertStatus - - class UsageAlertEvaluationCounts(BaseModel, frozen=True): pending: int = 0 complete: int = 0 @@ -107,73 +102,28 @@ class UsageAlertDefaultSummary(BaseModel, frozen=True): evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts) -class UsageAlertRuleList(BaseModel, frozen=True): - data: tuple[UsageAlertRule, ...] +class AlertRuleList(BaseModel, frozen=True): + data: tuple[AlertRule, ...] total: int limit: int offset: int default_summary: UsageAlertDefaultSummary = Field(default_factory=UsageAlertDefaultSummary) -class UsageAlertList(BaseModel, frozen=True): - data: tuple[UsageAlert, ...] +class PerformanceSignalList(BaseModel, frozen=True): + data: tuple[PerformanceSignal, ...] total: int limit: int offset: int -class UsageAlertGroupMetric(BaseModel, frozen=True): - metric: UsageAlertMetric - incident_count: int - representative: UsageAlert - - -class UsageAlertGroup(BaseModel, frozen=True): - id: str - scope_type: UsageAlertScope - scope_id: str | None - scope_label: str | None - day: date - incident_count: int - metrics: tuple[UsageAlertGroupMetric, ...] = Field(max_length=5) - activity_path: str - - -class UsageAlertGroupList(BaseModel, frozen=True): - data: tuple[UsageAlertGroup, ...] - total: int - limit: int - offset: int - - -class UsageAlertGroupUpdate(BaseModel, frozen=True): - model_config = ConfigDict(extra="forbid") - scope_type: UsageAlertScope - scope_id: str | None = Field(default=None, min_length=1, max_length=1024) - day: date - expected_status: UsageAlertStatus - status: UsageAlertStatus - - @model_validator(mode="after") - def validate_group(self) -> Self: - if (self.scope_type == "deployment") != (self.scope_id is None): - raise ValueError("Specify a resource ID except for deployment alerts") - if self.expected_status == self.status: - raise ValueError("The new status must differ from the expected status") - return self - - -class UsageAlertGroupUpdateResult(BaseModel, frozen=True): - updated_count: int - - -class UsageAlertModelOption(BaseModel, frozen=True): +class AlertModelOption(BaseModel, frozen=True): scope_id: str label: str -class UsageAlertModelList(BaseModel, frozen=True): - data: tuple[UsageAlertModelOption, ...] +class AlertModelList(BaseModel, frozen=True): + data: tuple[AlertModelOption, ...] limit: int next_cursor: str | None start_date: date diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index dac4ade7016..ac766c66ff3 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -25,28 +25,30 @@ from litellm.repositories.issue_activity import IssueActivityRepository from litellm.repositories.issues import IssuesRepository from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( - IssueActivityResponse, - IssueCohort, - IssueComparison, - IssueDeliveryBinding, - IssueDeliveryInput, - IssueFamilyPolicyUpdate, - IssueFamilyUpdate, - IssueInput, - IssueList, - IssueModelPair, - IssueSignal, - IssueSignalInput, - IssueSignalList, + AlertActivityResponse, + AlertCohort, + AlertComparison, + AlertDeliveryBinding, + AlertDeliveryInput, + AlertDeprecation, + AlertFamilyPolicyUpdate, + AlertFamilyUpdate, + AlertInput, + AlertList, + AlertModelPair, + AlertSignal, + AlertSignalInput, + AlertSignalList, ) +from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo from litellm.types.proxy.usage_alert_defaults import default_usage_rule_id, default_usage_rules -from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertObservation, UsageAlertRule, UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import AlertRule, AlertRuleInput, PerformanceSignal, UsageAlertObservation -def _issue(cohort: IssueCohort) -> IssueInput: +def _issue(cohort: AlertCohort) -> AlertInput: identity: Final = str(uuid4()) day: Final = date.today() - timedelta(days=3) - return IssueInput( + return AlertInput( id=identity, cohort_key=identity, kind="performance", @@ -60,7 +62,7 @@ def _issue(cohort: IssueCohort) -> IssueInput: grouping_reason="same_resource", explanation="Requests increased for this resource.", comparisons=( - IssueComparison( + AlertComparison( metric="requests", direction="increase", observed_value=420, @@ -74,12 +76,12 @@ def _issue(cohort: IssueCohort) -> IssueInput: ) -def _signals(issue_id: str, inputs: tuple[IssueSignalInput, ...]) -> tuple[IssueSignal, ...]: - return tuple(IssueSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in inputs) +def _signals(issue_id: str, inputs: tuple[AlertSignalInput, ...]) -> tuple[AlertSignal, ...]: + return tuple(AlertSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in inputs) -def _delivery(data: IssueDeliveryInput, rule: UsageAlertRule, source: UsageAlert) -> IssueDeliveryBinding: - return IssueDeliveryBinding( +def _delivery(data: AlertDeliveryInput, rule: AlertRule, source: PerformanceSignal) -> AlertDeliveryBinding: + return AlertDeliveryBinding( delivery=data, rule_id=rule.id, rule_revision=rule.updated_at.isoformat(), source_id=source.id ) @@ -114,11 +116,11 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") inference_key: Final = scenario.key(user_id=owner, key_type="llm_api", key_alias="Example checkout") reporting_id: Final = sha256(inference_key.encode()).hexdigest() - first: Final = _issue(IssueCohort(scope_type="key", scope_id=reporting_id)) + first: Final = _issue(AlertCohort(scope_type="key", scope_id=reporting_id)) second: Final = _issue(first.cohort) - global_issue: Final = _issue(IssueCohort(scope_type="provider", scope_id="example-provider")) + global_issue: Final = _issue(AlertCohort(scope_type="provider", scope_id="example-provider")) rule: Final = await alerts.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Visibility fixture", scope_type="key", scope_id=reporting_id, @@ -158,18 +160,18 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa _signals( first.id, ( - IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source), - IssueSignalInput( - source="usage_alert", + AlertSignalInput(source="performance", source_id=source.id, performance=source), + AlertSignalInput( + source="performance", source_id=broader.id, - usage_alert=broader, + performance=broader, grouping_reason="dominant_key", explanation="Parent contribution evidence.", ), ), ) ) - path: Final = f"/issues/{first.id}" + path: Final = f"/alerts/{first.id}" own: Final = gateway.request("GET", path, key=owner_key) assert own.status_code == 200 and own.json()["scope_label"] == "Example checkout" assert own.json()["can_update_status"] is False, "Broader source scopes require their own write permission" @@ -186,23 +188,23 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa pages: Final = tuple( gateway.request( "GET", - "/issues", + "/alerts", key=owner_key, params={"scope_type": "key", "scope_id": reporting_id, "limit": "1", "offset": str(offset)}, ) for offset in (0, 1, 2) ) assert all(page.status_code == 200 for page in pages) - for offset, parsed in enumerate(IssueList.model_validate(page.json()) for page in pages): + for offset, parsed in enumerate(AlertList.model_validate(page.json()) for page in pages): assert parsed.total == 2 and len(parsed.data) == (1 if offset < 2 else 0) - assert gateway.request("GET", "/issues", key=owner_key).status_code == 403 + assert gateway.request("GET", "/alerts", key=owner_key).status_code == 403 for suffix in ("", "/signals", "/activity", "/keys", "/users", "/impact"): - assert gateway.request("GET", f"/issues/{global_issue.id}{suffix}", key=owner_key).status_code == 403 + assert gateway.request("GET", f"/alerts/{global_issue.id}{suffix}", key=owner_key).status_code == 403 assert gateway.request("GET", path + suffix, key=outsider_key).status_code == 403 assert gateway.request("GET", path + suffix, key=inference_key).status_code == 403 public_signals: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1"}) assert public_signals.status_code == 200 - visible: Final = IssueSignalList.model_validate(public_signals.json()) + visible: Final = AlertSignalList.model_validate(public_signals.json()) assert visible.total == 1 and tuple(row.source_id for row in visible.data) == (source.id,) assert "99999" not in public_signals.text end: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1", "offset": "1"}) @@ -223,11 +225,11 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa ) assert gateway.request("PATCH", path, {"status": "dismissed", "expected_status": "open"}).status_code == 200 assert gateway.request("PATCH", path, {"status": "resolved", "expected_status": "open"}).status_code == 409 - assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["status"] == "open" - assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["can_update_status"] is True + assert gateway.request("GET", f"/alerts/{second.id}", key=owner_key).json()["status"] == "open" + assert gateway.request("GET", f"/alerts/{second.id}", key=owner_key).json()["can_update_status"] is True assert ( gateway.request( - "PATCH", f"/issues/{second.id}", {"status": "dismissed", "expected_status": "open"}, key=owner_key + "PATCH", f"/alerts/{second.id}", {"status": "dismissed", "expected_status": "open"}, key=owner_key ).status_code == 200 ) @@ -258,11 +260,11 @@ async def test_issue_detail_reports_team_status_permission(gateway: Gateway, db: key_type="management", metadata={"service_account_id": "example-activity-service"}, ) - issue: Final = _issue(IssueCohort(scope_type="team", scope_id=team)) - permitted_issue: Final = _issue(IssueCohort(scope_type="team", scope_id=permitted_team)) + issue: Final = _issue(AlertCohort(scope_type="team", scope_id=team)) + permitted_issue: Final = _issue(AlertCohort(scope_type="team", scope_id=permitted_team)) try: await store.upsert_issues((issue, permitted_issue)) - path: Final = f"/issues/{issue.id}" + path: Final = f"/alerts/{issue.id}" assert gateway.request("GET", path, key=admin_key).json()["can_update_status"] is True assert gateway.request("GET", path, key=admin_key).json()["can_view_activity"] is True assert gateway.request("GET", path, key=member_key).json()["can_update_status"] is False @@ -273,10 +275,10 @@ async def test_issue_detail_reports_team_status_permission(gateway: Gateway, db: for suffix in ("/activity", "/keys", "/impact"): assert gateway.request("GET", path + suffix, key=admin_key).status_code == 200 assert ( - gateway.request("GET", f"/issues/{permitted_issue.id}" + suffix, key=member_key).status_code == 200 + gateway.request("GET", f"/alerts/{permitted_issue.id}" + suffix, key=member_key).status_code == 200 ) assert ( - gateway.request("GET", f"/issues/{permitted_issue.id}", key=member_key).json()["can_view_activity"] + gateway.request("GET", f"/alerts/{permitted_issue.id}", key=member_key).json()["can_view_activity"] is True ) assert ( @@ -307,7 +309,7 @@ async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gatew provider: Final = f"example-provider-{identity}" day: Final = date.today() - timedelta(days=5) model_count: Final = 137 - cohort: Final = IssueCohort(scope_type="key", scope_id=key) + cohort: Final = AlertCohort(scope_type="key", scope_id=key) issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day + timedelta(days=2)}) await _query( db, @@ -330,9 +332,9 @@ async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gatew await store.upsert_issues((issue,)) scope: Final = issue_activity_scope(cohort, day, day + timedelta(days=2)) original: Final = await get_daily_activity_aggregated(repository.activity, scope, api_key_limit=20) - response: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + response: Final = gateway.request("GET", f"/alerts/{issue.id}/activity") assert response.status_code == 200 - bounded: Final = IssueActivityResponse.model_validate(response.json()) + bounded: Final = AlertActivityResponse.model_validate(response.json()) assert bounded.metadata.model_dump(exclude={"api_key_limit"}) == original.metadata.model_dump( exclude={"api_key_limit"} ) @@ -353,30 +355,30 @@ async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gatew await repository.database.query_raw(query.sql, *query.params) ) assert len(raw) == 3 + 1 + 100 + 1 - largest: Final = gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "100"}) + largest: Final = gateway.request("GET", f"/alerts/{issue.id}/activity", params={"model_limit": "100"}) assert largest.status_code == 200 - large_models: Final = IssueActivityResponse.model_validate(largest.json()).model_breakdown + large_models: Final = AlertActivityResponse.model_validate(largest.json()).model_breakdown assert large_models is not None and len(large_models.models) == 100 assert large_models.other is not None and large_models.other.model_count == model_count - 100 - assert gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "101"}).status_code == 422 + assert gateway.request("GET", f"/alerts/{issue.id}/activity", params={"model_limit": "101"}).status_code == 422 assert ( gateway.request( "GET", - f"/issues/{issue.id}/activity", + f"/alerts/{issue.id}/activity", params={"start_date": day.isoformat(), "end_date": (day + timedelta(days=366)).isoformat()}, ).status_code == 422 ) non_key: Final = await repository.daily_activity( - IssueCohort(scope_type="provider", scope_id=provider), day, day + timedelta(days=2) + AlertCohort(scope_type="provider", scope_id=provider), day, day + timedelta(days=2) ) assert non_key.model_breakdown is None and non_key.metadata == bounded.metadata assert len(non_key.results) == 3 async with db.prisma.writer_db.tx(timeout=timedelta(seconds=10)) as blocker: await blocker.execute_raw('LOCK TABLE "LiteLLM_DailyUserSpend" IN ACCESS EXCLUSIVE MODE') - busy: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + busy: Final = gateway.request("GET", f"/alerts/{issue.id}/activity") assert busy.status_code == 503 and busy.headers["Retry-After"] == "5" - assert gateway.request("GET", f"/issues/{issue.id}/activity").status_code == 200 + assert gateway.request("GET", f"/alerts/{issue.id}/activity").status_code == 200 finally: await _remove_issues(db, (issue.id,)) await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1 RETURNING id', identity) @@ -424,22 +426,22 @@ async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( RETURNING id""", json.dumps(entries), ) - cohort: Final = IssueCohort( + cohort: Final = AlertCohort( scope_type="provider", scope_id=provider, - model_pairs=(IssueModelPair(provider=provider, model="example-model-a"),), + model_pairs=(AlertModelPair(provider=provider, model="example-model-a"),), ) store: Final = IssuesRepository.from_prisma(db.prisma) issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day}) try: await store.upsert_issues((issue,)) - activity_http: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + activity_http: Final = gateway.request("GET", f"/alerts/{issue.id}/activity") assert activity_http.status_code == 200 and activity_http.json()["metadata"]["total_api_requests"] == 100 - keys_http: Final = gateway.request("GET", f"/issues/{issue.id}/keys", params={"limit": "1"}) + keys_http: Final = gateway.request("GET", f"/alerts/{issue.id}/keys", params={"limit": "1"}) assert keys_http.status_code == 200 and keys_http.json()["api_keys"][0]["metrics"]["api_requests"] == 100 - users_http: Final = gateway.request("GET", f"/issues/{issue.id}/users") + users_http: Final = gateway.request("GET", f"/alerts/{issue.id}/users") assert users_http.status_code == 200 and users_http.json()["total_users"] == 1 - impact_http: Final = gateway.request("GET", f"/issues/{issue.id}/impact") + impact_http: Final = gateway.request("GET", f"/alerts/{issue.id}/impact") assert impact_http.status_code == 200 and impact_http.json()["requests"] == 100 daily: Final = await repository.daily_activity(cohort, day, day) assert daily.metadata.total_api_requests == 100 and daily.metadata.total_spend == 10 @@ -453,15 +455,15 @@ async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( assert users.total_users == 1 and users.users[0].api_requests == 100 and users.users[0].timed_requests == 100 impact: Final = await repository.impact(cohort, day, day) assert (impact.keys, impact.users, impact.models, impact.requests) == (1, 1, 1, 100) - pairs: Final = IssueCohort( + pairs: Final = AlertCohort( scope_type="deployment", model_pairs=( - IssueModelPair(provider=provider, model="example-model-a"), - IssueModelPair(provider=other, model="example-model-b"), + AlertModelPair(provider=provider, model="example-model-a"), + AlertModelPair(provider=other, model="example-model-b"), ), ) assert (await repository.daily_activity(pairs, day, day)).metadata.total_api_requests == 100 - injected: Final = IssueCohort(scope_type="provider", scope_id=provider + "' OR true --") + injected: Final = AlertCohort(scope_type="provider", scope_id=provider + "' OR true --") assert (await repository.impact(injected, day, day)).requests == 0 from litellm.repositories.issue_activity import issue_activity_scope from litellm.types.repositories.daily_activity import ExportType @@ -488,8 +490,8 @@ async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: ManagementDatabase) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - original: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) - definition: Final = UsageAlertRuleInput( + original: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + definition: Final = AlertRuleInput( name="Delivery fixture", scope_type="user", scope_id=original.cohort.scope_id, @@ -503,8 +505,8 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma observation: Final = UsageAlertObservation( entity_id=rule.scope_id, day=original.observed_on, observed=420, baseline=120, threshold=360, breached=True ) - delivery: Final = IssueDeliveryInput(id=str(uuid4()), issue_id=original.id, channel="slack") - invalid: Final = IssueDeliveryInput( + delivery: Final = AlertDeliveryInput(id=str(uuid4()), issue_id=original.id, channel="slack") + invalid: Final = AlertDeliveryInput( id=str(uuid4()), issue_id=original.id, channel="email", email_recipients=("outside@example.com",) ) next_issue_id: Final = str(uuid4()) @@ -520,7 +522,7 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma assert await store.active_signals(original.observed_on, scope_ids=()) == () assert await store.active_signals(original.observed_on, scope_ids=("other-example-user",)) == () await store.add_signal_batch( - _signals(initial.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),)) + _signals(initial.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),)) ) await store.queue_deliveries((_delivery(delivery, rule, source),)) await store.queue_deliveries((_delivery(invalid, rule, source),)) @@ -658,7 +660,7 @@ async def test_late_default_override_stops_queued_delivery_and_recovers_on_fresh await alerts.ensure_default_rules(default_usage_rules(("slack",))) rule: Final = await alerts.get_rule(default_usage_rule_id("key", "requests", "increase")) assert rule is not None - issues: Final = tuple(_issue(IssueCohort(scope_type="key", scope_id=key)) for key in ("key-a", "key-b")) + issues: Final = tuple(_issue(AlertCohort(scope_type="key", scope_id=key)) for key in ("key-a", "key-b")) observations: Final = tuple( UsageAlertObservation( entity_id=issue.cohort.scope_id, @@ -673,15 +675,15 @@ async def test_late_default_override_stops_queued_delivery_and_recovers_on_fresh sources: Final = await alerts.record_observations(rule, observations) await store.upsert_issues(issues) for issue in issues: - source: UsageAlert = next(source for source in sources if source.scope_id == issue.cohort.scope_id) + source: PerformanceSignal = next(source for source in sources if source.scope_id == issue.cohort.scope_id) await store.add_signal_batch( - _signals(issue.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),)) + _signals(issue.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),)) ) await store.queue_deliveries( - (_delivery(IssueDeliveryInput(id=issue.id, issue_id=issue.id, channel="slack"), rule, source),) + (_delivery(AlertDeliveryInput(id=issue.id, issue_id=issue.id, channel="slack"), rule, source),) ) assert await store.pending_delivery(issue.id) is not None - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Project A threshold", scope_type="key", scope_id="key-a", @@ -728,7 +730,7 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren store: Final = IssuesRepository(borrowed) await alerts.ensure_default_rules(default_usage_rules()) custom: Final = await alerts.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Custom spend", scope_type="key", scope_id="example-key", metric="spend", direction="increase" ), "qa-admin", @@ -738,8 +740,8 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren await alerts.delete_rule(removed_id) result: Final = await store.update_family( "spend", - IssueFamilyUpdate( - enabled=False, policies=(IssueFamilyPolicyUpdate(direction="increase", multiplier=5, min_value=25),) + AlertFamilyUpdate( + enabled=False, policies=(AlertFamilyPolicyUpdate(direction="increase", multiplier=5, min_value=25),) ), ) assert result is not None and result.total_rules == 4 and result.enabled_rules == 0 @@ -750,8 +752,8 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren with pytest.raises(ValidationError, match="min_history_days"): await store.update_family( "spend", - IssueFamilyUpdate( - policies=(IssueFamilyPolicyUpdate(direction="increase", baseline_days=1, min_history_days=7),) + AlertFamilyUpdate( + policies=(AlertFamilyPolicyUpdate(direction="increase", baseline_days=1, min_history_days=7),) ), ) assert await store.family_settings() == baseline @@ -766,7 +768,7 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren .validate_python(await transaction.query_raw("SELECT pg_backend_pid() AS pid"))[0] .pid ) - update: Final = tasks.create_task(store.update_family("spend", IssueFamilyUpdate(enabled=True))) + update: Final = tasks.create_task(store.update_family("spend", AlertFamilyUpdate(enabled=True))) async with asyncio.timeout(10): while ( not TypeAdapter(tuple[_Blocked, ...]) @@ -805,7 +807,7 @@ async def test_family_edit_requires_a_new_evaluation(db: ManagementDatabase) -> assert rule is not None await alerts.record_evaluation_status(rule, date.today() - timedelta(days=1), "complete") result: Final = await store.update_family( - "spend", IssueFamilyUpdate(policies=(IssueFamilyPolicyUpdate(direction="increase", multiplier=5),)) + "spend", AlertFamilyUpdate(policies=(AlertFamilyPolicyUpdate(direction="increase", multiplier=5),)) ) saved: Final = await alerts.get_rule(rule.id) assert result is not None and saved is not None @@ -825,7 +827,7 @@ async def test_deleted_key_owner_retains_historical_issue_access(gateway: Gatewa owner_key: Final = scenario.key(user_id=owner, key_type="management") outsider_key: Final = scenario.key(user_id=outsider, key_type="management") reporting_id: Final = sha256(uuid4().bytes).hexdigest() - issue: Final = _issue(IssueCohort(scope_type="key", scope_id=reporting_id)) + issue: Final = _issue(AlertCohort(scope_type="key", scope_id=reporting_id)) await _query( db, 'INSERT INTO "LiteLLM_DeletedVerificationToken" (id,token,user_id,models) VALUES ($1,$1,$2,ARRAY[]::text[])', @@ -834,7 +836,7 @@ async def test_deleted_key_owner_retains_historical_issue_access(gateway: Gatewa ) try: await store.upsert_issues((issue,)) - path: Final = f"/issues/{issue.id}" + path: Final = f"/alerts/{issue.id}" detail: Final = gateway.request("GET", path, key=owner_key) assert detail.status_code == 200, "Deleted key owner must retain historical issue access" assert detail.json()["can_update_status"] is True @@ -863,9 +865,9 @@ async def test_deleted_key_owner_retains_historical_issue_access(gateway: Gatewa async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: ManagementDatabase) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + issue: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) rule: Final = await alerts.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Scope guard", scope_type="user", scope_id=issue.cohort.scope_id, @@ -893,13 +895,13 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana )[0] await store.upsert_issues((issue,)) await store.add_signal_batch( - _signals(issue.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),)) + _signals(issue.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),)) ) authorized: Final = await store.status_scopes(issue.id) assert authorized == (issue.cohort,) assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=authorized) is not None context: Final = source.model_copy(update={"id": str(uuid4()), "metric": "spend"}) - same_scope: Final = (IssueSignalInput(source="usage_alert", source_id=context.id, usage_alert=context),) + same_scope: Final = (AlertSignalInput(source="performance", source_id=context.id, performance=context),) await store.add_signal_batch(_signals(issue.id, same_scope)) unchanged: Final = await store.get_issue(issue.id) assert unchanged is not None and unchanged.status == "dismissed" @@ -907,7 +909,7 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana assert await store.get_issue(issue.id) == unchanged broader: Final = source.model_copy(update={"id": str(uuid4()), "scope_type": "deployment", "scope_id": None}) await store.add_signal_batch( - _signals(issue.id, (IssueSignalInput(source="usage_alert", source_id=broader.id, usage_alert=broader),)) + _signals(issue.id, (AlertSignalInput(source="performance", source_id=broader.id, performance=broader),)) ) assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=authorized) is None current: Final = await store.get_issue(issue.id) @@ -918,7 +920,7 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana await store.add_signal_batch( _signals( issue.id, - (IssueSignalInput(source="usage_alert", source_id=broader_context.id, usage_alert=broader_context),), + (AlertSignalInput(source="performance", source_id=broader_context.id, performance=broader_context),), ) ) repeated_scope: Final = await store.get_issue(issue.id) @@ -954,7 +956,7 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana ) await locked.add_signal_batch( _signals( - issue.id, (IssueSignalInput(source="usage_alert", source_id=racing.id, usage_alert=racing),) + issue.id, (AlertSignalInput(source="performance", source_id=racing.id, performance=racing),) ) ) assert update.result() is None @@ -964,10 +966,10 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana _signals( issue.id, tuple( - IssueSignalInput( - source="usage_alert", + AlertSignalInput( + source="performance", source_id=f"scope-{index}", - usage_alert=source.model_copy( + performance=source.model_copy( update={"id": f"scope-{index}", "scope_id": f"other-user-{index}"} ), ) @@ -989,22 +991,22 @@ async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: Mana async def test_resource_recovery_ignores_broader_and_traffic_context(db: ManagementDatabase) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - fixture: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + fixture: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) issue: Final = fixture.model_copy( update={"family": "spend", "comparisons": (fixture.comparisons[0].model_copy(update={"metric": "spend"}),)} ) definitions: Final = ( - UsageAlertRuleInput( + AlertRuleInput( name="Primary cost", scope_type="user", scope_id=issue.cohort.scope_id, metric="spend", direction="increase" ), - UsageAlertRuleInput( + AlertRuleInput( name="Traffic context", scope_type="user", scope_id=issue.cohort.scope_id, metric="requests", direction="increase", ), - UsageAlertRuleInput(name="Broader cost", scope_type="deployment", metric="spend", direction="increase"), + AlertRuleInput(name="Broader cost", scope_type="deployment", metric="spend", direction="increase"), ) rules: Final = tuple([await alerts.save_rule(definition, "qa-admin") for definition in definitions]) assert all(rule is not None for rule in rules) @@ -1035,7 +1037,7 @@ async def test_resource_recovery_ignores_broader_and_traffic_context(db: Managem _signals( issue.id, tuple( - IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source) + AlertSignalInput(source="performance", source_id=source.id, performance=source) for source in sources ), ) @@ -1072,7 +1074,7 @@ async def test_resource_recovery_ignores_broader_and_traffic_context(db: Managem async def test_provider_recovery_pages_saved_cohorts_and_rejects_stale_observations(db: ManagementDatabase) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) prefix: Final = f"fixture-{uuid4()}" - initial: Final = _issue(IssueCohort(scope_type="provider", scope_id="example-provider")) + initial: Final = _issue(AlertCohort(scope_type="provider", scope_id="example-provider")) template: Final = initial.model_copy(update={"observed_at": initial.observed_at.replace(microsecond=123789)}) inputs: Final = tuple( template.model_copy(update={"id": prefix + suffix, "cohort_key": prefix + suffix}) @@ -1111,11 +1113,11 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - expired: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + expired: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) open_issue: Final = _issue(expired.cohort) recent: Final = _issue(expired.cohort) rule: Final = await alerts.save_rule( - UsageAlertRuleInput( + AlertRuleInput( name="Prune fixture", scope_type="user", scope_id=expired.cohort.scope_id, @@ -1126,7 +1128,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) assert rule is not None deliveries: Final = tuple( - IssueDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") for _ in range(2) + AlertDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") for _ in range(2) ) try: await store.upsert_issues((expired, open_issue, recent)) @@ -1146,7 +1148,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) )[0] await store.add_signal_batch( - _signals(expired.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),)) + _signals(expired.id, (AlertSignalInput(source="performance", source_id=source.id, performance=source),)) ) for delivery in deliveries: await store.queue_deliveries((_delivery(delivery, rule, source),)) @@ -1191,6 +1193,61 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) +@pytest.mark.asyncio(loop_scope="session") +async def test_prune_preserves_retirement_reopened_by_new_usage(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + original: Final = _issue(AlertCohort(scope_type="model", scope_id=f"retirement-{uuid4()}")) + name: Final = original.cohort.scope_id or "example-model" + retirement: Final = original.observed_on + timedelta(days=10) + details: Final = AlertDeprecation( + model=ModelDeprecationInfo( + model_name=name, deprecation_date=retirement, days_until_deprecation=10, status="imminent" + ), + usage_model=name, + requests=10, + last_used_date=original.observed_on, + last_day_requests=10, + state=DeprecationAlertState(model=name, deprecation_date=retirement), + ) + issue: Final = original.model_copy( + update={ + "kind": "model_deprecation", + "family": "retirement", + "deprecation": details, + "grouping_reason": "model_deprecation", + "comparisons": (), + } + ) + try: + await store.upsert_issues((issue,)) + assert await store.set_status(issue.id, "dismissed", "open") is not None + await store.upsert_issues( + ( + issue.model_copy( + update={ + "deprecation": details.model_copy(update={"last_day_requests": 11, "requests": 11}), + "observed_at": original.observed_at + timedelta(minutes=1), + } + ), + ) + ) + await _query( + db, "UPDATE \"LiteLLM_Issue\" SET updated_at=TIMESTAMP '2000-01-01' WHERE id=$1 RETURNING id", issue.id + ) + reopened: Final = await store.get_issue(issue.id) + assert reopened is not None and reopened.status == "open" + await store.prune(retention_days=90, limit=10) + assert await store.get_issue(issue.id) == reopened + finally: + await _query( + db, + 'DELETE FROM "LiteLLM_Config" WHERE param_name IN ' + '(SELECT ack_key FROM "LiteLLM_Issue" WHERE id=$1) RETURNING param_name', + issue.id, + ) + await _remove_issues(db, (issue.id,)) + + class _MeasuredActivityDatabase: def __init__(self, database: DailyActivityQueryDatabase) -> None: self.database = database @@ -1215,7 +1272,7 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( day: Final = date.today() - timedelta(days=3) with gateway.scenario() as scenario: owner: Final = scenario.user(user_role="internal_user") - issue: Final = _issue(IssueCohort(scope_type="provider", scope_id=identity)).model_copy( + issue: Final = _issue(AlertCohort(scope_type="provider", scope_id=identity)).model_copy( update={"range_start": day, "range_end": day} ) try: @@ -1266,7 +1323,7 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( totals: Final = await repository.daily_activity(issue.cohort, day, day) assert totals.metadata.total_api_requests == 30 for endpoint in ("keys", "users"): - response: Final = gateway.request("GET", f"/issues/{issue.id}/{endpoint}", params={"limit": "1"}) + response: Final = gateway.request("GET", f"/alerts/{issue.id}/{endpoint}", params={"limit": "1"}) assert response.status_code == 200 and len(response.content) < 2048 assert "padding" not in response.text await _query( @@ -1289,8 +1346,8 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: ManagementDatabase) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) - definition: Final = UsageAlertRuleInput( + issue: Final = _issue(AlertCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + definition: Final = AlertRuleInput( name="Primary traffic", scope_type="user", scope_id=issue.cohort.scope_id, @@ -1315,7 +1372,7 @@ async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: M _signals( issue.id, tuple( - IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source) + AlertSignalInput(source="performance", source_id=source.id, performance=source) for source in sources ), ) diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index 5ebe5808014..b477f07d447 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -9,12 +9,11 @@ from integration._support.client import Gateway, string_value from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.proxy.usage_alerts import ( - UsageAlertGroupList, + AlertModelList, + AlertRule, + AlertRuleInput, UsageAlertMetric, - UsageAlertModelList, UsageAlertObservation, - UsageAlertRule, - UsageAlertRuleInput, ) @@ -22,7 +21,7 @@ from litellm.types.proxy.usage_alerts import ( async def test_alert_resource_labels_bound_legacy_display_values(db: ManagementDatabase) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) identity: Final = "label-bound-" + uuid4().hex - scope: Final = UsageAlertRuleInput( + scope: Final = AlertRuleInput( name="Display label", scope_type="user", scope_id=identity, metric="spend", direction="increase" ) await repository.database.query_raw( @@ -43,7 +42,7 @@ async def test_alert_resource_labels_bound_legacy_display_values(db: ManagementD async def test_deleted_key_labels_use_the_latest_alias_and_prefer_live_keys(db: ManagementDatabase) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) identity: Final = sha256(uuid4().bytes).hexdigest() - scope: Final = UsageAlertRuleInput( + scope: Final = AlertRuleInput( name="Deleted key", scope_type="key", scope_id=identity, metric="spend", direction="increase" ) try: @@ -63,7 +62,7 @@ async def test_deleted_key_labels_use_the_latest_alias_and_prefer_live_keys(db: await repository.database.query_raw('DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', identity) -@pytest.mark.parametrize("grant", ["management_routes", "/usage", "/usage/alert-rules"]) +@pytest.mark.parametrize("grant", ["management_routes", "/alerts", "/alerts/rules"]) def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gateway: Gateway, grant: str) -> None: with gateway.scenario() as scenario: owner: Final = scenario.user(user_role="internal_user") @@ -81,10 +80,10 @@ def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gate "direction": "increase", "multiplier": 3, } - created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + created: Final = gateway.request("POST", "/alerts/rules", config, key=owner_key) assert created.status_code == 201, created.text rule_id: Final = string_value(created.json()["id"]) - path: Final = f"/usage/alert-rules/{rule_id}" + path: Final = f"/alerts/rules/{rule_id}" try: assert gateway.request("GET", path, key=owner_key).status_code == 200 assert gateway.request("GET", path, key=outsider_key).status_code == 403 @@ -93,14 +92,13 @@ def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gate assert gateway.request("PUT", path, config, key=viewer_key).status_code == 403 assert gateway.request("PUT", path, config, key=outsider_key).status_code == 403 assert ( - gateway.request("POST", "/usage/alert-rules", {**config, "scope_id": None}, key=owner_key).status_code - == 403 + gateway.request("POST", "/alerts/rules", {**config, "scope_id": None}, key=owner_key).status_code == 403 ) disabled: Final = gateway.request("PUT", path, {**config, "enabled": False}, key=owner_key) assert disabled.status_code == 200 and disabled.json()["enabled"] is False page: Final = gateway.request( "GET", - "/usage/alert-rules", + "/alerts/rules", key=owner_key, params={"scope_type": "key", "scope_id": str(config["scope_id"]), "limit": "1"}, ) @@ -122,9 +120,9 @@ def test_admin_edit_preserves_owner_rule_access(gateway: Gateway) -> None: "metric": "spend", "direction": "increase", } - created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + created: Final = gateway.request("POST", "/alerts/rules", config, key=owner_key) assert created.status_code == 201, created.text - path: Final = f"/usage/alert-rules/{created.json()['id']}" + path: Final = f"/alerts/rules/{created.json()['id']}" try: edited: Final = gateway.request("PUT", path, {**config, "name": "Renamed by admin"}) assert edited.status_code == 200, edited.text @@ -151,10 +149,10 @@ def test_team_admin_can_manage_team_alerts_but_member_can_only_read(gateway: Gat "direction": "decrease", "multiplier": 2, } - created: Final = gateway.request("POST", "/usage/alert-rules", config, key=admin_key) + created: Final = gateway.request("POST", "/alerts/rules", config, key=admin_key) assert created.status_code == 201, created.text rule_id: Final = string_value(created.json()["id"]) - path: Final = f"/usage/alert-rules/{rule_id}" + path: Final = f"/alerts/rules/{rule_id}" try: assert gateway.request("GET", path, key=member_key).status_code == 200 assert gateway.request("PUT", path, config, key=member_key).status_code == 403 @@ -177,21 +175,21 @@ def test_info_only_credentials_cannot_mutate_alerts(gateway: Gateway) -> None: "metric": "spend", "direction": "increase", } - created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + created: Final = gateway.request("POST", "/alerts/rules", config, key=owner_key) assert created.status_code == 201 - path: Final = f"/usage/alert-rules/{created.json()['id']}" + path: Final = f"/alerts/rules/{created.json()['id']}" try: assert gateway.request("GET", path, key=reader).status_code == 200 - assert gateway.request("POST", "/usage/alert-rules", config, key=reader).status_code == 403 + assert gateway.request("POST", "/alerts/rules", config, key=reader).status_code == 403 assert gateway.request("PUT", path, config, key=reader).status_code == 403 assert gateway.request("DELETE", path, key=reader).status_code == 403 assert ( - gateway.request("PATCH", "/issues/families/spend", {"enabled": False}, key=admin_reader).status_code + gateway.request("PATCH", "/alerts/defaults/spend", {"enabled": False}, key=admin_reader).status_code == 403 ) assert ( gateway.request( - "PATCH", "/issues/unavailable", {"status": "resolved", "expected_status": "open"}, key=admin_reader + "PATCH", "/alerts/unavailable", {"status": "resolved", "expected_status": "open"}, key=admin_reader ).status_code == 403 ) @@ -215,23 +213,19 @@ def test_only_admins_can_configure_operator_notifications(gateway: Gateway) -> N {"notification_channels": ["email"], "email_recipients": ["ops@example.com"]}, ): assert ( - gateway.request("POST", "/usage/alert-rules", {**config, **notification}, key=owner_key).status_code - == 403 + gateway.request("POST", "/alerts/rules", {**config, **notification}, key=owner_key).status_code == 403 ) assert ( - gateway.request( - "POST", "/usage/alert-rules", {**config, "overrides_defaults": True}, key=owner_key - ).status_code + gateway.request("POST", "/alerts/rules", {**config, "overrides_defaults": True}, key=owner_key).status_code == 422 ) assert ( - gateway.request("POST", "/usage/alert-rules", {**config, "name": "Spend\r\nInjected: header"}).status_code - == 422 + gateway.request("POST", "/alerts/rules", {**config, "name": "Spend\r\nInjected: header"}).status_code == 422 ) - created: Final = gateway.request("POST", "/usage/alert-rules", {**config, "notification_channels": ["slack"]}) + created: Final = gateway.request("POST", "/alerts/rules", {**config, "notification_channels": ["slack"]}) assert created.status_code == 201 assert "overrides_defaults" not in created.json() - path: Final = f"/usage/alert-rules/{created.json()['id']}" + path: Final = f"/alerts/rules/{created.json()['id']}" try: assert gateway.request("PUT", path, config, key=owner_key).status_code == 403 assert gateway.request("DELETE", path, key=owner_key).status_code == 403 @@ -252,7 +246,7 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") created: Final = gateway.request( "POST", - "/usage/alert-rules", + "/alerts/rules", { "name": "Incident API fixture", "scope_type": "user", @@ -264,7 +258,7 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew key=owner_key, ) assert created.status_code == 201 - rule: Final = UsageAlertRule.model_validate(created.json()) + rule: Final = AlertRule.model_validate(created.json()) try: incident: Final = ( await repository.record_observations( @@ -282,9 +276,9 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew ), ) )[0] - path: Final = f"/usage/alerts/{incident.id}" + path: Final = f"/alerts/signals/{incident.id}" params: Final = {"scope_type": "user", "scope_id": owner, "limit": "1"} - page: Final = gateway.request("GET", "/usage/alerts", key=owner_key, params=params) + page: Final = gateway.request("GET", "/alerts/signals", key=owner_key, params=params) assert page.status_code == 200 and page.json()["data"][0]["id"] == incident.id detail: Final = gateway.request("GET", path, key=owner_key) assert detail.status_code == 200 @@ -293,22 +287,19 @@ async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gatew assert detail.json()["threshold"] == 360 assert detail.json()["baseline_sample_days"] == 7 assert "detail_tab=activity" in detail.json()["activity_path"] - assert gateway.request("GET", "/usage/alerts", key=outsider_key, params=params).status_code == 403 + assert gateway.request("GET", "/alerts/signals", key=outsider_key, params=params).status_code == 403 assert gateway.request("GET", path, key=outsider_key).status_code == 403 - assert gateway.request("PATCH", path, {"status": "dismissed"}, key=outsider_key).status_code == 403 assert gateway.request("GET", path, key=viewer_key).status_code == 200 - assert gateway.request("PATCH", path, {"status": "dismissed"}, key=viewer_key).status_code == 403 - dismissed: Final = gateway.request("PATCH", path, {"status": "dismissed"}, key=owner_key) - assert dismissed.status_code == 200 and dismissed.json()["status"] == "dismissed" - assert gateway.request("GET", path, key=owner_key).json()["status"] == "dismissed" + assert gateway.request("PATCH", path, {"status": "dismissed"}, key=owner_key).status_code == 405 + assert gateway.request("GET", path, key=owner_key).json()["status"] == "open" finally: - assert gateway.request("DELETE", f"/usage/alert-rules/{rule.id}", key=owner_key).status_code == 204 + assert gateway.request("DELETE", f"/alerts/rules/{rule.id}", key=owner_key).status_code == 204 @pytest.mark.asyncio(loop_scope="session") async def test_ongoing_episode_preserves_its_original_evidence_and_activity_link(db: ManagementDatabase) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Original episode", scope_type="user", scope_id="episode-example", metric="requests", direction="increase" ) rule: Final = await repository.save_rule(definition, "qa-admin") @@ -358,7 +349,7 @@ async def test_ongoing_episode_preserves_its_original_evidence_and_activity_link @pytest.mark.asyncio(loop_scope="session") async def test_rule_scope_changes_reject_stale_updates_and_deletes(db: ManagementDatabase) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Concurrent scope edit", scope_type="user", scope_id="original-owner", metric="spend", direction="increase" ) original: Final = await repository.save_rule(definition, "qa-admin") @@ -394,7 +385,7 @@ async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency( db: ManagementDatabase, changed_field: str, new_value: str ) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) - definition: Final = UsageAlertRuleInput( + definition: Final = AlertRuleInput( name="Semantic identity", scope_type="team", scope_id="semantic-example", @@ -410,7 +401,7 @@ async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency( try: original: Final = (await repository.record_observations(rule, (initial,)))[0] edited: Final = await repository.save_rule( - UsageAlertRuleInput.model_validate({**definition.model_dump(), changed_field: new_value}), "qa-admin", rule + AlertRuleInput.model_validate({**definition.model_dump(), changed_field: new_value}), "qa-admin", rule ) assert edited is not None observation: Final = initial.model_copy(update={"entity_id": edited.scope_id}) @@ -437,190 +428,12 @@ async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency( await repository.delete_rule(rule.id) -@pytest.mark.asyncio(loop_scope="session") -async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_actions( - gateway: Gateway, db: ManagementDatabase -) -> None: - repository: Final = UsageAlertsRepository.from_prisma(db.prisma) - day: Final = date(2026, 11, 4) - with gateway.scenario() as scenario: - owner: Final = scenario.user(user_role="internal_user", user_alias="Checkout operator") - outsider: Final = scenario.user(user_role="internal_user") - viewer: Final = scenario.user(user_role="proxy_admin_viewer") - owner_key: Final = scenario.key(user_id=owner, key_type="management") - outsider_key: Final = scenario.key(user_id=outsider, key_type="management") - viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") - inference_key: Final = scenario.key(user_id=owner, key_type="llm_api") - metrics: Final[tuple[UsageAlertMetric, ...]] = ( - *(("requests",) * 27), - "spend", - "errors", - "latency", - "error_rate", - ) - rules: Final = tuple( - [ - await repository.save_rule( - UsageAlertRuleInput( - name=f"Group rule {index}", - scope_type="user", - scope_id=owner, - metric=metric, - direction="increase", - baseline_days=7 if index % 2 else 14, - ), - owner, - ) - for index, metric in enumerate(metrics) - ] - ) - assert all(rule is not None for rule in rules) - incidents: Final = tuple( - [ - ( - await repository.record_observations( - rule, - ( - UsageAlertObservation( - entity_id=owner, - day=day, - observed=14 if rule.metric == "error_rate" else 420, - baseline=4 if rule.metric == "error_rate" else 0 if index == 0 else 120, - threshold=12 if rule.metric == "error_rate" else 100 if index == 0 else 360, - breached=True, - ), - ), - ) - )[0] - for index, rule in enumerate(rules) - if rule is not None - ] - ) - old_rule: Final = await repository.save_rule( - UsageAlertRuleInput( - name="Earlier day", scope_type="user", scope_id=owner, metric="requests", direction="increase" - ), - owner, - ) - assert old_rule is not None - old: Final = ( - await repository.record_observations( - old_rule, - ( - UsageAlertObservation( - entity_id=owner, - day=day - timedelta(days=1), - observed=420, - baseline=120, - threshold=360, - breached=True, - ), - ), - ) - )[0] - other_rule: Final = await repository.save_rule( - UsageAlertRuleInput( - name="Other resource", scope_type="user", scope_id=outsider, metric="requests", direction="increase" - ), - outsider, - ) - assert other_rule is not None - other: Final = ( - await repository.record_observations( - other_rule, - ( - UsageAlertObservation( - entity_id=outsider, - day=day, - observed=420, - baseline=120, - threshold=360, - breached=True, - ), - ), - ) - )[0] - params: Final = {"scope_type": "user", "scope_id": owner, "status": "open", "limit": "1"} - try: - response: Final = gateway.request("GET", "/usage/alert-groups", key=owner_key, params=params) - assert response.status_code == 200, response.text - page: Final = UsageAlertGroupList.model_validate(response.json()) - assert page.total == 2 and len(page.data) == 1 - group: Final = page.data[0] - assert group.day == day and group.scope_label == "Checkout operator" - assert group.incident_count == 31 and len(group.metrics) == 5 - assert {metric.metric: metric.incident_count for metric in group.metrics} == { - "requests": 27, - "spend": 1, - "errors": 1, - "latency": 1, - "error_rate": 1, - } - assert all(metric.representative.id in {incident.id for incident in incidents} for metric in group.metrics) - assert group.activity_path in {incident.activity_path for incident in incidents} - second: Final = UsageAlertGroupList.model_validate( - gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "1"}).json() - ) - assert second.total == 2 and second.data[0].day == old.day - empty: Final = UsageAlertGroupList.model_validate( - gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "2"}).json() - ) - assert empty.total == 2 and empty.data == () - history: Final = gateway.request( - "GET", - "/usage/alerts", - key=owner_key, - params={**params, "start_date": day.isoformat(), "end_date": day.isoformat(), "metric": "requests"}, - ) - assert history.status_code == 200 and history.json()["total"] == 27 - await repository.set_status(incidents[0].id, "dismissed") - body: Final = { - "scope_type": "user", - "scope_id": owner, - "day": day.isoformat(), - "expected_status": "open", - "status": "resolved", - } - for forbidden in (outsider_key, inference_key): - assert gateway.request("GET", "/usage/alert-groups", key=forbidden, params=params).status_code == 403 - for forbidden in (outsider_key, inference_key, viewer_key): - assert gateway.request("PATCH", "/usage/alert-groups", body, key=forbidden).status_code == 403 - assert gateway.request("GET", "/usage/alert-groups", key=viewer_key, params=params).status_code == 200 - updated: Final = gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key) - assert updated.status_code == 200 and updated.json()["updated_count"] == 30 - assert gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key).json()["updated_count"] == 0 - assert await repository.get_alert(old.id) == old - assert await repository.get_alert(other.id) == other - dismissed: Final = await repository.get_alert(incidents[0].id) - assert dismissed is not None and dismissed.status == "dismissed" and dismissed.baseline_value == 0 - resolved: Final = await repository.get_alert(incidents[-1].id) - assert resolved == incidents[-1].model_copy(update={"status": "resolved"}) - assert ( - gateway.request( - "GET", "/usage/alert-groups", params={"scope_type": "user", "scope_id": "x' OR 1=1 --"} - ).json()["total"] - == 0 - ) - assert ( - gateway.request( - "GET", "/usage/alert-groups", params={"start_date": "2026-11-05", "end_date": "2026-11-04"} - ).status_code - == 422 - ) - finally: - for rule in (*rules, old_rule, other_rule): - if rule is not None: - await repository.delete_rule(rule.id) - deleted: Final = await repository.list_groups(scope_type="user", scope_id=owner) - assert deleted.total == 2 and all(group.scope_label is None and group.scope_id == owner for group in deleted.data) - - @pytest.mark.asyncio(loop_scope="session") async def test_default_rule_summary_and_kind_filters_do_not_require_loading_every_rule( gateway: Gateway, db: ManagementDatabase ) -> None: repository: Final = UsageAlertsRepository.from_prisma(db.prisma) - default: Final = UsageAlertRuleInput( + default: Final = AlertRuleInput( name="Built-in daily model requests", scope_type="model", metric="requests", direction="increase" ) await repository.ensure_default_rules((default,)) @@ -645,12 +458,12 @@ async def test_default_rule_summary_and_kind_filters_do_not_require_loading_ever stored: Final = await repository.get_rule(builtins.data[0].id) assert stored is not None and stored.is_default filtered: Final = gateway.request( - "GET", "/usage/alert-rules", params={"scope_type": "model", "kind": "custom", "limit": "1"} + "GET", "/alerts/rules", params={"scope_type": "model", "kind": "custom", "limit": "1"} ) assert filtered.status_code == 200 and filtered.json()["total"] == customs.total assert all(not rule["is_default"] for rule in filtered.json()["data"]) - configuration: Final = stored.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields)) - path: Final = f"/usage/alert-rules/{stored.id}" + configuration: Final = stored.model_dump(mode="json", include=frozenset(AlertRuleInput.model_fields)) + path: Final = f"/alerts/rules/{stored.id}" for change in ( {"scope_type": "key"}, {"scope_id": "model-example"}, @@ -665,9 +478,9 @@ async def test_default_rule_summary_and_kind_filters_do_not_require_loading_ever assert updated.json()["min_value"] == stored.min_value + 1 finally: assert gateway.request("PUT", path, configuration).status_code == 200 - custom_input: Final = custom.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields)) + custom_input: Final = custom.model_dump(mode="json", include=frozenset(AlertRuleInput.model_fields)) assert ( - gateway.request("PUT", f"/usage/alert-rules/{custom.id}", {**custom_input, "metric": "latency"}).status_code + gateway.request("PUT", f"/alerts/rules/{custom.id}", {**custom_input, "metric": "latency"}).status_code == 200 ) finally: @@ -704,9 +517,9 @@ async def test_reported_model_search_preserves_exact_identity_and_cursor( prefix, f"{prefix}/old", ) - response: Final = gateway.request("GET", "/usage/alert-models", params={"search": prefix, "limit": "1"}) + response: Final = gateway.request("GET", "/alerts/models", params={"search": prefix, "limit": "1"}) assert response.status_code == 200, response.text - first: Final = UsageAlertModelList.model_validate(response.json()) + first: Final = AlertModelList.model_validate(response.json()) assert first.data[0].scope_id == models[0] and first.next_cursor == models[0] second: Final = await repository.list_reported_models(prefix, first.next_cursor, 1) third: Final = await repository.list_reported_models(prefix, second.next_cursor, 1) @@ -720,6 +533,6 @@ async def test_reported_model_search_preserves_exact_identity_and_cursor( with gateway.scenario() as scenario: user: Final = scenario.user(user_role="internal_user") key: Final = scenario.key(user_id=user, key_type="management") - assert gateway.request("GET", "/usage/alert-models", key=key).status_code == 403 + assert gateway.request("GET", "/alerts/models", key=key).status_code == 403 finally: await db.prisma.writer_db.execute_raw('DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', prefix) diff --git a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py index 239151f9ce1..5107e55743a 100644 --- a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py +++ b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py @@ -21,20 +21,20 @@ class NoDatabase: def test_info_grants_do_not_authorize_mutations(role: LitellmUserRoles, key_type: str | None) -> None: user: Final = UserAPIKeyAuth(user_role=role, key_type=key_type, allowed_routes=["info_routes", "openai_routes"]) with pytest.raises(HTTPException) as error: - require_write_access(user, "/usage/alert-rules/example") + require_write_access(user, "/alerts/rules/example") assert error.value.status_code == 403 -@pytest.mark.parametrize("grant", ["management_routes", "/usage", "/usage/alert-rules", "/usage/*"]) +@pytest.mark.parametrize("grant", ["management_routes", "/alerts", "/alerts/rules", "/alerts/*"]) def test_explicit_route_grants_authorize_alert_mutations(grant: str) -> None: - assert _allows_write(grant, "/usage/alert-rules/example") is True + assert _allows_write(grant, "/alerts/rules/example") is True -@pytest.mark.parametrize("grant", ["/usag", "/usage/alert-rule", "/usage/alerts"]) +@pytest.mark.parametrize("grant", ["/alert", "/alerts/rule", "/alerts/signals"]) def test_route_grants_require_a_complete_path_segment(grant: str) -> None: user: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, allowed_routes=[grant]) with pytest.raises(HTTPException) as error: - require_write_access(user, "/usage/alert-rules/example") + require_write_access(user, "/alerts/rules/example") assert error.value.status_code == 403 @@ -91,10 +91,10 @@ def test_global_activity_link_uses_the_usage_page_date_contract() -> None: @pytest.mark.asyncio async def test_no_database_returns_an_actionable_service_error() -> None: - from litellm.proxy.alerting_endpoints.usage_alerts import list_usage_alerts + from litellm.proxy.alerting_endpoints.usage_alerts import list_performance_signals user: Final = UserAPIKeyAuth(user_id="qa-admin", user_role=LitellmUserRoles.PROXY_ADMIN) with pytest.raises(HTTPException) as error: - await list_usage_alerts(user, None) + await list_performance_signals(user, None) assert error.value.status_code == 503 - assert error.value.detail == "Usage alerts require a database connection" + assert error.value.detail == "Alert signals require a database connection" diff --git a/tests/unit/repositories/test_issue_families.py b/tests/unit/repositories/test_issue_families.py index a22159e9b5a..200fe0c3fee 100644 --- a/tests/unit/repositories/test_issue_families.py +++ b/tests/unit/repositories/test_issue_families.py @@ -8,14 +8,14 @@ from litellm.repositories.issue_families import family_settings from litellm.types.proxy.usage_alerts import ( UsageAlertEvaluationStatus, UsageAlertMetric, - UsageAlertRule, + AlertRule, UsageAlertScope, ) @dataclass class RuleRows: - rules: tuple[UsageAlertRule, ...] + rules: tuple[AlertRule, ...] calls: int = 0 async def query_raw(self, query: str, *args: object) -> object: @@ -36,7 +36,7 @@ async def test_family_counts_only_its_enabled_rules_without_an_extra_query(statu ("model", "latency", True), ) rules: Final = tuple( - UsageAlertRule( + AlertRule( id=f"test-{scope}-{metric}", name="Daily monitoring", scope_type=scope, diff --git a/tests/unit/types/proxy/test_usage_alerts.py b/tests/unit/types/proxy/test_usage_alerts.py index 3ee4618d619..6ca22300fbb 100644 --- a/tests/unit/types/proxy/test_usage_alerts.py +++ b/tests/unit/types/proxy/test_usage_alerts.py @@ -3,7 +3,7 @@ from typing import Final import pytest from pydantic import ValidationError -from litellm.types.proxy.usage_alerts import UsageAlertGroupUpdate, UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import AlertRuleInput RULE: Final = { "name": "Daily spend", @@ -34,31 +34,13 @@ RULE: Final = { ) def test_invalid_rules_cannot_be_stored(changes: dict[str, object]) -> None: with pytest.raises(ValidationError): - UsageAlertRuleInput.model_validate({**RULE, **changes}) + AlertRuleInput.model_validate({**RULE, **changes}) def test_zero_request_floor_and_daily_drop_multiplier_are_supported() -> None: - rule: Final = UsageAlertRuleInput.model_validate( + rule: Final = AlertRuleInput.model_validate( {**RULE, "metric": "requests", "direction": "decrease", "multiplier": 2, "min_requests": 0} ) assert rule.direction == "decrease" assert rule.multiplier == 2 assert rule.min_requests == 0 - - -@pytest.mark.parametrize( - "changes", - [{"scope_id": None}, {"scope_type": "deployment"}, {"status": "open"}, {"status": "unknown"}], -) -def test_group_updates_require_exact_resource_and_a_different_valid_status(changes: dict[str, object]) -> None: - with pytest.raises(ValidationError): - UsageAlertGroupUpdate.model_validate( - { - "scope_type": "team", - "scope_id": "team-example", - "day": "2026-11-04", - "expected_status": "open", - "status": "resolved", - **changes, - } - ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 60d3f878e55..19d51588b1e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1002,6 +1002,251 @@ export interface paths { patch?: never; trace?: never; }; + "/alerts": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Alerts */ + get: operations["list_alerts_alerts_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/defaults": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Alert Defaults */ + get: operations["list_alert_defaults_alerts_defaults_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/defaults/{family}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Alert Default */ + patch: operations["update_alert_default_alerts_defaults__family__patch"]; + trace?: never; + }; + "/alerts/models": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Alert Models */ + get: operations["list_alert_models_alerts_models_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/rules": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Alert Rules */ + get: operations["list_alert_rules_alerts_rules_get"]; + put?: never; + /** + * Create Alert Rule + * @description Create a daily performance rule for a scope the caller can manage. + */ + post: operations["create_alert_rule_alerts_rules_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/rules/{rule_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Alert Rule */ + get: operations["get_alert_rule_alerts_rules__rule_id__get"]; + /** Update Alert Rule */ + put: operations["update_alert_rule_alerts_rules__rule_id__put"]; + post?: never; + /** Delete Alert Rule */ + delete: operations["delete_alert_rule_alerts_rules__rule_id__delete"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/signals": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Performance Signals */ + get: operations["list_performance_signals_alerts_signals_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/signals/{signal_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Performance Signal */ + get: operations["get_performance_signal_alerts_signals__signal_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/{alert_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Alert */ + get: operations["get_alert_alerts__alert_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Alert */ + patch: operations["update_alert_alerts__alert_id__patch"]; + trace?: never; + }; + "/alerts/{alert_id}/activity": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Alert Activity */ + get: operations["get_alert_activity_alerts__alert_id__activity_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/{alert_id}/impact": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Alert Impact */ + get: operations["get_alert_impact_alerts__alert_id__impact_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/{alert_id}/keys": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Alert Keys */ + get: operations["get_alert_keys_alerts__alert_id__keys_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/{alert_id}/signals": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Alert Signals */ + get: operations["list_alert_signals_alerts__alert_id__signals_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/{alert_id}/users": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Alert Users */ + get: operations["get_alert_users_alerts__alert_id__users_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/anthropic/{endpoint}": { parameters: { query?: never; @@ -8369,160 +8614,6 @@ export interface paths { patch?: never; trace?: never; }; - "/issues": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** List Issues */ - get: operations["list_issues_issues_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/issues/families": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** List Issue Families */ - get: operations["list_issue_families_issues_families_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/issues/families/{family}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - /** Update Issue Family */ - patch: operations["update_issue_family_issues_families__family__patch"]; - trace?: never; - }; - "/issues/{issue_id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Get Issue */ - get: operations["get_issue_issues__issue_id__get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - /** Update Issue */ - patch: operations["update_issue_issues__issue_id__patch"]; - trace?: never; - }; - "/issues/{issue_id}/activity": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Get Issue Activity */ - get: operations["get_issue_activity_issues__issue_id__activity_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/issues/{issue_id}/impact": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Get Issue Impact */ - get: operations["get_issue_impact_issues__issue_id__impact_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/issues/{issue_id}/keys": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Get Issue Keys */ - get: operations["get_issue_keys_issues__issue_id__keys_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/issues/{issue_id}/signals": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** List Issue Signals */ - get: operations["list_issue_signals_issues__issue_id__signals_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/issues/{issue_id}/users": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Get Issue Users */ - get: operations["get_issue_users_issues__issue_id__users_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/jwt/key/mapping/delete": { parameters: { query?: never; @@ -19242,116 +19333,6 @@ export interface paths { patch?: never; trace?: never; }; - "/usage/alert-groups": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** List Usage Alert Groups */ - get: operations["list_usage_alert_groups_usage_alert_groups_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - /** Update Usage Alert Group */ - patch: operations["update_usage_alert_group_usage_alert_groups_patch"]; - trace?: never; - }; - "/usage/alert-models": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** List Usage Alert Models */ - get: operations["list_usage_alert_models_usage_alert_models_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/usage/alert-rules": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** List Usage Alert Rules */ - get: operations["list_usage_alert_rules_usage_alert_rules_get"]; - put?: never; - /** - * Create Usage Alert Rule - * @description Create a daily usage alert rule for a scope the caller can manage. - */ - post: operations["create_usage_alert_rule_usage_alert_rules_post"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/usage/alert-rules/{rule_id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Get Usage Alert Rule */ - get: operations["get_usage_alert_rule_usage_alert_rules__rule_id__get"]; - /** Update Usage Alert Rule */ - put: operations["update_usage_alert_rule_usage_alert_rules__rule_id__put"]; - post?: never; - /** Delete Usage Alert Rule */ - delete: operations["delete_usage_alert_rule_usage_alert_rules__rule_id__delete"]; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/usage/alerts": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** List Usage Alerts */ - get: operations["list_usage_alerts_usage_alerts_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/usage/alerts/{alert_id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Get Usage Alert */ - get: operations["get_usage_alert_usage_alerts__alert_id__get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - /** Update Usage Alert */ - patch: operations["update_usage_alert_usage_alerts__alert_id__patch"]; - trace?: never; - }; "/user/available_roles": { parameters: { query?: never; @@ -27290,12 +27271,526 @@ export interface components { /** Url */ url: string; }; + /** Alert */ + Alert: { + /** + * Active + * @default true + */ + active: boolean; + /** + * Can Update Status + * @description Current reader's status permission, evaluated by the issue-detail endpoint + */ + can_update_status?: boolean | null; + /** + * Can View Activity + * @description Current reader's permission to inspect Activity, keys, users and impact + */ + can_view_activity?: boolean | null; + cohort: components["schemas"]["AlertCohort"]; + /** Cohort Key */ + cohort_key: string; + /** + * Comparisons + * @default [] + */ + comparisons: components["schemas"]["AlertComparison"][]; + deprecation?: components["schemas"]["AlertDeprecation"] | null; + /** + * Explanation + * @default + */ + explanation: string; + /** + * Family + * @enum {string} + */ + family: "retirement" | "latency" | "reliability" | "spend" | "traffic"; + /** + * First Seen + * Format: date-time + */ + first_seen: string; + /** + * Grouping Reason + * @enum {string} + */ + grouping_reason: "same_resource" | "dominant_key" | "provider_cohort" | "model_deprecation"; + /** Id */ + id: string; + impact?: components["schemas"]["AlertImpact"]; + /** + * Kind + * @enum {string} + */ + kind: "performance" | "model_deprecation"; + /** + * Last Seen + * Format: date-time + */ + last_seen: string; + /** + * Observed At + * Format: date-time + */ + observed_at: string; + /** + * Observed On + * Format: date + */ + observed_on: string; + /** + * Range End + * Format: date + */ + range_end: string; + /** + * Range Start + * Format: date + */ + range_start: string; + /** Scope Label */ + scope_label?: string | null; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + /** Title */ + title: string; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; + /** AlertCohort */ + AlertCohort: { + /** + * Model Pairs + * @default [] + */ + model_pairs: components["schemas"]["AlertModelPair"][]; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model" | "provider"; + }; + /** AlertComparison */ + AlertComparison: { + /** + * Baseline End + * Format: date + */ + baseline_end: string; + /** Baseline Sample Days */ + baseline_sample_days?: number | null; + /** + * Baseline Start + * Format: date + */ + baseline_start: string; + /** Baseline Value */ + baseline_value: number; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + /** Observed Value */ + observed_value: number; + /** Threshold */ + threshold?: number | null; + }; + /** AlertDeprecation */ + AlertDeprecation: { + /** Last Day Requests */ + last_day_requests: number; + /** + * Last Used Date + * Format: date + */ + last_used_date: string; + model: components["schemas"]["ModelDeprecationInfo"]; + /** Requests */ + requests: number; + state: components["schemas"]["DeprecationAlertState"]; + /** Usage Model */ + usage_model: string; + }; + /** AlertFamilyList */ + AlertFamilyList: { + /** Data */ + data: components["schemas"]["AlertFamilySetting"][]; + /** + * Projection Pending + * @default false + */ + projection_pending: boolean; + }; + /** AlertFamilyPolicy */ + AlertFamilyPolicy: { + /** Baseline Days */ + baseline_days: number | null; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** Enabled */ + enabled: boolean | null; + /** Enabled Rules */ + enabled_rules: number; + /** Min History Days */ + min_history_days: number | null; + /** Min Requests */ + min_requests: number | null; + /** Min Value */ + min_value: number | null; + /** Mixed */ + mixed: boolean; + /** Multiplier */ + multiplier: number | null; + /** Total Rules */ + total_rules: number; + }; + /** AlertFamilyPolicyUpdate */ + AlertFamilyPolicyUpdate: { + /** Baseline Days */ + baseline_days?: number | null; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** Enabled */ + enabled?: boolean | null; + /** Min History Days */ + min_history_days?: number | null; + /** Min Requests */ + min_requests?: number | null; + /** Min Value */ + min_value?: number | null; + /** Multiplier */ + multiplier?: number | null; + }; + /** AlertFamilySetting */ + AlertFamilySetting: { + /** Enabled */ + enabled: boolean | null; + /** Enabled Rules */ + enabled_rules: number; + evaluation_status_counts?: components["schemas"]["UsageAlertEvaluationCounts"]; + /** + * Family + * @enum {string} + */ + family: "spend" | "latency" | "reliability" | "traffic"; + /** Policies */ + policies: components["schemas"]["AlertFamilyPolicy"][]; + /** Total Rules */ + total_rules: number; + }; + /** AlertFamilyUpdate */ + AlertFamilyUpdate: { + /** Enabled */ + enabled?: boolean | null; + /** + * Policies + * @default [] + */ + policies: components["schemas"]["AlertFamilyPolicyUpdate"][]; + }; + /** AlertImpact */ + AlertImpact: { + /** Keys */ + keys?: number | null; + /** Measured Keys */ + measured_keys?: number | null; + /** Measured Models */ + measured_models?: number | null; + /** Models */ + models?: number | null; + /** Requests */ + requests?: number | null; + /** Teams */ + teams?: number | null; + /** Users */ + users?: number | null; + }; + /** AlertList */ + AlertList: { + /** Data */ + data: components["schemas"]["Alert"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** AlertModelList */ + AlertModelList: { + /** Data */ + data: components["schemas"]["AlertModelOption"][]; + /** + * End Date + * Format: date + */ + end_date: string; + /** Limit */ + limit: number; + /** Next Cursor */ + next_cursor: string | null; + /** + * Start Date + * Format: date + */ + start_date: string; + }; + /** AlertModelOption */ + AlertModelOption: { + /** Label */ + label: string; + /** Scope Id */ + scope_id: string; + }; + /** AlertModelPair */ + AlertModelPair: { + /** Model */ + model: string; + /** Provider */ + provider: string; + }; + /** AlertRule */ + AlertRule: { + /** + * Baseline Days + * @default 7 + */ + baseline_days: number; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** Created By */ + created_by: string | null; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * Email Recipients + * @default [] + */ + email_recipients: string[]; + /** + * Enabled + * @default true + */ + enabled: boolean; + /** + * Evaluation Status + * @default pending + * @enum {string} + */ + evaluation_status: "pending" | "complete" | "insufficient_coverage" | "insufficient_history"; + /** Id */ + id: string; + /** + * Is Default + * @default false + */ + is_default: boolean; + /** Last Evaluated Day */ + last_evaluated_day?: string | null; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + /** + * Min History Days + * @default 3 + */ + min_history_days: number; + /** + * Min Requests + * @default 100 + */ + min_requests: number; + /** + * Min Value + * @description Minimum absolute change in the metric's units + * @default 0 + */ + min_value: number; + /** + * Multiplier + * @default 3 + */ + multiplier: number; + /** Name */ + name: string; + /** + * Notification Channels + * @default [] + */ + notification_channels: ("email" | "slack")[]; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; + /** AlertRuleInput */ + AlertRuleInput: { + /** + * Baseline Days + * @default 7 + */ + baseline_days: number; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * Email Recipients + * @default [] + */ + email_recipients: string[]; + /** + * Enabled + * @default true + */ + enabled: boolean; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + /** + * Min History Days + * @default 3 + */ + min_history_days: number; + /** + * Min Requests + * @default 100 + */ + min_requests: number; + /** + * Min Value + * @description Minimum absolute change in the metric's units + * @default 0 + */ + min_value: number; + /** + * Multiplier + * @default 3 + */ + multiplier: number; + /** Name */ + name: string; + /** + * Notification Channels + * @default [] + */ + notification_channels: ("email" | "slack")[]; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + }; + /** AlertRuleList */ + AlertRuleList: { + /** Data */ + data: components["schemas"]["AlertRule"][]; + default_summary?: components["schemas"]["UsageAlertDefaultSummary"]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** AlertSignal */ + AlertSignal: { + /** Alert Id */ + alert_id: string; + deprecation?: components["schemas"]["AlertDeprecation"] | null; + /** + * Explanation + * @default + */ + explanation: string; + /** + * Grouping Reason + * @default same_resource + * @enum {string} + */ + grouping_reason: "same_resource" | "same_key_traffic" | "dominant_key" | "provider_cohort" | "volume_explains_spend" | "model_deprecation"; + performance?: components["schemas"]["PerformanceSignal"] | null; + /** + * Source + * @enum {string} + */ + source: "performance" | "model_deprecation"; + /** Source Id */ + source_id: string; + }; + /** AlertSignalList */ + AlertSignalList: { + /** Data */ + data: components["schemas"]["AlertSignal"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; /** * AlertType * @description Enum for alert types and management event types * @enum {string} */ AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted" | "mcp_tool_description_blocked" | "mcp_pinned_tools_changed"; + /** AlertUpdate */ + AlertUpdate: { + /** + * Expected Status + * @enum {string} + */ + expected_status: "open" | "resolved" | "dismissed"; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + }; /** AllowedVectorStoreIndexItem */ AllowedVectorStoreIndexItem: { /** Index Name */ @@ -35203,329 +35698,6 @@ export interface components { /** Is Accepted */ is_accepted: boolean; }; - /** Issue */ - Issue: { - /** - * Active - * @default true - */ - active: boolean; - /** - * Can Update Status - * @description Current reader's status permission, evaluated by the issue-detail endpoint - */ - can_update_status?: boolean | null; - /** - * Can View Activity - * @description Current reader's permission to inspect Activity, keys, users and impact - */ - can_view_activity?: boolean | null; - cohort: components["schemas"]["IssueCohort"]; - /** Cohort Key */ - cohort_key: string; - /** - * Comparisons - * @default [] - */ - comparisons: components["schemas"]["IssueComparison"][]; - deprecation?: components["schemas"]["IssueDeprecation"] | null; - /** - * Explanation - * @default - */ - explanation: string; - /** - * Family - * @enum {string} - */ - family: "retirement" | "latency" | "reliability" | "spend" | "traffic"; - /** - * First Seen - * Format: date-time - */ - first_seen: string; - /** - * Grouping Reason - * @enum {string} - */ - grouping_reason: "same_resource" | "dominant_key" | "provider_cohort" | "model_deprecation"; - /** Id */ - id: string; - impact?: components["schemas"]["IssueImpact"]; - /** - * Kind - * @enum {string} - */ - kind: "performance" | "model_deprecation"; - /** - * Last Seen - * Format: date-time - */ - last_seen: string; - /** - * Observed At - * Format: date-time - */ - observed_at: string; - /** - * Observed On - * Format: date - */ - observed_on: string; - /** - * Range End - * Format: date - */ - range_end: string; - /** - * Range Start - * Format: date - */ - range_start: string; - /** Scope Label */ - scope_label?: string | null; - /** - * Status - * @enum {string} - */ - status: "open" | "resolved" | "dismissed"; - /** Title */ - title: string; - /** - * Updated At - * Format: date-time - */ - updated_at: string; - }; - /** IssueCohort */ - IssueCohort: { - /** - * Model Pairs - * @default [] - */ - model_pairs: components["schemas"]["IssueModelPair"][]; - /** Scope Id */ - scope_id?: string | null; - /** - * Scope Type - * @enum {string} - */ - scope_type: "deployment" | "key" | "user" | "team" | "model" | "provider"; - }; - /** IssueComparison */ - IssueComparison: { - /** - * Baseline End - * Format: date - */ - baseline_end: string; - /** Baseline Sample Days */ - baseline_sample_days?: number | null; - /** - * Baseline Start - * Format: date - */ - baseline_start: string; - /** Baseline Value */ - baseline_value: number; - /** - * Direction - * @enum {string} - */ - direction: "increase" | "decrease"; - /** - * Metric - * @enum {string} - */ - metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; - /** Observed Value */ - observed_value: number; - /** Threshold */ - threshold?: number | null; - }; - /** IssueDeprecation */ - IssueDeprecation: { - /** Last Day Requests */ - last_day_requests: number; - /** - * Last Used Date - * Format: date - */ - last_used_date: string; - model: components["schemas"]["ModelDeprecationInfo"]; - /** Requests */ - requests: number; - state: components["schemas"]["DeprecationAlertState"]; - /** Usage Model */ - usage_model: string; - }; - /** IssueFamilyList */ - IssueFamilyList: { - /** Data */ - data: components["schemas"]["IssueFamilySetting"][]; - /** - * Projection Pending - * @default false - */ - projection_pending: boolean; - }; - /** IssueFamilyPolicy */ - IssueFamilyPolicy: { - /** Baseline Days */ - baseline_days: number | null; - /** - * Direction - * @enum {string} - */ - direction: "increase" | "decrease"; - /** Enabled */ - enabled: boolean | null; - /** Enabled Rules */ - enabled_rules: number; - /** Min History Days */ - min_history_days: number | null; - /** Min Requests */ - min_requests: number | null; - /** Min Value */ - min_value: number | null; - /** Mixed */ - mixed: boolean; - /** Multiplier */ - multiplier: number | null; - /** Total Rules */ - total_rules: number; - }; - /** IssueFamilyPolicyUpdate */ - IssueFamilyPolicyUpdate: { - /** Baseline Days */ - baseline_days?: number | null; - /** - * Direction - * @enum {string} - */ - direction: "increase" | "decrease"; - /** Enabled */ - enabled?: boolean | null; - /** Min History Days */ - min_history_days?: number | null; - /** Min Requests */ - min_requests?: number | null; - /** Min Value */ - min_value?: number | null; - /** Multiplier */ - multiplier?: number | null; - }; - /** IssueFamilySetting */ - IssueFamilySetting: { - /** Enabled */ - enabled: boolean | null; - /** Enabled Rules */ - enabled_rules: number; - evaluation_status_counts?: components["schemas"]["UsageAlertEvaluationCounts"]; - /** - * Family - * @enum {string} - */ - family: "spend" | "latency" | "reliability" | "traffic"; - /** Policies */ - policies: components["schemas"]["IssueFamilyPolicy"][]; - /** Total Rules */ - total_rules: number; - }; - /** IssueFamilyUpdate */ - IssueFamilyUpdate: { - /** Enabled */ - enabled?: boolean | null; - /** - * Policies - * @default [] - */ - policies: components["schemas"]["IssueFamilyPolicyUpdate"][]; - }; - /** IssueImpact */ - IssueImpact: { - /** Keys */ - keys?: number | null; - /** Measured Keys */ - measured_keys?: number | null; - /** Measured Models */ - measured_models?: number | null; - /** Models */ - models?: number | null; - /** Requests */ - requests?: number | null; - /** Teams */ - teams?: number | null; - /** Users */ - users?: number | null; - }; - /** IssueList */ - IssueList: { - /** Data */ - data: components["schemas"]["Issue"][]; - /** Limit */ - limit: number; - /** Offset */ - offset: number; - /** Total */ - total: number; - }; - /** IssueModelPair */ - IssueModelPair: { - /** Model */ - model: string; - /** Provider */ - provider: string; - }; - /** IssueSignal */ - IssueSignal: { - deprecation?: components["schemas"]["IssueDeprecation"] | null; - /** - * Explanation - * @default - */ - explanation: string; - /** - * Grouping Reason - * @default same_resource - * @enum {string} - */ - grouping_reason: "same_resource" | "same_key_traffic" | "dominant_key" | "provider_cohort" | "volume_explains_spend" | "model_deprecation"; - /** Issue Id */ - issue_id: string; - /** - * Source - * @enum {string} - */ - source: "usage_alert" | "model_deprecation"; - /** Source Id */ - source_id: string; - usage_alert?: components["schemas"]["UsageAlert"] | null; - }; - /** IssueSignalList */ - IssueSignalList: { - /** Data */ - data: components["schemas"]["IssueSignal"][]; - /** Limit */ - limit: number; - /** Offset */ - offset: number; - /** Total */ - total: number; - }; - /** IssueUpdate */ - IssueUpdate: { - /** - * Expected Status - * @enum {string} - */ - expected_status: "open" | "resolved" | "dismissed"; - /** - * Status - * @enum {string} - */ - status: "open" | "resolved" | "dismissed"; - }; /** * ItemReference * @description An internal identifier for an item to reference. @@ -42669,6 +42841,83 @@ export interface components { /** User Id */ user_id: string; }; + /** PerformanceSignal */ + PerformanceSignal: { + /** Activity Path */ + activity_path: string; + /** + * Baseline End + * Format: date + */ + baseline_end: string; + /** Baseline Sample Days */ + baseline_sample_days?: number | null; + /** + * Baseline Start + * Format: date + */ + baseline_start: string; + /** Baseline Value */ + baseline_value: number; + /** + * Day + * Format: date + */ + day: string; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * First Seen + * Format: date-time + */ + first_seen: string; + /** Id */ + id: string; + /** + * Last Seen + * Format: date-time + */ + last_seen: string; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + /** Observed Value */ + observed_value: number; + /** Rule Id */ + rule_id: string; + /** Rule Name */ + rule_name: string; + /** Scope Id */ + scope_id: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + /** Threshold */ + threshold: number; + }; + /** PerformanceSignalList */ + PerformanceSignalList: { + /** Data */ + data: components["schemas"]["PerformanceSignal"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; /** * PiiAction * @enum {string} @@ -51654,72 +51903,6 @@ export interface components { */ version: string | null; }; - /** UsageAlert */ - UsageAlert: { - /** Activity Path */ - activity_path: string; - /** - * Baseline End - * Format: date - */ - baseline_end: string; - /** Baseline Sample Days */ - baseline_sample_days?: number | null; - /** - * Baseline Start - * Format: date - */ - baseline_start: string; - /** Baseline Value */ - baseline_value: number; - /** - * Day - * Format: date - */ - day: string; - /** - * Direction - * @enum {string} - */ - direction: "increase" | "decrease"; - /** - * First Seen - * Format: date-time - */ - first_seen: string; - /** Id */ - id: string; - /** - * Last Seen - * Format: date-time - */ - last_seen: string; - /** - * Metric - * @enum {string} - */ - metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; - /** Observed Value */ - observed_value: number; - /** Rule Id */ - rule_id: string; - /** Rule Name */ - rule_name: string; - /** Scope Id */ - scope_id: string | null; - /** - * Scope Type - * @enum {string} - */ - scope_type: "deployment" | "key" | "user" | "team" | "model"; - /** - * Status - * @enum {string} - */ - status: "open" | "resolved" | "dismissed"; - /** Threshold */ - threshold: number; - }; /** UsageAlertDefaultSummary */ UsageAlertDefaultSummary: { /** @@ -51757,293 +51940,6 @@ export interface components { */ pending: number; }; - /** UsageAlertGroup */ - UsageAlertGroup: { - /** Activity Path */ - activity_path: string; - /** - * Day - * Format: date - */ - day: string; - /** Id */ - id: string; - /** Incident Count */ - incident_count: number; - /** Metrics */ - metrics: components["schemas"]["UsageAlertGroupMetric"][]; - /** Scope Id */ - scope_id: string | null; - /** Scope Label */ - scope_label: string | null; - /** - * Scope Type - * @enum {string} - */ - scope_type: "deployment" | "key" | "user" | "team" | "model"; - }; - /** UsageAlertGroupList */ - UsageAlertGroupList: { - /** Data */ - data: components["schemas"]["UsageAlertGroup"][]; - /** Limit */ - limit: number; - /** Offset */ - offset: number; - /** Total */ - total: number; - }; - /** UsageAlertGroupMetric */ - UsageAlertGroupMetric: { - /** Incident Count */ - incident_count: number; - /** - * Metric - * @enum {string} - */ - metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; - representative: components["schemas"]["UsageAlert"]; - }; - /** UsageAlertGroupUpdate */ - UsageAlertGroupUpdate: { - /** - * Day - * Format: date - */ - day: string; - /** - * Expected Status - * @enum {string} - */ - expected_status: "open" | "resolved" | "dismissed"; - /** Scope Id */ - scope_id?: string | null; - /** - * Scope Type - * @enum {string} - */ - scope_type: "deployment" | "key" | "user" | "team" | "model"; - /** - * Status - * @enum {string} - */ - status: "open" | "resolved" | "dismissed"; - }; - /** UsageAlertGroupUpdateResult */ - UsageAlertGroupUpdateResult: { - /** Updated Count */ - updated_count: number; - }; - /** UsageAlertList */ - UsageAlertList: { - /** Data */ - data: components["schemas"]["UsageAlert"][]; - /** Limit */ - limit: number; - /** Offset */ - offset: number; - /** Total */ - total: number; - }; - /** UsageAlertModelList */ - UsageAlertModelList: { - /** Data */ - data: components["schemas"]["UsageAlertModelOption"][]; - /** - * End Date - * Format: date - */ - end_date: string; - /** Limit */ - limit: number; - /** Next Cursor */ - next_cursor: string | null; - /** - * Start Date - * Format: date - */ - start_date: string; - }; - /** UsageAlertModelOption */ - UsageAlertModelOption: { - /** Label */ - label: string; - /** Scope Id */ - scope_id: string; - }; - /** UsageAlertRule */ - UsageAlertRule: { - /** - * Baseline Days - * @default 7 - */ - baseline_days: number; - /** - * Created At - * Format: date-time - */ - created_at: string; - /** Created By */ - created_by: string | null; - /** - * Direction - * @enum {string} - */ - direction: "increase" | "decrease"; - /** - * Email Recipients - * @default [] - */ - email_recipients: string[]; - /** - * Enabled - * @default true - */ - enabled: boolean; - /** - * Evaluation Status - * @default pending - * @enum {string} - */ - evaluation_status: "pending" | "complete" | "insufficient_coverage" | "insufficient_history"; - /** Id */ - id: string; - /** - * Is Default - * @default false - */ - is_default: boolean; - /** Last Evaluated Day */ - last_evaluated_day?: string | null; - /** - * Metric - * @enum {string} - */ - metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; - /** - * Min History Days - * @default 3 - */ - min_history_days: number; - /** - * Min Requests - * @default 100 - */ - min_requests: number; - /** - * Min Value - * @description Minimum absolute change in the metric's units - * @default 0 - */ - min_value: number; - /** - * Multiplier - * @default 3 - */ - multiplier: number; - /** Name */ - name: string; - /** - * Notification Channels - * @default [] - */ - notification_channels: ("email" | "slack")[]; - /** Scope Id */ - scope_id?: string | null; - /** - * Scope Type - * @enum {string} - */ - scope_type: "deployment" | "key" | "user" | "team" | "model"; - /** - * Updated At - * Format: date-time - */ - updated_at: string; - }; - /** UsageAlertRuleInput */ - UsageAlertRuleInput: { - /** - * Baseline Days - * @default 7 - */ - baseline_days: number; - /** - * Direction - * @enum {string} - */ - direction: "increase" | "decrease"; - /** - * Email Recipients - * @default [] - */ - email_recipients: string[]; - /** - * Enabled - * @default true - */ - enabled: boolean; - /** - * Metric - * @enum {string} - */ - metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; - /** - * Min History Days - * @default 3 - */ - min_history_days: number; - /** - * Min Requests - * @default 100 - */ - min_requests: number; - /** - * Min Value - * @description Minimum absolute change in the metric's units - * @default 0 - */ - min_value: number; - /** - * Multiplier - * @default 3 - */ - multiplier: number; - /** Name */ - name: string; - /** - * Notification Channels - * @default [] - */ - notification_channels: ("email" | "slack")[]; - /** Scope Id */ - scope_id?: string | null; - /** - * Scope Type - * @enum {string} - */ - scope_type: "deployment" | "key" | "user" | "team" | "model"; - }; - /** UsageAlertRuleList */ - UsageAlertRuleList: { - /** Data */ - data: components["schemas"]["UsageAlertRule"][]; - default_summary?: components["schemas"]["UsageAlertDefaultSummary"]; - /** Limit */ - limit: number; - /** Offset */ - offset: number; - /** Total */ - total: number; - }; - /** UsageAlertUpdate */ - UsageAlertUpdate: { - /** - * Status - * @enum {string} - */ - status: "open" | "resolved" | "dismissed"; - }; /** UsageChartPoint */ UsageChartPoint: { /** Blocked */ @@ -55370,6 +55266,604 @@ export interface operations { }; }; }; + list_alerts_alerts_get: { + parameters: { + query?: { + kind?: ("performance" | "model_deprecation") | null; + family?: ("retirement" | "latency" | "reliability" | "spend" | "traffic") | null; + status?: ("open" | "resolved" | "dismissed") | null; + scope_type?: ("deployment" | "key" | "user" | "team" | "model" | "provider") | null; + scope_id?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_alert_defaults_alerts_defaults_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertFamilyList"]; + }; + }; + }; + }; + update_alert_default_alerts_defaults__family__patch: { + parameters: { + query?: never; + header?: never; + path: { + family: "spend" | "latency" | "reliability" | "traffic"; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AlertFamilyUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertFamilySetting"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_alert_models_alerts_models_get: { + parameters: { + query?: { + search?: string | null; + cursor?: string | null; + limit?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertModelList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_alert_rules_alerts_rules_get: { + parameters: { + query?: { + scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; + scope_id?: string | null; + limit?: number; + offset?: number; + kind?: "all" | "default" | "custom"; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertRuleList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + create_alert_rule_alerts_rules_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AlertRuleInput"]; + }; + }; + responses: { + /** @description Successful Response */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertRule"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_alert_rule_alerts_rules__rule_id__get: { + parameters: { + query?: never; + header?: never; + path: { + rule_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertRule"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_alert_rule_alerts_rules__rule_id__put: { + parameters: { + query?: never; + header?: never; + path: { + rule_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AlertRuleInput"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertRule"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + delete_alert_rule_alerts_rules__rule_id__delete: { + parameters: { + query?: never; + header?: never; + path: { + rule_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_performance_signals_alerts_signals_get: { + parameters: { + query?: { + scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; + scope_id?: string | null; + metric?: ("spend" | "requests" | "latency" | "errors" | "error_rate") | null; + status?: ("open" | "resolved" | "dismissed") | null; + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PerformanceSignalList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_performance_signal_alerts_signals__signal_id__get: { + parameters: { + query?: never; + header?: never; + path: { + signal_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PerformanceSignal"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_alert_alerts__alert_id__get: { + parameters: { + query?: never; + header?: never; + path: { + alert_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Alert"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_alert_alerts__alert_id__patch: { + parameters: { + query?: never; + header?: never; + path: { + alert_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AlertUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Alert"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_alert_activity_alerts__alert_id__activity_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + model_limit?: number; + }; + header?: never; + path: { + alert_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BoundedDailyActivityResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_alert_impact_alerts__alert_id__impact_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + }; + header?: never; + path: { + alert_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertImpact"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_alert_keys_alerts__alert_id__keys_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path: { + alert_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DailyActivityKeyPageResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_alert_signals_alerts__alert_id__signals_get: { + parameters: { + query?: { + limit?: number; + offset?: number; + }; + header?: never; + path: { + alert_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertSignalList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_alert_users_alerts__alert_id__users_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path: { + alert_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DailyActivityUserPageResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; anthropic_proxy_route_anthropic__endpoint__get: { parameters: { query?: never; @@ -66272,339 +66766,6 @@ export interface operations { }; }; }; - list_issues_issues_get: { - parameters: { - query?: { - kind?: ("performance" | "model_deprecation") | null; - family?: ("retirement" | "latency" | "reliability" | "spend" | "traffic") | null; - status?: ("open" | "resolved" | "dismissed") | null; - scope_type?: ("deployment" | "key" | "user" | "team" | "model" | "provider") | null; - scope_id?: string | null; - limit?: number; - offset?: number; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["IssueList"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - list_issue_families_issues_families_get: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["IssueFamilyList"]; - }; - }; - }; - }; - update_issue_family_issues_families__family__patch: { - parameters: { - query?: never; - header?: never; - path: { - family: "spend" | "latency" | "reliability" | "traffic"; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["IssueFamilyUpdate"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["IssueFamilySetting"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - get_issue_issues__issue_id__get: { - parameters: { - query?: never; - header?: never; - path: { - issue_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["Issue"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - update_issue_issues__issue_id__patch: { - parameters: { - query?: never; - header?: never; - path: { - issue_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["IssueUpdate"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["Issue"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - get_issue_activity_issues__issue_id__activity_get: { - parameters: { - query?: { - start_date?: string | null; - end_date?: string | null; - model_limit?: number; - }; - header?: never; - path: { - issue_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["BoundedDailyActivityResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - get_issue_impact_issues__issue_id__impact_get: { - parameters: { - query?: { - start_date?: string | null; - end_date?: string | null; - }; - header?: never; - path: { - issue_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["IssueImpact"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - get_issue_keys_issues__issue_id__keys_get: { - parameters: { - query?: { - start_date?: string | null; - end_date?: string | null; - limit?: number; - offset?: number; - }; - header?: never; - path: { - issue_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DailyActivityKeyPageResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - list_issue_signals_issues__issue_id__signals_get: { - parameters: { - query?: { - limit?: number; - offset?: number; - }; - header?: never; - path: { - issue_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["IssueSignalList"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - get_issue_users_issues__issue_id__users_get: { - parameters: { - query?: { - start_date?: string | null; - end_date?: string | null; - limit?: number; - offset?: number; - }; - header?: never; - path: { - issue_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["DailyActivityUserPageResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; delete_jwt_key_mapping_jwt_key_mapping_delete_post: { parameters: { query?: never; @@ -80762,376 +80923,6 @@ export interface operations { }; }; }; - list_usage_alert_groups_usage_alert_groups_get: { - parameters: { - query?: { - scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; - scope_id?: string | null; - status?: ("open" | "resolved" | "dismissed") | null; - start_date?: string | null; - end_date?: string | null; - limit?: number; - offset?: number; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UsageAlertGroupList"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - update_usage_alert_group_usage_alert_groups_patch: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["UsageAlertGroupUpdate"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UsageAlertGroupUpdateResult"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - list_usage_alert_models_usage_alert_models_get: { - parameters: { - query?: { - search?: string | null; - cursor?: string | null; - limit?: number; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UsageAlertModelList"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - list_usage_alert_rules_usage_alert_rules_get: { - parameters: { - query?: { - scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; - scope_id?: string | null; - limit?: number; - offset?: number; - kind?: "all" | "default" | "custom"; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UsageAlertRuleList"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - create_usage_alert_rule_usage_alert_rules_post: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["UsageAlertRuleInput"]; - }; - }; - responses: { - /** @description Successful Response */ - 201: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UsageAlertRule"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - get_usage_alert_rule_usage_alert_rules__rule_id__get: { - parameters: { - query?: never; - header?: never; - path: { - rule_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UsageAlertRule"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - update_usage_alert_rule_usage_alert_rules__rule_id__put: { - parameters: { - query?: never; - header?: never; - path: { - rule_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["UsageAlertRuleInput"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UsageAlertRule"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - delete_usage_alert_rule_usage_alert_rules__rule_id__delete: { - parameters: { - query?: never; - header?: never; - path: { - rule_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 204: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - list_usage_alerts_usage_alerts_get: { - parameters: { - query?: { - scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; - scope_id?: string | null; - metric?: ("spend" | "requests" | "latency" | "errors" | "error_rate") | null; - status?: ("open" | "resolved" | "dismissed") | null; - start_date?: string | null; - end_date?: string | null; - limit?: number; - offset?: number; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UsageAlertList"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - get_usage_alert_usage_alerts__alert_id__get: { - parameters: { - query?: never; - header?: never; - path: { - alert_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UsageAlert"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - update_usage_alert_usage_alerts__alert_id__patch: { - parameters: { - query?: never; - header?: never; - path: { - alert_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["UsageAlertUpdate"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UsageAlert"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; ui_get_available_role_user_available_roles_get: { parameters: { query?: never;