From f82c79a6f05046c268b89c98fe2d18ba31b8a237 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 11:41:02 +0800 Subject: [PATCH 01/38] feat: add scoped daily usage alert API and storage --- .../migration.sql | 24 + .../litellm_proxy_extras/schema.prisma | 40 ++ litellm/proxy/_types.py | 12 + .../proxy/alerting_endpoints/usage_alerts.py | 175 +++++++ litellm/proxy/proxy_server.py | 2 + litellm/proxy/schema.prisma | 40 ++ litellm/repositories/usage_alerts.py | 460 ++++++++++++++++++ litellm/types/proxy/usage_alerts.py | 116 +++++ schema.prisma | 40 ++ 9 files changed, 909 insertions(+) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql create mode 100644 litellm/proxy/alerting_endpoints/usage_alerts.py create mode 100644 litellm/repositories/usage_alerts.py create mode 100644 litellm/types/proxy/usage_alerts.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql new file mode 100644 index 00000000000..1e459468f84 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql @@ -0,0 +1,24 @@ +CREATE TABLE "LiteLLM_UsageAlertRule" ( + "id" TEXT PRIMARY KEY, "scope_type" TEXT NOT NULL, "scope_id" TEXT, + "enabled" BOOLEAN NOT NULL DEFAULT true, "record" JSONB NOT NULL, + "deleted" BOOLEAN NOT NULL DEFAULT false +); +CREATE INDEX "LiteLLM_UsageAlertRule_enabled_deleted_id_idx" ON "LiteLLM_UsageAlertRule" ("enabled", "deleted", "id"); +CREATE INDEX "LiteLLM_UsageAlertRule_scope_type_scope_id_deleted_idx" ON "LiteLLM_UsageAlertRule" ("scope_type", "scope_id", "deleted"); +CREATE TABLE "LiteLLM_UsageAlertIncident" ( + "id" TEXT PRIMARY KEY, "rule_id" TEXT NOT NULL, "scope_key" TEXT NOT NULL, + "scope_type" TEXT NOT NULL, "scope_id" TEXT, "metric" TEXT NOT NULL, + "status" TEXT NOT NULL, "day" TEXT NOT NULL, "evaluation_day" TEXT NOT NULL, + "breached" BOOLEAN NOT NULL, "record" JSONB NOT NULL, + "notified_channels" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX "LiteLLM_UsageAlertIncident_rule_id_scope_key_evaluation_day_idx" ON "LiteLLM_UsageAlertIncident" ("rule_id", "scope_key", "evaluation_day"); +CREATE INDEX "LiteLLM_UsageAlertIncident_scope_type_scope_id_day_idx" ON "LiteLLM_UsageAlertIncident" ("scope_type", "scope_id", "day"); +CREATE INDEX "LiteLLM_UsageAlertIncident_status_day_idx" ON "LiteLLM_UsageAlertIncident" ("status", "day"); +CREATE INDEX "LiteLLM_UsageAlertIncident_updated_at_idx" ON "LiteLLM_UsageAlertIncident" ("updated_at"); +CREATE TABLE "LiteLLM_UsageAlertCheckpoint" ( + "id" TEXT PRIMARY KEY, "completed" BOOLEAN NOT NULL DEFAULT false, + "owner" TEXT, "expires_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX "LiteLLM_UsageAlertCheckpoint_expires_at_idx" ON "LiteLLM_UsageAlertCheckpoint" ("expires_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 78f16d9f1fd..46ab28530a9 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2109,3 +2109,43 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +// Daily alerts consume existing aggregate tables; no request-path writes. +model LiteLLM_UsageAlertRule { + id String @id + scope_type String + scope_id String? + enabled Boolean @default(true) + record Json + deleted Boolean @default(false) + @@index([enabled, deleted, id]) + @@index([scope_type, scope_id, deleted]) +} + +model LiteLLM_UsageAlertIncident { + id String @id + rule_id String + scope_key String + scope_type String + scope_id String? + metric String + status String + day String + evaluation_day String + breached Boolean + record Json + notified_channels String[] @default([]) + updated_at DateTime @default(now()) + @@index([rule_id, scope_key, evaluation_day]) + @@index([scope_type, scope_id, day]) + @@index([status, day]) + @@index([updated_at]) +} + +model LiteLLM_UsageAlertCheckpoint { + id String @id + completed Boolean @default(false) + owner String? + expires_at DateTime @default(now()) + @@index([expires_at]) +} diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 25849634764..14e71a0f292 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -646,6 +646,10 @@ class LiteLLMRoutes(enum.Enum): management_routes = ( [ + "/usage/alert-rules", + "/usage/alert-rules/{rule_id}", + "/usage/alerts", + "/usage/alerts/{alert_id}", # user "/user/new", "/management/v1/users/bulk", @@ -855,7 +859,15 @@ class LiteLLMRoutes(enum.Enum): ] ) + usage_alert_routes = [ + "/usage/alert-rules", + "/usage/alert-rules/{rule_id}", + "/usage/alerts", + "/usage/alerts/{alert_id}", + ] + self_managed_routes = [ + *usage_alert_routes, "/lens", "/lens/{path:path}", "/telemetry/ui_events", diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py new file mode 100644 index 00000000000..3a4c4112581 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -0,0 +1,175 @@ +"""Public management API for daily usage alert rules and incidents.""" + +from datetime import date +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, HTTPException, Query, Response + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.deps.database import DatabaseDep +from litellm.repositories.daily_activity_buckets import api_key_from_bucket +from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.usage_alerts import ( + UsageAlert, + UsageAlertList, + UsageAlertMetric, + UsageAlertRule, + UsageAlertRuleInput, + UsageAlertRuleList, + UsageAlertScope, + UsageAlertStatus, + UsageAlertUpdate, +) + +router: Final = APIRouter(prefix="/usage", tags=["usage alerts"]) + + +def usage_alert_repository(database: DatabaseDep) -> UsageAlertsRepository: + if database is None: + raise HTTPException(status_code=503, detail="Usage alerts require a database connection") + return UsageAlertsRepository.from_prisma(database.prisma) + + +User = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] +Repository = Annotated[UsageAlertsRepository, Depends(usage_alert_repository)] +Limit = Annotated[int, Query(ge=1, le=100)] +Offset = Annotated[int, Query(ge=0, le=100000)] +ScopeId = Annotated[str | None, Query(min_length=1, max_length=1024)] + + +async def require_scope( + user: UserAPIKeyAuth, + repository: UsageAlertsRepository, + scope_type: UsageAlertScope | None, + scope_id: str | None, + *, + write: bool, +) -> None: + if user.key_type == "llm_api" or (write and user.key_type == "read_only"): + raise HTTPException(status_code=403, detail="A management-capable credential is required") + if user.user_role == LitellmUserRoles.PROXY_ADMIN: + return + if user.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: + if not write: + return + raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts") + if user.user_role not in ( + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + LitellmUserRoles.ORG_ADMIN, + LitellmUserRoles.TEAM, + ): + raise HTTPException(status_code=403, detail="Usage alerts require a management role") + if write and user.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY: + raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts") + if scope_id is None: + raise HTTPException(status_code=403, detail="Specify a scope you can access") + if scope_type == "user" and user.user_id is not None and scope_id == user.user_id: + return + if scope_type == "team": + role: Final = await repository.team_role(scope_id, user.user_id) if user.user_id else None + if role == "admin" or ( + not write and (role == "user" or (user.user_role == LitellmUserRoles.TEAM and user.team_id == scope_id)) + ): + return + if scope_type == "key": + ownership: Final = await repository.key_ownership(api_key_from_bucket(scope_id)) + if ownership is not None: + owner_id, team_id = ownership + if user.user_id is not None and owner_id == user.user_id: + return + if user.user_id and team_id and await repository.team_role(team_id, user.user_id) == "admin": + return + raise HTTPException(status_code=403, detail="You cannot access this alert scope") + + +@router.get("/alert-rules", response_model=UsageAlertRuleList) +async def list_usage_alert_rules( + user: User, + repository: Repository, + scope_type: UsageAlertScope | None = None, + scope_id: ScopeId = None, + limit: Limit = 50, + offset: Offset = 0, +) -> UsageAlertRuleList: + await require_scope(user, repository, scope_type, scope_id, write=False) + return await repository.list_rules(scope_type, scope_id, limit, offset) + + +@router.post("/alert-rules", response_model=UsageAlertRule, status_code=201) +async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, repository: Repository) -> UsageAlertRule: + await require_scope(user, repository, data.scope_type, data.scope_id, write=True) + return await repository.save_rule(data, user.user_id) + + +@router.get("/alert-rules/{rule_id}", response_model=UsageAlertRule) +async def get_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> UsageAlertRule: + rule: Final = await repository.get_rule(rule_id) + if rule is None: + raise HTTPException(status_code=404, detail="Alert rule not found") + await require_scope(user, repository, rule.scope_type, rule.scope_id, write=False) + return rule + + +@router.put("/alert-rules/{rule_id}", response_model=UsageAlertRule) +async def update_usage_alert_rule( + rule_id: str, data: UsageAlertRuleInput, user: User, repository: Repository +) -> UsageAlertRule: + rule: Final = await repository.get_rule(rule_id) + if rule is None: + raise HTTPException(status_code=404, detail="Alert rule not found") + await require_scope(user, repository, rule.scope_type, rule.scope_id, write=True) + await require_scope(user, repository, data.scope_type, data.scope_id, write=True) + return await repository.save_rule(data, user.user_id, existing=rule) + + +@router.delete("/alert-rules/{rule_id}", status_code=204) +async def delete_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> Response: + rule: Final = await repository.get_rule(rule_id) + if rule is None: + raise HTTPException(status_code=404, detail="Alert rule not found") + await require_scope(user, repository, rule.scope_type, rule.scope_id, write=True) + await repository.delete_rule(rule_id) + return Response(status_code=204) + + +@router.get("/alerts", response_model=UsageAlertList) +async def list_usage_alerts( + user: User, + repository: Repository, + scope_type: UsageAlertScope | None = None, + scope_id: ScopeId = None, + metric: UsageAlertMetric | None = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: Limit = 50, + offset: Offset = 0, +) -> UsageAlertList: + await require_scope(user, repository, scope_type, scope_id, write=False) + if start_date is not None and end_date is not None and start_date > end_date: + raise HTTPException(status_code=422, detail="start_date must not be after end_date") + return await repository.list_alerts(scope_type, scope_id, metric, status, start_date, end_date, limit, offset) + + +@router.get("/alerts/{alert_id}", response_model=UsageAlert) +async def get_usage_alert(alert_id: str, user: User, repository: Repository) -> UsageAlert: + alert: Final = await repository.get_alert(alert_id) + if alert is None: + raise HTTPException(status_code=404, detail="Usage alert not found") + await require_scope(user, repository, alert.scope_type, alert.scope_id, write=False) + return alert + + +@router.patch("/alerts/{alert_id}", response_model=UsageAlert) +async def update_usage_alert(alert_id: str, data: UsageAlertUpdate, user: User, repository: Repository) -> UsageAlert: + alert: Final = await repository.get_alert(alert_id) + if alert is None: + raise HTTPException(status_code=404, detail="Usage alert not found") + await require_scope(user, repository, alert.scope_type, alert.scope_id, write=True) + updated: Final = await repository.set_status(alert_id, data.status) + if updated is None: + raise HTTPException(status_code=404, detail="Usage alert not found") + return updated diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index debefec4a30..d876ce011a4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -427,6 +427,7 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolv from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( router as model_deprecation_alerts_router, ) +from litellm.proxy.alerting_endpoints.usage_alerts import router as usage_alerts_router from litellm.proxy.analytics_endpoints.analytics_endpoints import ( router as analytics_router, ) @@ -22438,6 +22439,7 @@ app.include_router(router_settings_router) app.include_router(fallback_management_router) app.include_router(cache_settings_router) app.include_router(model_deprecation_alerts_router) +app.include_router(usage_alerts_router) app.include_router(coordination_redis_settings_router) app.include_router(user_agent_analytics_router) app.include_router(gateway_request_router) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 78f16d9f1fd..46ab28530a9 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2109,3 +2109,43 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +// Daily alerts consume existing aggregate tables; no request-path writes. +model LiteLLM_UsageAlertRule { + id String @id + scope_type String + scope_id String? + enabled Boolean @default(true) + record Json + deleted Boolean @default(false) + @@index([enabled, deleted, id]) + @@index([scope_type, scope_id, deleted]) +} + +model LiteLLM_UsageAlertIncident { + id String @id + rule_id String + scope_key String + scope_type String + scope_id String? + metric String + status String + day String + evaluation_day String + breached Boolean + record Json + notified_channels String[] @default([]) + updated_at DateTime @default(now()) + @@index([rule_id, scope_key, evaluation_day]) + @@index([scope_type, scope_id, day]) + @@index([status, day]) + @@index([updated_at]) +} + +model LiteLLM_UsageAlertCheckpoint { + id String @id + completed Boolean @default(false) + owner String? + expires_at DateTime @default(now()) + @@index([expires_at]) +} diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py new file mode 100644 index 00000000000..0a42657c478 --- /dev/null +++ b/litellm/repositories/usage_alerts.py @@ -0,0 +1,460 @@ +"""Bounded, writer-consistent storage for daily alert rules and episodes.""" + +import json +from datetime import date, datetime, timedelta, timezone +from typing import TYPE_CHECKING, Final, Protocol, cast +from urllib.parse import urlencode +from uuid import NAMESPACE_URL, uuid4, uuid5 + +from pydantic import BaseModel, TypeAdapter + +from litellm.types.proxy.usage_alerts import ( + UsageAlert, + UsageAlertChannel, + UsageAlertEvaluationStatus, + UsageAlertList, + UsageAlertMetric, + UsageAlertNotification, + UsageAlertObservation, + UsageAlertRule, + UsageAlertRuleInput, + UsageAlertRuleList, + UsageAlertScope, + UsageAlertStatus, +) + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +class AlertDatabase(Protocol): + async def query_raw(self, query: str, *args: object) -> object: ... + + +class _Record(BaseModel): + record: dict[str, object] + + +class _Count(BaseModel): + count: int + + +class _Episode(_Record): + evaluation_day: date + breached: bool + + +class _Pending(_Record): + rule: dict[str, object] + notified_channels: tuple[UsageAlertChannel, ...] + + +class _Ownership(BaseModel): + user_id: str | None + team_id: str | None + + +def activity_path( + scope: UsageAlertScope, entity_id: str | None, day: date, alert_id: str, baseline_days: int = 7 +) -> str: + routes: Final = { + "key": ("/api-keys", "key"), + "team": ("/teams", "team"), + "user": ("/users", "user"), + "model": ("/models-and-endpoints", "activity_model"), + "deployment": ("/usage", ""), + } + route, parameter = routes[scope] + if scope == "deployment": + return ( + route + + "?" + + urlencode( + { + "range": "custom", + "from": (day - timedelta(days=baseline_days)).isoformat(), + "to": day.isoformat(), + "alert": alert_id, + } + ) + ) + query: Final = { + "activity_from": (day - timedelta(days=baseline_days)).isoformat(), + "activity_to": day.isoformat(), + "alert": alert_id, + } + if parameter and entity_id and not entity_id.startswith("sk-"): + query[parameter] = entity_id + query["detail_tab"] = "activity" + return route + "?" + urlencode(query) + + +class UsageAlertsRepository: + def __init__(self, database: AlertDatabase): + self.database: Final = database + + @classmethod + def from_prisma(cls, prisma: "PrismaClient") -> "UsageAlertsRepository": + return cls(cast(AlertDatabase, prisma.writer_db)) + + async def get_rule(self, rule_id: str) -> UsageAlertRule | None: + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND NOT deleted', rule_id + ) + ) + return UsageAlertRule.model_validate(rows[0].record) if rows else None + + async def save_rule( + self, data: UsageAlertRuleInput, created_by: str | None, existing: UsageAlertRule | None = None + ) -> UsageAlertRule: + now: Final = datetime.now(timezone.utc) + rule: Final = UsageAlertRule( + **data.model_dump(), + id=existing.id if existing else str(uuid4()), + created_by=existing.created_by if existing else created_by, + created_at=existing.created_at if existing else now, + updated_at=now, + ) + rows: Final = await self.database.query_raw( + """INSERT INTO "LiteLLM_UsageAlertRule" (id,scope_type,scope_id,enabled,record) VALUES ($1,$2,$3,$4,$5::jsonb) + ON CONFLICT (id) DO UPDATE SET scope_type=EXCLUDED.scope_type,scope_id=EXCLUDED.scope_id,enabled=EXCLUDED.enabled,record=EXCLUDED.record + WHERE NOT "LiteLLM_UsageAlertRule".deleted RETURNING record""", + rule.id, + rule.scope_type, + rule.scope_id, + rule.enabled, + rule.model_dump_json(), + ) + return UsageAlertRule.model_validate(TypeAdapter(tuple[_Record, ...]).validate_python(rows)[0].record) + + async def delete_rule(self, rule_id: str) -> None: + await self.database.query_raw( + 'UPDATE "LiteLLM_UsageAlertRule" SET deleted=true,enabled=false WHERE id=$1 RETURNING id', rule_id + ) + + async def list_rules( + self, scope_type: UsageAlertScope | None = None, scope_id: str | None = None, limit: int = 50, offset: int = 0 + ) -> UsageAlertRuleList: + args: Final = (scope_type, scope_id) + where: Final = "NOT deleted AND ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2)" + count: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + f'SELECT count(*)::int AS count FROM "LiteLLM_UsageAlertRule" WHERE {where}', *args + ) + ) + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + f'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE {where} ORDER BY id LIMIT $3 OFFSET $4', + *args, + limit, + offset, + ) + ) + return UsageAlertRuleList( + data=tuple(UsageAlertRule.model_validate(row.record) for row in rows), + total=count[0].count, + limit=limit, + offset=offset, + ) + + async def list_enabled_rules(self, limit: int = 100, offset: int = 0) -> tuple[UsageAlertRule, ...]: + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE enabled AND NOT deleted ORDER BY id LIMIT $1 OFFSET $2', + limit, + offset, + ) + ) + return tuple(UsageAlertRule.model_validate(row.record) for row in rows) + + async def get_alert(self, alert_id: str) -> UsageAlert | None: + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw('SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE id=$1', alert_id) + ) + return UsageAlert.model_validate(rows[0].record) if rows else None + + async def list_alerts( + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + metric: UsageAlertMetric | None = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: int = 50, + offset: int = 0, + ) -> UsageAlertList: + args: Final = ( + scope_type, + scope_id, + metric, + status, + start_date.isoformat() if start_date else None, + end_date.isoformat() if end_date else None, + ) + where: Final = "($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2) AND ($3::text IS NULL OR metric=$3) AND ($4::text IS NULL OR status=$4) AND ($5::text IS NULL OR day >= $5) AND ($6::text IS NULL OR day <= $6)" + count: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + f'SELECT count(*)::int AS count FROM "LiteLLM_UsageAlertIncident" WHERE {where}', *args + ) + ) + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + f'SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE {where} ORDER BY day DESC,id LIMIT $7 OFFSET $8', + *args, + limit, + offset, + ) + ) + return UsageAlertList( + data=tuple(UsageAlert.model_validate(row.record) for row in rows), + total=count[0].count, + limit=limit, + offset=offset, + ) + + async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None: + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + """UPDATE "LiteLLM_UsageAlertIncident" SET status=$2, record=jsonb_set(record,'{status}',to_jsonb($2::text)),updated_at=CURRENT_TIMESTAMP WHERE id=$1 RETURNING record""", + alert_id, + status, + ) + ) + return UsageAlert.model_validate(rows[0].record) if rows else None + + async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None: + rows: Final = TypeAdapter(tuple[_Ownership, ...]).validate_python( + await self.database.query_raw( + 'SELECT user_id,team_id FROM "LiteLLM_VerificationToken" WHERE token=$1', key_id + ) + ) + return (rows[0].user_id, rows[0].team_id) if rows else None + + async def claim_job(self, owner: str, lease_seconds: int = 900) -> bool: + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + """INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,owner,expires_at) VALUES ('worker',$1,CURRENT_TIMESTAMP+($2::int * INTERVAL '1 second')) + ON CONFLICT(id) DO UPDATE SET owner=EXCLUDED.owner,expires_at=EXCLUDED.expires_at WHERE "LiteLLM_UsageAlertCheckpoint".expires_at None: + await self.database.query_raw( + "DELETE FROM \"LiteLLM_UsageAlertCheckpoint\" WHERE id='worker' AND owner=$1 RETURNING id", owner + ) + + @staticmethod + def _checkpoint(rule: UsageAlertRule, day: date) -> str: + return f"{rule.id}:{rule.updated_at.isoformat()}:{day.isoformat()}" + + async def is_day_complete(self, rule: UsageAlertRule, day: date) -> bool: + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + 'SELECT 1 AS count FROM "LiteLLM_UsageAlertCheckpoint" WHERE id=$1 AND completed', + self._checkpoint(rule, day), + ) + ) + return bool(rows) + + async def finish_day(self, rule: UsageAlertRule, day: date) -> None: + await self.database.query_raw( + 'INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed) VALUES ($1,true) ON CONFLICT(id) DO NOTHING RETURNING id', + self._checkpoint(rule, day), + ) + + async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None: + for rule in rules: + rule_id: Final = str( + uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}") + ) + now: Final = datetime.now(timezone.utc) + record: Final = UsageAlertRule( + **rule.model_dump(), id=rule_id, created_at=now, updated_at=now, created_by=None + ) + await self.database.query_raw( + 'INSERT INTO "LiteLLM_UsageAlertRule" (id,scope_type,scope_id,enabled,record) VALUES ($1,$2,$3,$4,$5::jsonb) ON CONFLICT(id) DO NOTHING RETURNING id', + rule_id, + rule.scope_type, + rule.scope_id, + rule.enabled, + record.model_dump_json(), + ) + + async def list_breached_entity_ids(self, rule_id: str, limit: int = 500, offset: int = 0) -> tuple[str | None, ...]: + rows: Final = TypeAdapter(tuple[dict[str, str | None], ...]).validate_python( + await self.database.query_raw( + 'SELECT DISTINCT scope_id FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND breached ORDER BY scope_id NULLS FIRST LIMIT $2 OFFSET $3', + rule_id, + limit, + offset, + ) + ) + return tuple(row["scope_id"] for row in rows) + + async def record_evaluation_status( + self, rule: UsageAlertRule, day: date, status: UsageAlertEvaluationStatus + ) -> None: + await self.database.query_raw( + """UPDATE "LiteLLM_UsageAlertRule" SET record=record || jsonb_build_object('last_evaluated_day',$2::text,'evaluation_status',$3::text) WHERE id=$1 AND record->>'updated_at'=$4 AND NOT deleted RETURNING id""", + rule.id, + day.isoformat(), + status, + rule.model_dump(mode="json")["updated_at"], + ) + + async def record_observation( + self, + rule: UsageAlertRule, + entity_id: str | None, + day: date, + observed: float, + baseline: float, + threshold: float, + breached: bool, + ) -> UsageAlert | None: + observation: Final = UsageAlertObservation( + entity_id=entity_id, day=day, observed=observed, baseline=baseline, threshold=threshold, breached=breached + ) + changed: Final = await self.record_observations(rule, (observation,)) + if changed: + return changed[0] + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=$2 ORDER BY evaluation_day DESC,id DESC LIMIT 1', + rule.id, + entity_id or "", + ) + ) + return UsageAlert.model_validate(rows[0].record) if rows else None + + async def record_observations( + self, rule: UsageAlertRule, observations: tuple[UsageAlertObservation, ...] + ) -> tuple[UsageAlert, ...]: + if not observations: + return () + if len(observations) > 500 or len({item.entity_id for item in observations}) != len(observations): + raise ValueError("An observation batch must contain at most 500 distinct entities") + now: Final = datetime.now(timezone.utc) + records: Final = [] + for item in observations: + alert_id: Final = str( + uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.id}:{item.entity_id or ''}:{item.day.isoformat()}") + ) + incident: Final = UsageAlert( + id=alert_id, + rule_id=rule.id, + rule_name=rule.name, + scope_type=rule.scope_type, + scope_id=item.entity_id, + metric=rule.metric, + direction=rule.direction, + status="open", + day=item.day, + baseline_start=item.day - timedelta(days=rule.baseline_days), + baseline_end=item.day - timedelta(days=1), + observed_value=item.observed, + baseline_value=item.baseline, + threshold=item.threshold, + first_seen=now, + last_seen=now, + activity_path=activity_path(rule.scope_type, item.entity_id, item.day, alert_id, rule.baseline_days), + ) + records.append( + { + "scope_key": item.entity_id or "", + "breached": item.breached, + "day": item.day.isoformat(), + "record": incident.model_dump(mode="json"), + } + ) + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + """ + WITH input AS (SELECT * FROM jsonb_to_recordset($2::jsonb) AS x(scope_key text, breached boolean, day text, record jsonb)), + prior AS ( + SELECT x.*,p.id AS prior_id,p.evaluation_day,p.breached AS prior_breached,p.record AS prior_record + FROM input x LEFT JOIN LATERAL ( + SELECT * FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=x.scope_key ORDER BY evaluation_day DESC,id DESC LIMIT 1 + ) p ON true + WHERE EXISTS (SELECT 1 FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND enabled AND NOT deleted AND record->>'updated_at'=$3) + ), updated AS ( + UPDATE "LiteLLM_UsageAlertIncident" i SET + evaluation_day=p.day, breached=p.breached, + day=CASE WHEN p.breached THEN p.day ELSE i.day END, + record=(CASE WHEN p.breached THEN p.record || jsonb_build_object( + 'id',i.id,'first_seen',i.record->'first_seen', + 'activity_path',replace(p.record->>'activity_path',p.record->>'id',i.id)) ELSE i.record END) + || jsonb_build_object('status',CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END), + status=CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END, + updated_at=CURRENT_TIMESTAMP + FROM prior p WHERE i.id=p.prior_id AND p.evaluation_day>'id',$1,p.scope_key,p.record->>'scope_type',p.record->>'scope_id',p.record->>'metric','open',p.day,p.day,true,p.record + FROM prior p WHERE p.breached AND (p.prior_id IS NULL OR (NOT p.prior_breached AND p.evaluation_day tuple[UsageAlertNotification, ...]: + rows: Final = TypeAdapter(tuple[_Pending, ...]).validate_python( + await self.database.query_raw( + """SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id + WHERE i.status='open' AND r.enabled AND NOT r.deleted AND EXISTS ( + SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels))) + ORDER BY i.updated_at,i.id LIMIT $1""", + limit, + ) + ) + return tuple( + UsageAlertNotification( + incident=UsageAlert.model_validate(row.record), + rule=UsageAlertRule.model_validate(row.rule), + pending_channels=tuple( + channel + for channel in UsageAlertRule.model_validate(row.rule).notification_channels + if channel not in row.notified_channels + ), + ) + for row in rows + ) + + async def mark_notified(self, incident_id: str, channel: UsageAlertChannel) -> None: + await self.database.query_raw( + 'UPDATE "LiteLLM_UsageAlertIncident" SET notified_channels=array_append(notified_channels,$2::text) WHERE id=$1 AND NOT ($2::text=ANY(notified_channels)) RETURNING id', + incident_id, + channel, + ) + + async def prune(self, retention_days: int = 90, limit: int = 500) -> None: + await self.database.query_raw( + """DELETE FROM "LiteLLM_UsageAlertIncident" WHERE id IN (SELECT id FROM "LiteLLM_UsageAlertIncident" WHERE status<>'open' AND updated_at str | None: + rows: Final = TypeAdapter(tuple[dict[str, str], ...]).validate_python( + await self.database.query_raw( + """SELECT member->>'role' AS role FROM "LiteLLM_TeamTable", jsonb_array_elements(members_with_roles) member WHERE team_id=$1 AND member->>'user_id'=$2 LIMIT 1""", + team_id, + user_id, + ) + ) + return rows[0]["role"] if rows else None diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py new file mode 100644 index 00000000000..b0bd8d148ea --- /dev/null +++ b/litellm/types/proxy/usage_alerts.py @@ -0,0 +1,116 @@ +"""Daily usage alert rules and incidents shared by the API and evaluator.""" + +from datetime import date, datetime +from typing import Literal, Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +UsageAlertScope = Literal["deployment", "key", "user", "team", "model"] +UsageAlertMetric = Literal["spend", "requests", "latency", "errors"] +UsageAlertDirection = Literal["increase", "decrease"] +UsageAlertStatus = Literal["open", "resolved", "dismissed"] +UsageAlertChannel = Literal["email", "slack"] +UsageAlertEvaluationStatus = Literal["pending", "complete", "insufficient_coverage", "insufficient_history"] + + +class UsageAlertRuleInput(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + name: str = Field(min_length=1, max_length=200) + scope_type: UsageAlertScope + scope_id: str | None = Field(default=None, min_length=1, max_length=1024) + metric: UsageAlertMetric + direction: UsageAlertDirection + multiplier: float = Field(default=3, gt=1, le=1000) + baseline_days: int = Field(default=7, ge=1, le=30) + min_history_days: int = Field(default=3, ge=1, le=30) + min_requests: int = Field(default=100, ge=0) + min_value: float = Field(default=0, ge=0, description="Minimum absolute change in the metric's units") + enabled: bool = True + notification_channels: tuple[UsageAlertChannel, ...] = Field(default=(), max_length=2) + email_recipients: tuple[str, ...] = Field(default=(), max_length=20) + + @model_validator(mode="after") + def validate_rule(self) -> Self: + if self.min_history_days > self.baseline_days: + raise ValueError("min_history_days must not exceed baseline_days") + if self.scope_type == "deployment" and self.scope_id is not None: + raise ValueError("Deployment alerts cover the whole gateway; omit scope_id") + if self.scope_type == "key" and self.scope_id is not None and self.scope_id.startswith("sk-"): + raise ValueError("Use the key's reporting ID, never the secret key") + if len(set(self.notification_channels)) != len(self.notification_channels): + raise ValueError("notification_channels must be unique") + if "email" in self.notification_channels and not self.email_recipients: + raise ValueError("Email notifications require at least one recipient") + if any( + "@" not in email or "\n" in email or "\r" in email or len(email) > 254 for email in self.email_recipients + ): + raise ValueError("Invalid email recipient") + return self + + +class UsageAlertRule(UsageAlertRuleInput): + id: str + created_at: datetime + updated_at: datetime + created_by: str | None + last_evaluated_day: date | None = None + evaluation_status: UsageAlertEvaluationStatus = "pending" + + +class UsageAlert(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + id: str + rule_id: str + rule_name: str + scope_type: UsageAlertScope + scope_id: str | None + metric: UsageAlertMetric + direction: UsageAlertDirection + status: UsageAlertStatus + day: date + baseline_start: date + baseline_end: date + observed_value: float + baseline_value: float + threshold: float + first_seen: datetime + last_seen: datetime + activity_path: str + + +class UsageAlertUpdate(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + status: UsageAlertStatus + + +class UsageAlertRuleList(BaseModel): + data: tuple[UsageAlertRule, ...] + total: int + limit: int + offset: int + + +class UsageAlertList(BaseModel): + data: tuple[UsageAlert, ...] + total: int + limit: int + offset: int + + +class UsageAlertNotification(BaseModel): + model_config = ConfigDict(frozen=True) + incident: UsageAlert + rule: UsageAlertRule + pending_channels: tuple[UsageAlertChannel, ...] + + +class UsageAlertObservation(BaseModel): + model_config = ConfigDict(frozen=True, allow_inf_nan=False) + entity_id: str | None + day: date + observed: float + baseline: float + threshold: float + breached: bool diff --git a/schema.prisma b/schema.prisma index 78f16d9f1fd..46ab28530a9 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2109,3 +2109,43 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +// Daily alerts consume existing aggregate tables; no request-path writes. +model LiteLLM_UsageAlertRule { + id String @id + scope_type String + scope_id String? + enabled Boolean @default(true) + record Json + deleted Boolean @default(false) + @@index([enabled, deleted, id]) + @@index([scope_type, scope_id, deleted]) +} + +model LiteLLM_UsageAlertIncident { + id String @id + rule_id String + scope_key String + scope_type String + scope_id String? + metric String + status String + day String + evaluation_day String + breached Boolean + record Json + notified_channels String[] @default([]) + updated_at DateTime @default(now()) + @@index([rule_id, scope_key, evaluation_day]) + @@index([scope_type, scope_id, day]) + @@index([status, day]) + @@index([updated_at]) +} + +model LiteLLM_UsageAlertCheckpoint { + id String @id + completed Boolean @default(false) + owner String? + expires_at DateTime @default(now()) + @@index([expires_at]) +} From c741d1271df132f7c22dabc7e9575bb2328daf42 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 11:49:51 +0800 Subject: [PATCH 02/38] test: validate daily alerts and document the public API --- .github/workflows/test-unit.yml | 1 + docs/features/usage-alerts.md | 71 +++ litellm/proxy/_types.py | 4 + .../proxy/alerting_endpoints/usage_alerts.py | 22 +- litellm/repositories/usage_alerts.py | 162 ++--- litellm/types/proxy/usage_alerts.py | 22 +- .../integration/database/test_usage_alerts.py | 78 +++ .../alerting_endpoints/test_usage_alerts.py | 67 ++ tests/unit/types/proxy/test_usage_alerts.py | 43 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 582 ++++++++++++++++++ 10 files changed, 958 insertions(+), 94 deletions(-) create mode 100644 docs/features/usage-alerts.md create mode 100644 tests/integration/database/test_usage_alerts.py create mode 100644 tests/unit/proxy/alerting_endpoints/test_usage_alerts.py create mode 100644 tests/unit/types/proxy/test_usage_alerts.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f1456b1fde8..b85238ce58c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -537,6 +537,7 @@ jobs: tests/unit/proxy/management_endpoints tests/unit/proxy/management_helpers tests/unit/proxy/list_api + tests/unit/proxy/alerting_endpoints tests/unit/proxy/analytics_endpoints tests/unit/proxy/decisions_endpoints tests/unit/proxy/memory diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md new file mode 100644 index 00000000000..d596d17f1a4 --- /dev/null +++ b/docs/features/usage-alerts.md @@ -0,0 +1,71 @@ +# Daily usage alerts + +Daily usage alerts compare a completed UTC day with earlier recorded days. They cover spend, request count, average latency, and failed-request count for the gateway, a key, a user, a team, or a model. Open an incident to inspect its saved comparison and follow its Activity link to the corresponding usage view. + +This feature requires a database. Apply the normal LiteLLM database migrations when deploying a version that includes usage alerts. The API and dashboard use the same persisted rules and incidents; no enterprise license is required. + +## Read incidents + +Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the key in your environment. + +```bash +curl "$LITELLM_API_BASE/usage/alerts?status=open&limit=50" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +``` + +The response has `data`, `total`, `limit`, and `offset`. Each incident includes its scope, metric, direction, status, UTC day, observed value, baseline value, threshold, baseline date range, and `activity_path`. The Activity path is relative to the dashboard mount: for example, `/api-keys?...` is opened below the dashboard's `/ui` mount. + +`GET /usage/alerts/{alert_id}` returns one incident. List filters are `scope_type`, `scope_id`, `metric`, `status`, `start_date`, and `end_date`. Dates use `YYYY-MM-DD`; date filters include both endpoints. The maximum page size is 100. + +## Create a rule + +This example alerts when a team's daily spend is at least three times its baseline and the absolute change is at least $10. Replace `team-example` with the team's ID. + +```bash +curl "$LITELLM_API_BASE/usage/alert-rules" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Team daily spend", + "scope_type": "team", + "scope_id": "team-example", + "metric": "spend", + "direction": "increase", + "multiplier": 3, + "baseline_days": 7, + "min_history_days": 3, + "min_requests": 100, + "min_value": 10, + "enabled": true + }' +``` + +The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value at or below half the baseline. `min_value` is the minimum absolute difference in the metric's own units. + +| Metric | Unit | +| --- | --- | +| `spend` | USD | +| `requests` | Request count | +| `latency` | Average response time in milliseconds | +| `errors` | Failed-request count | + +The evaluator runs against daily aggregates. An hourly scheduler does not make these rolling hourly alerts. A complete UTC day and sufficient ingestion coverage are required before detecting drops; a missing flush must not become a false traffic-drop incident. Rule responses expose `last_evaluated_day` and `evaluation_status` so you can distinguish a completed evaluation from insufficient history or coverage. + +Use `scope_type: "deployment"` with no `scope_id` for the gateway. For `model`, use the underlying model identity recorded in Activity. For `key`, use its reporting ID, never the secret API key. Omitting `scope_id` for key, user, team, or model evaluates all entities of that type and requires a gateway administrator. + +## Manage rules and incidents + +- `GET /usage/alert-rules` lists rules; `GET /usage/alert-rules/{rule_id}` reads one. +- `PUT /usage/alert-rules/{rule_id}` replaces the rule configuration. Send the configuration fields from the create example, including any notification settings you want to retain. Set `enabled` to `false` to pause it. +- `DELETE /usage/alert-rules/{rule_id}` removes a rule from evaluation and preserves existing incidents. Deleted defaults stay deleted. +- `PATCH /usage/alerts/{alert_id}` with `{"status":"dismissed"}` dismisses an incident. `open` and `resolved` are also accepted. + +Consecutive breached days update one incident. Re-evaluating the same day does not reopen a dismissed or resolved incident. After recovery, a later breach starts a new incident. Editing a rule does not rewrite historical comparisons. + +Rules can specify `notification_channels: ["slack", "email"]` and `email_recipients: ["alerts@example.com"]`. Notification transports must already be configured on the gateway. An empty channel list keeps incidents available through the API and dashboard. Successful delivery is recorded per channel so a failed channel can retry independently. + +## Access + +Gateway administrators can manage every scope. Admin viewers can read every scope. Internal users can manage their own user and key scopes; team administrators can manage their team's scope and keys. Team members can read their team's incidents. Read-only users cannot change rules or incident state, and inference-only keys cannot access these endpoints. + +Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` indicates that the gateway has no database connection. diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 14e71a0f292..de12eb00c6f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -579,6 +579,10 @@ class LiteLLMRoutes(enum.Enum): ) llm_api_routes_without_mcp = sorted(set(llm_api_routes) - set(mcp_inference_routes)) info_routes = [ + "/usage/alert-rules", + "/usage/alert-rules/{rule_id}", + "/usage/alerts", + "/usage/alerts/{alert_id}", "/key/info", "/key/health", "/team/info", diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py index 3a4c4112581..a13e927ddb3 100644 --- a/litellm/proxy/alerting_endpoints/usage_alerts.py +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -1,7 +1,7 @@ """Public management API for daily usage alert rules and incidents.""" from datetime import date -from typing import Annotated, Final +from typing import Annotated, Final, TypeAlias from fastapi import APIRouter, Depends, HTTPException, Query, Response @@ -32,11 +32,11 @@ def usage_alert_repository(database: DatabaseDep) -> UsageAlertsRepository: return UsageAlertsRepository.from_prisma(database.prisma) -User = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] -Repository = Annotated[UsageAlertsRepository, Depends(usage_alert_repository)] -Limit = Annotated[int, Query(ge=1, le=100)] -Offset = Annotated[int, Query(ge=0, le=100000)] -ScopeId = Annotated[str | None, Query(min_length=1, max_length=1024)] +User: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] +Repository: TypeAlias = Annotated[UsageAlertsRepository, Depends(usage_alert_repository)] +Limit: TypeAlias = Annotated[int, Query(ge=1, le=100)] +Offset: TypeAlias = Annotated[int, Query(ge=0, le=100000)] +ScopeId: TypeAlias = Annotated[str | None, Query(min_length=1, max_length=1024)] async def require_scope( @@ -101,7 +101,10 @@ async def list_usage_alert_rules( @router.post("/alert-rules", response_model=UsageAlertRule, status_code=201) async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, repository: Repository) -> UsageAlertRule: await require_scope(user, repository, data.scope_type, data.scope_id, write=True) - return await repository.save_rule(data, user.user_id) + saved: Final = await repository.save_rule(data, user.user_id) + if saved is None: + raise HTTPException(status_code=409, detail="Alert rule could not be saved") + return saved @router.get("/alert-rules/{rule_id}", response_model=UsageAlertRule) @@ -122,7 +125,10 @@ async def update_usage_alert_rule( raise HTTPException(status_code=404, detail="Alert rule not found") await require_scope(user, repository, rule.scope_type, rule.scope_id, write=True) await require_scope(user, repository, data.scope_type, data.scope_id, write=True) - return await repository.save_rule(data, user.user_id, existing=rule) + saved: Final = await repository.save_rule(data, user.user_id, existing=rule) + if saved is None: + raise HTTPException(status_code=404, detail="Alert rule not found") + return saved @router.delete("/alert-rules/{rule_id}", status_code=204) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 0a42657c478..130f9fb0549 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -1,8 +1,14 @@ """Bounded, writer-consistent storage for daily alert rules and episodes.""" import json +from collections.abc import Mapping from datetime import date, datetime, timedelta, timezone -from typing import TYPE_CHECKING, Final, Protocol, cast +from typing import ( # PrismaWrapper forwards generated client methods dynamically + TYPE_CHECKING, + Final, + Protocol, + cast, # noqa: TID251 # PrismaWrapper forwards generated client methods dynamically +) from urllib.parse import urlencode from uuid import NAMESPACE_URL, uuid4, uuid5 @@ -31,25 +37,20 @@ class AlertDatabase(Protocol): async def query_raw(self, query: str, *args: object) -> object: ... -class _Record(BaseModel): - record: dict[str, object] +class _Record(BaseModel, frozen=True): + record: Mapping[str, object] -class _Count(BaseModel): +class _Count(BaseModel, frozen=True): count: int -class _Episode(_Record): - evaluation_day: date - breached: bool - - -class _Pending(_Record): - rule: dict[str, object] +class _Pending(_Record, frozen=True): + rule: Mapping[str, object] notified_channels: tuple[UsageAlertChannel, ...] -class _Ownership(BaseModel): +class _Ownership(BaseModel, frozen=True): user_id: str | None team_id: str | None @@ -71,7 +72,7 @@ def activity_path( + "?" + urlencode( { - "range": "custom", + "tab": "keys", "from": (day - timedelta(days=baseline_days)).isoformat(), "to": day.isoformat(), "alert": alert_id, @@ -89,13 +90,47 @@ def activity_path( return route + "?" + urlencode(query) +def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now: datetime) -> Mapping[str, object]: + alert_id: Final = str( + uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.id}:{item.entity_id or ''}:{item.day.isoformat()}") + ) + incident: Final = UsageAlert( + id=alert_id, + rule_id=rule.id, + rule_name=rule.name, + scope_type=rule.scope_type, + scope_id=item.entity_id, + metric=rule.metric, + direction=rule.direction, + status="open", + day=item.day, + baseline_start=item.day - timedelta(days=rule.baseline_days), + baseline_end=item.day - timedelta(days=1), + baseline_sample_days=item.baseline_sample_days, + observed_value=item.observed, + baseline_value=item.baseline, + threshold=item.threshold, + first_seen=now, + last_seen=now, + activity_path=activity_path(rule.scope_type, item.entity_id, item.day, alert_id, rule.baseline_days), + ) + return { + "scope_key": item.entity_id or "", + "breached": item.breached, + "day": item.day.isoformat(), + "record": incident.model_dump(mode="json"), + } + + class UsageAlertsRepository: - def __init__(self, database: AlertDatabase): + def __init__(self, database: AlertDatabase) -> None: self.database: Final = database @classmethod def from_prisma(cls, prisma: "PrismaClient") -> "UsageAlertsRepository": - return cls(cast(AlertDatabase, prisma.writer_db)) + return cls( + cast(AlertDatabase, prisma.writer_db) # cast-ok: PrismaWrapper forwards query_raw to its generated client + ) async def get_rule(self, rule_id: str) -> UsageAlertRule | None: rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( @@ -107,14 +142,16 @@ class UsageAlertsRepository: async def save_rule( self, data: UsageAlertRuleInput, created_by: str | None, existing: UsageAlertRule | None = None - ) -> UsageAlertRule: + ) -> UsageAlertRule | None: now: Final = datetime.now(timezone.utc) - rule: Final = UsageAlertRule( - **data.model_dump(), - id=existing.id if existing else str(uuid4()), - created_by=existing.created_by if existing else created_by, - created_at=existing.created_at if existing else now, - updated_at=now, + rule: Final = UsageAlertRule.model_validate( + { + **data.model_dump(), + "id": existing.id if existing else str(uuid4()), + "created_by": existing.created_by if existing else created_by, + "created_at": existing.created_at if existing else now, + "updated_at": now, + } ) rows: Final = await self.database.query_raw( """INSERT INTO "LiteLLM_UsageAlertRule" (id,scope_type,scope_id,enabled,record) VALUES ($1,$2,$3,$4,$5::jsonb) @@ -126,7 +163,8 @@ class UsageAlertsRepository: rule.enabled, rule.model_dump_json(), ) - return UsageAlertRule.model_validate(TypeAdapter(tuple[_Record, ...]).validate_python(rows)[0].record) + saved: Final = TypeAdapter(tuple[_Record, ...]).validate_python(rows) + return UsageAlertRule.model_validate(saved[0].record) if saved else None async def delete_rule(self, rule_id: str) -> None: await self.database.query_raw( @@ -269,24 +307,27 @@ class UsageAlertsRepository: async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None: for rule in rules: - rule_id: Final = str( - uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}") - ) - now: Final = datetime.now(timezone.utc) - record: Final = UsageAlertRule( - **rule.model_dump(), id=rule_id, created_at=now, updated_at=now, created_by=None - ) - await self.database.query_raw( - 'INSERT INTO "LiteLLM_UsageAlertRule" (id,scope_type,scope_id,enabled,record) VALUES ($1,$2,$3,$4,$5::jsonb) ON CONFLICT(id) DO NOTHING RETURNING id', - rule_id, - rule.scope_type, - rule.scope_id, - rule.enabled, - record.model_dump_json(), - ) + await self._ensure_default_rule(rule) + + async def _ensure_default_rule(self, rule: UsageAlertRuleInput) -> None: + rule_id: Final = str( + uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}") + ) + now: Final = datetime.now(timezone.utc) + record: Final = UsageAlertRule.model_validate( + {**rule.model_dump(), "id": rule_id, "created_at": now, "updated_at": now, "created_by": None} + ) + await self.database.query_raw( + 'INSERT INTO "LiteLLM_UsageAlertRule" (id,scope_type,scope_id,enabled,record) VALUES ($1,$2,$3,$4,$5::jsonb) ON CONFLICT(id) DO NOTHING RETURNING id', + rule_id, + rule.scope_type, + rule.scope_id, + rule.enabled, + record.model_dump_json(), + ) async def list_breached_entity_ids(self, rule_id: str, limit: int = 500, offset: int = 0) -> tuple[str | None, ...]: - rows: Final = TypeAdapter(tuple[dict[str, str | None], ...]).validate_python( + rows: Final = TypeAdapter(tuple[Mapping[str, str | None], ...]).validate_python( await self.database.query_raw( 'SELECT DISTINCT scope_id FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND breached ORDER BY scope_id NULLS FIRST LIMIT $2 OFFSET $3', rule_id, @@ -300,11 +341,11 @@ class UsageAlertsRepository: self, rule: UsageAlertRule, day: date, status: UsageAlertEvaluationStatus ) -> None: await self.database.query_raw( - """UPDATE "LiteLLM_UsageAlertRule" SET record=record || jsonb_build_object('last_evaluated_day',$2::text,'evaluation_status',$3::text) WHERE id=$1 AND record->>'updated_at'=$4 AND NOT deleted RETURNING id""", + """UPDATE "LiteLLM_UsageAlertRule" SET record=record || jsonb_build_object('last_evaluated_day',$2::text,'evaluation_status',$3::text) WHERE id=$1 AND (record->>'updated_at')::timestamptz=$4::timestamptz AND NOT deleted RETURNING id""", rule.id, day.isoformat(), status, - rule.model_dump(mode="json")["updated_at"], + rule.updated_at.isoformat(), ) async def record_observation( @@ -340,38 +381,7 @@ class UsageAlertsRepository: if len(observations) > 500 or len({item.entity_id for item in observations}) != len(observations): raise ValueError("An observation batch must contain at most 500 distinct entities") now: Final = datetime.now(timezone.utc) - records: Final = [] - for item in observations: - alert_id: Final = str( - uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.id}:{item.entity_id or ''}:{item.day.isoformat()}") - ) - incident: Final = UsageAlert( - id=alert_id, - rule_id=rule.id, - rule_name=rule.name, - scope_type=rule.scope_type, - scope_id=item.entity_id, - metric=rule.metric, - direction=rule.direction, - status="open", - day=item.day, - baseline_start=item.day - timedelta(days=rule.baseline_days), - baseline_end=item.day - timedelta(days=1), - observed_value=item.observed, - baseline_value=item.baseline, - threshold=item.threshold, - first_seen=now, - last_seen=now, - activity_path=activity_path(rule.scope_type, item.entity_id, item.day, alert_id, rule.baseline_days), - ) - records.append( - { - "scope_key": item.entity_id or "", - "breached": item.breached, - "day": item.day.isoformat(), - "record": incident.model_dump(mode="json"), - } - ) + records: Final = tuple(_observation_payload(rule, item, now) for item in observations) rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( await self.database.query_raw( """ @@ -381,7 +391,7 @@ class UsageAlertsRepository: FROM input x LEFT JOIN LATERAL ( SELECT * FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=x.scope_key ORDER BY evaluation_day DESC,id DESC LIMIT 1 ) p ON true - WHERE EXISTS (SELECT 1 FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND enabled AND NOT deleted AND record->>'updated_at'=$3) + WHERE EXISTS (SELECT 1 FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND enabled AND NOT deleted AND (record->>'updated_at')::timestamptz=$3::timestamptz) ), updated AS ( UPDATE "LiteLLM_UsageAlertIncident" i SET evaluation_day=p.day, breached=p.breached, @@ -403,7 +413,7 @@ class UsageAlertsRepository: """, rule.id, json.dumps(records), - rule.model_dump(mode="json")["updated_at"], + rule.updated_at.isoformat(), ) ) return tuple(UsageAlert.model_validate(row.record) for row in rows) @@ -450,7 +460,7 @@ class UsageAlertsRepository: ) async def team_role(self, team_id: str, user_id: str) -> str | None: - rows: Final = TypeAdapter(tuple[dict[str, str], ...]).validate_python( + rows: Final = TypeAdapter(tuple[Mapping[str, str], ...]).validate_python( await self.database.query_raw( """SELECT member->>'role' AS role FROM "LiteLLM_TeamTable", jsonb_array_elements(members_with_roles) member WHERE team_id=$1 AND member->>'user_id'=$2 LIMIT 1""", team_id, diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index b0bd8d148ea..0654fc3022b 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -1,16 +1,16 @@ """Daily usage alert rules and incidents shared by the API and evaluator.""" from datetime import date, datetime -from typing import Literal, Self +from typing import Literal, Self, TypeAlias from pydantic import BaseModel, ConfigDict, Field, model_validator -UsageAlertScope = Literal["deployment", "key", "user", "team", "model"] -UsageAlertMetric = Literal["spend", "requests", "latency", "errors"] -UsageAlertDirection = Literal["increase", "decrease"] -UsageAlertStatus = Literal["open", "resolved", "dismissed"] -UsageAlertChannel = Literal["email", "slack"] -UsageAlertEvaluationStatus = Literal["pending", "complete", "insufficient_coverage", "insufficient_history"] +UsageAlertScope: TypeAlias = Literal["deployment", "key", "user", "team", "model"] +UsageAlertMetric: TypeAlias = Literal["spend", "requests", "latency", "errors"] +UsageAlertDirection: TypeAlias = Literal["increase", "decrease"] +UsageAlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"] +UsageAlertChannel: TypeAlias = Literal["email", "slack"] +UsageAlertEvaluationStatus: TypeAlias = Literal["pending", "complete", "insufficient_coverage", "insufficient_history"] class UsageAlertRuleInput(BaseModel): @@ -36,7 +36,7 @@ class UsageAlertRuleInput(BaseModel): raise ValueError("min_history_days must not exceed baseline_days") if self.scope_type == "deployment" and self.scope_id is not None: raise ValueError("Deployment alerts cover the whole gateway; omit scope_id") - if self.scope_type == "key" and self.scope_id is not None and self.scope_id.startswith("sk-"): + if self.scope_type == "key" and self.scope_id is not None and self.scope_id.removeprefix("raw:").startswith("sk-"): raise ValueError("Use the key's reporting ID, never the secret key") if len(set(self.notification_channels)) != len(self.notification_channels): raise ValueError("notification_channels must be unique") @@ -72,6 +72,7 @@ class UsageAlert(BaseModel): day: date baseline_start: date baseline_end: date + baseline_sample_days: int | None = Field(default=None, ge=1, le=30) observed_value: float baseline_value: float threshold: float @@ -85,14 +86,14 @@ class UsageAlertUpdate(BaseModel): status: UsageAlertStatus -class UsageAlertRuleList(BaseModel): +class UsageAlertRuleList(BaseModel, frozen=True): data: tuple[UsageAlertRule, ...] total: int limit: int offset: int -class UsageAlertList(BaseModel): +class UsageAlertList(BaseModel, frozen=True): data: tuple[UsageAlert, ...] total: int limit: int @@ -112,5 +113,6 @@ class UsageAlertObservation(BaseModel): day: date observed: float baseline: float + baseline_sample_days: int | None = Field(default=None, ge=1, le=30) threshold: float breached: bool diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py new file mode 100644 index 00000000000..0bf6025130c --- /dev/null +++ b/tests/integration/database/test_usage_alerts.py @@ -0,0 +1,78 @@ +from hashlib import sha256 +from typing import Final + +from integration._support.client import Gateway, string_value + + +def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + outsider: Final = scenario.user(user_role="internal_user") + viewer: Final = scenario.user(user_role="proxy_admin_viewer") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + outsider_key: Final = scenario.key(user_id=outsider, key_type="management") + viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") + inference_key: Final = scenario.key(user_id=owner, key_type="llm_api") + config: Final = { + "name": "Daily key spend", + "scope_type": "key", + "scope_id": sha256(inference_key.encode()).hexdigest(), + "metric": "spend", + "direction": "increase", + "multiplier": 3, + } + created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + assert created.status_code == 201, created.text + rule_id: Final = string_value(created.json()["id"]) + path: Final = f"/usage/alert-rules/{rule_id}" + try: + assert gateway.request("GET", path, key=owner_key).status_code == 200 + assert gateway.request("GET", path, key=outsider_key).status_code == 403 + assert gateway.request("GET", path, key=inference_key).status_code == 403 + assert gateway.request("GET", path, key=viewer_key).status_code == 200 + assert gateway.request("PUT", path, config, key=viewer_key).status_code == 403 + assert gateway.request("PUT", path, config, key=outsider_key).status_code == 403 + assert ( + gateway.request("POST", "/usage/alert-rules", {**config, "scope_id": None}, key=owner_key).status_code + == 403 + ) + disabled: Final = gateway.request("PUT", path, {**config, "enabled": False}, key=owner_key) + assert disabled.status_code == 200 and disabled.json()["enabled"] is False + page: Final = gateway.request( + "GET", + "/usage/alert-rules", + key=owner_key, + params={"scope_type": "key", "scope_id": str(config["scope_id"]), "limit": "1"}, + ) + assert page.status_code == 200 and page.json()["total"] == 1 + assert page.json()["data"][0]["id"] == rule_id + finally: + assert gateway.request("DELETE", path, key=owner_key).status_code == 204 + assert gateway.request("GET", path, key=owner_key).status_code == 404 + + +def test_team_admin_can_manage_team_alerts_but_member_can_only_read(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team_id: Final = scenario.team() + admin_id: Final = scenario.member(team_id, role="admin") + member_id: Final = scenario.member(team_id, role="user") + admin_key: Final = scenario.key(user_id=admin_id, key_type="management") + member_key: Final = scenario.key(user_id=member_id, key_type="management") + config: Final = { + "name": "Daily team requests", + "scope_type": "team", + "scope_id": team_id, + "metric": "requests", + "direction": "decrease", + "multiplier": 2, + } + created: Final = gateway.request("POST", "/usage/alert-rules", config, key=admin_key) + assert created.status_code == 201, created.text + rule_id: Final = string_value(created.json()["id"]) + path: Final = f"/usage/alert-rules/{rule_id}" + try: + assert gateway.request("GET", path, key=member_key).status_code == 200 + assert gateway.request("PUT", path, config, key=member_key).status_code == 403 + assert gateway.request("DELETE", path, key=member_key).status_code == 403 + finally: + assert gateway.request("DELETE", path, key=admin_key).status_code == 204 diff --git a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py new file mode 100644 index 00000000000..1e26104494d --- /dev/null +++ b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py @@ -0,0 +1,67 @@ +from datetime import date +from typing import Final +from urllib.parse import parse_qs, urlsplit + +import pytest +from fastapi import HTTPException + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.alerting_endpoints.usage_alerts import require_scope +from litellm.repositories.usage_alerts import UsageAlertsRepository, activity_path +from litellm.types.proxy.auth.user_roles import LitellmUserRoles + + +class NoDatabase: + async def query_raw(self, query: str, *args: object) -> object: + raise AssertionError("Unauthorized callers must not reach the database") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "role,key_type,write", + [ + (LitellmUserRoles.PROXY_ADMIN, "llm_api", False), + (LitellmUserRoles.PROXY_ADMIN, "read_only", True), + (LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, None, True), + (LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, None, True), + (LitellmUserRoles.INTERNAL_USER, None, False), + (LitellmUserRoles.CUSTOMER, None, False), + ], +) +async def test_global_access_rejects_inference_keys_and_unprivileged_roles( + role: LitellmUserRoles, key_type: str | None, write: bool +) -> None: + user: Final = UserAPIKeyAuth(user_id="qa-user", user_role=role, key_type=key_type) + with pytest.raises(HTTPException) as error: + await require_scope(user, UsageAlertsRepository(NoDatabase()), "deployment", None, write=write) + assert error.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_internal_user_can_read_and_manage_only_their_user_scope() -> None: + user: Final = UserAPIKeyAuth(user_id="qa-user", user_role=LitellmUserRoles.INTERNAL_USER) + repository: Final = UsageAlertsRepository(NoDatabase()) + await require_scope(user, repository, "user", "qa-user", write=True) + with pytest.raises(HTTPException) as error: + await require_scope(user, repository, "user", "other-user", write=False) + assert error.value.status_code == 403 + + +def test_activity_link_escapes_entity_identity_and_includes_baseline() -> None: + path: Final = activity_path("model", "provider/model?foo=bar&other=1", date(2026, 11, 4), "qa-alert", 7) + parsed: Final = urlsplit(path) + query: Final = parse_qs(parsed.query) + assert parsed.path == "/models-and-endpoints" + assert query["activity_model"] == ["provider/model?foo=bar&other=1"] + assert query["activity_from"] == ["2026-10-28"] + assert query["activity_to"] == ["2026-11-04"] + assert query["alert"] == ["qa-alert"] + assert query["detail_tab"] == ["activity"] + + +def test_global_activity_link_uses_the_usage_page_date_contract() -> None: + query: Final = parse_qs(urlsplit(activity_path("deployment", None, date(2026, 11, 4), "qa-alert")).query) + assert query["tab"] == ["keys"] + assert "range" not in query + assert query["from"] == ["2026-10-28"] + assert query["to"] == ["2026-11-04"] diff --git a/tests/unit/types/proxy/test_usage_alerts.py b/tests/unit/types/proxy/test_usage_alerts.py new file mode 100644 index 00000000000..ad8aaaf493e --- /dev/null +++ b/tests/unit/types/proxy/test_usage_alerts.py @@ -0,0 +1,43 @@ +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.types.proxy.usage_alerts import UsageAlertRuleInput + +RULE: Final = { + "name": "Daily spend", + "scope_type": "team", + "scope_id": "qa-team", + "metric": "spend", + "direction": "increase", +} + + +@pytest.mark.parametrize( + "changes", + [ + {"multiplier": 1}, + {"multiplier": float("inf")}, + {"min_value": float("nan")}, + {"baseline_days": 2, "min_history_days": 3}, + {"baseline_days": 31}, + {"scope_type": "deployment"}, + {"scope_type": "key", "scope_id": "sk-test-placeholder"}, + {"notification_channels": ["email"]}, + {"notification_channels": ["slack", "slack"]}, + {"email_recipients": ["ops@example.com\r\nBcc: other@example.com"]}, + ], +) +def test_invalid_rules_cannot_be_stored(changes: dict[str, object]) -> None: + with pytest.raises(ValidationError): + UsageAlertRuleInput.model_validate({**RULE, **changes}) + + +def test_zero_request_floor_and_daily_drop_multiplier_are_supported() -> None: + rule: Final = UsageAlertRuleInput.model_validate( + {**RULE, "metric": "requests", "direction": "decrease", "multiplier": 2, "min_requests": 0} + ) + assert rule.direction == "decrease" + assert rule.multiplier == 2 + assert rule.min_requests == 0 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index da96e9ffa01..98f7c623c20 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -18834,6 +18834,78 @@ export interface paths { patch?: never; trace?: never; }; + "/usage/alert-rules": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Usage Alert Rules */ + get: operations["list_usage_alert_rules_usage_alert_rules_get"]; + put?: never; + /** Create Usage Alert Rule */ + post: operations["create_usage_alert_rule_usage_alert_rules_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/usage/alert-rules/{rule_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Usage Alert Rule */ + get: operations["get_usage_alert_rule_usage_alert_rules__rule_id__get"]; + /** Update Usage Alert Rule */ + put: operations["update_usage_alert_rule_usage_alert_rules__rule_id__put"]; + post?: never; + /** Delete Usage Alert Rule */ + delete: operations["delete_usage_alert_rule_usage_alert_rules__rule_id__delete"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/usage/alerts": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Usage Alerts */ + get: operations["list_usage_alerts_usage_alerts_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/usage/alerts/{alert_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Usage Alert */ + get: operations["get_usage_alert_usage_alerts__alert_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Usage Alert */ + patch: operations["update_usage_alert_usage_alerts__alert_id__patch"]; + trace?: never; + }; "/user/available_roles": { parameters: { query?: never; @@ -50331,6 +50403,250 @@ export interface components { */ version: string | null; }; + /** UsageAlert */ + UsageAlert: { + /** Activity Path */ + activity_path: string; + /** + * Baseline End + * Format: date + */ + baseline_end: string; + /** Baseline Sample Days */ + baseline_sample_days?: number | null; + /** + * Baseline Start + * Format: date + */ + baseline_start: string; + /** Baseline Value */ + baseline_value: number; + /** + * Day + * Format: date + */ + day: string; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * First Seen + * Format: date-time + */ + first_seen: string; + /** Id */ + id: string; + /** + * Last Seen + * Format: date-time + */ + last_seen: string; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors"; + /** Observed Value */ + observed_value: number; + /** Rule Id */ + rule_id: string; + /** Rule Name */ + rule_name: string; + /** Scope Id */ + scope_id: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + /** Threshold */ + threshold: number; + }; + /** UsageAlertList */ + UsageAlertList: { + /** Data */ + data: components["schemas"]["UsageAlert"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** UsageAlertRule */ + UsageAlertRule: { + /** + * Baseline Days + * @default 7 + */ + baseline_days: number; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** Created By */ + created_by: string | null; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * Email Recipients + * @default [] + */ + email_recipients: string[]; + /** + * Enabled + * @default true + */ + enabled: boolean; + /** + * Evaluation Status + * @default pending + * @enum {string} + */ + evaluation_status: "pending" | "complete" | "insufficient_coverage" | "insufficient_history"; + /** Id */ + id: string; + /** Last Evaluated Day */ + last_evaluated_day?: string | null; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors"; + /** + * Min History Days + * @default 3 + */ + min_history_days: number; + /** + * Min Requests + * @default 100 + */ + min_requests: number; + /** + * Min Value + * @description Minimum absolute change in the metric's units + * @default 0 + */ + min_value: number; + /** + * Multiplier + * @default 3 + */ + multiplier: number; + /** Name */ + name: string; + /** + * Notification Channels + * @default [] + */ + notification_channels: ("email" | "slack")[]; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; + /** UsageAlertRuleInput */ + UsageAlertRuleInput: { + /** + * Baseline Days + * @default 7 + */ + baseline_days: number; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * Email Recipients + * @default [] + */ + email_recipients: string[]; + /** + * Enabled + * @default true + */ + enabled: boolean; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors"; + /** + * Min History Days + * @default 3 + */ + min_history_days: number; + /** + * Min Requests + * @default 100 + */ + min_requests: number; + /** + * Min Value + * @description Minimum absolute change in the metric's units + * @default 0 + */ + min_value: number; + /** + * Multiplier + * @default 3 + */ + multiplier: number; + /** Name */ + name: string; + /** + * Notification Channels + * @default [] + */ + notification_channels: ("email" | "slack")[]; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + }; + /** UsageAlertRuleList */ + UsageAlertRuleList: { + /** Data */ + data: components["schemas"]["UsageAlertRule"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** UsageAlertUpdate */ + UsageAlertUpdate: { + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + }; /** UsageChartPoint */ UsageChartPoint: { /** Blocked */ @@ -78242,6 +78558,272 @@ export interface operations { }; }; }; + list_usage_alert_rules_usage_alert_rules_get: { + parameters: { + query?: { + scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; + scope_id?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertRuleList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + create_usage_alert_rule_usage_alert_rules_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UsageAlertRuleInput"]; + }; + }; + responses: { + /** @description Successful Response */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertRule"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_usage_alert_rule_usage_alert_rules__rule_id__get: { + parameters: { + query?: never; + header?: never; + path: { + rule_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertRule"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_usage_alert_rule_usage_alert_rules__rule_id__put: { + parameters: { + query?: never; + header?: never; + path: { + rule_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UsageAlertRuleInput"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertRule"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + delete_usage_alert_rule_usage_alert_rules__rule_id__delete: { + parameters: { + query?: never; + header?: never; + path: { + rule_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_usage_alerts_usage_alerts_get: { + parameters: { + query?: { + scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; + scope_id?: string | null; + metric?: ("spend" | "requests" | "latency" | "errors") | null; + status?: ("open" | "resolved" | "dismissed") | null; + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_usage_alert_usage_alerts__alert_id__get: { + parameters: { + query?: never; + header?: never; + path: { + alert_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlert"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_usage_alert_usage_alerts__alert_id__patch: { + parameters: { + query?: never; + header?: never; + path: { + alert_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UsageAlertUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlert"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; ui_get_available_role_user_available_roles_get: { parameters: { query?: never; From efaa14a9477316a118faed8ec2bce425a1bacc5c Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 11:52:54 +0800 Subject: [PATCH 03/38] refactor: provide usage alert storage through domain interface --- docs/features/usage-alerts.md | 6 +- litellm/models/usage_alerts.py | 47 ++++++++++++++ .../proxy/alerting_endpoints/usage_alerts.py | 62 +++++++++++-------- litellm/proxy/deps/repositories.py | 9 +++ .../integration/database/test_usage_alerts.py | 8 ++- .../alerting_endpoints/test_usage_alerts.py | 11 ++++ 6 files changed, 111 insertions(+), 32 deletions(-) create mode 100644 litellm/models/usage_alerts.py diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index d596d17f1a4..842709b66bc 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -19,7 +19,7 @@ The response has `data`, `total`, `limit`, and `offset`. Each incident includes ## Create a rule -This example alerts when a team's daily spend is at least three times its baseline and the absolute change is at least $10. Replace `team-example` with the team's ID. +This example alerts when a team's daily spend is more than three times its baseline and the absolute change is at least $10. Replace `team-example` with the team's ID. ```bash curl "$LITELLM_API_BASE/usage/alert-rules" \ @@ -40,7 +40,7 @@ curl "$LITELLM_API_BASE/usage/alert-rules" \ }' ``` -The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value at or below half the baseline. `min_value` is the minimum absolute difference in the metric's own units. +The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value below half the baseline. `min_value` is the minimum absolute difference in the metric's own units. | Metric | Unit | | --- | --- | @@ -49,6 +49,8 @@ The baseline is the mean across recorded days in the preceding window. Missing r | `latency` | Average response time in milliseconds | | `errors` | Failed-request count | +Counts come from the existing Activity/spend aggregates, rather than every gateway HTTP outcome. Latency averages successful timed, non-internal requests; it is not time to first token and does not include failed requests. Gateway incidents link to Key Activity, which uses the same aggregates. + The evaluator runs against daily aggregates. An hourly scheduler does not make these rolling hourly alerts. A complete UTC day and sufficient ingestion coverage are required before detecting drops; a missing flush must not become a false traffic-drop incident. Rule responses expose `last_evaluated_day` and `evaluation_status` so you can distinguish a completed evaluation from insufficient history or coverage. Use `scope_type: "deployment"` with no `scope_id` for the gateway. For `model`, use the underlying model identity recorded in Activity. For `key`, use its reporting ID, never the secret API key. Omitting `scope_id` for key, user, team, or model evaluates all entities of that type and requires a gateway administrator. diff --git a/litellm/models/usage_alerts.py b/litellm/models/usage_alerts.py new file mode 100644 index 00000000000..9a10be2d409 --- /dev/null +++ b/litellm/models/usage_alerts.py @@ -0,0 +1,47 @@ +from datetime import date +from typing import Protocol + +from litellm.types.proxy.usage_alerts import ( + UsageAlert, + UsageAlertList, + UsageAlertMetric, + UsageAlertRule, + UsageAlertRuleInput, + UsageAlertRuleList, + UsageAlertScope, + UsageAlertStatus, +) + + +class UsageAlertStore(Protocol): + async def get_rule(self, rule_id: str) -> UsageAlertRule | None: ... + + async def save_rule( + self, data: UsageAlertRuleInput, created_by: str | None, existing: UsageAlertRule | None = None + ) -> UsageAlertRule | None: ... + + async def delete_rule(self, rule_id: str) -> None: ... + + async def list_rules( + self, scope_type: UsageAlertScope | None = None, scope_id: str | None = None, limit: int = 50, offset: int = 0 + ) -> UsageAlertRuleList: ... + + async def get_alert(self, alert_id: str) -> UsageAlert | None: ... + + async def list_alerts( + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + metric: UsageAlertMetric | None = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: int = 50, + offset: int = 0, + ) -> UsageAlertList: ... + + async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None: ... + + async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None: ... + + async def team_role(self, team_id: str, user_id: str) -> str | None: ... diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py index a13e927ddb3..92ff63072dc 100644 --- a/litellm/proxy/alerting_endpoints/usage_alerts.py +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -5,11 +5,11 @@ from typing import Annotated, Final, TypeAlias from fastapi import APIRouter, Depends, HTTPException, Query, Response +from litellm.models.usage_alerts import UsageAlertStore from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.deps.database import DatabaseDep +from litellm.proxy.deps.repositories import UsageAlertsRepositoryDep from litellm.repositories.daily_activity_buckets import api_key_from_bucket -from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.usage_alerts import ( UsageAlert, @@ -26,14 +26,14 @@ from litellm.types.proxy.usage_alerts import ( router: Final = APIRouter(prefix="/usage", tags=["usage alerts"]) -def usage_alert_repository(database: DatabaseDep) -> UsageAlertsRepository: - if database is None: +def require_repository(repository: UsageAlertStore | None) -> UsageAlertStore: + if repository is None: raise HTTPException(status_code=503, detail="Usage alerts require a database connection") - return UsageAlertsRepository.from_prisma(database.prisma) + return repository User: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] -Repository: TypeAlias = Annotated[UsageAlertsRepository, Depends(usage_alert_repository)] +Repository: TypeAlias = UsageAlertsRepositoryDep Limit: TypeAlias = Annotated[int, Query(ge=1, le=100)] Offset: TypeAlias = Annotated[int, Query(ge=0, le=100000)] ScopeId: TypeAlias = Annotated[str | None, Query(min_length=1, max_length=1024)] @@ -41,7 +41,7 @@ ScopeId: TypeAlias = Annotated[str | None, Query(min_length=1, max_length=1024)] async def require_scope( user: UserAPIKeyAuth, - repository: UsageAlertsRepository, + repository: UsageAlertStore, scope_type: UsageAlertScope | None, scope_id: str | None, *, @@ -94,14 +94,16 @@ async def list_usage_alert_rules( limit: Limit = 50, offset: Offset = 0, ) -> UsageAlertRuleList: - await require_scope(user, repository, scope_type, scope_id, write=False) - return await repository.list_rules(scope_type, scope_id, limit, offset) + store: Final = require_repository(repository) + await require_scope(user, store, scope_type, scope_id, write=False) + return await store.list_rules(scope_type, scope_id, limit, offset) @router.post("/alert-rules", response_model=UsageAlertRule, status_code=201) async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, repository: Repository) -> UsageAlertRule: - await require_scope(user, repository, data.scope_type, data.scope_id, write=True) - saved: Final = await repository.save_rule(data, user.user_id) + store: Final = require_repository(repository) + await require_scope(user, store, data.scope_type, data.scope_id, write=True) + saved: Final = await store.save_rule(data, user.user_id) if saved is None: raise HTTPException(status_code=409, detail="Alert rule could not be saved") return saved @@ -109,10 +111,11 @@ async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, reposit @router.get("/alert-rules/{rule_id}", response_model=UsageAlertRule) async def get_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> UsageAlertRule: - rule: Final = await repository.get_rule(rule_id) + store: Final = require_repository(repository) + rule: Final = await store.get_rule(rule_id) if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") - await require_scope(user, repository, rule.scope_type, rule.scope_id, write=False) + await require_scope(user, store, rule.scope_type, rule.scope_id, write=False) return rule @@ -120,12 +123,13 @@ async def get_usage_alert_rule(rule_id: str, user: User, repository: Repository) async def update_usage_alert_rule( rule_id: str, data: UsageAlertRuleInput, user: User, repository: Repository ) -> UsageAlertRule: - rule: Final = await repository.get_rule(rule_id) + store: Final = require_repository(repository) + rule: Final = await store.get_rule(rule_id) if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") - await require_scope(user, repository, rule.scope_type, rule.scope_id, write=True) - await require_scope(user, repository, data.scope_type, data.scope_id, write=True) - saved: Final = await repository.save_rule(data, user.user_id, existing=rule) + await require_scope(user, store, rule.scope_type, rule.scope_id, write=True) + await require_scope(user, store, data.scope_type, data.scope_id, write=True) + saved: Final = await store.save_rule(data, user.user_id, existing=rule) if saved is None: raise HTTPException(status_code=404, detail="Alert rule not found") return saved @@ -133,11 +137,12 @@ async def update_usage_alert_rule( @router.delete("/alert-rules/{rule_id}", status_code=204) async def delete_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> Response: - rule: Final = await repository.get_rule(rule_id) + store: Final = require_repository(repository) + rule: Final = await store.get_rule(rule_id) if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") - await require_scope(user, repository, rule.scope_type, rule.scope_id, write=True) - await repository.delete_rule(rule_id) + await require_scope(user, store, rule.scope_type, rule.scope_id, write=True) + await store.delete_rule(rule_id) return Response(status_code=204) @@ -154,28 +159,31 @@ async def list_usage_alerts( limit: Limit = 50, offset: Offset = 0, ) -> UsageAlertList: - await require_scope(user, repository, scope_type, scope_id, write=False) + store: Final = require_repository(repository) + await require_scope(user, store, scope_type, scope_id, write=False) if start_date is not None and end_date is not None and start_date > end_date: raise HTTPException(status_code=422, detail="start_date must not be after end_date") - return await repository.list_alerts(scope_type, scope_id, metric, status, start_date, end_date, limit, offset) + return await store.list_alerts(scope_type, scope_id, metric, status, start_date, end_date, limit, offset) @router.get("/alerts/{alert_id}", response_model=UsageAlert) async def get_usage_alert(alert_id: str, user: User, repository: Repository) -> UsageAlert: - alert: Final = await repository.get_alert(alert_id) + store: Final = require_repository(repository) + alert: Final = await store.get_alert(alert_id) if alert is None: raise HTTPException(status_code=404, detail="Usage alert not found") - await require_scope(user, repository, alert.scope_type, alert.scope_id, write=False) + await require_scope(user, store, alert.scope_type, alert.scope_id, write=False) return alert @router.patch("/alerts/{alert_id}", response_model=UsageAlert) async def update_usage_alert(alert_id: str, data: UsageAlertUpdate, user: User, repository: Repository) -> UsageAlert: - alert: Final = await repository.get_alert(alert_id) + store: Final = require_repository(repository) + alert: Final = await store.get_alert(alert_id) if alert is None: raise HTTPException(status_code=404, detail="Usage alert not found") - await require_scope(user, repository, alert.scope_type, alert.scope_id, write=True) - updated: Final = await repository.set_status(alert_id, data.status) + await require_scope(user, store, alert.scope_type, alert.scope_id, write=True) + updated: Final = await store.set_status(alert_id, data.status) if updated is None: raise HTTPException(status_code=404, detail="Usage alert not found") return updated diff --git a/litellm/proxy/deps/repositories.py b/litellm/proxy/deps/repositories.py index 8f929119dee..b3b3aeafe28 100644 --- a/litellm/proxy/deps/repositories.py +++ b/litellm/proxy/deps/repositories.py @@ -5,11 +5,13 @@ from fastapi import Depends from litellm.models.deployment_warning import DeploymentWarningIgnoreStore from litellm.models.gateway_activity import GatewayActivityStore from litellm.models.plugin_upload import PluginUploadStore +from litellm.models.usage_alerts import UsageAlertStore from litellm.models.user_banner import UserBannerStore from litellm.proxy.deps.database import DatabaseDep from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository from litellm.repositories.gateway_activity_repository import GatewayActivityRepository from litellm.repositories.plugin_upload_repository import PluginUploadRepository +from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.repositories.user_banner_repository import UserBannerRepository @@ -43,3 +45,10 @@ async def get_plugin_upload_repository(database: DatabaseDep) -> PluginUploadSto PluginUploadRepositoryDep: TypeAlias = Annotated[PluginUploadStore | None, Depends(get_plugin_upload_repository)] + + +async def get_usage_alerts_repository(database: DatabaseDep) -> UsageAlertStore | None: + return None if database is None else UsageAlertsRepository.from_prisma(database.prisma) + + +UsageAlertsRepositoryDep: TypeAlias = Annotated[UsageAlertStore | None, Depends(get_usage_alerts_repository)] diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index 0bf6025130c..88c7634544a 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -53,9 +53,11 @@ def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gate def test_team_admin_can_manage_team_alerts_but_member_can_only_read(gateway: Gateway) -> None: with gateway.scenario() as scenario: - team_id: Final = scenario.team() - admin_id: Final = scenario.member(team_id, role="admin") - member_id: Final = scenario.member(team_id, role="user") + admin_id: Final = scenario.user(user_role="internal_user") + member_id: Final = scenario.user(user_role="internal_user") + team_id: Final = scenario.team( + members_with_roles=[{"user_id": admin_id, "role": "admin"}, {"user_id": member_id, "role": "user"}] + ) admin_key: Final = scenario.key(user_id=admin_id, key_type="management") member_key: Final = scenario.key(user_id=member_id, key_type="management") config: Final = { diff --git a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py index 1e26104494d..e8221599f8b 100644 --- a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py +++ b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py @@ -65,3 +65,14 @@ def test_global_activity_link_uses_the_usage_page_date_contract() -> None: assert "range" not in query assert query["from"] == ["2026-10-28"] assert query["to"] == ["2026-11-04"] + + +@pytest.mark.asyncio +async def test_no_database_returns_an_actionable_service_error() -> None: + from litellm.proxy.alerting_endpoints.usage_alerts import list_usage_alerts + + user: Final = UserAPIKeyAuth(user_id="qa-admin", user_role=LitellmUserRoles.PROXY_ADMIN) + with pytest.raises(HTTPException) as error: + await list_usage_alerts(user, None) + assert error.value.status_code == 503 + assert error.value.detail == "Usage alerts require a database connection" From 68b03b7eb88a3702b6737f008ed5b7c7cf682fa1 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 11:57:44 +0800 Subject: [PATCH 04/38] fix: preserve alert identity when rule meaning changes --- litellm/repositories/usage_alerts.py | 29 ++++++++-- .../integration/database/test_usage_alerts.py | 58 +++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 130f9fb0549..e9dbf977a8a 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -3,7 +3,7 @@ import json from collections.abc import Mapping from datetime import date, datetime, timedelta, timezone -from typing import ( # PrismaWrapper forwards generated client methods dynamically +from typing import ( TYPE_CHECKING, Final, Protocol, @@ -92,7 +92,20 @@ def activity_path( def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now: datetime) -> Mapping[str, object]: alert_id: Final = str( - uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.id}:{item.entity_id or ''}:{item.day.isoformat()}") + uuid5( + NAMESPACE_URL, + json.dumps( + [ + "litellm:usage-alert", + rule.id, + rule.scope_type, + item.entity_id, + rule.metric, + rule.direction, + item.day.isoformat(), + ] + ), + ) ) incident: Final = UsageAlert( id=alert_id, @@ -329,7 +342,7 @@ class UsageAlertsRepository: async def list_breached_entity_ids(self, rule_id: str, limit: int = 500, offset: int = 0) -> tuple[str | None, ...]: rows: Final = TypeAdapter(tuple[Mapping[str, str | None], ...]).validate_python( await self.database.query_raw( - 'SELECT DISTINCT scope_id FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND breached ORDER BY scope_id NULLS FIRST LIMIT $2 OFFSET $3', + """SELECT DISTINCT i.scope_id FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id WHERE i.rule_id=$1 AND i.breached AND i.scope_type=r.scope_type AND i.metric=r.record->>'metric' AND i.record->>'direction'=r.record->>'direction' AND (r.scope_id IS NULL OR i.scope_id=r.scope_id) ORDER BY i.scope_id NULLS FIRST LIMIT $2 OFFSET $3""", rule_id, limit, offset, @@ -366,9 +379,12 @@ class UsageAlertsRepository: return changed[0] rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( await self.database.query_raw( - 'SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=$2 ORDER BY evaluation_day DESC,id DESC LIMIT 1', + """SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=$2 AND scope_type=$3 AND metric=$4 AND record->>'direction'=$5 ORDER BY evaluation_day DESC,id DESC LIMIT 1""", rule.id, entity_id or "", + rule.scope_type, + rule.metric, + rule.direction, ) ) return UsageAlert.model_validate(rows[0].record) if rows else None @@ -389,7 +405,10 @@ class UsageAlertsRepository: prior AS ( SELECT x.*,p.id AS prior_id,p.evaluation_day,p.breached AS prior_breached,p.record AS prior_record FROM input x LEFT JOIN LATERAL ( - SELECT * FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=x.scope_key ORDER BY evaluation_day DESC,id DESC LIMIT 1 + SELECT * FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=x.scope_key + AND scope_type=x.record->>'scope_type' AND scope_id IS NOT DISTINCT FROM x.record->>'scope_id' + AND metric=x.record->>'metric' AND record->>'direction'=x.record->>'direction' + ORDER BY evaluation_day DESC,id DESC LIMIT 1 ) p ON true WHERE EXISTS (SELECT 1 FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND enabled AND NOT deleted AND (record->>'updated_at')::timestamptz=$3::timestamptz) ), updated AS ( diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index 88c7634544a..9fd94214dd1 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -1,8 +1,14 @@ +from datetime import date from hashlib import sha256 from typing import Final +import pytest +from integration._support.actors import ManagementDatabase from integration._support.client import Gateway, string_value +from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRuleInput + def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gateway: Gateway) -> None: with gateway.scenario() as scenario: @@ -78,3 +84,55 @@ def test_team_admin_can_manage_team_alerts_but_member_can_only_read(gateway: Gat assert gateway.request("DELETE", path, key=member_key).status_code == 403 finally: assert gateway.request("DELETE", path, key=admin_key).status_code == 204 + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.parametrize( + "changed_field,new_value", + [("metric", "latency"), ("direction", "decrease"), ("scope_type", "user"), ("scope_id", "semantic-other")], +) +async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency( + db: ManagementDatabase, changed_field: str, new_value: str +) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + definition: Final = UsageAlertRuleInput( + name="Semantic identity", + scope_type="team", + scope_id="semantic-example", + metric="spend", + direction="increase", + notification_channels=("slack",), + ) + rule: Final = await repository.save_rule(definition, "qa-admin") + assert rule is not None + initial: Final = UsageAlertObservation( + entity_id=rule.scope_id, day=date(2026, 11, 4), observed=40, baseline=5, threshold=15, breached=True + ) + try: + original: Final = (await repository.record_observations(rule, (initial,)))[0] + edited: Final = await repository.save_rule( + UsageAlertRuleInput.model_validate({**definition.model_dump(), changed_field: new_value}), "qa-admin", rule + ) + assert edited is not None + observation: Final = initial.model_copy(update={"entity_id": edited.scope_id}) + replacement: Final = (await repository.record_observations(edited, (observation,)))[0] + assert replacement.id != original.id + assert await repository.get_alert(original.id) == original + before_page: Final = await repository.list_alerts( + scope_type=original.scope_type, scope_id=original.scope_id, metric=original.metric + ) + after_page: Final = await repository.list_alerts( + scope_type=replacement.scope_type, scope_id=replacement.scope_id, metric=replacement.metric + ) + assert original.id in {item.id for item in before_page.data} + assert replacement.id in {item.id for item in after_page.data} + assert all( + item.scope_type == original.scope_type and item.metric == original.metric for item in before_page.data + ) + healthy: Final = observation.model_copy(update={"day": date(2026, 11, 5), "breached": False}) + await repository.record_observations(edited, (healthy,)) + assert (await repository.get_alert(original.id)) == original + resolved: Final = await repository.get_alert(replacement.id) + assert resolved is not None and resolved.status == "resolved" + finally: + await repository.delete_rule(rule.id) From 491f45d4c565629a96d4622e9f8b295bfd52b8ae Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 11:59:42 +0800 Subject: [PATCH 05/38] test: exercise scoped incident API over HTTP --- .../integration/database/test_usage_alerts.py | 68 ++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index 9fd94214dd1..0dfd1ed0b15 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -7,7 +7,7 @@ from integration._support.actors import ManagementDatabase from integration._support.client import Gateway, string_value from litellm.repositories.usage_alerts import UsageAlertsRepository -from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRule, UsageAlertRuleInput def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gateway: Gateway) -> None: @@ -86,6 +86,72 @@ def test_team_admin_can_manage_team_alerts_but_member_can_only_read(gateway: Gat assert gateway.request("DELETE", path, key=admin_key).status_code == 204 +@pytest.mark.asyncio(loop_scope="session") +async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gateway, db: ManagementDatabase) -> None: + """Populate an explicit storage fixture, then exercise the real incident HTTP API.""" + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + outsider: Final = scenario.user(user_role="internal_user") + viewer: Final = scenario.user(user_role="proxy_admin_viewer") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + outsider_key: Final = scenario.key(user_id=outsider, key_type="management") + viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") + created: Final = gateway.request( + "POST", + "/usage/alert-rules", + { + "name": "Incident API fixture", + "scope_type": "user", + "scope_id": owner, + "metric": "requests", + "direction": "increase", + "multiplier": 3, + }, + key=owner_key, + ) + assert created.status_code == 201 + rule: Final = UsageAlertRule.model_validate(created.json()) + try: + incident: Final = ( + await repository.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=owner, + day=date(2026, 11, 4), + observed=420, + baseline=120, + baseline_sample_days=7, + threshold=360, + breached=True, + ), + ), + ) + )[0] + path: Final = f"/usage/alerts/{incident.id}" + params: Final = {"scope_type": "user", "scope_id": owner, "limit": "1"} + page: Final = gateway.request("GET", "/usage/alerts", key=owner_key, params=params) + assert page.status_code == 200 and page.json()["data"][0]["id"] == incident.id + detail: Final = gateway.request("GET", path, key=owner_key) + assert detail.status_code == 200 + assert detail.json()["observed_value"] == 420 + assert detail.json()["baseline_value"] == 120 + assert detail.json()["threshold"] == 360 + assert detail.json()["baseline_sample_days"] == 7 + assert "detail_tab=activity" in detail.json()["activity_path"] + assert gateway.request("GET", "/usage/alerts", key=outsider_key, params=params).status_code == 403 + assert gateway.request("GET", path, key=outsider_key).status_code == 403 + assert gateway.request("PATCH", path, {"status": "dismissed"}, key=outsider_key).status_code == 403 + assert gateway.request("GET", path, key=viewer_key).status_code == 200 + assert gateway.request("PATCH", path, {"status": "dismissed"}, key=viewer_key).status_code == 403 + dismissed: Final = gateway.request("PATCH", path, {"status": "dismissed"}, key=owner_key) + assert dismissed.status_code == 200 and dismissed.json()["status"] == "dismissed" + assert gateway.request("GET", path, key=owner_key).json()["status"] == "dismissed" + finally: + assert gateway.request("DELETE", f"/usage/alert-rules/{rule.id}", key=owner_key).status_code == 204 + + @pytest.mark.asyncio(loop_scope="session") @pytest.mark.parametrize( "changed_field,new_value", From f8e681bf612d3b5f1e5ffcb22ae8363f4415df2e Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 12:03:14 +0800 Subject: [PATCH 06/38] fix: preserve the original evidence for an alert episode --- litellm/repositories/usage_alerts.py | 8 ++- .../integration/database/test_usage_alerts.py | 50 +++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index e9dbf977a8a..d2e0c38f7e3 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -414,11 +414,9 @@ class UsageAlertsRepository: ), updated AS ( UPDATE "LiteLLM_UsageAlertIncident" i SET evaluation_day=p.day, breached=p.breached, - day=CASE WHEN p.breached THEN p.day ELSE i.day END, - record=(CASE WHEN p.breached THEN p.record || jsonb_build_object( - 'id',i.id,'first_seen',i.record->'first_seen', - 'activity_path',replace(p.record->>'activity_path',p.record->>'id',i.id)) ELSE i.record END) - || jsonb_build_object('status',CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END), + record=i.record || jsonb_build_object( + 'last_seen',CASE WHEN p.breached THEN p.record->'last_seen' ELSE i.record->'last_seen' END, + 'status',CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END), status=CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END, updated_at=CURRENT_TIMESTAMP FROM prior p WHERE i.id=p.prior_id AND p.evaluation_day None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + definition: Final = UsageAlertRuleInput( + name="Original episode", scope_type="user", scope_id="episode-example", metric="requests", direction="increase" + ) + rule: Final = await repository.save_rule(definition, "qa-admin") + assert rule is not None + initial: Final = UsageAlertObservation( + entity_id=rule.scope_id, + day=date(2026, 11, 2), + observed=420, + baseline=120, + baseline_sample_days=7, + threshold=360, + breached=True, + ) + try: + original: Final = (await repository.record_observations(rule, (initial,)))[0] + edited: Final = await repository.save_rule( + definition.model_copy(update={"name": "Edited rule", "multiplier": 4}), "qa-admin", rule + ) + assert edited is not None + later: Final = initial.model_copy( + update={ + "day": date(2026, 11, 3), + "observed": 600, + "baseline": 100, + "threshold": 400, + "baseline_sample_days": 4, + } + ) + ongoing: Final = (await repository.record_observations(edited, (later,)))[0] + assert ongoing.last_seen >= original.last_seen + assert ongoing.model_copy(update={"last_seen": original.last_seen}) == original + stored: Final = await repository.get_alert(original.id) + assert stored == ongoing + healthy: Final = later.model_copy(update={"day": date(2026, 11, 4), "breached": False}) + resolved: Final = (await repository.record_observations(edited, (healthy,)))[0] + assert resolved.model_copy(update={"status": "open", "last_seen": original.last_seen}) == original + repeated: Final = later.model_copy(update={"day": date(2026, 11, 5)}) + new_episode: Final = (await repository.record_observations(edited, (repeated,)))[0] + assert new_episode.id != original.id + assert new_episode.day == repeated.day and new_episode.observed_value == 600 + assert new_episode.threshold == 400 and new_episode.rule_name == "Edited rule" + assert new_episode.activity_path != original.activity_path + finally: + await repository.delete_rule(rule.id) + + @pytest.mark.asyncio(loop_scope="session") @pytest.mark.parametrize( "changed_field,new_value", From 4f95407a1bfdc0d63b5c4e78c95f63ef4a92a00e Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 12:19:21 +0800 Subject: [PATCH 07/38] fix(api): document and validate created alert rules --- litellm/proxy/_types.py | 4 ++-- .../proxy/alerting_endpoints/usage_alerts.py | 1 + litellm/types/proxy/usage_alerts.py | 6 +++++- scripts/check_openapi_docs.py | 5 +++-- scripts/openapi_docs_gate.py | 4 ++-- tests/unit/test_check_openapi_docs.py | 17 +++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 ++++- 7 files changed, 34 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index de12eb00c6f..796e334f3cc 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -863,12 +863,12 @@ class LiteLLMRoutes(enum.Enum): ] ) - usage_alert_routes = [ + usage_alert_routes = ( "/usage/alert-rules", "/usage/alert-rules/{rule_id}", "/usage/alerts", "/usage/alerts/{alert_id}", - ] + ) self_managed_routes = [ *usage_alert_routes, diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py index 92ff63072dc..64f06fa62ed 100644 --- a/litellm/proxy/alerting_endpoints/usage_alerts.py +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -101,6 +101,7 @@ async def list_usage_alert_rules( @router.post("/alert-rules", response_model=UsageAlertRule, status_code=201) async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, repository: Repository) -> UsageAlertRule: + """Create a daily usage alert rule for a scope the caller can manage.""" store: Final = require_repository(repository) await require_scope(user, store, data.scope_type, data.scope_id, write=True) saved: Final = await store.save_rule(data, user.user_id) diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index 0654fc3022b..070586fef5b 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -36,7 +36,11 @@ class UsageAlertRuleInput(BaseModel): raise ValueError("min_history_days must not exceed baseline_days") if self.scope_type == "deployment" and self.scope_id is not None: raise ValueError("Deployment alerts cover the whole gateway; omit scope_id") - if self.scope_type == "key" and self.scope_id is not None and self.scope_id.removeprefix("raw:").startswith("sk-"): + if ( + self.scope_type == "key" + and self.scope_id is not None + and self.scope_id.removeprefix("raw:").startswith("sk-") + ): raise ValueError("Use the key's reporting ID, never the secret key") if len(set(self.notification_channels)) != len(self.notification_channels): raise ValueError("notification_channels must be unique") diff --git a/scripts/check_openapi_docs.py b/scripts/check_openapi_docs.py index 219ef912952..c6c7224e35d 100755 --- a/scripts/check_openapi_docs.py +++ b/scripts/check_openapi_docs.py @@ -7,7 +7,7 @@ count them per rule the same delta-vs-base way as the other lint gates. The rule OA001 no request body schema (the handler reads ``request.body()`` and never told FastAPI what is in it, see ``inline_request_body`` in custom_openapi_spec.py for the fix) OA002 no description (the handler has no docstring and the route no ``description=``) -OA003 the 200 response has no schema (no ``response_model`` and no ``responses={200: ...}``) +OA003 the 200 or 201 response has no schema (no ``response_model`` or explicit response schema) """ from __future__ import annotations @@ -45,7 +45,8 @@ def _operation_codes(operation: Mapping[str, object]) -> Iterator[str]: description: Final = operation.get("description") if not isinstance(description, str) or not description.strip(): yield "OA002" - if not _content_schema(_as_mapping(operation.get("responses")).get("200")): + responses: Final = _as_mapping(operation.get("responses")) + if not _content_schema(responses.get("200", responses.get("201"))): yield "OA003" diff --git a/scripts/openapi_docs_gate.py b/scripts/openapi_docs_gate.py index de340bf84e7..7b4a7e19e57 100755 --- a/scripts/openapi_docs_gate.py +++ b/scripts/openapi_docs_gate.py @@ -5,7 +5,7 @@ Sibling of scripts/test_quality_gate.py, pointed at the proxy's OpenAPI document instead of a source tree. Each rule is counted over every POST operation the proxy serves at HEAD and at the merge-base with the branch this change merges into, and the gate fails only when a rule grew past the merge-base count, so a new endpoint has to -ship with a request schema, a description and a 200 schema while the undocumented +ship with a request schema, a description and a 200/201 schema while the undocumented routes that already exist are only ever allowed to shrink. The merge-base counts come from scripts/lint_base_counts.py: the disk cache, then the @@ -167,7 +167,7 @@ def cmd_check(base: str) -> None: print( "Give every new POST route a request body schema (a typed body parameter, or " "`openapi_extra={'requestBody': inline_request_body(Model, example)}` when the handler reads " - "`request.body()`), a docstring, and a 200 schema (`response_model=` or `responses={200: {'model': ...}}`), " + "`request.body()`), a docstring, and a 200/201 schema (`response_model=` or an explicit response schema), " "or document an equal number of existing routes; the ceiling is the merge-base count. Run " "`python scripts/check_openapi_docs.py` to see every finding." ) diff --git a/tests/unit/test_check_openapi_docs.py b/tests/unit/test_check_openapi_docs.py index c6ea0cd936e..7d5f09c522b 100644 --- a/tests/unit/test_check_openapi_docs.py +++ b/tests/unit/test_check_openapi_docs.py @@ -40,6 +40,23 @@ def test_each_missing_piece_is_its_own_rule() -> None: assert codes == {"OA001", "OA002", "OA003"} +def test_created_response_schema_satisfies_the_success_contract() -> None: + operation: Final = {**_DOCUMENTED, "responses": {"201": _RESPONSES["200"]}} + assert checker.violations({"paths": {"/v1/thing": {"post": operation}}}) == () + + +@pytest.mark.parametrize("responses", [ + {"201": {"description": "Created"}}, + {"400": _RESPONSES["200"]}, + {"200": {}, "201": _RESPONSES["200"]}, +]) +def test_success_response_must_still_have_a_schema(responses: Mapping[str, object]) -> None: + operation: Final = {**_DOCUMENTED, "responses": responses} + assert checker.violations({"paths": {"/v1/thing": {"post": operation}}}) == ( + checker.Violation("POST", "/v1/thing", "OA003"), + ) + + @pytest.mark.parametrize("body", [{}, {"content": {"multipart/form-data": {}}}]) def test_only_the_missing_piece_is_reported(body: Mapping[str, object]) -> None: no_body: Final = {**_DOCUMENTED, "requestBody": body} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 98f7c623c20..cd55938d7c3 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -18844,7 +18844,10 @@ export interface paths { /** List Usage Alert Rules */ get: operations["list_usage_alert_rules_usage_alert_rules_get"]; put?: never; - /** Create Usage Alert Rule */ + /** + * Create Usage Alert Rule + * @description Create a daily usage alert rule for a scope the caller can manage. + */ post: operations["create_usage_alert_rule_usage_alert_rules_post"]; delete?: never; options?: never; From 549e2639844d2d60b425d31c6016f60ea485c9f6 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 12:30:14 +0800 Subject: [PATCH 08/38] fix: align usage alert migration with Prisma array nullability --- .../20261105000000_add_daily_usage_alerts/migration.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql index 1e459468f84..d68707b4395 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql @@ -10,7 +10,7 @@ CREATE TABLE "LiteLLM_UsageAlertIncident" ( "scope_type" TEXT NOT NULL, "scope_id" TEXT, "metric" TEXT NOT NULL, "status" TEXT NOT NULL, "day" TEXT NOT NULL, "evaluation_day" TEXT NOT NULL, "breached" BOOLEAN NOT NULL, "record" JSONB NOT NULL, - "notified_channels" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + "notified_channels" TEXT[] DEFAULT ARRAY[]::TEXT[], "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX "LiteLLM_UsageAlertIncident_rule_id_scope_key_evaluation_day_idx" ON "LiteLLM_UsageAlertIncident" ("rule_id", "scope_key", "evaluation_day"); From 949cc1e2eed9ca7a21071562527ecc4e368c7a60 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 13:20:47 +0800 Subject: [PATCH 09/38] feat: group Performance alerts by resource and day --- docs/features/usage-alerts.md | 37 ++- litellm/models/usage_alerts.py | 27 +- litellm/proxy/_types.py | 6 + .../proxy/alerting_endpoints/usage_alerts.py | 58 +++- litellm/repositories/usage_alerts.py | 250 ++++++++++++++- litellm/types/proxy/usage_alerts.py | 74 +++++ .../integration/database/test_usage_alerts.py | 286 +++++++++++++++++- tests/unit/types/proxy/test_usage_alerts.py | 20 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 285 +++++++++++++++++ 9 files changed, 1023 insertions(+), 20 deletions(-) diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index 842709b66bc..804a8b44aec 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -1,19 +1,40 @@ -# Daily usage alerts +# Daily Performance alerts -Daily usage alerts compare a completed UTC day with earlier recorded days. They cover spend, request count, average latency, and failed-request count for the gateway, a key, a user, a team, or a model. Open an incident to inspect its saved comparison and follow its Activity link to the corresponding usage view. +Performance alerts compare a completed UTC day with earlier recorded days. They cover spend, request count, average latency, and failed-request count for the gateway, a key, a user, a team, or a model. The Performance tab groups alerts for the same resource and original UTC day. Open a resource to inspect its saved comparison and Activity. The API paths remain under `/usage`. This feature requires a database. Apply the normal LiteLLM database migrations when deploying a version that includes usage alerts. The API and dashboard use the same persisted rules and incidents; no enterprise license is required. -## Read incidents +## Read resource groups Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the key in your environment. ```bash -curl "$LITELLM_API_BASE/usage/alerts?status=open&limit=50" \ +curl "$LITELLM_API_BASE/usage/alert-groups?status=open&limit=50" \ -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" ``` -The response has `data`, `total`, `limit`, and `offset`. Each incident includes its scope, metric, direction, status, UTC day, observed value, baseline value, threshold, baseline date range, and `activity_path`. The Activity path is relative to the dashboard mount: for example, `/api-keys?...` is opened below the dashboard's `/ui` mount. +The response has `data`, `total`, `limit`, and `offset`. `total` counts resource/day groups, including when the requested offset is beyond the last page. Filters are `scope_type`, `scope_id`, `status`, `start_date`, and `end_date`. Status filters apply to individual incidents before grouping, so a group with open and resolved incidents can appear under either filter with the corresponding members. No default date cutoff hides older open incidents. + +Each group includes its stable `id`, `scope_type`, exact `scope_id`, current `scope_label` when available, original UTC `day`, `incident_count`, and `activity_path`. Its `metrics` list contains up to four entries with a `metric`, `incident_count`, and the most recently detected matching `representative` incident. These representatives are summaries, not the entire group. Missing or deleted resources keep their original IDs. Names are current display labels, not historical evidence. + +To inspect every matching rule, use `GET /usage/alerts` with the group's scope and `start_date` and `end_date` both set to its `day`. Add `metric` and `status` as needed and paginate that response. Rules can have different baseline windows and thresholds, so their comparisons must not be combined into an error rate or a cost-per-request estimate. + +The group Activity link uses a representative's saved date range and alert context. The path is relative to the dashboard mount: for example, `/api-keys?...` is opened below the dashboard's `/ui` mount. A continuing episode retains its original day and evidence; the grouped row does not describe today's traffic. + +Resolve or dismiss all matching incidents in one resource/day group with an expected current status: + +```bash +curl -X PATCH "$LITELLM_API_BASE/usage/alert-groups" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \ + -H "Content-Type: application/json" \ + -d '{"scope_type":"team","scope_id":"team-example","day":"2026-11-04","expected_status":"open","status":"resolved"}' +``` + +The response contains `updated_count`. Only incidents in that exact group that still have `expected_status` are updated atomically. Other statuses remain unchanged; repeating the same request returns zero after all matching incidents have changed. Use `scope_type: "deployment"` with `scope_id: null` for the gateway. + +## Read individual incidents + +`GET /usage/alerts` retains the individual-incident view. Each incident includes its scope, metric, direction, status, UTC day, observed value, baseline value, threshold, baseline date range, and `activity_path`. `GET /usage/alerts/{alert_id}` returns one incident. List filters are `scope_type`, `scope_id`, `metric`, `status`, `start_date`, and `end_date`. Dates use `YYYY-MM-DD`; date filters include both endpoints. The maximum page size is 100. @@ -55,10 +76,12 @@ The evaluator runs against daily aggregates. An hourly scheduler does not make t Use `scope_type: "deployment"` with no `scope_id` for the gateway. For `model`, use the underlying model identity recorded in Activity. For `key`, use its reporting ID, never the secret API key. Omitting `scope_id` for key, user, team, or model evaluates all entities of that type and requires a gateway administrator. +`GET /usage/alert-models?limit=20&search=example` returns paginated model identities recorded in the most recent 30 UTC days. Its `data` entries contain `scope_id` and `label`; `start_date` and `end_date` state the searched window. Pass `next_cursor` as `cursor` to read the next page. A null cursor ends the result. Search matches literal text, not SQL wildcards. This endpoint requires gateway admin or admin-viewer access. Configured aliases and deployment IDs are not reporting-model identities. An existing rule's saved model remains valid even when it falls outside the options window. + ## Manage rules and incidents -- `GET /usage/alert-rules` lists rules; `GET /usage/alert-rules/{rule_id}` reads one. -- `PUT /usage/alert-rules/{rule_id}` replaces the rule configuration. Send the configuration fields from the create example, including any notification settings you want to retain. Set `enabled` to `false` to pause it. +- `GET /usage/alert-rules` lists rules; `GET /usage/alert-rules/{rule_id}` reads one. Use `kind=custom`, `kind=default`, or `kind=all` to choose which rules to list. Each rule exposes `is_default`. The list's `default_summary` counts total and enabled defaults within the requested scope, independent of `kind`, and reports evaluation-status counts among enabled defaults. +- `PUT /usage/alert-rules/{rule_id}` replaces the rule configuration. Send the configuration fields from the create example, including any notification settings you want to retain. Set `enabled` to `false` to pause it. Built-in rules keep their scope, metric, and direction; create a custom rule to change those fields. Their names, thresholds, notification settings, and enabled state remain editable. - `DELETE /usage/alert-rules/{rule_id}` removes a rule from evaluation and preserves existing incidents. Deleted defaults stay deleted. - `PATCH /usage/alerts/{alert_id}` with `{"status":"dismissed"}` dismisses an incident. `open` and `resolved` are also accepted. diff --git a/litellm/models/usage_alerts.py b/litellm/models/usage_alerts.py index 9a10be2d409..8858835eafc 100644 --- a/litellm/models/usage_alerts.py +++ b/litellm/models/usage_alerts.py @@ -3,10 +3,15 @@ from typing import Protocol from litellm.types.proxy.usage_alerts import ( UsageAlert, + UsageAlertGroupList, + UsageAlertGroupUpdate, + UsageAlertGroupUpdateResult, UsageAlertList, UsageAlertMetric, + UsageAlertModelList, UsageAlertRule, UsageAlertRuleInput, + UsageAlertRuleKind, UsageAlertRuleList, UsageAlertScope, UsageAlertStatus, @@ -23,7 +28,12 @@ class UsageAlertStore(Protocol): async def delete_rule(self, rule_id: str) -> None: ... async def list_rules( - self, scope_type: UsageAlertScope | None = None, scope_id: str | None = None, limit: int = 50, offset: int = 0 + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + limit: int = 50, + offset: int = 0, + kind: UsageAlertRuleKind = "all", ) -> UsageAlertRuleList: ... async def get_alert(self, alert_id: str) -> UsageAlert | None: ... @@ -42,6 +52,21 @@ class UsageAlertStore(Protocol): async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None: ... + async def list_groups( + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: int = 50, + offset: int = 0, + ) -> UsageAlertGroupList: ... + + async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult: ... + + async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList: ... + async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None: ... async def team_role(self, team_id: str, user_id: str) -> str | None: ... diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 796e334f3cc..7254a0fb6b7 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -579,6 +579,8 @@ class LiteLLMRoutes(enum.Enum): ) llm_api_routes_without_mcp = sorted(set(llm_api_routes) - set(mcp_inference_routes)) info_routes = [ + "/usage/alert-groups", + "/usage/alert-models", "/usage/alert-rules", "/usage/alert-rules/{rule_id}", "/usage/alerts", @@ -650,6 +652,8 @@ class LiteLLMRoutes(enum.Enum): management_routes = ( [ + "/usage/alert-groups", + "/usage/alert-models", "/usage/alert-rules", "/usage/alert-rules/{rule_id}", "/usage/alerts", @@ -864,6 +868,8 @@ class LiteLLMRoutes(enum.Enum): ) usage_alert_routes = ( + "/usage/alert-groups", + "/usage/alert-models", "/usage/alert-rules", "/usage/alert-rules/{rule_id}", "/usage/alerts", diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py index 64f06fa62ed..e6b1b22cc00 100644 --- a/litellm/proxy/alerting_endpoints/usage_alerts.py +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -13,10 +13,15 @@ from litellm.repositories.daily_activity_buckets import api_key_from_bucket from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.usage_alerts import ( UsageAlert, + UsageAlertGroupList, + UsageAlertGroupUpdate, + UsageAlertGroupUpdateResult, UsageAlertList, UsageAlertMetric, + UsageAlertModelList, UsageAlertRule, UsageAlertRuleInput, + UsageAlertRuleKind, UsageAlertRuleList, UsageAlertScope, UsageAlertStatus, @@ -93,10 +98,52 @@ async def list_usage_alert_rules( scope_id: ScopeId = None, limit: Limit = 50, offset: Offset = 0, + kind: UsageAlertRuleKind = "all", ) -> UsageAlertRuleList: store: Final = require_repository(repository) await require_scope(user, store, scope_type, scope_id, write=False) - return await store.list_rules(scope_type, scope_id, limit, offset) + return await store.list_rules(scope_type, scope_id, limit, offset, kind) + + +@router.get("/alert-groups", response_model=UsageAlertGroupList) +async def list_usage_alert_groups( + user: User, + repository: Repository, + scope_type: UsageAlertScope | None = None, + scope_id: ScopeId = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: Limit = 50, + offset: Offset = 0, +) -> UsageAlertGroupList: + store: Final = require_repository(repository) + await require_scope(user, store, scope_type, scope_id, write=False) + if start_date is not None and end_date is not None and start_date > end_date: + raise HTTPException(status_code=422, detail="start_date must not be after end_date") + return await store.list_groups(scope_type, scope_id, status, start_date, end_date, limit, offset) + + +@router.patch("/alert-groups", response_model=UsageAlertGroupUpdateResult) +async def update_usage_alert_group( + data: UsageAlertGroupUpdate, user: User, repository: Repository +) -> UsageAlertGroupUpdateResult: + store: Final = require_repository(repository) + await require_scope(user, store, data.scope_type, data.scope_id, write=True) + return await store.set_group_status(data) + + +@router.get("/alert-models", response_model=UsageAlertModelList) +async def list_usage_alert_models( + user: User, + repository: Repository, + search: ScopeId = None, + cursor: ScopeId = None, + limit: Limit = 20, +) -> UsageAlertModelList: + store: Final = require_repository(repository) + await require_scope(user, store, "model", None, write=False) + return await store.list_reported_models(search, cursor, limit) @router.post("/alert-rules", response_model=UsageAlertRule, status_code=201) @@ -129,6 +176,15 @@ async def update_usage_alert_rule( if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") await require_scope(user, store, rule.scope_type, rule.scope_id, write=True) + if rule.is_default and (rule.scope_type, rule.scope_id, rule.metric, rule.direction) != ( + data.scope_type, + data.scope_id, + data.metric, + data.direction, + ): + raise HTTPException( + status_code=422, detail="Create a custom rule to change a built-in rule's scope, metric, or direction" + ) await require_scope(user, store, data.scope_type, data.scope_id, write=True) saved: Final = await store.save_rule(data, user.user_id, existing=rule) if saved is None: diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index d2e0c38f7e3..c7efc5c77f8 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -3,6 +3,8 @@ import json from collections.abc import Mapping from datetime import date, datetime, timedelta, timezone +from itertools import product +from types import MappingProxyType from typing import ( TYPE_CHECKING, Final, @@ -14,16 +16,26 @@ from uuid import NAMESPACE_URL, uuid4, uuid5 from pydantic import BaseModel, TypeAdapter +from litellm.repositories.daily_activity_buckets import api_key_from_bucket from litellm.types.proxy.usage_alerts import ( UsageAlert, UsageAlertChannel, + UsageAlertDefaultSummary, UsageAlertEvaluationStatus, + UsageAlertGroup, + UsageAlertGroupList, + UsageAlertGroupMetric, + UsageAlertGroupUpdate, + UsageAlertGroupUpdateResult, UsageAlertList, UsageAlertMetric, + UsageAlertModelList, + UsageAlertModelOption, UsageAlertNotification, UsageAlertObservation, UsageAlertRule, UsageAlertRuleInput, + UsageAlertRuleKind, UsageAlertRuleList, UsageAlertScope, UsageAlertStatus, @@ -45,6 +57,37 @@ class _Count(BaseModel, frozen=True): count: int +class _RuleCounts(_Count, frozen=True): + default_summary: UsageAlertDefaultSummary + + +class _GroupRow(BaseModel, frozen=True): + scope_type: UsageAlertScope + scope_id: str | None + day: date + incident_count: int + metrics: tuple[UsageAlertGroupMetric, ...] + + +class _GroupPage(BaseModel, frozen=True): + total: int + data: tuple[_GroupRow, ...] + + +class _ResourceLabel(BaseModel, frozen=True): + scope_type: UsageAlertScope + scope_id: str | None + label: str | None + + +class _ModelIdentity(BaseModel, frozen=True): + model: str + + +class _Identifier(BaseModel, frozen=True): + id: str + + class _Pending(_Record, frozen=True): rule: Mapping[str, object] notified_channels: tuple[UsageAlertChannel, ...] @@ -55,6 +98,20 @@ class _Ownership(BaseModel, frozen=True): team_id: str | None +_DEFAULT_RULE_IDS: Final = frozenset( + str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{scope}:{metric}:{direction}")) + for scope, metric, direction in product( + ("deployment", "key", "user", "team", "model"), + ("spend", "requests", "latency", "errors"), + ("increase", "decrease"), + ) +) + + +def _rule(record: Mapping[str, object]) -> UsageAlertRule: + return UsageAlertRule.model_validate({**record, "is_default": str(record.get("id")) in _DEFAULT_RULE_IDS}) + + def activity_path( scope: UsageAlertScope, entity_id: str | None, day: date, alert_id: str, baseline_days: int = 7 ) -> str: @@ -151,7 +208,7 @@ class UsageAlertsRepository: 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND NOT deleted', rule_id ) ) - return UsageAlertRule.model_validate(rows[0].record) if rows else None + return _rule(rows[0].record) if rows else None async def save_rule( self, data: UsageAlertRuleInput, created_by: str | None, existing: UsageAlertRule | None = None @@ -177,7 +234,7 @@ class UsageAlertsRepository: rule.model_dump_json(), ) saved: Final = TypeAdapter(tuple[_Record, ...]).validate_python(rows) - return UsageAlertRule.model_validate(saved[0].record) if saved else None + return _rule(saved[0].record) if saved else None async def delete_rule(self, rule_id: str) -> None: await self.database.query_raw( @@ -185,28 +242,46 @@ class UsageAlertsRepository: ) async def list_rules( - self, scope_type: UsageAlertScope | None = None, scope_id: str | None = None, limit: int = 50, offset: int = 0 + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + limit: int = 50, + offset: int = 0, + kind: UsageAlertRuleKind = "all", ) -> UsageAlertRuleList: - args: Final = (scope_type, scope_id) + args: Final = (scope_type, scope_id, json.dumps(tuple(sorted(_DEFAULT_RULE_IDS))), kind) where: Final = "NOT deleted AND ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2)" - count: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + builtin: Final = "id IN (SELECT jsonb_array_elements_text($3::jsonb))" + count: Final = TypeAdapter(tuple[_RuleCounts, ...]).validate_python( await self.database.query_raw( - f'SELECT count(*)::int AS count FROM "LiteLLM_UsageAlertRule" WHERE {where}', *args + f"""WITH scoped AS ( + SELECT enabled,record,({builtin}) AS builtin FROM "LiteLLM_UsageAlertRule" WHERE {where} + ) SELECT count(*) FILTER (WHERE $4='all' OR builtin=($4='default'))::int AS count, + jsonb_build_object('total',count(*) FILTER (WHERE builtin), + 'enabled',count(*) FILTER (WHERE builtin AND enabled), + 'evaluation_status_counts',jsonb_build_object( + 'pending',count(*) FILTER (WHERE builtin AND enabled AND COALESCE(record->>'evaluation_status','pending')='pending'), + 'complete',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='complete'), + 'insufficient_coverage',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='insufficient_coverage'), + 'insufficient_history',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='insufficient_history') + )) AS default_summary FROM scoped""", + *args, ) ) rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( await self.database.query_raw( - f'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE {where} ORDER BY id LIMIT $3 OFFSET $4', + f"SELECT record FROM \"LiteLLM_UsageAlertRule\" WHERE {where} AND ($4='all' OR ({builtin})=($4='default')) ORDER BY id LIMIT $5 OFFSET $6", *args, limit, offset, ) ) return UsageAlertRuleList( - data=tuple(UsageAlertRule.model_validate(row.record) for row in rows), + data=tuple(_rule(row.record) for row in rows), total=count[0].count, limit=limit, offset=offset, + default_summary=count[0].default_summary, ) async def list_enabled_rules(self, limit: int = 100, offset: int = 0) -> tuple[UsageAlertRule, ...]: @@ -275,6 +350,165 @@ class UsageAlertsRepository: ) return UsageAlert.model_validate(rows[0].record) if rows else None + async def list_groups( + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: int = 50, + offset: int = 0, + ) -> UsageAlertGroupList: + rows: Final = TypeAdapter(tuple[_GroupPage, ...]).validate_python( + await self.database.query_raw( + """WITH groups AS MATERIALIZED ( + SELECT scope_type,scope_id,day,count(*)::int AS incident_count FROM "LiteLLM_UsageAlertIncident" + WHERE ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2) + AND ($3::text IS NULL OR status=$3) AND ($4::text IS NULL OR day >= $4) + AND ($5::text IS NULL OR day <= $5) + GROUP BY scope_type,scope_id,day + ), page AS MATERIALIZED ( + SELECT * FROM groups ORDER BY day DESC,scope_type,scope_id NULLS FIRST LIMIT $6 OFFSET $7 + ), selected AS ( + SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i + ON i.scope_type=p.scope_type AND i.scope_id=p.scope_id AND i.day=p.day + WHERE ($3::text IS NULL OR i.status=$3) + UNION ALL + SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i + ON i.scope_type=p.scope_type AND i.scope_id IS NULL AND i.day=p.day + WHERE p.scope_id IS NULL AND ($3::text IS NULL OR i.status=$3) + ), ranked AS ( + SELECT scope_type,scope_id,day,metric,record, + count(*) OVER (PARTITION BY scope_type,scope_id,day,metric)::int AS incident_count, + row_number() OVER (PARTITION BY scope_type,scope_id,day,metric + ORDER BY record->>'first_seen' DESC,id) AS rank + FROM selected + ), packed AS ( + SELECT p.scope_type,p.scope_id,p.day,p.incident_count, + jsonb_agg(jsonb_build_object('metric',r.metric,'incident_count',r.incident_count, + 'representative',r.record) ORDER BY CASE r.metric + WHEN 'requests' THEN 0 WHEN 'spend' THEN 1 WHEN 'errors' THEN 2 ELSE 3 END) AS metrics + FROM page p JOIN ranked r ON p.scope_type=r.scope_type + AND p.scope_id IS NOT DISTINCT FROM r.scope_id AND p.day=r.day AND r.rank=1 + GROUP BY p.scope_type,p.scope_id,p.day,p.incident_count + ) SELECT (SELECT count(*)::int FROM groups) AS total, + COALESCE((SELECT jsonb_agg(to_jsonb(packed) ORDER BY day DESC,scope_type,scope_id NULLS FIRST) + FROM packed),'[]'::jsonb) AS data""", + scope_type, + scope_id, + status, + start_date.isoformat() if start_date else None, + end_date.isoformat() if end_date else None, + limit, + offset, + ) + ) + page: Final = rows[0] + labels: Final[Mapping[tuple[UsageAlertScope, str | None], str | None]] = ( + await self._group_labels(page.data) if page.data else {} + ) + return UsageAlertGroupList( + data=tuple( + UsageAlertGroup( + id=str( + uuid5( + NAMESPACE_URL, + json.dumps( + ["litellm:performance-group", row.scope_type, row.scope_id, row.day.isoformat()] + ), + ) + ), + scope_type=row.scope_type, + scope_id=row.scope_id, + scope_label=labels.get((row.scope_type, row.scope_id)), + day=row.day, + incident_count=row.incident_count, + metrics=row.metrics, + activity_path=row.metrics[0].representative.activity_path, + ) + for row in page.data + ), + total=page.total, + limit=limit, + offset=offset, + ) + + async def _group_labels( + self, groups: tuple[_GroupRow, ...] + ) -> Mapping[tuple[UsageAlertScope, str | None], str | None]: + rows: Final = TypeAdapter(tuple[_ResourceLabel, ...]).validate_python( + await self.database.query_raw( + """WITH scopes AS ( + SELECT DISTINCT scope_type,scope_id,key_id FROM jsonb_to_recordset($1::jsonb) + AS x(scope_type text,scope_id text,key_id text) + ) SELECT s.scope_type,s.scope_id,CASE s.scope_type + WHEN 'key' THEN COALESCE(NULLIF(k.key_alias,''),NULLIF(k.key_name,'')) + WHEN 'team' THEN NULLIF(t.team_alias,'') + WHEN 'user' THEN COALESCE(NULLIF(u.user_alias,''),NULLIF(u.user_email,'')) + WHEN 'model' THEN s.scope_id WHEN 'deployment' THEN 'Gateway' END AS label + FROM scopes s LEFT JOIN "LiteLLM_VerificationToken" k ON s.scope_type='key' AND k.token=s.key_id + LEFT JOIN "LiteLLM_TeamTable" t ON s.scope_type='team' AND t.team_id=s.scope_id + LEFT JOIN "LiteLLM_UserTable" u ON s.scope_type='user' AND u.user_id=s.scope_id""", + json.dumps( + tuple( + { + "scope_type": row.scope_type, + "scope_id": row.scope_id, + "key_id": api_key_from_bucket(row.scope_id) + if row.scope_type == "key" and row.scope_id + else None, + } + for row in groups + ) + ), + ) + ) + return MappingProxyType({(row.scope_type, row.scope_id): row.label for row in rows}) + + async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult: + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + """WITH changed AS ( + UPDATE "LiteLLM_UsageAlertIncident" SET status=$5, + record=jsonb_set(record,'{status}',to_jsonb($5::text)),updated_at=CURRENT_TIMESTAMP + WHERE scope_type=$1 AND scope_id IS NOT DISTINCT FROM $2::text AND day=$3 AND status=$4 + RETURNING id + ) SELECT count(*)::int AS count FROM changed""", + data.scope_type, + data.scope_id, + data.day.isoformat(), + data.expected_status, + data.status, + ) + ) + return UsageAlertGroupUpdateResult(updated_count=rows[0].count) + + async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList: + today: Final = datetime.now(timezone.utc).date() + rows: Final = TypeAdapter(tuple[_ModelIdentity, ...]).validate_python( + await self.database.query_raw( + """SELECT DISTINCT model FROM "LiteLLM_DailyUserSpend" + WHERE date >= $1 AND date <= $2 AND model IS NOT NULL AND model <> '' + AND ($3::text IS NULL OR strpos(lower(model),lower($3))>0) + AND ($4::text IS NULL OR model>$4) + ORDER BY model LIMIT $5""", + (today - timedelta(days=29)).isoformat(), + today.isoformat(), + search, + cursor, + limit + 1, + ) + ) + selected: Final = rows[:limit] + return UsageAlertModelList( + data=tuple(UsageAlertModelOption(scope_id=row.model, label=row.model) for row in selected), + limit=limit, + next_cursor=selected[-1].model if len(rows) > limit else None, + start_date=today - timedelta(days=29), + end_date=today, + ) + async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None: rows: Final = TypeAdapter(tuple[_Ownership, ...]).validate_python( await self.database.query_raw( diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index 070586fef5b..bb719ea48a2 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -11,6 +11,7 @@ UsageAlertDirection: TypeAlias = Literal["increase", "decrease"] UsageAlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"] UsageAlertChannel: TypeAlias = Literal["email", "slack"] UsageAlertEvaluationStatus: TypeAlias = Literal["pending", "complete", "insufficient_coverage", "insufficient_history"] +UsageAlertRuleKind: TypeAlias = Literal["all", "default", "custom"] class UsageAlertRuleInput(BaseModel): @@ -60,6 +61,7 @@ class UsageAlertRule(UsageAlertRuleInput): created_by: str | None last_evaluated_day: date | None = None evaluation_status: UsageAlertEvaluationStatus = "pending" + is_default: bool = False class UsageAlert(BaseModel): @@ -90,11 +92,25 @@ class UsageAlertUpdate(BaseModel): status: UsageAlertStatus +class UsageAlertEvaluationCounts(BaseModel, frozen=True): + pending: int = 0 + complete: int = 0 + insufficient_coverage: int = 0 + insufficient_history: int = 0 + + +class UsageAlertDefaultSummary(BaseModel, frozen=True): + total: int = 0 + enabled: int = 0 + evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts) + + class UsageAlertRuleList(BaseModel, frozen=True): data: tuple[UsageAlertRule, ...] total: int limit: int offset: int + default_summary: UsageAlertDefaultSummary = Field(default_factory=UsageAlertDefaultSummary) class UsageAlertList(BaseModel, frozen=True): @@ -104,6 +120,64 @@ class UsageAlertList(BaseModel, frozen=True): offset: int +class UsageAlertGroupMetric(BaseModel, frozen=True): + metric: UsageAlertMetric + incident_count: int + representative: UsageAlert + + +class UsageAlertGroup(BaseModel, frozen=True): + id: str + scope_type: UsageAlertScope + scope_id: str | None + scope_label: str | None + day: date + incident_count: int + metrics: tuple[UsageAlertGroupMetric, ...] = Field(max_length=4) + activity_path: str + + +class UsageAlertGroupList(BaseModel, frozen=True): + data: tuple[UsageAlertGroup, ...] + total: int + limit: int + offset: int + + +class UsageAlertGroupUpdate(BaseModel, frozen=True): + model_config = ConfigDict(extra="forbid") + scope_type: UsageAlertScope + scope_id: str | None = Field(default=None, min_length=1, max_length=1024) + day: date + expected_status: UsageAlertStatus + status: UsageAlertStatus + + @model_validator(mode="after") + def validate_group(self) -> Self: + if (self.scope_type == "deployment") != (self.scope_id is None): + raise ValueError("Specify a resource ID except for deployment alerts") + if self.expected_status == self.status: + raise ValueError("The new status must differ from the expected status") + return self + + +class UsageAlertGroupUpdateResult(BaseModel, frozen=True): + updated_count: int + + +class UsageAlertModelOption(BaseModel, frozen=True): + scope_id: str + label: str + + +class UsageAlertModelList(BaseModel, frozen=True): + data: tuple[UsageAlertModelOption, ...] + limit: int + next_cursor: str | None + start_date: date + end_date: date + + class UsageAlertNotification(BaseModel): model_config = ConfigDict(frozen=True) incident: UsageAlert diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index 6bf08bfd4a9..a43b5858fe0 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -1,13 +1,21 @@ -from datetime import date +from datetime import date, timedelta from hashlib import sha256 from typing import Final +from uuid import uuid4 import pytest from integration._support.actors import ManagementDatabase from integration._support.client import Gateway, string_value from litellm.repositories.usage_alerts import UsageAlertsRepository -from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRule, UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import ( + UsageAlertGroupList, + UsageAlertMetric, + UsageAlertModelList, + UsageAlertObservation, + UsageAlertRule, + UsageAlertRuleInput, +) def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gateway: Gateway) -> None: @@ -252,3 +260,277 @@ async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency( assert resolved is not None and resolved.status == "resolved" finally: await repository.delete_rule(rule.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_actions( + gateway: Gateway, db: ManagementDatabase +) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + day: Final = date(2026, 11, 4) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user", user_alias="Checkout operator") + outsider: Final = scenario.user(user_role="internal_user") + viewer: Final = scenario.user(user_role="proxy_admin_viewer") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + outsider_key: Final = scenario.key(user_id=outsider, key_type="management") + viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") + inference_key: Final = scenario.key(user_id=owner, key_type="llm_api") + metrics: Final[tuple[UsageAlertMetric, ...]] = (*(("requests",) * 27), "spend", "errors", "latency") + rules: Final = tuple( + [ + await repository.save_rule( + UsageAlertRuleInput( + name=f"Group rule {index}", + scope_type="user", + scope_id=owner, + metric=metric, + direction="increase", + baseline_days=7 if index % 2 else 14, + ), + owner, + ) + for index, metric in enumerate(metrics) + ] + ) + assert all(rule is not None for rule in rules) + incidents: Final = tuple( + [ + ( + await repository.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=owner, + day=day, + observed=420, + baseline=0 if index == 0 else 120, + threshold=100 if index == 0 else 360, + breached=True, + ), + ), + ) + )[0] + for index, rule in enumerate(rules) + if rule is not None + ] + ) + old_rule: Final = await repository.save_rule( + UsageAlertRuleInput( + name="Earlier day", scope_type="user", scope_id=owner, metric="requests", direction="increase" + ), + owner, + ) + assert old_rule is not None + old: Final = ( + await repository.record_observations( + old_rule, + ( + UsageAlertObservation( + entity_id=owner, + day=day - timedelta(days=1), + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + other_rule: Final = await repository.save_rule( + UsageAlertRuleInput( + name="Other resource", scope_type="user", scope_id=outsider, metric="requests", direction="increase" + ), + outsider, + ) + assert other_rule is not None + other: Final = ( + await repository.record_observations( + other_rule, + ( + UsageAlertObservation( + entity_id=outsider, + day=day, + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + params: Final = {"scope_type": "user", "scope_id": owner, "status": "open", "limit": "1"} + try: + response: Final = gateway.request("GET", "/usage/alert-groups", key=owner_key, params=params) + assert response.status_code == 200, response.text + page: Final = UsageAlertGroupList.model_validate(response.json()) + assert page.total == 2 and len(page.data) == 1 + group: Final = page.data[0] + assert group.day == day and group.scope_label == "Checkout operator" + assert group.incident_count == 30 and len(group.metrics) == 4 + assert {metric.metric: metric.incident_count for metric in group.metrics} == { + "requests": 27, + "spend": 1, + "errors": 1, + "latency": 1, + } + assert all(metric.representative.id in {incident.id for incident in incidents} for metric in group.metrics) + assert group.activity_path in {incident.activity_path for incident in incidents} + second: Final = UsageAlertGroupList.model_validate( + gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "1"}).json() + ) + assert second.total == 2 and second.data[0].day == old.day + empty: Final = UsageAlertGroupList.model_validate( + gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "2"}).json() + ) + assert empty.total == 2 and empty.data == () + history: Final = gateway.request( + "GET", + "/usage/alerts", + key=owner_key, + params={**params, "start_date": day.isoformat(), "end_date": day.isoformat(), "metric": "requests"}, + ) + assert history.status_code == 200 and history.json()["total"] == 27 + await repository.set_status(incidents[0].id, "dismissed") + body: Final = { + "scope_type": "user", + "scope_id": owner, + "day": day.isoformat(), + "expected_status": "open", + "status": "resolved", + } + for forbidden in (outsider_key, inference_key): + assert gateway.request("GET", "/usage/alert-groups", key=forbidden, params=params).status_code == 403 + for forbidden in (outsider_key, inference_key, viewer_key): + assert gateway.request("PATCH", "/usage/alert-groups", body, key=forbidden).status_code == 403 + assert gateway.request("GET", "/usage/alert-groups", key=viewer_key, params=params).status_code == 200 + updated: Final = gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key) + assert updated.status_code == 200 and updated.json()["updated_count"] == 29 + assert gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key).json()["updated_count"] == 0 + assert await repository.get_alert(old.id) == old + assert await repository.get_alert(other.id) == other + dismissed: Final = await repository.get_alert(incidents[0].id) + assert dismissed is not None and dismissed.status == "dismissed" and dismissed.baseline_value == 0 + resolved: Final = await repository.get_alert(incidents[-1].id) + assert resolved == incidents[-1].model_copy(update={"status": "resolved"}) + assert ( + gateway.request( + "GET", "/usage/alert-groups", params={"scope_type": "user", "scope_id": "x' OR 1=1 --"} + ).json()["total"] + == 0 + ) + assert ( + gateway.request( + "GET", "/usage/alert-groups", params={"start_date": "2026-11-05", "end_date": "2026-11-04"} + ).status_code + == 422 + ) + finally: + for rule in (*rules, old_rule, other_rule): + if rule is not None: + await repository.delete_rule(rule.id) + deleted: Final = await repository.list_groups(scope_type="user", scope_id=owner) + assert deleted.total == 2 and all(group.scope_label is None and group.scope_id == owner for group in deleted.data) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_default_rule_summary_and_kind_filters_do_not_require_loading_every_rule( + gateway: Gateway, db: ManagementDatabase +) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + default: Final = UsageAlertRuleInput( + name="Built-in daily model requests", scope_type="model", metric="requests", direction="increase" + ) + await repository.ensure_default_rules((default,)) + custom: Final = await repository.save_rule( + default.model_copy(update={"scope_id": f"model-{uuid4().hex}"}), "qa-admin" + ) + assert custom is not None + try: + all_rules: Final = await repository.list_rules(scope_type="model", limit=1) + builtins: Final = await repository.list_rules(scope_type="model", limit=1, kind="default") + customs: Final = await repository.list_rules(scope_type="model", limit=1, kind="custom") + assert builtins.total >= 1 and all(rule.is_default for rule in builtins.data) + assert customs.total >= 1 and all(not rule.is_default for rule in customs.data) + assert all_rules.total == builtins.total + customs.total + assert all_rules.default_summary == builtins.default_summary == customs.default_summary + assert builtins.default_summary.total == builtins.total + assert ( + sum(builtins.default_summary.evaluation_status_counts.model_dump().values()) + == builtins.default_summary.enabled + ) + assert (await repository.list_rules(scope_type="model", kind="default", offset=builtins.total)).data == () + stored: Final = await repository.get_rule(builtins.data[0].id) + assert stored is not None and stored.is_default + filtered: Final = gateway.request( + "GET", "/usage/alert-rules", params={"scope_type": "model", "kind": "custom", "limit": "1"} + ) + assert filtered.status_code == 200 and filtered.json()["total"] == customs.total + assert all(not rule["is_default"] for rule in filtered.json()["data"]) + configuration: Final = stored.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields)) + path: Final = f"/usage/alert-rules/{stored.id}" + for change in ( + {"scope_type": "key"}, + {"scope_id": "model-example"}, + {"metric": "requests" if stored.metric != "requests" else "spend"}, + {"direction": "increase" if stored.direction == "decrease" else "decrease"}, + ): + assert gateway.request("PUT", path, {**configuration, **change}).status_code == 422 + assert await repository.get_rule(stored.id) == stored + try: + updated: Final = gateway.request("PUT", path, {**configuration, "min_value": stored.min_value + 1}) + assert updated.status_code == 200 and updated.json()["is_default"] is True + assert updated.json()["min_value"] == stored.min_value + 1 + finally: + assert gateway.request("PUT", path, configuration).status_code == 200 + custom_input: Final = custom.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields)) + assert ( + gateway.request("PUT", f"/usage/alert-rules/{custom.id}", {**custom_input, "metric": "latency"}).status_code + == 200 + ) + finally: + await repository.delete_rule(custom.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_reported_model_search_preserves_exact_identity_and_cursor( + gateway: Gateway, db: ManagementDatabase +) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + prefix: Final = f"reported-{uuid4().hex}" + initial: Final = await repository.list_reported_models(prefix, None, 1) + assert initial.data == () + models: Final = (f"{prefix}/first", f"{prefix}/literal%quote'", f"{prefix}/second") + try: + for index, model in enumerate((*models, models[0])): + await db.prisma.writer_db.execute_raw( + 'INSERT INTO "LiteLLM_DailyUserSpend" (id,date,api_key,model,updated_at) VALUES ($1,$2,$3,$4,CURRENT_TIMESTAMP)', + f"{prefix}-{index}", + initial.end_date.isoformat(), + prefix, + model, + ) + await db.prisma.writer_db.execute_raw( + 'INSERT INTO "LiteLLM_DailyUserSpend" (id,date,api_key,model,updated_at) VALUES ($1,$2,$3,$4,CURRENT_TIMESTAMP)', + f"{prefix}-old", + (initial.start_date - timedelta(days=1)).isoformat(), + prefix, + f"{prefix}/old", + ) + response: Final = gateway.request("GET", "/usage/alert-models", params={"search": prefix, "limit": "1"}) + assert response.status_code == 200, response.text + first: Final = UsageAlertModelList.model_validate(response.json()) + assert first.data[0].scope_id == models[0] and first.next_cursor == models[0] + second: Final = await repository.list_reported_models(prefix, first.next_cursor, 1) + third: Final = await repository.list_reported_models(prefix, second.next_cursor, 1) + assert second.data[0].scope_id == models[1] and third.data[0].scope_id == models[2] + assert third.next_cursor is None + literal: Final = await repository.list_reported_models(f"{prefix}/literal%quote'", None, 20) + assert tuple(option.scope_id for option in literal.data) == (models[1],) + assert literal.data[0].label == models[1] + with gateway.scenario() as scenario: + user: Final = scenario.user(user_role="internal_user") + key: Final = scenario.key(user_id=user, key_type="management") + assert gateway.request("GET", "/usage/alert-models", key=key).status_code == 403 + finally: + await db.prisma.writer_db.execute_raw('DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', prefix) diff --git a/tests/unit/types/proxy/test_usage_alerts.py b/tests/unit/types/proxy/test_usage_alerts.py index ad8aaaf493e..57d7e133438 100644 --- a/tests/unit/types/proxy/test_usage_alerts.py +++ b/tests/unit/types/proxy/test_usage_alerts.py @@ -3,7 +3,7 @@ from typing import Final import pytest from pydantic import ValidationError -from litellm.types.proxy.usage_alerts import UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import UsageAlertGroupUpdate, UsageAlertRuleInput RULE: Final = { "name": "Daily spend", @@ -41,3 +41,21 @@ def test_zero_request_floor_and_daily_drop_multiplier_are_supported() -> None: assert rule.direction == "decrease" assert rule.multiplier == 2 assert rule.min_requests == 0 + + +@pytest.mark.parametrize( + "changes", + [{"scope_id": None}, {"scope_type": "deployment"}, {"status": "open"}, {"status": "unknown"}], +) +def test_group_updates_require_exact_resource_and_a_different_valid_status(changes: dict[str, object]) -> None: + with pytest.raises(ValidationError): + UsageAlertGroupUpdate.model_validate( + { + "scope_type": "team", + "scope_id": "team-example", + "day": "2026-11-04", + "expected_status": "open", + "status": "resolved", + **changes, + } + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index cd55938d7c3..42ae1107a2e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -18834,6 +18834,41 @@ export interface paths { patch?: never; trace?: never; }; + "/usage/alert-groups": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Usage Alert Groups */ + get: operations["list_usage_alert_groups_usage_alert_groups_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Usage Alert Group */ + patch: operations["update_usage_alert_group_usage_alert_groups_patch"]; + trace?: never; + }; + "/usage/alert-models": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Usage Alert Models */ + get: operations["list_usage_alert_models_usage_alert_models_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/usage/alert-rules": { parameters: { query?: never; @@ -50472,6 +50507,120 @@ export interface components { /** Threshold */ threshold: number; }; + /** UsageAlertDefaultSummary */ + UsageAlertDefaultSummary: { + /** + * Enabled + * @default 0 + */ + enabled: number; + evaluation_status_counts?: components["schemas"]["UsageAlertEvaluationCounts"]; + /** + * Total + * @default 0 + */ + total: number; + }; + /** UsageAlertEvaluationCounts */ + UsageAlertEvaluationCounts: { + /** + * Complete + * @default 0 + */ + complete: number; + /** + * Insufficient Coverage + * @default 0 + */ + insufficient_coverage: number; + /** + * Insufficient History + * @default 0 + */ + insufficient_history: number; + /** + * Pending + * @default 0 + */ + pending: number; + }; + /** UsageAlertGroup */ + UsageAlertGroup: { + /** Activity Path */ + activity_path: string; + /** + * Day + * Format: date + */ + day: string; + /** Id */ + id: string; + /** Incident Count */ + incident_count: number; + /** Metrics */ + metrics: components["schemas"]["UsageAlertGroupMetric"][]; + /** Scope Id */ + scope_id: string | null; + /** Scope Label */ + scope_label: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + }; + /** UsageAlertGroupList */ + UsageAlertGroupList: { + /** Data */ + data: components["schemas"]["UsageAlertGroup"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** UsageAlertGroupMetric */ + UsageAlertGroupMetric: { + /** Incident Count */ + incident_count: number; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors"; + representative: components["schemas"]["UsageAlert"]; + }; + /** UsageAlertGroupUpdate */ + UsageAlertGroupUpdate: { + /** + * Day + * Format: date + */ + day: string; + /** + * Expected Status + * @enum {string} + */ + expected_status: "open" | "resolved" | "dismissed"; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + }; + /** UsageAlertGroupUpdateResult */ + UsageAlertGroupUpdateResult: { + /** Updated Count */ + updated_count: number; + }; /** UsageAlertList */ UsageAlertList: { /** Data */ @@ -50483,6 +50632,32 @@ export interface components { /** Total */ total: number; }; + /** UsageAlertModelList */ + UsageAlertModelList: { + /** Data */ + data: components["schemas"]["UsageAlertModelOption"][]; + /** + * End Date + * Format: date + */ + end_date: string; + /** Limit */ + limit: number; + /** Next Cursor */ + next_cursor: string | null; + /** + * Start Date + * Format: date + */ + start_date: string; + }; + /** UsageAlertModelOption */ + UsageAlertModelOption: { + /** Label */ + label: string; + /** Scope Id */ + scope_id: string; + }; /** UsageAlertRule */ UsageAlertRule: { /** @@ -50520,6 +50695,11 @@ export interface components { evaluation_status: "pending" | "complete" | "insufficient_coverage" | "insufficient_history"; /** Id */ id: string; + /** + * Is Default + * @default false + */ + is_default: boolean; /** Last Evaluated Day */ last_evaluated_day?: string | null; /** @@ -50635,6 +50815,7 @@ export interface components { UsageAlertRuleList: { /** Data */ data: components["schemas"]["UsageAlertRule"][]; + default_summary?: components["schemas"]["UsageAlertDefaultSummary"]; /** Limit */ limit: number; /** Offset */ @@ -78561,6 +78742,109 @@ export interface operations { }; }; }; + list_usage_alert_groups_usage_alert_groups_get: { + parameters: { + query?: { + scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; + scope_id?: string | null; + status?: ("open" | "resolved" | "dismissed") | null; + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertGroupList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_usage_alert_group_usage_alert_groups_patch: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UsageAlertGroupUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertGroupUpdateResult"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_usage_alert_models_usage_alert_models_get: { + parameters: { + query?: { + search?: string | null; + cursor?: string | null; + limit?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertModelList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; list_usage_alert_rules_usage_alert_rules_get: { parameters: { query?: { @@ -78568,6 +78852,7 @@ export interface operations { scope_id?: string | null; limit?: number; offset?: number; + kind?: "all" | "default" | "custom"; }; header?: never; path?: never; From f334a32717273e12d26993048de1ffc0f934b1ce Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 14:22:59 +0800 Subject: [PATCH 10/38] feat(alerts): share retirement issue state and batched acknowledgments --- .../model_deprecation_alerts.py | 15 ++++------- .../proxy/common_utils/model_deprecation.py | 7 +++++ litellm/repositories/config_repository.py | 14 +++++++++- litellm/types/proxy/model_deprecation.py | 12 +++++++++ .../repositories/test_config_repository.py | 26 +++++++++++++++++++ 5 files changed, 63 insertions(+), 11 deletions(-) create mode 100644 tests/unit/repositories/test_config_repository.py diff --git a/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py index a69dd6ee025..f0a057c4be2 100644 --- a/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py +++ b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py @@ -1,7 +1,5 @@ """Shared acknowledgement state for model migration alerts.""" -import hashlib -import json from datetime import date, datetime, timezone from typing import Annotated, Final @@ -10,6 +8,7 @@ from pydantic import TypeAdapter from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key from litellm.proxy.deps.database import DatabaseDep from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket @@ -32,12 +31,6 @@ def alert_tag_repository(database: DatabaseDep) -> DailyTagSpendRepository: return DailyTagSpendRepository(database.prisma) -def _state_key(model: str, deprecation_date: date) -> str: - identity: Final = json.dumps([model, deprecation_date.isoformat()]) - digest: Final = hashlib.sha256(identity.encode()).hexdigest() - return f"model_deprecation_alert:{digest}" - - def _require_admin(user: UserAPIKeyAuth, *, write: bool) -> None: if user.user_role == LitellmUserRoles.PROXY_ADMIN: return @@ -54,7 +47,7 @@ async def get_deprecation_alert_state( repository: Annotated[ConfigRepository, Depends(alert_state_repository)], ) -> DeprecationAlertState: _require_admin(user, write=False) - record: Final = await repository.get_param(_state_key(model, deprecation_date)) + record: Final = await repository.get_param(deprecation_alert_state_key(model, deprecation_date)) if record is None: return DeprecationAlertState(model=model, deprecation_date=deprecation_date) return DeprecationAlertState.model_validate(record.param_value) @@ -70,7 +63,9 @@ async def update_deprecation_alert_state( state: Final = DeprecationAlertState( **data.model_dump(), updated_at=datetime.now(timezone.utc), updated_by=user.user_id ) - await repository.set_param(_state_key(data.model, data.deprecation_date), state.model_dump(mode="json")) + await repository.set_param( + deprecation_alert_state_key(data.model, data.deprecation_date), state.model_dump(mode="json") + ) return state diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index 8a91c7b0a0c..a210ba6eb69 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -1,5 +1,7 @@ from __future__ import annotations +import hashlib +import json import logging from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass @@ -28,6 +30,11 @@ _SHARED_SCHEDULE_PROVIDER: Final[Mapping[str, str]] = MappingProxyType({"azure_a LoadedCatalog: TypeAlias = Callable[[], Mapping[str, Mapping[str, object]]] +def deprecation_alert_state_key(model: str, deprecation_date: date) -> str: + identity: Final = json.dumps([model, deprecation_date.isoformat()]) + return f"model_deprecation_alert:{hashlib.sha256(identity.encode()).hexdigest()}" + + @dataclass(frozen=True, slots=True) class _ResolvedDeprecation: deprecation_date: date diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py index 36c6ab7f711..5cf5dafcce5 100644 --- a/litellm/repositories/config_repository.py +++ b/litellm/repositories/config_repository.py @@ -30,7 +30,7 @@ class _ConfigRow(Protocol): class _ConfigTable(Protocol): async def find_unique(self, *, where: Mapping[str, str]) -> _ConfigRow | None: ... - async def find_many(self) -> Sequence[_ConfigRow]: ... + async def find_many(self, *, where: Mapping[str, object] | None = None) -> Sequence[_ConfigRow]: ... async def upsert(self, *, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]]) -> _ConfigRow: ... @@ -83,6 +83,18 @@ class ConfigRepository: param_value = _decoded_json(param_value) return ConfigParam(param_name=param_name, param_value=param_value) + async def get_params(self, param_names: Sequence[str]) -> tuple[ConfigParam, ...]: + if not param_names: + return () + records: Final = await self._config_table.find_many(where={"param_name": {"in": tuple(param_names)}}) + return tuple( + ConfigParam( + record.param_name, + _decoded_json(record.param_value) if isinstance(record.param_value, str) else record.param_value, + ) + for record in records + ) + async def set_param(self, param_name: str, param_value: object) -> ConfigParam: """Set a config parameter in the database.""" value_json: Final = json.dumps(param_value) if not isinstance(param_value, str) else param_value diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index 37c49d737af..33f9b942595 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -112,3 +112,15 @@ class DeprecationTagUsage(LiteLLMBaseModel): api_key: str tag: str + + +class ModelDeprecationUsage(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model: str + requests: int = Field(ge=0) + last_used_date: date + last_day_requests: int = Field(ge=0) + keys: int = Field(ge=0) + users: int = Field(ge=0) + teams: int = Field(ge=0) diff --git a/tests/unit/repositories/test_config_repository.py b/tests/unit/repositories/test_config_repository.py new file mode 100644 index 00000000000..0856aebf0ff --- /dev/null +++ b/tests/unit/repositories/test_config_repository.py @@ -0,0 +1,26 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from litellm.repositories.config_repository import ConfigRepository + + +@pytest.mark.asyncio +async def test_batch_reads_decode_legacy_json_and_current_objects_from_writer(): + find_many = AsyncMock( + return_value=( + SimpleNamespace(param_name="first", param_value='{"status":"resolved"}'), + SimpleNamespace(param_name="second", param_value={"status": "dismissed"}), + ) + ) + prisma = SimpleNamespace(writer_db=SimpleNamespace(litellm_config=SimpleNamespace(find_many=find_many))) + repository = ConfigRepository(prisma, use_writer=True) + rows = await repository.get_params(("first", "second", "absent")) + assert {row.param_name: row.param_value for row in rows} == { + "first": {"status": "resolved"}, + "second": {"status": "dismissed"}, + } + find_many.assert_awaited_once_with(where={"param_name": {"in": ("first", "second", "absent")}}) + assert await repository.get_params(()) == () + assert find_many.await_count == 1 From 5ab3a0a7399b44f2c5c5fa6d5932ced536a90caf Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 14:50:43 +0800 Subject: [PATCH 11/38] feat: add unified issue storage and investigation APIs --- .../20261105200000_issues_inbox/migration.sql | 49 ++ litellm/models/issues.py | 134 +++ litellm/proxy/_types.py | 27 + litellm/proxy/alerting_endpoints/issues.py | 235 +++++ litellm/proxy/deps/repositories.py | 17 + .../common_daily_activity.py | 2 +- .../daily_activity_routes.py | 4 + litellm/proxy/proxy_server.py | 2 + .../repositories/daily_activity_repository.py | 2 + litellm/repositories/daily_activity_sql.py | 36 +- litellm/repositories/issue_activity.py | 154 ++++ litellm/repositories/issue_families.py | 149 ++++ litellm/repositories/issues.py | 504 +++++++++++ litellm/repositories/usage_alerts.py | 34 +- litellm/types/proxy/issues.py | 260 ++++++ .../common_daily_activity.py | 4 + litellm/types/proxy/usage_alert_defaults.py | 75 ++ litellm/types/proxy/usage_alerts.py | 2 +- litellm/types/repositories/daily_activity.py | 6 + schema.prisma | 54 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 812 +++++++++++++++++- 21 files changed, 2537 insertions(+), 25 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql create mode 100644 litellm/models/issues.py create mode 100644 litellm/proxy/alerting_endpoints/issues.py create mode 100644 litellm/repositories/issue_activity.py create mode 100644 litellm/repositories/issue_families.py create mode 100644 litellm/repositories/issues.py create mode 100644 litellm/types/proxy/issues.py create mode 100644 litellm/types/proxy/usage_alert_defaults.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql new file mode 100644 index 00000000000..3044320e8e8 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql @@ -0,0 +1,49 @@ +CREATE TABLE "LiteLLM_Issue" ( + "id" TEXT NOT NULL, + "cohort_key" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "family" TEXT NOT NULL, + "scope_type" TEXT NOT NULL, + "scope_id" TEXT, + "status" TEXT NOT NULL, + "active" BOOLEAN NOT NULL DEFAULT true, + "ack_key" TEXT, + "record" JSONB NOT NULL, + "first_seen" TIMESTAMP(3) NOT NULL, + "last_seen" TIMESTAMP(3) NOT NULL, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "LiteLLM_Issue_pkey" PRIMARY KEY ("id") +); +CREATE INDEX "LiteLLM_Issue_status_last_seen_id_idx" ON "LiteLLM_Issue"("status", "last_seen", "id"); +CREATE INDEX "LiteLLM_Issue_kind_family_last_seen_id_idx" ON "LiteLLM_Issue"("kind", "family", "last_seen", "id"); +CREATE INDEX "LiteLLM_Issue_scope_type_scope_id_last_seen_id_idx" ON "LiteLLM_Issue"("scope_type", "scope_id", "last_seen", "id"); +CREATE INDEX "LiteLLM_Issue_cohort_key_active_idx" ON "LiteLLM_Issue"("cohort_key", "active"); +CREATE TABLE "LiteLLM_IssueSignal" ( + "issue_id" TEXT NOT NULL, + "source" TEXT NOT NULL, + "source_id" TEXT NOT NULL, + "record" JSONB NOT NULL, + CONSTRAINT "LiteLLM_IssueSignal_pkey" PRIMARY KEY ("issue_id", "source", "source_id") +); +CREATE INDEX "LiteLLM_IssueSignal_source_source_id_idx" ON "LiteLLM_IssueSignal"("source", "source_id"); +CREATE TABLE "LiteLLM_IssueDelivery" ( + "id" TEXT NOT NULL, + "issue_id" TEXT NOT NULL, + "record" JSONB NOT NULL, + "sent" BOOLEAN NOT NULL DEFAULT false, + "next_attempt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "LiteLLM_IssueDelivery_pkey" PRIMARY KEY ("id") +); +CREATE INDEX "LiteLLM_IssueDelivery_sent_next_attempt_id_idx" ON "LiteLLM_IssueDelivery"("sent", "next_attempt", "id"); +CREATE INDEX "LiteLLM_IssueDelivery_issue_id_idx" ON "LiteLLM_IssueDelivery"("issue_id"); +CREATE TABLE "LiteLLM_IssueDeliveryRule" ( + "delivery_id" TEXT NOT NULL, + "rule_id" TEXT NOT NULL, + "source_id" TEXT NOT NULL, + "rule_revision" TEXT NOT NULL, + CONSTRAINT "LiteLLM_IssueDeliveryRule_pkey" PRIMARY KEY ("delivery_id", "rule_id", "source_id") +); +ALTER TABLE "LiteLLM_UsageAlertIncident" ADD COLUMN "evaluation_revision" TEXT NOT NULL DEFAULT ''; +CREATE INDEX "LiteLLM_UsageAlertIncident_evaluation_day_breached_id_idx" + ON "LiteLLM_UsageAlertIncident"("evaluation_day", "breached", "id"); diff --git a/litellm/models/issues.py b/litellm/models/issues.py new file mode 100644 index 00000000000..7d0692d26db --- /dev/null +++ b/litellm/models/issues.py @@ -0,0 +1,134 @@ +from datetime import date, datetime +from typing import Literal, Protocol + +from litellm.types.proxy.issues import ( + Issue, + IssueCohort, + IssueDeliveryBinding, + IssueDeliveryInput, + IssueFamily, + IssueFamilyList, + IssueFamilySetting, + IssueFamilyUpdate, + IssueImpact, + IssueInput, + IssueKind, + IssueList, + IssuePendingDelivery, + IssueProjectionSignal, + IssueScope, + IssueSignal, + IssueSignalInput, + IssueSignalList, +) +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailyActivityKeyPageResponse, + DailyActivityUserPageResponse, + SpendAnalyticsPaginatedResponse, +) +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus + + +class IssueStore(Protocol): + async def get_issue(self, issue_id: str) -> Issue | None: ... + + async def list_issues( + self, + kind: IssueKind | None = None, + family: IssueFamily | None = None, + status: UsageAlertStatus | None = None, + scope_type: IssueScope | None = None, + scope_id: str | None = None, + limit: int = 50, + offset: int = 0, + ) -> IssueList: ... + + async def set_status( + self, + issue_id: str, + status: UsageAlertStatus, + expected_status: UsageAlertStatus, + actor: str | None = None, + ) -> Issue | None: ... + + async def upsert_issue( + self, + data: IssueInput, + reopen: bool = False, + status: UsageAlertStatus | None = None, + ) -> Issue: ... + + async def upsert_issues( + self, + inputs: tuple[IssueInput, ...], + reopen: bool = False, + status: UsageAlertStatus | None = None, + ) -> tuple[Issue, ...]: ... + + async def find_active_issue(self, cohort_key: str) -> Issue | None: ... + + async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: ... + + async def add_signals(self, issue_id: str, signals: tuple[IssueSignalInput, ...]) -> None: ... + + async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: ... + + async def list_signals(self, issue_id: str, limit: int = 50, offset: int = 0) -> IssueSignalList: ... + + async def active_signals( + self, + day: date, + limit: int = 500, + cursor: str | None = None, + scope_type: IssueScope | None = None, + metric: UsageAlertMetric | None = None, + ) -> tuple[IssueProjectionSignal, ...]: ... + + async def recover_issues(self, day: date) -> int: ... + + async def projection_completed(self, day: date, version: str) -> bool: ... + + async def complete_projection(self, day: date, version: str) -> None: ... + + async def finish_deprecation_projection(self, observed_at: datetime) -> int: ... + + async def family_settings(self) -> IssueFamilyList: ... + + async def update_family( + self, family: PerformanceIssueFamily, data: IssueFamilyUpdate + ) -> IssueFamilySetting | None: ... + + async def queue_delivery( + self, + data: IssueDeliveryInput, + rule_id: str, + rule_revision: str, + source_id: str, + ) -> None: ... + + async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: ... + + async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: ... + + async def mark_delivery_sent(self, delivery_id: str) -> None: ... + + async def defer_delivery( + self, delivery_id: str, error: Literal["delivery_failed", "transport_not_configured"] + ) -> None: ... + + async def prune(self, retention_days: int = 90, limit: int = 500) -> None: ... + + +class IssueActivityStore(Protocol): + async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse: ... + + async def keys( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityKeyPageResponse: ... + + async def users( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityUserPageResponse: ... + + async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: ... diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7254a0fb6b7..633768fa9c3 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -585,6 +585,15 @@ class LiteLLMRoutes(enum.Enum): "/usage/alert-rules/{rule_id}", "/usage/alerts", "/usage/alerts/{alert_id}", + "/issues", + "/issues/families", + "/issues/families/{family}", + "/issues/{issue_id}", + "/issues/{issue_id}/signals", + "/issues/{issue_id}/activity", + "/issues/{issue_id}/keys", + "/issues/{issue_id}/users", + "/issues/{issue_id}/impact", "/key/info", "/key/health", "/team/info", @@ -658,6 +667,15 @@ class LiteLLMRoutes(enum.Enum): "/usage/alert-rules/{rule_id}", "/usage/alerts", "/usage/alerts/{alert_id}", + "/issues", + "/issues/families", + "/issues/families/{family}", + "/issues/{issue_id}", + "/issues/{issue_id}/signals", + "/issues/{issue_id}/activity", + "/issues/{issue_id}/keys", + "/issues/{issue_id}/users", + "/issues/{issue_id}/impact", # user "/user/new", "/management/v1/users/bulk", @@ -874,6 +892,15 @@ class LiteLLMRoutes(enum.Enum): "/usage/alert-rules/{rule_id}", "/usage/alerts", "/usage/alerts/{alert_id}", + "/issues", + "/issues/families", + "/issues/families/{family}", + "/issues/{issue_id}", + "/issues/{issue_id}/signals", + "/issues/{issue_id}/activity", + "/issues/{issue_id}/keys", + "/issues/{issue_id}/users", + "/issues/{issue_id}/impact", ) self_managed_routes = [ diff --git a/litellm/proxy/alerting_endpoints/issues.py b/litellm/proxy/alerting_endpoints/issues.py new file mode 100644 index 00000000000..30f3d44afe7 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/issues.py @@ -0,0 +1,235 @@ +"""Unified issue inbox and investigations backed by existing daily aggregates.""" + +from datetime import date +from typing import Final + +from fastapi import APIRouter, HTTPException + +from litellm.models.issues import IssueActivityStore, IssueStore +from litellm.proxy.alerting_endpoints.usage_alerts import ( + Limit, + Offset, + ScopeId, + User, + require_repository, + require_scope, +) +from litellm.proxy.deps.repositories import IssueActivityRepositoryDep, IssuesRepositoryDep, UsageAlertsRepositoryDep +from litellm.types.proxy.issues import ( + Issue, + IssueFamily, + IssueFamilyList, + IssueFamilySetting, + IssueFamilyUpdate, + IssueImpact, + IssueKind, + IssueList, + IssueScope, + IssueSignalList, + IssueUpdate, +) +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailyActivityKeyPageResponse, + DailyActivityUserPageResponse, + SpendAnalyticsPaginatedResponse, +) +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import UsageAlertStatus + +router: Final = APIRouter(prefix="/issues", tags=["issues"]) + + +def require_issues(repository: IssueStore | None) -> IssueStore: + if repository is None: + raise HTTPException(status_code=503, detail="Issues require a database connection") + return repository + + +def require_activity(repository: IssueActivityStore | None) -> IssueActivityStore: + if repository is None: + raise HTTPException(status_code=503, detail="Issue Activity requires a database connection") + return repository + + +async def authorized_issue( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + *, + write: bool = False, +) -> Issue: + issue: Final = await require_issues(repository).get_issue(issue_id) + if issue is None: + raise HTTPException(status_code=404, detail="Issue not found") + await require_scope( + user, + require_repository(alerts), + "model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type, + issue.cohort.scope_id, + write=write, + ) + return issue + + +def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]: + start: Final = start_date or issue.range_start + end: Final = end_date or issue.range_end + if start > end or (end - start).days > 365: + raise HTTPException(status_code=422, detail="Choose an ordered date range of at most 366 days") + return start, end + + +@router.get("", response_model=IssueList) +async def list_issues( + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + kind: IssueKind | None = None, + family: IssueFamily | None = None, + status: UsageAlertStatus | None = None, + scope_type: IssueScope | None = None, + scope_id: ScopeId = None, + limit: Limit = 50, + offset: Offset = 0, +) -> IssueList: + await require_scope( + user, require_repository(alerts), "model" if scope_type == "provider" else scope_type, scope_id, write=False + ) + return await require_issues(repository).list_issues(kind, family, status, scope_type, scope_id, limit, offset) + + +@router.get("/families", response_model=IssueFamilyList) +async def list_issue_families( + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> IssueFamilyList: + await require_scope(user, require_repository(alerts), "deployment", None, write=False) + return await require_issues(repository).family_settings() + + +@router.patch("/families/{family}", response_model=IssueFamilySetting) +async def update_issue_family( + family: PerformanceIssueFamily, + data: IssueFamilyUpdate, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> IssueFamilySetting: + await require_scope(user, require_repository(alerts), "deployment", None, write=True) + try: + result: Final = await require_issues(repository).update_family(family, data) + except ValueError as exc: + raise HTTPException( + status_code=422, detail="Invalid family policy; check direction and history bounds" + ) from exc + if result is None: + raise HTTPException(status_code=409, detail="Built-in rules changed or are unavailable; reload and retry") + return result + + +@router.get("/{issue_id}", response_model=Issue) +async def get_issue( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> Issue: + return await authorized_issue(issue_id, user, repository, alerts) + + +@router.patch("/{issue_id}", response_model=Issue) +async def update_issue( + issue_id: str, + data: IssueUpdate, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> Issue: + await authorized_issue(issue_id, user, repository, alerts, write=True) + changed: Final = await require_issues(repository).set_status( + issue_id, data.status, data.expected_status, user.user_id + ) + if changed is None: + raise HTTPException(status_code=409, detail="Issue status changed; reload and retry") + return changed + + +@router.get("/{issue_id}/signals", response_model=IssueSignalList) +async def list_issue_signals( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + limit: Limit = 50, + offset: Offset = 0, +) -> IssueSignalList: + await authorized_issue(issue_id, user, repository, alerts) + return await require_issues(repository).list_signals(issue_id, limit, offset) + + +@router.get("/{issue_id}/activity", response_model=SpendAnalyticsPaginatedResponse) +async def get_issue_activity( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, +) -> SpendAnalyticsPaginatedResponse: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + start, end = activity_range(issue, start_date, end_date) + return await require_activity(activity).daily_activity(issue.cohort, start, end) + + +@router.get("/{issue_id}/keys", response_model=DailyActivityKeyPageResponse) +async def get_issue_keys( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, + limit: Limit = 20, + offset: Offset = 0, +) -> DailyActivityKeyPageResponse: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + start, end = activity_range(issue, start_date, end_date) + return await require_activity(activity).keys(issue.cohort, start, end, limit, offset) + + +@router.get("/{issue_id}/users", response_model=DailyActivityUserPageResponse) +async def get_issue_users( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, + limit: Limit = 20, + offset: Offset = 0, +) -> DailyActivityUserPageResponse: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + if issue.cohort.scope_type == "team": + raise HTTPException(status_code=422, detail="Historical team aggregates do not retain user membership") + start, end = activity_range(issue, start_date, end_date) + return await require_activity(activity).users(issue.cohort, start, end, limit, offset) + + +@router.get("/{issue_id}/impact", response_model=IssueImpact) +async def get_issue_impact( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, +) -> IssueImpact: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + start, end = activity_range(issue, start_date, end_date) + return await require_activity(activity).impact(issue.cohort, start, end) diff --git a/litellm/proxy/deps/repositories.py b/litellm/proxy/deps/repositories.py index b3b3aeafe28..ca12c6ff87c 100644 --- a/litellm/proxy/deps/repositories.py +++ b/litellm/proxy/deps/repositories.py @@ -4,12 +4,15 @@ from fastapi import Depends from litellm.models.deployment_warning import DeploymentWarningIgnoreStore from litellm.models.gateway_activity import GatewayActivityStore +from litellm.models.issues import IssueActivityStore, IssueStore from litellm.models.plugin_upload import PluginUploadStore from litellm.models.usage_alerts import UsageAlertStore from litellm.models.user_banner import UserBannerStore from litellm.proxy.deps.database import DatabaseDep from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository from litellm.repositories.gateway_activity_repository import GatewayActivityRepository +from litellm.repositories.issue_activity import IssueActivityRepository +from litellm.repositories.issues import IssuesRepository from litellm.repositories.plugin_upload_repository import PluginUploadRepository from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.repositories.user_banner_repository import UserBannerRepository @@ -52,3 +55,17 @@ async def get_usage_alerts_repository(database: DatabaseDep) -> UsageAlertStore UsageAlertsRepositoryDep: TypeAlias = Annotated[UsageAlertStore | None, Depends(get_usage_alerts_repository)] + + +async def get_issues_repository(database: DatabaseDep) -> IssueStore | None: + return None if database is None else IssuesRepository.from_prisma(database.prisma) + + +IssuesRepositoryDep: TypeAlias = Annotated[IssueStore | None, Depends(get_issues_repository)] + + +async def get_issue_activity_repository(database: DatabaseDep) -> IssueActivityStore | None: + return None if database is None else IssueActivityRepository.from_prisma(database.prisma) + + +IssueActivityRepositoryDep: TypeAlias = Annotated[IssueActivityStore | None, Depends(get_issue_activity_repository)] diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 6e4ea0bd0cb..1fd3b8bae12 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -1128,7 +1128,7 @@ async def get_daily_activity_aggregated( scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit ), _guardrail_spend_or_empty(repository, scope) - if include_guardrail_breakdown and not scope.model + if include_guardrail_breakdown and not scope.model and scope.provider is None and not scope.model_pairs else asyncio.sleep(0, result=()), ) records: Final = aggregated_rows.grouping_rows diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 1a3445aae52..6cf44e10109 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -136,6 +136,8 @@ def _key_activity_row(row: KeySpendRow, metadata: Mapping[str, KeyMetadataRow]) failed_requests=row.failed_requests, cache_read_input_tokens=row.cache_read_input_tokens, cache_creation_input_tokens=row.cache_creation_input_tokens, + total_response_time_ms=row.total_response_time_ms, + timed_requests=row.timed_requests, ), metadata=_key_metadata(row.api_key, metadata), ) @@ -169,6 +171,8 @@ def _user_activity_row(row: UserSpendRow, metadata: Mapping[str, UserMetadataRow api_requests=row.api_requests, successful_requests=row.successful_requests, failed_requests=row.failed_requests, + total_response_time_ms=row.total_response_time_ms, + timed_requests=row.timed_requests, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d876ce011a4..ad8a1bc67c0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -428,6 +428,7 @@ from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( router as model_deprecation_alerts_router, ) from litellm.proxy.alerting_endpoints.usage_alerts import router as usage_alerts_router +from litellm.proxy.alerting_endpoints.issues import router as issues_router from litellm.proxy.analytics_endpoints.analytics_endpoints import ( router as analytics_router, ) @@ -22440,6 +22441,7 @@ app.include_router(fallback_management_router) app.include_router(cache_settings_router) app.include_router(model_deprecation_alerts_router) app.include_router(usage_alerts_router) +app.include_router(issues_router) app.include_router(coordination_redis_settings_router) app.include_router(user_agent_analytics_router) app.include_router(gateway_request_router) diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 321468c4976..262eb9d7ec1 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -324,6 +324,8 @@ class DailyActivityRepository: ) async def daily_rows(self, scope: DailyActivityScope, *, page: int, page_size: int) -> DailyRowsPage: + if scope.provider is not None or scope.model_pairs: + raise ValueError("Provider cohorts require the aggregated Activity API") table: Final = _daily_rows_table(self._prisma_client, scope.table) adjusted_start, adjusted_end = adjust_dates_for_timezone( scope.start_date, diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 4bb52767de6..c480af53272 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -1,3 +1,4 @@ +import json from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime, timedelta, timezone @@ -75,6 +76,8 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu exclusion_index: Final = entity_index + int(has_entity_array) model_index: Final = exclusion_index + int(bool(scope.exclude_entity_ids)) api_keys_index: Final = model_index + int(bool(scope.model)) + provider_index: Final = api_keys_index + int(bool(scope.api_keys)) + pairs_index: Final = provider_index + int(scope.provider is not None) conditions: Final = ( f"date >= ${start_index}", f"date <= ${start_index + 1}", @@ -85,6 +88,21 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu if has_entity_array else () ), + *( + (f"COALESCE(NULLIF(custom_llm_provider, ''), 'unknown') = ${provider_index}",) + if scope.provider is not None + else () + ), + *( + ( + f"EXISTS (SELECT 1 FROM jsonb_to_recordset(${pairs_index}::jsonb) " + f"AS cohort(match_provider text, match_model text) WHERE " + f"cohort.match_provider=COALESCE(NULLIF(custom_llm_provider, ''), 'unknown') " + f"AND cohort.match_model={_MODEL_EXPR})", + ) + if scope.model_pairs + else () + ), *( ( f'("{scope.entity_id_field}" IS NULL ' @@ -113,6 +131,16 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu *((list(scope.exclude_entity_ids),) if scope.exclude_entity_ids else ()), *((scope.model,) if scope.model else ()), *((list(scope.api_keys),) if scope.api_keys else ()), + *((scope.provider,) if scope.provider is not None else ()), + *( + ( + json.dumps( + tuple({"match_provider": provider, "match_model": model} for provider, model in scope.model_pairs) + ), + ) + if scope.model_pairs + else () + ), ) return " AND ".join(conditions), params @@ -301,7 +329,9 @@ def _key_spend_select() -> str: COALESCE(SUM(successful_requests), 0)::bigint AS successful_requests, COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests, COALESCE(SUM(cache_read_input_tokens), 0)::bigint AS cache_read_input_tokens, - COALESCE(SUM(cache_creation_input_tokens), 0)::bigint AS cache_creation_input_tokens""" + COALESCE(SUM(cache_creation_input_tokens), 0)::bigint AS cache_creation_input_tokens, + COALESCE(SUM(total_response_time_ms), 0)::bigint AS total_response_time_ms, + COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests""" def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery: @@ -339,7 +369,9 @@ def _user_spend_select() -> str: (COALESCE(SUM(prompt_tokens), 0) + COALESCE(SUM(completion_tokens), 0))::bigint AS total_tokens, COALESCE(SUM(api_requests), 0)::bigint AS api_requests, COALESCE(SUM(successful_requests), 0)::bigint AS successful_requests, - COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests""" + COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests, + COALESCE(SUM(total_response_time_ms), 0)::bigint AS total_response_time_ms, + COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests""" def build_user_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery: diff --git a/litellm/repositories/issue_activity.py b/litellm/repositories/issue_activity.py new file mode 100644 index 00000000000..fed59a82cbc --- /dev/null +++ b/litellm/repositories/issue_activity.py @@ -0,0 +1,154 @@ +"""Issue investigations reuse the same aggregate filters as entity Activity.""" + +from dataclasses import asdict, replace +from datetime import date +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # generated Prisma structural database interface + +from pydantic import BaseModel, TypeAdapter + +from litellm.constants import PTU_SENTINEL_API_KEY +from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket +from litellm.repositories.daily_activity_repository import DailyActivityRepository +from litellm.repositories.daily_activity_sql import PRISMA_TO_PG_TABLE, build_where_clause +from litellm.repositories.usage_alerts import AlertDatabase +from litellm.types.proxy.issues import IssueCohort, IssueImpact +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailyActivityKeyPageResponse, + DailyActivityUserPageResponse, + KeyMetadata, + KeySpendActivityRow, + KeySpendMetrics, + SpendAnalyticsPaginatedResponse, + UserActivityRow, +) +from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +class _Counts(BaseModel, frozen=True): + keys: int + entities: int + models: int + requests: int + + +def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyActivityScope: + if start > end or (end - start).days > 365: + raise ValueError("Choose an ordered date range of at most 366 days") + return DailyActivityScope( + table=DailyActivityTable.TEAM if cohort.scope_type == "team" else DailyActivityTable.USER, + entity_id_field="team_id" if cohort.scope_type == "team" else "user_id", + entity_ids=(cohort.scope_id,) if cohort.scope_type in ("team", "user") and cohort.scope_id else None, + exclude_entity_ids=(), + api_keys=(api_key_from_bucket(cohort.scope_id),) if cohort.scope_type == "key" and cohort.scope_id else None, + start_date=start.isoformat(), + end_date=end.isoformat(), + model=cohort.scope_id if cohort.scope_type == "model" else None, + timezone_offset_minutes=0, + include_current_utc_day=True, + provider=cohort.scope_id if cohort.scope_type == "provider" else None, + model_pairs=tuple((pair.provider, pair.model) for pair in cohort.model_pairs), + ) + + +class IssueActivityRepository: + def __init__(self, activity: DailyActivityRepository, database: AlertDatabase) -> None: + self.activity = activity + self.database = database + + @classmethod + def from_prisma(cls, prisma: "PrismaClient") -> "IssueActivityRepository": + from litellm.proxy.management_endpoints.common_daily_activity import daily_activity_repository + + database: Final = cast(AlertDatabase, prisma.db) # cast-ok: generated Prisma query_raw + return cls(daily_activity_repository(prisma), database) + + async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse: + from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated + + return await get_daily_activity_aggregated( + self.activity, issue_activity_scope(cohort, start, end), api_key_limit=20 + ) + + async def keys( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityKeyPageResponse: + page: Final = await self.activity.key_page(issue_activity_scope(cohort, start, end), limit=limit, offset=offset) + metadata: Final = await self.activity.key_metadata( + frozenset(row.api_key for row in page.rows if row.api_key), None + ) + return DailyActivityKeyPageResponse( + api_keys=[ + KeySpendActivityRow( + api_key=api_key_bucket(row.api_key), + metrics=KeySpendMetrics.model_validate(asdict(row)), + metadata=KeyMetadata.model_validate(asdict(metadata[row.api_key])) + if row.api_key in metadata + else KeyMetadata(), + ) + for row in page.rows + ], + total_api_keys=page.total_api_keys, + limit=limit, + offset=offset, + ) + + async def users( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityUserPageResponse: + if cohort.scope_type == "team": + raise ValueError("Historical team aggregates do not retain user membership") + page: Final = await self.activity.user_page( + issue_activity_scope(cohort, start, end), limit=limit, offset=offset + ) + metadata: Final = await self.activity.user_metadata(frozenset(row.user_id for row in page.rows if row.user_id)) + return DailyActivityUserPageResponse( + users=tuple( + UserActivityRow.model_validate( + { + **asdict(row), + **( + asdict(metadata[row.user_id]) if row.user_id is not None and row.user_id in metadata else {} + ), + } + ) + for row in page.rows + ), + total_users=page.total_users, + limit=limit, + offset=offset, + ) + + async def _counts(self, scope: DailyActivityScope) -> _Counts: + where, params = build_where_clause(scope) + rows: Final = TypeAdapter(tuple[_Counts, ...]).validate_python( + await self.database.query_raw( + f'''SELECT COUNT(DISTINCT NULLIF(api_key,'')) FILTER(WHERE api_key<>${len(params) + 1})::int AS keys, + COUNT(DISTINCT NULLIF("{scope.entity_id_field}",'')) + FILTER(WHERE "{scope.entity_id_field}" NOT IN ('default_user_id','litellm-dashboard'))::int AS entities, + COUNT(DISTINCT NULLIF(model,''))::int AS models, + COALESCE(SUM(api_requests),0)::bigint AS requests + FROM "{PRISMA_TO_PG_TABLE[scope.table]}" WHERE {where}''', + *params, + PTU_SENTINEL_API_KEY, + ) + ) + return rows[0] + + async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: + scope: Final = issue_activity_scope(cohort, start, end) + counts: Final = await self._counts(scope) + teams: Final = ( + await self._counts(replace(scope, table=DailyActivityTable.TEAM, entity_id_field="team_id")) + if cohort.scope_type not in ("user", "team") + else None + ) + return IssueImpact( + keys=counts.keys, + models=counts.models, + requests=counts.requests, + users=counts.entities if cohort.scope_type != "team" else None, + teams=counts.entities if cohort.scope_type == "team" else teams.entities if teams else None, + ) diff --git a/litellm/repositories/issue_families.py b/litellm/repositories/issue_families.py new file mode 100644 index 00000000000..05b1e938206 --- /dev/null +++ b/litellm/repositories/issue_families.py @@ -0,0 +1,149 @@ +"""Bounded settings for the built-in performance policies.""" + +import json +from datetime import datetime, timezone +from itertools import chain, product +from typing import Final + +from pydantic import BaseModel, TypeAdapter + +from litellm.repositories.usage_alerts import AlertDatabase +from litellm.types.proxy.issues import ( + IssueFamilyList, + IssueFamilyPolicy, + IssueFamilySetting, + IssueFamilyUpdate, +) +from litellm.types.proxy.usage_alert_defaults import ( + DEFAULT_USAGE_POLICIES, + DEFAULT_USAGE_RULE_IDS, + USAGE_ALERT_SCOPES, + PerformanceIssueFamily, + default_usage_rule_id, +) +from litellm.types.proxy.usage_alerts import UsageAlertRule + + +class _RuleRow(BaseModel, frozen=True): + record: UsageAlertRule + + +class _Count(BaseModel, frozen=True): + count: int + + +def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) -> IssueFamilySetting: + policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family) + groups: Final = tuple( + (policy, tuple(rule for rule in rules if rule.metric == policy.metric and rule.direction == policy.direction)) + for policy in policies + ) + values: Final = tuple( + IssueFamilyPolicy( + direction=policy.direction, + enabled=group[0].enabled if group and len({rule.enabled for rule in group}) == 1 else None, + multiplier=group[0].multiplier if group and len({rule.multiplier for rule in group}) == 1 else None, + min_value=group[0].min_value if group and len({rule.min_value for rule in group}) == 1 else None, + min_requests=group[0].min_requests if group and len({rule.min_requests for rule in group}) == 1 else None, + baseline_days=group[0].baseline_days + if group and len({rule.baseline_days for rule in group}) == 1 + else None, + min_history_days=group[0].min_history_days + if group and len({rule.min_history_days for rule in group}) == 1 + else None, + mixed=len( + { + (r.enabled, r.multiplier, r.min_value, r.min_requests, r.baseline_days, r.min_history_days) + for r in group + } + ) + > 1, + total_rules=len(group), + enabled_rules=sum(rule.enabled for rule in group), + ) + for policy, group in groups + ) + family_rules: Final = tuple(chain.from_iterable(group for _, group in groups)) + return IssueFamilySetting( + family=family, + enabled=family_rules[0].enabled if family_rules and len({r.enabled for r in family_rules}) == 1 else None, + total_rules=len(family_rules), + enabled_rules=sum(rule.enabled for rule in family_rules), + policies=values, + ) + + +async def family_settings(database: AlertDatabase) -> IssueFamilyList: + rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python( + await database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted', + list(DEFAULT_USAGE_RULE_IDS), + ) + ) + return IssueFamilyList( + data=tuple( + _setting(family, tuple(row.record for row in rows)) + for family in ("spend", "latency", "reliability", "traffic") + ) + ) + + +async def update_family( + database: AlertDatabase, + family: PerformanceIssueFamily, + data: IssueFamilyUpdate, +) -> IssueFamilySetting | None: + policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family) + if any(update.direction not in {policy.direction for policy in policies} for update in data.policies): + raise ValueError("This family does not support that direction") + ids: Final = tuple( + default_usage_rule_id(scope, policy.metric, policy.direction) + for policy, scope in product(policies, USAGE_ALERT_SCOPES) + ) + rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python( + await database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted', + list(ids), + ) + ) + timestamp: Final = datetime.now(timezone.utc).isoformat() + updates: Final = { + policy.direction: policy.model_dump(exclude_none=True, exclude={"direction"}) for policy in data.policies + } + records: Final = tuple( + UsageAlertRule.model_validate( + { + **row.record.model_dump(), + **({"enabled": data.enabled} if data.enabled is not None else {}), + **updates.get(row.record.direction, {}), + "updated_at": timestamp, + } + ) + for row in rows + ) + if not records: + return None + changed: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await database.query_raw( + """WITH incoming AS MATERIALIZED ( + SELECT x->'record' AS record,x->>'revision' AS revision FROM jsonb_array_elements($1::jsonb) x + ), eligible AS MATERIALIZED ( + SELECT r.id FROM "LiteLLM_UsageAlertRule" r JOIN incoming x ON r.id=x.record->>'id' + WHERE NOT r.deleted AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz + ORDER BY r.id FOR UPDATE OF r + ), changed AS ( + UPDATE "LiteLLM_UsageAlertRule" r SET record=x.record,enabled=(x.record->>'enabled')::boolean, + updated_at=(x.record->>'updated_at')::timestamptz AT TIME ZONE 'UTC' + FROM incoming x WHERE r.id=x.record->>'id' AND NOT r.deleted + AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz + AND (SELECT count(*) FROM eligible)=(SELECT count(*) FROM incoming) RETURNING r.id + ) SELECT count(*)::int AS count FROM changed""", + json.dumps( + tuple( + {"record": record.model_dump(mode="json"), "revision": row.record.updated_at.isoformat()} + for record, row in zip(records, rows, strict=True) + ) + ), + ) + ) + return _setting(family, records) if changed[0].count == len(records) else None diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py new file mode 100644 index 00000000000..0f9ac3e90f3 --- /dev/null +++ b/litellm/repositories/issues.py @@ -0,0 +1,504 @@ +import json +from collections.abc import Mapping +from datetime import date, datetime, timezone +from typing import ( + TYPE_CHECKING, + Final, + Literal, + cast, # noqa: TID251 # generated Prisma client implements query_raw +) + +from pydantic import BaseModel, TypeAdapter + +from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key +from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository +from litellm.types.proxy.issues import ( + Issue, + IssueDeliveryBinding, + IssueDeliveryInput, + IssueFamily, + IssueFamilyList, + IssueFamilySetting, + IssueFamilyUpdate, + IssueInput, + IssueKind, + IssueList, + IssuePendingDelivery, + IssueProjectionSignal, + IssueScope, + IssueSignal, + IssueSignalInput, + IssueSignalList, +) +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +class _IssueRow(BaseModel, frozen=True): + record: Issue + + +class _SignalRow(BaseModel, frozen=True): + record: IssueSignal + + +class _Count(BaseModel, frozen=True): + count: int + + +class _Identifier(BaseModel, frozen=True): + id: str + + +_ACK: Final = "CASE WHEN jsonb_typeof(c.param_value)='string' THEN (c.param_value#>>'{}')::jsonb ELSE c.param_value END" +_STATUS: Final = f"""CASE WHEN i.kind <> 'model_deprecation' THEN i.status + WHEN NOT i.active THEN CASE WHEN ({_ACK})->>'status'='dismissed' THEN 'dismissed' ELSE 'resolved' END + WHEN COALESCE(({_ACK})->>'status','open')='open' + OR ({_ACK})->>'last_seen_date' IS NULL + OR i.record->'deprecation'->>'last_used_date' > ({_ACK})->>'last_seen_date' + OR (i.record->'deprecation'->>'last_used_date' = ({_ACK})->>'last_seen_date' + AND (i.record->'deprecation'->>'last_day_requests')::bigint + > COALESCE((({_ACK})->>'last_seen_requests')::bigint,0)) THEN 'open' + ELSE ({_ACK})->>'status' END""" +_JOIN: Final = 'FROM "LiteLLM_Issue" i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key' +_RECORD: Final = f"""i.record || jsonb_build_object('status',({_STATUS}),'active',i.active, + 'first_seen',i.first_seen AT TIME ZONE 'UTC','last_seen',i.last_seen AT TIME ZONE 'UTC', + 'updated_at',i.updated_at AT TIME ZONE 'UTC') || CASE WHEN i.kind='model_deprecation' + THEN jsonb_build_object('deprecation',i.record->'deprecation' || + jsonb_build_object('state',COALESCE(({_ACK}),i.record->'deprecation'->'state'))) + ELSE '{{}}'::jsonb END""" + + +def _utc(value: datetime) -> str: + return value.astimezone(timezone.utc).replace(tzinfo=None).isoformat() + + +def _issue_payload(data: IssueInput, status: UsageAlertStatus | None) -> Mapping[str, object]: + initial: Final = Issue.model_validate( + { + **data.model_dump(), + "status": status or "open", + "first_seen": data.observed_at, + "last_seen": data.observed_at, + "updated_at": data.observed_at, + } + ) + return { + "record": initial.model_dump(mode="json"), + "ack_key": deprecation_alert_state_key(data.deprecation.usage_model, data.deprecation.model.deprecation_date) + if data.deprecation + else None, + } + + +class IssuesRepository: + def __init__(self, database: AlertDatabase) -> None: + self.database = database + + @classmethod + def from_prisma(cls, prisma: "PrismaClient") -> "IssuesRepository": + return cls(cast(AlertDatabase, prisma.writer_db)) # cast-ok: generated Prisma delegate implements query_raw + + async def upsert_issue( + self, data: IssueInput, reopen: bool = False, status: UsageAlertStatus | None = None + ) -> Issue: + rows: Final = await self.upsert_issues((data,), reopen=reopen, status=status) + return rows[0] + + async def upsert_issues( + self, + inputs: tuple[IssueInput, ...], + reopen: bool = False, + status: UsageAlertStatus | None = None, + ) -> tuple[Issue, ...]: + if not inputs: + return () + if len(inputs) > 500 or len(frozenset(item.id for item in inputs)) != len(inputs): + raise ValueError("Write at most 500 distinct issues per batch") + payload: Final = tuple(_issue_payload(item, status) for item in inputs) + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw( + f"""WITH incoming AS ( + SELECT x->'record' AS record,x->>'ack_key' AS ack_key FROM jsonb_array_elements($1::jsonb) x + ), changed AS ( + INSERT INTO "LiteLLM_Issue" AS old + (id,cohort_key,kind,family,scope_type,scope_id,status,ack_key,record,first_seen,last_seen,updated_at) + SELECT record->>'id',record->>'cohort_key',record->>'kind',record->>'family', + record->'cohort'->>'scope_type',record->'cohort'->>'scope_id',record->>'status',ack_key,record, + (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC', + (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC', + (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC' FROM incoming + ON CONFLICT(id) DO UPDATE SET + record=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen + THEN EXCLUDED.record ELSE old.record END, + last_seen=GREATEST(old.last_seen,EXCLUDED.last_seen), + active=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen + THEN true ELSE old.active END, + status=CASE WHEN $2 AND old.active AND EXCLUDED.last_seen > old.updated_at + THEN 'open' ELSE old.status END, + updated_at=GREATEST(old.updated_at,EXCLUDED.last_seen) + WHERE old.kind=EXCLUDED.kind AND old.cohort_key=EXCLUDED.cohort_key + AND old.scope_type=EXCLUDED.scope_type AND old.scope_id IS NOT DISTINCT FROM EXCLUDED.scope_id + RETURNING * + ) SELECT {_RECORD} AS record FROM changed i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key""", + json.dumps(payload), + reopen, + ) + ) + if len(rows) != len(inputs): + raise ValueError("An issue ID already belongs to another cohort") + return tuple(row.record for row in rows) + + async def get_issue(self, issue_id: str) -> Issue | None: + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw(f"SELECT {_RECORD} AS record {_JOIN} WHERE i.id=$1", issue_id) + ) + return (await self._labels((rows[0].record,)))[0] if rows else None + + async def list_issues( + self, + kind: IssueKind | None = None, + family: IssueFamily | None = None, + status: UsageAlertStatus | None = None, + scope_type: IssueScope | None = None, + scope_id: str | None = None, + limit: int = 50, + offset: int = 0, + ) -> IssueList: + where: Final = f"""($1::text IS NULL OR i.kind=$1) AND ($2::text IS NULL OR i.family=$2) + AND ($3::text IS NULL OR ({_STATUS})=$3) AND ($4::text IS NULL OR i.scope_type=$4) + AND ($5::text IS NULL OR i.scope_id=$5)""" + args: Final = (kind, family, status, scope_type, scope_id) + count: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw(f"SELECT count(*)::int AS count {_JOIN} WHERE {where}", *args) + ) + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw( + f"SELECT {_RECORD} AS record {_JOIN} WHERE {where} ORDER BY i.last_seen DESC,i.id LIMIT $6 OFFSET $7", + *args, + limit, + offset, + ) + ) + return IssueList( + data=await self._labels(tuple(row.record for row in rows)), total=count[0].count, limit=limit, offset=offset + ) + + async def _labels(self, issues: tuple[Issue, ...]) -> tuple[Issue, ...]: + if not issues: + return () + labels: Final = await UsageAlertsRepository(self.database).resource_labels( + tuple(issue.cohort for issue in issues) + ) + return tuple( + issue.model_copy(update={"scope_label": labels.get((issue.cohort.scope_type, issue.cohort.scope_id))}) + for issue in issues + ) + + async def find_active_issue(self, cohort_key: str) -> Issue | None: + rows: Final = await self.find_active_issues((cohort_key,)) + return rows[0] if rows else None + + async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: + if len(cohort_keys) > 500: + raise ValueError("Read at most 500 issue cohorts per batch") + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw( + f"SELECT DISTINCT ON(i.cohort_key) {_RECORD} AS record {_JOIN} " + "WHERE i.cohort_key=ANY($1::text[]) AND i.active " + "ORDER BY i.cohort_key,i.first_seen DESC,i.id", + list(cohort_keys), + ) + ) + return tuple(row.record for row in rows) + + async def set_status( + self, + issue_id: str, + status: UsageAlertStatus, + expected_status: UsageAlertStatus, + actor: str | None = None, + ) -> Issue | None: + changed: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( + await self.database.query_raw( + f"""WITH candidate AS MATERIALIZED ( + SELECT i.*,({_STATUS}) AS effective_status {_JOIN} WHERE i.id=$1 + ), ack AS ( + INSERT INTO "LiteLLM_Config" AS prior (param_name,param_value) + SELECT ack_key,jsonb_build_object('model',record->'deprecation'->>'usage_model', + 'deprecation_date',record->'deprecation'->'model'->>'deprecation_date', + 'status',$2::text,'last_seen_date',record->'deprecation'->>'last_used_date', + 'last_seen_requests',record->'deprecation'->'last_day_requests', + 'updated_at',CURRENT_TIMESTAMP,'updated_by',$4::text) + FROM candidate WHERE kind='model_deprecation' AND effective_status=$3 + ON CONFLICT(param_name) DO UPDATE SET param_value=EXCLUDED.param_value + WHERE (CASE WHEN COALESCE((CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'status','open')='open' + OR (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' IS NULL + OR EXCLUDED.param_value->>'last_seen_date' > (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' + OR (EXCLUDED.param_value->>'last_seen_date'=(CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' + AND (EXCLUDED.param_value->>'last_seen_requests')::bigint + > COALESCE(((CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_requests')::bigint,0)) + THEN 'open' ELSE (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'status' END)=$3 + RETURNING param_name + ) UPDATE "LiteLLM_Issue" i SET status=$2,updated_at=CURRENT_TIMESTAMP + FROM candidate c WHERE i.id=c.id AND c.effective_status=$3 AND + ((c.kind<>'model_deprecation' AND i.status=$3) OR i.ack_key IN(SELECT param_name FROM ack)) + RETURNING i.id""", + issue_id, + status, + expected_status, + actor, + ) + ) + return await self.get_issue(issue_id) if changed else None + + async def add_signals(self, issue_id: str, signals: tuple[IssueSignalInput, ...]) -> None: + await self.add_signal_batch( + tuple(IssueSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in signals) + ) + + async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: + if not signals: + return + if len(signals) > 500: + raise ValueError("Write at most 500 issue signals per batch") + await self.database.query_raw( + """INSERT INTO "LiteLLM_IssueSignal" (issue_id,source,source_id,record) + SELECT DISTINCT ON(x->>'issue_id',x->>'source',x->>'source_id') + x->>'issue_id',x->>'source',x->>'source_id',x FROM jsonb_array_elements($1::jsonb) x + WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" i WHERE i.id=x->>'issue_id' + AND (i.scope_type NOT IN ('key','team','user') OR + (x->'usage_alert'->>'scope_type'=i.scope_type + AND x->'usage_alert'->>'scope_id'=i.scope_id))) + ON CONFLICT(issue_id,source,source_id) DO NOTHING RETURNING source_id""", + json.dumps(tuple(signal.model_dump(mode="json") for signal in signals)), + ) + + async def list_signals(self, issue_id: str, limit: int = 50, offset: int = 0) -> IssueSignalList: + counts: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + 'SELECT count(*)::int AS count FROM "LiteLLM_IssueSignal" WHERE issue_id=$1', + issue_id, + ) + ) + rows: Final = TypeAdapter(tuple[_SignalRow, ...]).validate_python( + await self.database.query_raw( + 'SELECT record FROM "LiteLLM_IssueSignal" WHERE issue_id=$1 ORDER BY source,source_id LIMIT $2 OFFSET $3', + issue_id, + limit, + offset, + ) + ) + return IssueSignalList( + data=tuple(row.record for row in rows), total=counts[0].count, limit=limit, offset=offset + ) + + async def active_signals( + self, + day: date, + limit: int = 500, + cursor: str | None = None, + scope_type: IssueScope | None = None, + metric: UsageAlertMetric | None = None, + ) -> tuple[IssueProjectionSignal, ...]: + return TypeAdapter(tuple[IssueProjectionSignal, ...]).validate_python( + await self.database.query_raw( + """SELECT a.record AS alert,r.record AS rule,a.evaluation_day + FROM "LiteLLM_UsageAlertIncident" a JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id + WHERE a.evaluation_day=$1 AND a.breached AND r.enabled AND NOT r.deleted + AND NULLIF(a.evaluation_revision,'')::timestamptz=(r.record->>'updated_at')::timestamptz + AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric' + AND a.record->>'direction'=r.record->>'direction' + AND (r.scope_id IS NULL OR a.scope_id=r.scope_id) + AND ($2::text IS NULL OR a.id>$2) AND ($4::text IS NULL OR a.scope_type=$4) + AND ($5::text IS NULL OR a.metric=$5) ORDER BY a.id LIMIT $3""", + day.isoformat(), + cursor, + limit, + scope_type, + metric, + ) + ) + + async def recover_issues(self, day: date) -> int: + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + """WITH changed AS ( + UPDATE "LiteLLM_Issue" i SET active=false, + status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP + WHERE kind='performance' AND active + AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id AND s.source='usage_alert') + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s + LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id + WHERE s.issue_id=i.id AND s.source='usage_alert' + AND (a.id IS NULL OR a.evaluation_day>$1 OR a.breached)) RETURNING i.id + ) SELECT count(*)::int AS count FROM changed""", + day.isoformat(), + ) + ) + return rows[0].count + + async def projection_completed(self, day: date, version: str) -> bool: + rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( + await self.database.query_raw( + 'SELECT id FROM "LiteLLM_UsageAlertCheckpoint" WHERE id=$1 AND completed', + f"issues:{version}:{day.isoformat()}", + ) + ) + return bool(rows) + + async def complete_projection(self, day: date, version: str) -> None: + await self.database.query_raw( + """INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed,expires_at) VALUES($1,true,CURRENT_TIMESTAMP) + ON CONFLICT(id) DO UPDATE SET completed=true,expires_at=CURRENT_TIMESTAMP RETURNING id""", + f"issues:{version}:{day.isoformat()}", + ) + + async def finish_deprecation_projection(self, observed_at: datetime) -> int: + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + """WITH changed AS ( + UPDATE "LiteLLM_Issue" SET active=false, + status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP + WHERE kind='model_deprecation' AND active AND last_seen<$1::timestamp RETURNING id + ) SELECT count(*)::int AS count FROM changed""", + _utc(observed_at), + ) + ) + return rows[0].count + + async def family_settings(self) -> "IssueFamilyList": + from litellm.repositories.issue_families import family_settings + + return await family_settings(self.database) + + async def update_family( + self, family: "PerformanceIssueFamily", data: "IssueFamilyUpdate" + ) -> "IssueFamilySetting | None": + from litellm.repositories.issue_families import update_family + + return await update_family(self.database, family, data) + + async def queue_delivery( + self, + data: IssueDeliveryInput, + rule_id: str, + rule_revision: str, + source_id: str, + ) -> None: + await self.queue_deliveries( + (IssueDeliveryBinding(delivery=data, rule_id=rule_id, rule_revision=rule_revision, source_id=source_id),) + ) + + async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: + if not bindings: + return + if len(bindings) > 500: + raise ValueError("Queue at most 500 delivery bindings per batch") + await self.database.query_raw( + """WITH incoming AS MATERIALIZED ( + SELECT x->'delivery' AS delivery,x->>'rule_id' AS rule_id, + x->>'rule_revision' AS revision,x->>'source_id' AS source_id + FROM jsonb_array_elements($1::jsonb) x + WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" WHERE id=x->'delivery'->>'issue_id') + ), saved AS ( + INSERT INTO "LiteLLM_IssueDelivery" (id,issue_id,record) + SELECT DISTINCT ON(delivery->>'id') delivery->>'id',delivery->>'issue_id',delivery + FROM incoming ON CONFLICT(id) DO NOTHING RETURNING id + ) INSERT INTO "LiteLLM_IssueDeliveryRule" (delivery_id,rule_id,source_id,rule_revision) + SELECT DISTINCT ON(delivery->>'id',rule_id,source_id) delivery->>'id',rule_id,source_id,revision + FROM incoming WHERE delivery->>'id' IN(SELECT id FROM saved) + OR EXISTS(SELECT 1 FROM "LiteLLM_IssueDelivery" d WHERE d.id=delivery->>'id' + AND d.issue_id=delivery->>'issue_id') + ON CONFLICT(delivery_id,rule_id,source_id) DO UPDATE SET rule_revision=EXCLUDED.rule_revision + RETURNING delivery_id""", + json.dumps(tuple(binding.model_dump(mode="json") for binding in bindings)), + ) + + async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: + if not 1 <= limit <= 500: + raise ValueError("Read between one and 500 deliveries") + return TypeAdapter(tuple[IssuePendingDelivery, ...]).validate_python( + await self.database.query_raw( + f"""SELECT d.record || jsonb_build_object('next_attempt',d.next_attempt AT TIME ZONE 'UTC') AS delivery, + {_RECORD} AS issue,bound.rule,bound.alert + FROM "LiteLLM_IssueDelivery" d JOIN "LiteLLM_Issue" i ON i.id=d.issue_id + LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key + JOIN LATERAL ( + SELECT r.record AS rule,a.record AS alert FROM "LiteLLM_IssueDeliveryRule" b + JOIN "LiteLLM_UsageAlertRule" r ON r.id=b.rule_id + JOIN "LiteLLM_UsageAlertIncident" a ON a.id=b.source_id AND a.rule_id=r.id + WHERE b.delivery_id=d.id AND r.enabled AND NOT r.deleted AND a.breached AND a.status='open' + AND (r.record->>'updated_at')::timestamptz=b.rule_revision::timestamptz + AND NULLIF(a.evaluation_revision,'')::timestamptz=b.rule_revision::timestamptz + AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric' + AND a.record->>'direction'=r.record->>'direction' + AND (r.scope_id IS NULL OR a.scope_id=r.scope_id) + AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id + AND s.source='usage_alert' AND s.source_id=a.id) + ORDER BY b.rule_id,b.source_id LIMIT 1 + ) bound ON true + WHERE NOT d.sent AND d.next_attempt<=CURRENT_TIMESTAMP AND i.active AND ({_STATUS})='open' + ORDER BY d.next_attempt,d.id LIMIT $1""", + limit, + ) + ) + + async def mark_delivery_sent(self, delivery_id: str) -> None: + await self.database.query_raw( + """UPDATE "LiteLLM_IssueDelivery" SET sent=true,updated_at=CURRENT_TIMESTAMP, + record=record || jsonb_build_object('last_error',NULL,'attempts', + COALESCE((record->>'attempts')::int,0)+1) + WHERE id=$1 AND NOT sent RETURNING id""", + delivery_id, + ) + + async def defer_delivery( + self, + delivery_id: str, + error: Literal["delivery_failed", "transport_not_configured"], + ) -> None: + await self.database.query_raw( + """UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP+interval '1 hour', + updated_at=CURRENT_TIMESTAMP,record=record || jsonb_build_object('last_error',$2::text, + 'attempts',COALESCE((record->>'attempts')::int,0)+1) + WHERE id=$1 AND NOT sent RETURNING id""", + delivery_id, + error, + ) + + async def prune(self, retention_days: int = 90, limit: int = 500) -> None: + if not 1 <= limit <= 500 or retention_days < 1: + raise ValueError("Retention and cleanup batch must be bounded positive values") + await self.database.query_raw( + """WITH expired AS MATERIALIZED ( + SELECT id FROM "LiteLLM_Issue" WHERE status<>'open' + AND updated_at$1) + ORDER BY updated_at,id LIMIT $2 + ), signals AS ( + DELETE FROM "LiteLLM_IssueSignal" WHERE (issue_id,source,source_id) IN ( + SELECT issue_id,source,source_id FROM "LiteLLM_IssueSignal" + WHERE issue_id IN(SELECT id FROM expired) ORDER BY issue_id,source,source_id LIMIT $2) + RETURNING issue_id + ), bindings AS ( + DELETE FROM "LiteLLM_IssueDeliveryRule" WHERE (delivery_id,rule_id,source_id) IN ( + SELECT b.delivery_id,b.rule_id,b.source_id FROM "LiteLLM_IssueDeliveryRule" b + JOIN "LiteLLM_IssueDelivery" d ON d.id=b.delivery_id + WHERE d.issue_id IN(SELECT id FROM expired) ORDER BY b.delivery_id,b.rule_id,b.source_id LIMIT $2) + RETURNING delivery_id + ), deliveries AS ( + DELETE FROM "LiteLLM_IssueDelivery" d WHERE d.id IN ( + SELECT id FROM "LiteLLM_IssueDelivery" d WHERE issue_id IN(SELECT id FROM expired) + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueDeliveryRule" b WHERE b.delivery_id=d.id) + ORDER BY id LIMIT $2) RETURNING issue_id + ), removed AS ( + DELETE FROM "LiteLLM_Issue" i WHERE id IN(SELECT id FROM expired) + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id) + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueDelivery" d WHERE d.issue_id=i.id) RETURNING id + ) SELECT count(*)::int AS count FROM removed""", + retention_days, + limit, + ) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index c7efc5c77f8..7eb80dae626 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -3,7 +3,6 @@ import json from collections.abc import Mapping from datetime import date, datetime, timedelta, timezone -from itertools import product from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -17,6 +16,7 @@ from uuid import NAMESPACE_URL, uuid4, uuid5 from pydantic import BaseModel, TypeAdapter from litellm.repositories.daily_activity_buckets import api_key_from_bucket +from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS from litellm.types.proxy.usage_alerts import ( UsageAlert, UsageAlertChannel, @@ -49,6 +49,14 @@ class AlertDatabase(Protocol): async def query_raw(self, query: str, *args: object) -> object: ... +class ResourceLabelScope(Protocol): + @property + def scope_type(self) -> str: ... + + @property + def scope_id(self) -> str | None: ... + + class _Record(BaseModel, frozen=True): record: Mapping[str, object] @@ -75,7 +83,7 @@ class _GroupPage(BaseModel, frozen=True): class _ResourceLabel(BaseModel, frozen=True): - scope_type: UsageAlertScope + scope_type: str scope_id: str | None label: str | None @@ -98,14 +106,7 @@ class _Ownership(BaseModel, frozen=True): team_id: str | None -_DEFAULT_RULE_IDS: Final = frozenset( - str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{scope}:{metric}:{direction}")) - for scope, metric, direction in product( - ("deployment", "key", "user", "team", "model"), - ("spend", "requests", "latency", "errors"), - ("increase", "decrease"), - ) -) +_DEFAULT_RULE_IDS: Final = DEFAULT_USAGE_RULE_IDS def _rule(record: Mapping[str, object]) -> UsageAlertRule: @@ -405,8 +406,8 @@ class UsageAlertsRepository: ) ) page: Final = rows[0] - labels: Final[Mapping[tuple[UsageAlertScope, str | None], str | None]] = ( - await self._group_labels(page.data) if page.data else {} + labels: Final[Mapping[tuple[str, str | None], str | None]] = ( + await self.resource_labels(page.data) if page.data else {} ) return UsageAlertGroupList( data=tuple( @@ -434,9 +435,9 @@ class UsageAlertsRepository: offset=offset, ) - async def _group_labels( - self, groups: tuple[_GroupRow, ...] - ) -> Mapping[tuple[UsageAlertScope, str | None], str | None]: + async def resource_labels( + self, groups: tuple[ResourceLabelScope, ...] + ) -> Mapping[tuple[str, str | None], str | None]: rows: Final = TypeAdapter(tuple[_ResourceLabel, ...]).validate_python( await self.database.query_raw( """WITH scopes AS ( @@ -446,7 +447,8 @@ class UsageAlertsRepository: WHEN 'key' THEN COALESCE(NULLIF(k.key_alias,''),NULLIF(k.key_name,'')) WHEN 'team' THEN NULLIF(t.team_alias,'') WHEN 'user' THEN COALESCE(NULLIF(u.user_alias,''),NULLIF(u.user_email,'')) - WHEN 'model' THEN s.scope_id WHEN 'deployment' THEN 'Gateway' END AS label + WHEN 'model' THEN s.scope_id WHEN 'provider' THEN s.scope_id + WHEN 'deployment' THEN 'Gateway' END AS label FROM scopes s LEFT JOIN "LiteLLM_VerificationToken" k ON s.scope_type='key' AND k.token=s.key_id LEFT JOIN "LiteLLM_TeamTable" t ON s.scope_type='team' AND t.team_id=s.scope_id LEFT JOIN "LiteLLM_UserTable" u ON s.scope_type='user' AND u.user_id=s.scope_id""", diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py new file mode 100644 index 00000000000..e0197e6edf2 --- /dev/null +++ b/litellm/types/proxy/issues.py @@ -0,0 +1,260 @@ +from datetime import date, datetime +from typing import Literal, Self, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import ( + UsageAlert, + UsageAlertChannel, + UsageAlertDirection, + UsageAlertMetric, + UsageAlertRule, + UsageAlertStatus, +) + +IssueKind: TypeAlias = Literal["performance", "model_deprecation"] +IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] +IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] +IssueGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"] + + +class IssueModelPair(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + provider: str = Field(min_length=1, max_length=256) + model: str = Field(min_length=1, max_length=1024) + + +class IssueCohort(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + scope_type: IssueScope + scope_id: str | None = Field(default=None, min_length=1, max_length=1024) + model_pairs: tuple[IssueModelPair, ...] = Field(default=(), max_length=100) + + @model_validator(mode="after") + def validate_scope(self) -> Self: + if (self.scope_type == "deployment") != (self.scope_id is None): + raise ValueError("Only a deployment cohort omits scope_id") + if self.scope_type == "key" and self.scope_id and self.scope_id.removeprefix("raw:").startswith("sk-"): + raise ValueError("Use a reporting key ID, never a secret key") + if len(frozenset((pair.provider, pair.model) for pair in self.model_pairs)) != len(self.model_pairs): + raise ValueError("model_pairs must be unique") + return self + + +class IssueComparison(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + metric: UsageAlertMetric + direction: UsageAlertDirection + observed_value: float = Field(ge=0) + baseline_value: float = Field(ge=0) + threshold: float + baseline_start: date + baseline_end: date + baseline_sample_days: int | None = Field(default=None, ge=1, le=30) + + @model_validator(mode="after") + def validate_comparison(self) -> Self: + if self.baseline_start > self.baseline_end: + raise ValueError("baseline_start must not follow baseline_end") + if self.metric == "error_rate" and (self.observed_value > 100 or self.baseline_value > 100): + raise ValueError("error_rate uses percentages from 0 to 100") + return self + + +class IssueImpact(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + keys: int | None = Field(default=None, ge=0) + users: int | None = Field(default=None, ge=0) + teams: int | None = Field(default=None, ge=0) + models: int | None = Field(default=None, ge=0) + requests: int | None = Field(default=None, ge=0) + measured_keys: int | None = Field(default=None, ge=0) + measured_models: int | None = Field(default=None, ge=0) + + +class IssueDeprecation(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + model: ModelDeprecationInfo + usage_model: str = Field(min_length=1, max_length=1024) + requests: int = Field(ge=0) + last_used_date: date + last_day_requests: int = Field(ge=0) + state: DeprecationAlertState + + +class IssueInput(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + id: str = Field(min_length=1, max_length=200) + cohort_key: str = Field(min_length=1, max_length=200) + kind: IssueKind + family: IssueFamily + title: str = Field(min_length=1, max_length=2048) + cohort: IssueCohort + observed_on: date + observed_at: datetime + range_start: date + range_end: date + grouping_reason: IssueGroupingReason + explanation: str = Field(default="", max_length=2000) + comparisons: tuple[IssueComparison, ...] = Field(default=(), max_length=10) + impact: IssueImpact = Field(default_factory=IssueImpact) + deprecation: IssueDeprecation | None = None + + @model_validator(mode="after") + def validate_issue(self) -> Self: + if self.range_start > self.range_end: + raise ValueError("range_start must not follow range_end") + if (self.kind == "model_deprecation") != (self.deprecation is not None): + raise ValueError("Model deprecation issues require deprecation details") + if (self.kind == "model_deprecation") != (self.family == "retirement"): + raise ValueError("Only model deprecation issues use the retirement family") + if self.observed_at.tzinfo is None: + raise ValueError("observed_at must include a timezone") + return self + + +class Issue(IssueInput, frozen=True): + status: UsageAlertStatus + first_seen: datetime + last_seen: datetime + updated_at: datetime + scope_label: str | None = None + active: bool = True + + +class IssueList(BaseModel, frozen=True): + data: tuple[Issue, ...] + total: int + limit: int + offset: int + + +class IssueUpdate(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: UsageAlertStatus + expected_status: UsageAlertStatus + + +class IssueSignalInput(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + source: Literal["usage_alert", "model_deprecation"] + source_id: str = Field(min_length=1, max_length=200) + usage_alert: UsageAlert | None = None + deprecation: IssueDeprecation | None = None + + @model_validator(mode="after") + def validate_signal(self) -> Self: + if self.source == "usage_alert": + if self.usage_alert is None or self.deprecation is not None or self.usage_alert.id != self.source_id: + raise ValueError("Usage signals require their matching source alert") + elif self.deprecation is None or self.usage_alert is not None: + raise ValueError("Deprecation signals require deprecation details") + return self + + +class IssueSignal(IssueSignalInput, frozen=True): + issue_id: str + + +class IssueSignalList(BaseModel, frozen=True): + data: tuple[IssueSignal, ...] + total: int + limit: int + offset: int + + +class IssueProjectionSignal(BaseModel, frozen=True): + alert: UsageAlert + rule: UsageAlertRule + evaluation_day: date + + +class IssueDeliveryInput(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + id: str = Field(min_length=1, max_length=200) + issue_id: str = Field(min_length=1, max_length=200) + channel: UsageAlertChannel + email_recipients: tuple[str, ...] = Field(default=(), max_length=20) + + +class IssueDeliveryBinding(BaseModel, frozen=True): + delivery: IssueDeliveryInput + rule_id: str + rule_revision: str + source_id: str + + +class IssueDelivery(IssueDeliveryInput, frozen=True): + attempts: int = 0 + next_attempt: datetime | None = None + last_error: Literal["delivery_failed", "transport_not_configured"] | None = None + + +class IssuePendingDelivery(BaseModel, frozen=True): + delivery: IssueDelivery + issue: Issue + rule: UsageAlertRule + alert: UsageAlert + + +class IssueFamilyPolicy(BaseModel, frozen=True): + direction: UsageAlertDirection + enabled: bool | None + multiplier: float | None + min_value: float | None + min_requests: int | None + baseline_days: int | None + min_history_days: int | None + mixed: bool + total_rules: int + enabled_rules: int + + +class IssueFamilySetting(BaseModel, frozen=True): + family: PerformanceIssueFamily + enabled: bool | None + total_rules: int + enabled_rules: int + policies: tuple[IssueFamilyPolicy, ...] + + +class IssueFamilyList(BaseModel, frozen=True): + data: tuple[IssueFamilySetting, ...] + + +class IssueFamilyPolicyUpdate(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + direction: UsageAlertDirection + enabled: bool | None = None + multiplier: float | None = Field(default=None, gt=1, le=1000) + min_value: float | None = Field(default=None, ge=0) + min_requests: int | None = Field(default=None, ge=0) + baseline_days: int | None = Field(default=None, ge=1, le=30) + min_history_days: int | None = Field(default=None, ge=1, le=30) + + +class IssueFamilyUpdate(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + enabled: bool | None = None + policies: tuple[IssueFamilyPolicyUpdate, ...] = Field(default=(), max_length=2) + + @model_validator(mode="after") + def validate_update(self) -> Self: + if len(frozenset(policy.direction for policy in self.policies)) != len(self.policies): + raise ValueError("Each direction can occur once") + if self.enabled is None and not self.policies: + raise ValueError("Specify enabled or a policy update") + return self diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 37f59711050..c95df8d6840 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -182,6 +182,8 @@ class KeySpendMetrics(LiteLLMBaseModel): failed_requests: int = 0 cache_read_input_tokens: int = 0 cache_creation_input_tokens: int = 0 + total_response_time_ms: int = 0 + timed_requests: int = 0 class KeySpendActivityRow(LiteLLMBaseModel): @@ -214,6 +216,8 @@ class UserActivityRow(LiteLLMBaseModel): api_requests: int = 0 successful_requests: int = 0 failed_requests: int = 0 + total_response_time_ms: int = 0 + timed_requests: int = 0 class DailyActivityUserPageResponse(LiteLLMBaseModel): diff --git a/litellm/types/proxy/usage_alert_defaults.py b/litellm/types/proxy/usage_alert_defaults.py new file mode 100644 index 00000000000..7b54ac3eb45 --- /dev/null +++ b/litellm/types/proxy/usage_alert_defaults.py @@ -0,0 +1,75 @@ +from dataclasses import dataclass +from itertools import product +from typing import Final, Literal, TypeAlias +from uuid import NAMESPACE_URL, uuid5 + +from litellm.types.proxy.usage_alerts import ( + UsageAlertChannel, + UsageAlertDirection, + UsageAlertMetric, + UsageAlertRule, + UsageAlertRuleInput, + UsageAlertScope, +) + +PerformanceIssueFamily: TypeAlias = Literal["spend", "latency", "reliability", "traffic"] +USAGE_ALERT_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("deployment", "key", "user", "team", "model") + + +@dataclass(frozen=True, slots=True) +class DefaultUsagePolicy: + family: PerformanceIssueFamily + metric: UsageAlertMetric + direction: UsageAlertDirection + multiplier: float + min_value: float + min_requests: int = 0 + min_history_days: int = 3 + + +DEFAULT_USAGE_POLICIES: Final = ( + DefaultUsagePolicy("spend", "spend", "increase", 3, 10), + DefaultUsagePolicy("latency", "latency", "increase", 3, 500, 100), + DefaultUsagePolicy("reliability", "error_rate", "increase", 3, 2, 500), + DefaultUsagePolicy("traffic", "requests", "increase", 10, 1000), + DefaultUsagePolicy("traffic", "requests", "decrease", 5, 0, 500, 7), +) + + +def default_usage_rule_id(scope: UsageAlertScope, metric: UsageAlertMetric, direction: UsageAlertDirection) -> str: + return str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{scope}:{metric}:{direction}")) + + +DEFAULT_USAGE_RULE_IDS: Final = frozenset( + default_usage_rule_id(scope, policy.metric, policy.direction) + for scope, policy in product(USAGE_ALERT_SCOPES, DEFAULT_USAGE_POLICIES) +) + + +def is_default_usage_rule(rule: UsageAlertRule) -> bool: + return rule.id in DEFAULT_USAGE_RULE_IDS and rule.id == default_usage_rule_id( + rule.scope_type, rule.metric, rule.direction + ) + + +def default_usage_rules( + channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False +) -> tuple[UsageAlertRuleInput, ...]: + return tuple( + UsageAlertRuleInput( + name=f"Daily {scope} {policy.metric.replace('_', ' ')} {policy.direction}", + scope_type=scope, + metric=policy.metric, + direction=policy.direction, + multiplier=policy.multiplier, + min_value=policy.min_value, + min_requests=policy.min_requests, + min_history_days=policy.min_history_days, + notification_channels=( + () + if legacy_user_spend_alerts and (scope, policy.metric, policy.direction) == ("user", "spend", "increase") + else channels + ), + ) + for scope, policy in product(USAGE_ALERT_SCOPES, DEFAULT_USAGE_POLICIES) + ) diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index bb719ea48a2..72ccc1f3ef7 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -6,7 +6,7 @@ from typing import Literal, Self, TypeAlias from pydantic import BaseModel, ConfigDict, Field, model_validator UsageAlertScope: TypeAlias = Literal["deployment", "key", "user", "team", "model"] -UsageAlertMetric: TypeAlias = Literal["spend", "requests", "latency", "errors"] +UsageAlertMetric: TypeAlias = Literal["spend", "requests", "latency", "errors", "error_rate"] UsageAlertDirection: TypeAlias = Literal["increase", "decrease"] UsageAlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"] UsageAlertChannel: TypeAlias = Literal["email", "slack"] diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index b013ce43113..938ef0467ed 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -39,6 +39,8 @@ class DailyActivityScope: model: str | None timezone_offset_minutes: int | None include_current_utc_day: bool = False + provider: str | None = None + model_pairs: tuple[tuple[str, str], ...] = () def __post_init__(self) -> None: if self.entity_id_field not in _ENTITY_FIELDS[self.table]: @@ -57,6 +59,8 @@ class KeySpendRow: failed_requests: int cache_read_input_tokens: int cache_creation_input_tokens: int + total_response_time_ms: int = 0 + timed_requests: int = 0 @dataclass(frozen=True, slots=True) @@ -75,6 +79,8 @@ class UserSpendRow: api_requests: int successful_requests: int failed_requests: int + total_response_time_ms: int = 0 + timed_requests: int = 0 @dataclass(frozen=True, slots=True) diff --git a/schema.prisma b/schema.prisma index 46ab28530a9..8a67d1b123a 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2132,11 +2132,13 @@ model LiteLLM_UsageAlertIncident { status String day String evaluation_day String + evaluation_revision String @default("") breached Boolean record Json notified_channels String[] @default([]) updated_at DateTime @default(now()) @@index([rule_id, scope_key, evaluation_day]) + @@index([evaluation_day, breached, id]) @@index([scope_type, scope_id, day]) @@index([status, day]) @@index([updated_at]) @@ -2149,3 +2151,55 @@ model LiteLLM_UsageAlertCheckpoint { expires_at DateTime @default(now()) @@index([expires_at]) } + +model LiteLLM_Issue { + id String @id + cohort_key String + kind String + family String + scope_type String + scope_id String? + status String + active Boolean @default(true) + ack_key String? + record Json + first_seen DateTime + last_seen DateTime + updated_at DateTime @default(now()) + + @@index([status, last_seen, id]) + @@index([kind, family, last_seen, id]) + @@index([scope_type, scope_id, last_seen, id]) + @@index([cohort_key, active]) +} + +model LiteLLM_IssueSignal { + issue_id String + source String + source_id String + record Json + + @@id([issue_id, source, source_id]) + @@index([source, source_id]) +} + +model LiteLLM_IssueDelivery { + id String @id + issue_id String + record Json + sent Boolean @default(false) + next_attempt DateTime @default(now()) + updated_at DateTime @default(now()) + + @@index([sent, next_attempt, id]) + @@index([issue_id]) +} + +model LiteLLM_IssueDeliveryRule { + delivery_id String + rule_id String + source_id String + rule_revision String + + @@id([delivery_id, rule_id, source_id]) +} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 42ae1107a2e..bbd4acaedf8 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8343,6 +8343,160 @@ export interface paths { patch?: never; trace?: never; }; + "/issues": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Issues */ + get: operations["list_issues_issues_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/families": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Issue Families */ + get: operations["list_issue_families_issues_families_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/families/{family}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Issue Family */ + patch: operations["update_issue_family_issues_families__family__patch"]; + trace?: never; + }; + "/issues/{issue_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue */ + get: operations["get_issue_issues__issue_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Issue */ + patch: operations["update_issue_issues__issue_id__patch"]; + trace?: never; + }; + "/issues/{issue_id}/activity": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Activity */ + get: operations["get_issue_activity_issues__issue_id__activity_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/impact": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Impact */ + get: operations["get_issue_impact_issues__issue_id__impact_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/keys": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Keys */ + get: operations["get_issue_keys_issues__issue_id__keys_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/signals": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Issue Signals */ + get: operations["list_issue_signals_issues__issue_id__signals_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/users": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Users */ + get: operations["get_issue_users_issues__issue_id__users_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/jwt/key/mapping/delete": { parameters: { query?: never; @@ -34710,6 +34864,302 @@ export interface components { /** Is Accepted */ is_accepted: boolean; }; + /** Issue */ + Issue: { + /** + * Active + * @default true + */ + active: boolean; + cohort: components["schemas"]["IssueCohort"]; + /** Cohort Key */ + cohort_key: string; + /** + * Comparisons + * @default [] + */ + comparisons: components["schemas"]["IssueComparison"][]; + deprecation?: components["schemas"]["IssueDeprecation"] | null; + /** + * Explanation + * @default + */ + explanation: string; + /** + * Family + * @enum {string} + */ + family: "retirement" | "latency" | "reliability" | "spend" | "traffic"; + /** + * First Seen + * Format: date-time + */ + first_seen: string; + /** + * Grouping Reason + * @enum {string} + */ + grouping_reason: "same_resource" | "dominant_key" | "provider_cohort" | "model_deprecation"; + /** Id */ + id: string; + impact?: components["schemas"]["IssueImpact"]; + /** + * Kind + * @enum {string} + */ + kind: "performance" | "model_deprecation"; + /** + * Last Seen + * Format: date-time + */ + last_seen: string; + /** + * Observed At + * Format: date-time + */ + observed_at: string; + /** + * Observed On + * Format: date + */ + observed_on: string; + /** + * Range End + * Format: date + */ + range_end: string; + /** + * Range Start + * Format: date + */ + range_start: string; + /** Scope Label */ + scope_label?: string | null; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + /** Title */ + title: string; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; + /** IssueCohort */ + IssueCohort: { + /** + * Model Pairs + * @default [] + */ + model_pairs: components["schemas"]["IssueModelPair"][]; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model" | "provider"; + }; + /** IssueComparison */ + IssueComparison: { + /** + * Baseline End + * Format: date + */ + baseline_end: string; + /** Baseline Sample Days */ + baseline_sample_days?: number | null; + /** + * Baseline Start + * Format: date + */ + baseline_start: string; + /** Baseline Value */ + baseline_value: number; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + /** Observed Value */ + observed_value: number; + /** Threshold */ + threshold: number; + }; + /** IssueDeprecation */ + IssueDeprecation: { + /** Last Day Requests */ + last_day_requests: number; + /** + * Last Used Date + * Format: date + */ + last_used_date: string; + model: components["schemas"]["ModelDeprecationInfo"]; + /** Requests */ + requests: number; + state: components["schemas"]["DeprecationAlertState"]; + /** Usage Model */ + usage_model: string; + }; + /** IssueFamilyList */ + IssueFamilyList: { + /** Data */ + data: components["schemas"]["IssueFamilySetting"][]; + }; + /** IssueFamilyPolicy */ + IssueFamilyPolicy: { + /** Baseline Days */ + baseline_days: number | null; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** Enabled */ + enabled: boolean | null; + /** Enabled Rules */ + enabled_rules: number; + /** Min History Days */ + min_history_days: number | null; + /** Min Requests */ + min_requests: number | null; + /** Min Value */ + min_value: number | null; + /** Mixed */ + mixed: boolean; + /** Multiplier */ + multiplier: number | null; + /** Total Rules */ + total_rules: number; + }; + /** IssueFamilyPolicyUpdate */ + IssueFamilyPolicyUpdate: { + /** Baseline Days */ + baseline_days?: number | null; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** Enabled */ + enabled?: boolean | null; + /** Min History Days */ + min_history_days?: number | null; + /** Min Requests */ + min_requests?: number | null; + /** Min Value */ + min_value?: number | null; + /** Multiplier */ + multiplier?: number | null; + }; + /** IssueFamilySetting */ + IssueFamilySetting: { + /** Enabled */ + enabled: boolean | null; + /** Enabled Rules */ + enabled_rules: number; + /** + * Family + * @enum {string} + */ + family: "spend" | "latency" | "reliability" | "traffic"; + /** Policies */ + policies: components["schemas"]["IssueFamilyPolicy"][]; + /** Total Rules */ + total_rules: number; + }; + /** IssueFamilyUpdate */ + IssueFamilyUpdate: { + /** Enabled */ + enabled?: boolean | null; + /** + * Policies + * @default [] + */ + policies: components["schemas"]["IssueFamilyPolicyUpdate"][]; + }; + /** IssueImpact */ + IssueImpact: { + /** Keys */ + keys?: number | null; + /** Measured Keys */ + measured_keys?: number | null; + /** Measured Models */ + measured_models?: number | null; + /** Models */ + models?: number | null; + /** Requests */ + requests?: number | null; + /** Teams */ + teams?: number | null; + /** Users */ + users?: number | null; + }; + /** IssueList */ + IssueList: { + /** Data */ + data: components["schemas"]["Issue"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** IssueModelPair */ + IssueModelPair: { + /** Model */ + model: string; + /** Provider */ + provider: string; + }; + /** IssueSignal */ + IssueSignal: { + deprecation?: components["schemas"]["IssueDeprecation"] | null; + /** Issue Id */ + issue_id: string; + /** + * Source + * @enum {string} + */ + source: "usage_alert" | "model_deprecation"; + /** Source Id */ + source_id: string; + usage_alert?: components["schemas"]["UsageAlert"] | null; + }; + /** IssueSignalList */ + IssueSignalList: { + /** Data */ + data: components["schemas"]["IssueSignal"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** IssueUpdate */ + IssueUpdate: { + /** + * Expected Status + * @enum {string} + */ + expected_status: "open" | "resolved" | "dismissed"; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + }; /** * ItemReference * @description An internal identifier for an item to reference. @@ -34868,6 +35318,16 @@ export interface components { * @default 0 */ successful_requests: number; + /** + * Timed Requests + * @default 0 + */ + timed_requests: number; + /** + * Total Response Time Ms + * @default 0 + */ + total_response_time_ms: number; /** * Total Tokens * @default 0 @@ -50485,7 +50945,7 @@ export interface components { * Metric * @enum {string} */ - metric: "spend" | "requests" | "latency" | "errors"; + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; /** Observed Value */ observed_value: number; /** Rule Id */ @@ -50588,7 +51048,7 @@ export interface components { * Metric * @enum {string} */ - metric: "spend" | "requests" | "latency" | "errors"; + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; representative: components["schemas"]["UsageAlert"]; }; /** UsageAlertGroupUpdate */ @@ -50706,7 +51166,7 @@ export interface components { * Metric * @enum {string} */ - metric: "spend" | "requests" | "latency" | "errors"; + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; /** * Min History Days * @default 3 @@ -50774,7 +51234,7 @@ export interface components { * Metric * @enum {string} */ - metric: "spend" | "requests" | "latency" | "errors"; + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; /** * Min History Days * @default 3 @@ -51349,6 +51809,16 @@ export interface components { * @default 0 */ successful_requests: number; + /** + * Timed Requests + * @default 0 + */ + timed_requests: number; + /** + * Total Response Time Ms + * @default 0 + */ + total_response_time_ms: number; /** * Total Tokens * @default 0 @@ -64994,6 +65464,338 @@ export interface operations { }; }; }; + list_issues_issues_get: { + parameters: { + query?: { + kind?: ("performance" | "model_deprecation") | null; + family?: ("retirement" | "latency" | "reliability" | "spend" | "traffic") | null; + status?: ("open" | "resolved" | "dismissed") | null; + scope_type?: ("deployment" | "key" | "user" | "team" | "model" | "provider") | null; + scope_id?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_issue_families_issues_families_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueFamilyList"]; + }; + }; + }; + }; + update_issue_family_issues_families__family__patch: { + parameters: { + query?: never; + header?: never; + path: { + family: "spend" | "latency" | "reliability" | "traffic"; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["IssueFamilyUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueFamilySetting"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_issues__issue_id__get: { + parameters: { + query?: never; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Issue"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_issue_issues__issue_id__patch: { + parameters: { + query?: never; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["IssueUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Issue"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_activity_issues__issue_id__activity_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_impact_issues__issue_id__impact_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueImpact"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_keys_issues__issue_id__keys_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DailyActivityKeyPageResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_issue_signals_issues__issue_id__signals_get: { + parameters: { + query?: { + limit?: number; + offset?: number; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueSignalList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_users_issues__issue_id__users_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DailyActivityUserPageResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; delete_jwt_key_mapping_jwt_key_mapping_delete_post: { parameters: { query?: never; @@ -79013,7 +79815,7 @@ export interface operations { query?: { scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; scope_id?: string | null; - metric?: ("spend" | "requests" | "latency" | "errors") | null; + metric?: ("spend" | "requests" | "latency" | "errors" | "error_rate") | null; status?: ("open" | "resolved" | "dismissed") | null; start_date?: string | null; end_date?: string | null; From df62ec53cd7b8d73884507866c3f1ace3dca3c93 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:13:01 +0800 Subject: [PATCH 12/38] fix: scope issue evidence and preserve retirement lifecycle --- docs/features/usage-alerts.md | 96 +++++- litellm/models/issues.py | 6 +- litellm/proxy/alerting_endpoints/issues.py | 18 +- litellm/repositories/issues.py | 57 +++- litellm/repositories/usage_alerts.py | 3 +- litellm/types/proxy/issues.py | 7 +- litellm/types/proxy/usage_alerts.py | 2 +- tests/integration/database/test_issues.py | 290 ++++++++++++++++++ .../integration/database/test_usage_alerts.py | 19 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 13 +- 10 files changed, 475 insertions(+), 36 deletions(-) create mode 100644 tests/integration/database/test_issues.py diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index 804a8b44aec..63c353b0fb6 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -1,9 +1,92 @@ -# Daily Performance alerts +# Issues and daily Performance alerts -Performance alerts compare a completed UTC day with earlier recorded days. They cover spend, request count, average latency, and failed-request count for the gateway, a key, a user, a team, or a model. The Performance tab groups alerts for the same resource and original UTC day. Open a resource to inspect its saved comparison and Activity. The API paths remain under `/usage`. +The Issues inbox combines model retirements with daily Performance alerts. Open an issue to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause. + +Performance comparisons cover completed UTC days. Retirement issues show recent model usage and the provider's retirement schedule, including today's usage when present. This feature requires a database. Apply the normal LiteLLM database migrations when deploying a version that includes usage alerts. The API and dashboard use the same persisted rules and incidents; no enterprise license is required. +## Start with the defaults + +Daily Performance defaults cover spend, latency, reliability, and traffic across the gateway, keys, users, teams, and models. They use the previous seven days and the following policies. Retirement uses the existing model-deprecation warning window. + +| Family | Default comparison | Minimum history and volume | +| --- | --- | --- | +| Spend | At least 3× daily spend and a $10 increase | Three recorded reference days with a positive spend baseline | +| Latency | At least 3× average latency and a 500 ms increase | Three recorded reference days and at least 100 timed requests in both periods | +| Reliability | At least 3× error rate and a two percentage-point increase | Three recorded reference days and at least 500 daily requests in both periods | +| Traffic increase | At least 10× requests and an increase of 1,000 requests | Three recorded reference days with a positive request baseline | +| Traffic decrease | At most one fifth of baseline requests | Seven recorded reference days and a baseline of at least 500 daily requests | + +New resources need history before they can produce a default spike. A zero spend, request, or latency baseline does not establish a spike. A zero error-rate baseline is usable only after sufficient historical traffic has been observed; the percentage-point floor still applies. + +Use the family settings to enable or disable a family or adjust its thresholds across the five scopes. Existing custom rules remain independent. An enabled custom rule replaces the matching default for its scope, metric, and direction. A custom wildcard rule replaces that default for every entity. Deleted defaults stay deleted. + +The scheduler wakes hourly and evaluates the latest completed UTC day after a one-hour grace period. A successful aggregate flush after the grace period is required before either direction is finalized. Completed rule versions are skipped on later ticks. The evaluator reads existing daily aggregates in pages of 500 entities; it adds no request-time counters or per-request database writes. + +Decrease rules need a successful background aggregate-flush sample for every hour of the target day and another sample after the grace period. Until that coverage exists, their status is `insufficient_coverage`. A key with no requests can then produce a request or spend drop; missing latency samples never become zero latency. Coverage shows that an aggregate writer was healthy each hour, not that every replica delivered every request. + +The baseline averages recorded reference days; it does not turn missing reference days into zero. Average latency is weighted by timed request count, using the same successful-request duration totals as Activity. `baseline_sample_days` records how many reference days contributed. Cost, traffic, and latency comparisons require a positive baseline. Reliability can compare against a recorded 0% error rate when historical and current request volume meet the rule's minimum. Missing history and low-volume comparisons remain `insufficient_history`. + +## Read and investigate issues + +Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the credential in your environment. + +```bash +curl "$LITELLM_API_BASE/issues?status=open&limit=50" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +``` + +The response contains `data`, `total`, `limit`, and `offset`. Pagination and filtering happen on the combined inbox. Filters are `kind`, `family`, `status`, `scope_type`, and `scope_id`. Families are `retirement`, `latency`, `reliability`, `spend`, and `traffic`; kinds are `model_deprecation` and `performance`. An offset beyond the last page still returns the correct total. There is no default history cutoff on open issues. + +Each issue has an ID, status, readable title, current resource label, typed `cohort`, date range, grouping reason, and saved comparison values. The cohort is one resource, one provider, or exact provider/model pairs. A provider/model pair is an intersection, not a list of independently interchangeable filters. Resource names can change; IDs remain the identity. + +Use the ID from the list to inspect an issue: + +```bash +ISSUE_ID="issue-example" +curl "$LITELLM_API_BASE/issues/$ISSUE_ID" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +curl "$LITELLM_API_BASE/issues/$ISSUE_ID/activity" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +curl "$LITELLM_API_BASE/issues/$ISSUE_ID/signals?limit=50" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +``` + +Replace `issue-example` with an ID returned by `/issues`. Activity uses the issue's saved date range by default. Optional `start_date` and `end_date` are inclusive UTC dates and can span at most 366 days. `/keys` and `/users` provide paginated Activity rows with `limit` and `offset`. Their latency totals are `total_response_time_ms` and `timed_requests`; divide the former by the latter only when timed requests are present. + +`/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown. + +Supporting signals have their own saved comparisons and relationship explanations. Their baseline windows can differ, so do not derive a combined error rate or unit cost by adding their displayed values. Administrators and admin viewers can inspect broader supporting signals. A scoped reader receives only signals for the exact key, user, or team they can access. That restriction applies before the signal count and pagination; broader totals are not disclosed. + +A continuing Performance issue retains its initial comparison and cohort while `last_seen` advances. New supporting observations do not rewrite the original evidence. After a healthy observation ends every linked source episode, a later breach starts a new issue. Retirement details refresh as usage and the provider schedule change. A retirement issue with no recent use becomes inactive; actual new traffic can reopen it. + +Resolve or dismiss an issue using the current status from the read response: + +```bash +curl -X PATCH "$LITELLM_API_BASE/issues/$ISSUE_ID" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \ + -H "Content-Type: application/json" \ + -d '{"expected_status":"open","status":"resolved"}' +``` + +A stale status returns `409`; reload before retrying. Performance acknowledgements do not change other issues that share a supporting observation. Retirement acknowledgements use the same state as the existing model-deprecation API. An inactive retirement cannot be manually reopened. + +## Configure default families + +`GET /issues/families` returns the four Performance families, their enabled counts, and common policy values. A null common value means the underlying scopes have different settings. This is separate from custom rules and the model-retirement warning window. + +This example sets the spend family to a 5× threshold and a $25 minimum absolute increase: + +```bash +curl -X PATCH "$LITELLM_API_BASE/issues/families/spend" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \ + -H "Content-Type: application/json" \ + -d '{"enabled":true,"policies":[{"direction":"increase","multiplier":5,"min_value":25}]}' +``` + +The update applies atomically to registered, non-deleted built-in rules in that family. Omitted settings remain unchanged. It does not change custom rules or recreate deleted defaults. A concurrent rule edit returns `409`; reload and retry. These controls require a gateway administrator. + ## Read resource groups Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the key in your environment. @@ -15,7 +98,7 @@ curl "$LITELLM_API_BASE/usage/alert-groups?status=open&limit=50" \ The response has `data`, `total`, `limit`, and `offset`. `total` counts resource/day groups, including when the requested offset is beyond the last page. Filters are `scope_type`, `scope_id`, `status`, `start_date`, and `end_date`. Status filters apply to individual incidents before grouping, so a group with open and resolved incidents can appear under either filter with the corresponding members. No default date cutoff hides older open incidents. -Each group includes its stable `id`, `scope_type`, exact `scope_id`, current `scope_label` when available, original UTC `day`, `incident_count`, and `activity_path`. Its `metrics` list contains up to four entries with a `metric`, `incident_count`, and the most recently detected matching `representative` incident. These representatives are summaries, not the entire group. Missing or deleted resources keep their original IDs. Names are current display labels, not historical evidence. +Each group includes its stable `id`, `scope_type`, exact `scope_id`, current `scope_label` when available, original UTC `day`, `incident_count`, and `activity_path`. Its `metrics` list contains up to five entries with a `metric`, `incident_count`, and the most recently detected matching `representative` incident. These representatives are summaries, not the entire group. Missing or deleted resources keep their original IDs. Names are current display labels, not historical evidence. To inspect every matching rule, use `GET /usage/alerts` with the group's scope and `start_date` and `end_date` both set to its `day`. Add `metric` and `status` as needed and paginate that response. Rules can have different baseline windows and thresholds, so their comparisons must not be combined into an error rate or a cost-per-request estimate. @@ -69,8 +152,9 @@ The baseline is the mean across recorded days in the preceding window. Missing r | `requests` | Request count | | `latency` | Average response time in milliseconds | | `errors` | Failed-request count | +| `error_rate` | Failed requests divided by requests, as a percentage from 0 to 100 | -Counts come from the existing Activity/spend aggregates, rather than every gateway HTTP outcome. Latency averages successful timed, non-internal requests; it is not time to first token and does not include failed requests. Gateway incidents link to Key Activity, which uses the same aggregates. +Counts come from the existing Activity/spend aggregates, rather than every gateway HTTP outcome. Latency averages successful timed, non-internal requests; it is not time to first token and does not include failed requests. Issue Activity and gateway source incidents use the same aggregates. An error-rate `min_value` of 2 means two percentage points, such as 1% to 3%, not a 2% relative increase. The evaluator runs against daily aggregates. An hourly scheduler does not make these rolling hourly alerts. A complete UTC day and sufficient ingestion coverage are required before detecting drops; a missing flush must not become a false traffic-drop incident. Rule responses expose `last_evaluated_day` and `evaluation_status` so you can distinguish a completed evaluation from insufficient history or coverage. @@ -87,10 +171,12 @@ Use `scope_type: "deployment"` with no `scope_id` for the gateway. For `model`, Consecutive breached days update one incident. Re-evaluating the same day does not reopen a dismissed or resolved incident. After recovery, a later breach starts a new incident. Editing a rule does not rewrite historical comparisons. -Rules can specify `notification_channels: ["slack", "email"]` and `email_recipients: ["alerts@example.com"]`. Notification transports must already be configured on the gateway. An empty channel list keeps incidents available through the API and dashboard. Successful delivery is recorded per channel so a failed channel can retry independently. +Rules can specify `notification_channels: ["slack", "email"]` and `email_recipients: ["alerts@example.com"]`. Notification transports must already be configured on the gateway. An empty channel list keeps incidents available through the API and dashboard. Issue delivery is recorded per channel and destination. A repeated projection does not resend a successful delivery. Failed deliveries retry in the background, and the worker rechecks the issue, source signal, and current rule immediately before transport. Disabling a rule or acknowledging an issue cannot recall a send already in flight. Delivery is at least once: an ambiguous transport timeout, or partial success across several configured Slack webhooks, can cause a retry after a destination accepted a message. Configure `PROXY_BASE_URL` for clickable links in email and Slack. Default rules notify through Slack only when `usage_alerts` is explicitly enabled in the gateway's alert types when those rules are first registered. Otherwise they start with in-app/API incidents. To enable notifications after the defaults already exist, edit their channels through the API or dashboard; changing the gateway alert types does not rewrite saved rules. Explicit rule channels use the configured transport without an additional alert-type switch. If legacy `user_spend_anomalies` notifications are enabled, the default daily user-spend increase rule stays in-app to avoid overlapping notifications. Email requires recipients on the rule. ## Access Gateway administrators can manage every scope. Admin viewers can read every scope. Internal users can manage their own user and key scopes; team administrators can manage their team's scope and keys. Team members can read their team's incidents. Read-only users cannot change rules or incident state, and inference-only keys cannot access these endpoints. Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` indicates that the gateway has no database connection. + +The dashboard's Issues inbox is available to administrators and admin viewers. Other authorized roles can use the scoped API. diff --git a/litellm/models/issues.py b/litellm/models/issues.py index 7d0692d26db..4e5aa4931c0 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -74,7 +74,9 @@ class IssueStore(Protocol): async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: ... - async def list_signals(self, issue_id: str, limit: int = 50, offset: int = 0) -> IssueSignalList: ... + async def list_signals( + self, issue_id: str, limit: int = 50, offset: int = 0, visible_scope: IssueCohort | None = None + ) -> IssueSignalList: ... async def active_signals( self, @@ -111,6 +113,8 @@ class IssueStore(Protocol): async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: ... + async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: ... + async def mark_delivery_sent(self, delivery_id: str) -> None: ... async def defer_delivery( diff --git a/litellm/proxy/alerting_endpoints/issues.py b/litellm/proxy/alerting_endpoints/issues.py index 30f3d44afe7..cca5c730bb5 100644 --- a/litellm/proxy/alerting_endpoints/issues.py +++ b/litellm/proxy/alerting_endpoints/issues.py @@ -15,6 +15,7 @@ from litellm.proxy.alerting_endpoints.usage_alerts import ( require_scope, ) from litellm.proxy.deps.repositories import IssueActivityRepositoryDep, IssuesRepositoryDep, UsageAlertsRepositoryDep +from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.issues import ( Issue, IssueFamily, @@ -147,7 +148,9 @@ async def update_issue( repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, ) -> Issue: - await authorized_issue(issue_id, user, repository, alerts, write=True) + issue: Final = await authorized_issue(issue_id, user, repository, alerts, write=True) + if issue.kind == "model_deprecation" and not issue.active and data.status == "open": + raise HTTPException(status_code=409, detail="This retirement issue reopens when the model receives new traffic") changed: Final = await require_issues(repository).set_status( issue_id, data.status, data.expected_status, user.user_id ) @@ -165,8 +168,17 @@ async def list_issue_signals( limit: Limit = 50, offset: Offset = 0, ) -> IssueSignalList: - await authorized_issue(issue_id, user, repository, alerts) - return await require_issues(repository).list_signals(issue_id, limit, offset) + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + visible_scope: Final = ( + None + if user.user_role + in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ) + else issue.cohort + ) + return await require_issues(repository).list_signals(issue_id, limit, offset, visible_scope) @router.get("/{issue_id}/activity", response_model=SpendAnalyticsPaginatedResponse) diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 0f9ac3e90f3..bec14f68417 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -14,6 +14,7 @@ from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( Issue, + IssueCohort, IssueDeliveryBinding, IssueDeliveryInput, IssueFamily, @@ -54,8 +55,13 @@ class _Identifier(BaseModel, frozen=True): _ACK: Final = "CASE WHEN jsonb_typeof(c.param_value)='string' THEN (c.param_value#>>'{}')::jsonb ELSE c.param_value END" +_ACK_CURRENT: Final = f"""(({_ACK})->>'last_seen_date' > i.record->'deprecation'->>'last_used_date' + OR (({_ACK})->>'last_seen_date' = i.record->'deprecation'->>'last_used_date' + AND COALESCE((({_ACK})->>'last_seen_requests')::bigint,0) + >= (i.record->'deprecation'->>'last_day_requests')::bigint))""" _STATUS: Final = f"""CASE WHEN i.kind <> 'model_deprecation' THEN i.status - WHEN NOT i.active THEN CASE WHEN ({_ACK})->>'status'='dismissed' THEN 'dismissed' ELSE 'resolved' END + WHEN NOT i.active THEN CASE WHEN ({_ACK})->>'status'='dismissed' AND ({_ACK_CURRENT}) + THEN 'dismissed' ELSE 'resolved' END WHEN COALESCE(({_ACK})->>'status','open')='open' OR ({_ACK})->>'last_seen_date' IS NULL OR i.record->'deprecation'->>'last_used_date' > ({_ACK})->>'last_seen_date' @@ -63,6 +69,7 @@ _STATUS: Final = f"""CASE WHEN i.kind <> 'model_deprecation' THEN i.status AND (i.record->'deprecation'->>'last_day_requests')::bigint > COALESCE((({_ACK})->>'last_seen_requests')::bigint,0)) THEN 'open' ELSE ({_ACK})->>'status' END""" +_PRIOR_STATUS: Final = _STATUS.replace(_ACK, _ACK.replace("c.param_value", "prior.param_value")) _JOIN: Final = 'FROM "LiteLLM_Issue" i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key' _RECORD: Final = f"""i.record || jsonb_build_object('status',({_STATUS}),'active',i.active, 'first_seen',i.first_seen AT TIME ZONE 'UTC','last_seen',i.last_seen AT TIME ZONE 'UTC', @@ -226,6 +233,7 @@ class IssuesRepository: await self.database.query_raw( f"""WITH candidate AS MATERIALIZED ( SELECT i.*,({_STATUS}) AS effective_status {_JOIN} WHERE i.id=$1 + AND NOT(i.kind='model_deprecation' AND NOT i.active AND $2='open') ), ack AS ( INSERT INTO "LiteLLM_Config" AS prior (param_name,param_value) SELECT ack_key,jsonb_build_object('model',record->'deprecation'->>'usage_model', @@ -235,13 +243,8 @@ class IssuesRepository: 'updated_at',CURRENT_TIMESTAMP,'updated_by',$4::text) FROM candidate WHERE kind='model_deprecation' AND effective_status=$3 ON CONFLICT(param_name) DO UPDATE SET param_value=EXCLUDED.param_value - WHERE (CASE WHEN COALESCE((CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'status','open')='open' - OR (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' IS NULL - OR EXCLUDED.param_value->>'last_seen_date' > (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' - OR (EXCLUDED.param_value->>'last_seen_date'=(CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' - AND (EXCLUDED.param_value->>'last_seen_requests')::bigint - > COALESCE(((CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_requests')::bigint,0)) - THEN 'open' ELSE (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'status' END)=$3 + WHERE EXISTS(SELECT 1 FROM candidate i WHERE i.ack_key=prior.param_name + AND ({_PRIOR_STATUS})=$3) RETURNING param_name ) UPDATE "LiteLLM_Issue" i SET status=$2,updated_at=CURRENT_TIMESTAMP FROM candidate c WHERE i.id=c.id AND c.effective_status=$3 AND @@ -269,25 +272,36 @@ class IssuesRepository: """INSERT INTO "LiteLLM_IssueSignal" (issue_id,source,source_id,record) SELECT DISTINCT ON(x->>'issue_id',x->>'source',x->>'source_id') x->>'issue_id',x->>'source',x->>'source_id',x FROM jsonb_array_elements($1::jsonb) x - WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" i WHERE i.id=x->>'issue_id' - AND (i.scope_type NOT IN ('key','team','user') OR - (x->'usage_alert'->>'scope_type'=i.scope_type - AND x->'usage_alert'->>'scope_id'=i.scope_id))) + WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" i WHERE i.id=x->>'issue_id') ON CONFLICT(issue_id,source,source_id) DO NOTHING RETURNING source_id""", json.dumps(tuple(signal.model_dump(mode="json") for signal in signals)), ) - async def list_signals(self, issue_id: str, limit: int = 50, offset: int = 0) -> IssueSignalList: + async def list_signals( + self, + issue_id: str, + limit: int = 50, + offset: int = 0, + visible_scope: IssueCohort | None = None, + ) -> IssueSignalList: + where: Final = """issue_id=$1 AND ($2::text IS NULL OR (source='usage_alert' + AND record->'usage_alert'->>'scope_type'=$2 + AND record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM $3::text))""" + args: Final = ( + issue_id, + visible_scope.scope_type if visible_scope else None, + visible_scope.scope_id if visible_scope else None, + ) counts: Final = TypeAdapter(tuple[_Count, ...]).validate_python( await self.database.query_raw( - 'SELECT count(*)::int AS count FROM "LiteLLM_IssueSignal" WHERE issue_id=$1', - issue_id, + f'SELECT count(*)::int AS count FROM "LiteLLM_IssueSignal" WHERE {where}', + *args, ) ) rows: Final = TypeAdapter(tuple[_SignalRow, ...]).validate_python( await self.database.query_raw( - 'SELECT record FROM "LiteLLM_IssueSignal" WHERE issue_id=$1 ORDER BY source,source_id LIMIT $2 OFFSET $3', - issue_id, + f'SELECT record FROM "LiteLLM_IssueSignal" WHERE {where} ORDER BY source,source_id LIMIT $4 OFFSET $5', + *args, limit, offset, ) @@ -419,6 +433,13 @@ class IssuesRepository: ) async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: + return await self._pending_deliveries(limit, None) + + async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: + rows: Final = await self._pending_deliveries(1, delivery_id) + return rows[0] if rows else None + + async def _pending_deliveries(self, limit: int, delivery_id: str | None) -> tuple[IssuePendingDelivery, ...]: if not 1 <= limit <= 500: raise ValueError("Read between one and 500 deliveries") return TypeAdapter(tuple[IssuePendingDelivery, ...]).validate_python( @@ -442,8 +463,10 @@ class IssuesRepository: ORDER BY b.rule_id,b.source_id LIMIT 1 ) bound ON true WHERE NOT d.sent AND d.next_attempt<=CURRENT_TIMESTAMP AND i.active AND ({_STATUS})='open' + AND ($2::text IS NULL OR d.id=$2) ORDER BY d.next_attempt,d.id LIMIT $1""", limit, + delivery_id, ) ) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 7eb80dae626..25d7a89d077 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -389,7 +389,8 @@ class UsageAlertsRepository: SELECT p.scope_type,p.scope_id,p.day,p.incident_count, jsonb_agg(jsonb_build_object('metric',r.metric,'incident_count',r.incident_count, 'representative',r.record) ORDER BY CASE r.metric - WHEN 'requests' THEN 0 WHEN 'spend' THEN 1 WHEN 'errors' THEN 2 ELSE 3 END) AS metrics + WHEN 'requests' THEN 0 WHEN 'spend' THEN 1 WHEN 'error_rate' THEN 2 + WHEN 'errors' THEN 3 ELSE 4 END) AS metrics FROM page p JOIN ranked r ON p.scope_type=r.scope_type AND p.scope_id IS NOT DISTINCT FROM r.scope_id AND p.day=r.day AND r.rank=1 GROUP BY p.scope_type,p.scope_id,p.day,p.incident_count diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py index e0197e6edf2..1a4104726fb 100644 --- a/litellm/types/proxy/issues.py +++ b/litellm/types/proxy/issues.py @@ -18,6 +18,9 @@ IssueKind: TypeAlias = Literal["performance", "model_deprecation"] IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] IssueGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"] +IssueSignalGroupingReason: TypeAlias = Literal[ + "same_resource", "same_key_traffic", "dominant_key", "provider_cohort", "volume_explains_spend", "model_deprecation" +] class IssueModelPair(BaseModel, frozen=True): @@ -52,7 +55,7 @@ class IssueComparison(BaseModel, frozen=True): direction: UsageAlertDirection observed_value: float = Field(ge=0) baseline_value: float = Field(ge=0) - threshold: float + threshold: float | None = None baseline_start: date baseline_end: date baseline_sample_days: int | None = Field(default=None, ge=1, le=30) @@ -151,6 +154,8 @@ class IssueSignalInput(BaseModel, frozen=True): source_id: str = Field(min_length=1, max_length=200) usage_alert: UsageAlert | None = None deprecation: IssueDeprecation | None = None + grouping_reason: IssueSignalGroupingReason = "same_resource" + explanation: str = Field(default="", max_length=2000) @model_validator(mode="after") def validate_signal(self) -> Self: diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index 72ccc1f3ef7..64336441fbe 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -133,7 +133,7 @@ class UsageAlertGroup(BaseModel, frozen=True): scope_label: str | None day: date incident_count: int - metrics: tuple[UsageAlertGroupMetric, ...] = Field(max_length=4) + metrics: tuple[UsageAlertGroupMetric, ...] = Field(max_length=5) activity_path: str diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py new file mode 100644 index 00000000000..c173cef9e75 --- /dev/null +++ b/tests/integration/database/test_issues.py @@ -0,0 +1,290 @@ +import json +from datetime import date, datetime, timedelta, timezone +from hashlib import sha256 +from typing import Final +from uuid import uuid4 + +import pytest +from integration._support.actors import ManagementDatabase +from integration._support.client import Gateway + +from litellm.repositories.issue_activity import IssueActivityRepository +from litellm.repositories.issues import IssuesRepository +from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.types.proxy.issues import ( + IssueCohort, + IssueComparison, + IssueInput, + IssueList, + IssueModelPair, + IssueSignalInput, + IssueSignalList, +) +from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRuleInput + + +def _issue(cohort: IssueCohort) -> IssueInput: + identity: Final = str(uuid4()) + day: Final = date.today() - timedelta(days=3) + return IssueInput( + id=identity, + cohort_key=identity, + kind="performance", + family="traffic", + title="Requests increased", + cohort=cohort, + observed_on=day, + observed_at=datetime.now(timezone.utc), + range_start=day - timedelta(days=7), + range_end=day, + grouping_reason="same_resource", + explanation="Requests increased for this resource.", + comparisons=( + IssueComparison( + metric="requests", + direction="increase", + observed_value=420, + baseline_value=120, + threshold=360, + baseline_start=day - timedelta(days=7), + baseline_end=day - timedelta(days=1), + baseline_sample_days=7, + ), + ), + ) + + +async def _remove_issues(db: ManagementDatabase, ids: tuple[str, ...]) -> None: + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_IssueDeliveryRule" WHERE delivery_id IN ' + '(SELECT id FROM "LiteLLM_IssueDelivery" WHERE issue_id=ANY($1::text[])) RETURNING delivery_id', + list(ids), + ) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_IssueDelivery" WHERE issue_id=ANY($1::text[]) RETURNING id', list(ids) + ) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_IssueSignal" WHERE issue_id=ANY($1::text[]) RETURNING source_id', list(ids) + ) + await db.prisma.writer_db.query_raw('DELETE FROM "LiteLLM_Issue" WHERE id=ANY($1::text[]) RETURNING id', list(ids)) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gateway, db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + outsider: Final = scenario.user(user_role="internal_user") + viewer: Final = scenario.user(user_role="proxy_admin_viewer") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + outsider_key: Final = scenario.key(user_id=outsider, key_type="management") + viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") + inference_key: Final = scenario.key(user_id=owner, key_type="llm_api", key_alias="Example checkout") + reporting_id: Final = sha256(inference_key.encode()).hexdigest() + first: Final = _issue(IssueCohort(scope_type="key", scope_id=reporting_id)) + second: Final = _issue(first.cohort) + global_issue: Final = _issue(IssueCohort(scope_type="provider", scope_id="example-provider")) + rule: Final = await alerts.save_rule( + UsageAlertRuleInput( + name="Visibility fixture", + scope_type="key", + scope_id=reporting_id, + metric="requests", + direction="increase", + ), + owner, + ) + assert rule is not None + try: + await store.upsert_issues((first, second, global_issue)) + source: Final = ( + await alerts.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=reporting_id, + day=first.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + broader: Final = source.model_copy( + update={ + "id": str(uuid4()), + "scope_type": "deployment", + "scope_id": None, + "observed_value": 99999, + "baseline_value": 9999, + } + ) + await store.add_signals( + first.id, + ( + IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source), + IssueSignalInput( + source="usage_alert", + source_id=broader.id, + usage_alert=broader, + grouping_reason="dominant_key", + explanation="Parent contribution evidence.", + ), + ), + ) + path: Final = f"/issues/{first.id}" + own: Final = gateway.request("GET", path, key=owner_key) + assert own.status_code == 200 and own.json()["scope_label"] == "Example checkout" + assert "99999" not in own.text and "9999" not in own.text + for offset in (0, 1, 2): + page: Final = gateway.request( + "GET", + "/issues", + key=owner_key, + params={"scope_type": "key", "scope_id": reporting_id, "limit": "1", "offset": str(offset)}, + ) + assert page.status_code == 200 + parsed: Final = IssueList.model_validate(page.json()) + assert parsed.total == 2 and len(parsed.data) == (1 if offset < 2 else 0) + assert gateway.request("GET", "/issues", key=owner_key).status_code == 403 + for suffix in ("", "/signals", "/activity", "/keys", "/users", "/impact"): + assert gateway.request("GET", f"/issues/{global_issue.id}{suffix}", key=owner_key).status_code == 403 + assert gateway.request("GET", path + suffix, key=outsider_key).status_code == 403 + assert gateway.request("GET", path + suffix, key=inference_key).status_code == 403 + public_signals: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1"}) + assert public_signals.status_code == 200 + visible: Final = IssueSignalList.model_validate(public_signals.json()) + assert visible.total == 1 and tuple(row.source_id for row in visible.data) == (source.id,) + assert "99999" not in public_signals.text + end: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1", "offset": "1"}) + assert end.json()["total"] == 1 and end.json()["data"] == [] + admin: Final = gateway.request("GET", path + "/signals", key=viewer_key) + assert admin.status_code == 200 and admin.json()["total"] == 2 and "99999" in admin.text + assert ( + gateway.request( + "PATCH", path, {"status": "dismissed", "expected_status": "open"}, key=viewer_key + ).status_code + == 403 + ) + assert ( + gateway.request( + "PATCH", path, {"status": "dismissed", "expected_status": "open"}, key=owner_key + ).status_code + == 200 + ) + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=owner_key + ).status_code + == 409 + ) + assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["status"] == "open" + finally: + await _remove_issues(db, (first.id, second.id, global_issue.id)) + await alerts.delete_rule(rule.id) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id + ) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( + gateway: Gateway, db: ManagementDatabase +) -> None: + repository: Final = IssueActivityRepository.from_prisma(db.prisma) + run_id: Final = str(uuid4()) + day: Final = date.today() - timedelta(days=3) + provider: Final = f"example-provider-{run_id}" + other: Final = f"other-provider-{run_id}" + key: Final = sha256(run_id.encode()).hexdigest() + entries: Final = tuple( + { + "id": f"{run_id}-{index}", + "user_id": f"user-{run_id}", + "date": day.isoformat(), + "api_key": key, + "model": model, + "custom_llm_provider": name, + "api_requests": requests, + "spend": spend, + "successful_requests": requests, + "total_response_time_ms": requests * latency, + "timed_requests": requests, + } + for index, (name, model, requests, spend, latency) in enumerate( + ( + (provider, "example-model-a", 100, 10, 200), + (other, "example-model-a", 1000, 100, 900), + (provider, "example-model-b", 50, 5, 400), + ) + ) + ) + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,api_requests,spend,successful_requests,total_response_time_ms,timed_requests,updated_at) + SELECT id,user_id,date,api_key,model,custom_llm_provider,api_requests,spend,successful_requests,total_response_time_ms,timed_requests,CURRENT_TIMESTAMP + FROM jsonb_to_recordset($1::jsonb) AS x(id text,user_id text,date text,api_key text,model text, + custom_llm_provider text,api_requests int,spend float,successful_requests int,total_response_time_ms bigint,timed_requests int) + RETURNING id""", + json.dumps(entries), + ) + cohort: Final = IssueCohort( + scope_type="provider", + scope_id=provider, + model_pairs=(IssueModelPair(provider=provider, model="example-model-a"),), + ) + store: Final = IssuesRepository.from_prisma(db.prisma) + issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day}) + try: + await store.upsert_issue(issue) + activity_http: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + assert activity_http.status_code == 200 and activity_http.json()["metadata"]["total_api_requests"] == 100 + keys_http: Final = gateway.request("GET", f"/issues/{issue.id}/keys", params={"limit": "1"}) + assert keys_http.status_code == 200 and keys_http.json()["api_keys"][0]["metrics"]["api_requests"] == 100 + users_http: Final = gateway.request("GET", f"/issues/{issue.id}/users") + assert users_http.status_code == 200 and users_http.json()["total_users"] == 1 + impact_http: Final = gateway.request("GET", f"/issues/{issue.id}/impact") + assert impact_http.status_code == 200 and impact_http.json()["requests"] == 100 + daily: Final = await repository.daily_activity(cohort, day, day) + assert daily.metadata.total_api_requests == 100 and daily.metadata.total_spend == 10 + assert daily.metadata.total_response_time_ms == 20000 and daily.metadata.total_timed_requests == 100 + keys: Final = await repository.keys(cohort, day, day, 1, 0) + assert keys.total_api_keys == 1 and keys.api_keys[0].metrics.api_requests == 100 + assert ( + keys.api_keys[0].metrics.total_response_time_ms == 20000 and keys.api_keys[0].metrics.timed_requests == 100 + ) + users: Final = await repository.users(cohort, day, day, 1, 0) + assert users.total_users == 1 and users.users[0].api_requests == 100 and users.users[0].timed_requests == 100 + impact: Final = await repository.impact(cohort, day, day) + assert (impact.keys, impact.users, impact.models, impact.requests) == (1, 1, 1, 100) + pairs: Final = IssueCohort( + scope_type="deployment", + model_pairs=( + IssueModelPair(provider=provider, model="example-model-a"), + IssueModelPair(provider=other, model="example-model-b"), + ), + ) + assert (await repository.daily_activity(pairs, day, day)).metadata.total_api_requests == 100 + injected: Final = IssueCohort(scope_type="provider", scope_id=provider + "' OR true --") + assert (await repository.impact(injected, day, day)).requests == 0 + from litellm.repositories.issue_activity import issue_activity_scope + from litellm.types.repositories.daily_activity import ExportType + + exported: Final = tuple( + [ + row + async for row in repository.activity.export_rows( + issue_activity_scope(cohort, day, day), export_type=ExportType.DAILY + ) + ] + ) + assert len(exported) == 1 and exported[0].api_requests == 100 + finally: + await _remove_issues(db, (issue.id,)) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id=ANY($1::text[]) RETURNING id', + [entry["id"] for entry in entries], + ) diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index a43b5858fe0..51978152ed2 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -276,7 +276,13 @@ async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_action outsider_key: Final = scenario.key(user_id=outsider, key_type="management") viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") inference_key: Final = scenario.key(user_id=owner, key_type="llm_api") - metrics: Final[tuple[UsageAlertMetric, ...]] = (*(("requests",) * 27), "spend", "errors", "latency") + metrics: Final[tuple[UsageAlertMetric, ...]] = ( + *(("requests",) * 27), + "spend", + "errors", + "latency", + "error_rate", + ) rules: Final = tuple( [ await repository.save_rule( @@ -303,9 +309,9 @@ async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_action UsageAlertObservation( entity_id=owner, day=day, - observed=420, - baseline=0 if index == 0 else 120, - threshold=100 if index == 0 else 360, + observed=14 if rule.metric == "error_rate" else 420, + baseline=4 if rule.metric == "error_rate" else 0 if index == 0 else 120, + threshold=12 if rule.metric == "error_rate" else 100 if index == 0 else 360, breached=True, ), ), @@ -367,12 +373,13 @@ async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_action assert page.total == 2 and len(page.data) == 1 group: Final = page.data[0] assert group.day == day and group.scope_label == "Checkout operator" - assert group.incident_count == 30 and len(group.metrics) == 4 + assert group.incident_count == 31 and len(group.metrics) == 5 assert {metric.metric: metric.incident_count for metric in group.metrics} == { "requests": 27, "spend": 1, "errors": 1, "latency": 1, + "error_rate": 1, } assert all(metric.representative.id in {incident.id for incident in incidents} for metric in group.metrics) assert group.activity_path in {incident.activity_path for incident in incidents} @@ -405,7 +412,7 @@ async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_action assert gateway.request("PATCH", "/usage/alert-groups", body, key=forbidden).status_code == 403 assert gateway.request("GET", "/usage/alert-groups", key=viewer_key, params=params).status_code == 200 updated: Final = gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key) - assert updated.status_code == 200 and updated.json()["updated_count"] == 29 + assert updated.status_code == 200 and updated.json()["updated_count"] == 30 assert gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key).json()["updated_count"] == 0 assert await repository.get_alert(old.id) == old assert await repository.get_alert(other.id) == other diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bbd4acaedf8..3517ddb0aef 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -34992,7 +34992,7 @@ export interface components { /** Observed Value */ observed_value: number; /** Threshold */ - threshold: number; + threshold?: number | null; }; /** IssueDeprecation */ IssueDeprecation: { @@ -35125,6 +35125,17 @@ export interface components { /** IssueSignal */ IssueSignal: { deprecation?: components["schemas"]["IssueDeprecation"] | null; + /** + * Explanation + * @default + */ + explanation: string; + /** + * Grouping Reason + * @default same_resource + * @enum {string} + */ + grouping_reason: "same_resource" | "same_key_traffic" | "dominant_key" | "provider_cohort" | "volume_explains_spend" | "model_deprecation"; /** Issue Id */ issue_id: string; /** From 5322b1ac8815b7207f9d9ad0483cf07b5952325d Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:33:35 +0800 Subject: [PATCH 13/38] fix: harden issue recovery and family updates --- docs/features/usage-alerts.md | 2 +- litellm/models/issues.py | 5 + litellm/repositories/issue_families.py | 3 +- litellm/repositories/issues.py | 70 +++- tests/integration/database/test_issues.py | 396 ++++++++++++++++++++-- 5 files changed, 445 insertions(+), 31 deletions(-) diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index 63c353b0fb6..c14aa564221 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -59,7 +59,7 @@ Replace `issue-example` with an ID returned by `/issues`. Activity uses the issu Supporting signals have their own saved comparisons and relationship explanations. Their baseline windows can differ, so do not derive a combined error rate or unit cost by adding their displayed values. Administrators and admin viewers can inspect broader supporting signals. A scoped reader receives only signals for the exact key, user, or team they can access. That restriction applies before the signal count and pagination; broader totals are not disclosed. -A continuing Performance issue retains its initial comparison and cohort while `last_seen` advances. New supporting observations do not rewrite the original evidence. After a healthy observation ends every linked source episode, a later breach starts a new issue. Retirement details refresh as usage and the provider schedule change. A retirement issue with no recent use becomes inactive; actual new traffic can reopen it. +A continuing Performance issue retains its initial comparison and cohort while `last_seen` advances. New supporting observations do not rewrite the original evidence. Resource issues recover when their primary metric recovers for that resource; broader supporting observations do not keep them open. Provider issues require a fresh comparison of the saved provider/model cohort. Missing history or coverage does not establish recovery. A later breach starts a new issue. Retirement details refresh as usage and the provider schedule change. A retirement issue with no recent use becomes inactive; actual new traffic can reopen it. Resolve or dismiss an issue using the current status from the read response: diff --git a/litellm/models/issues.py b/litellm/models/issues.py index 4e5aa4931c0..cd3f9767638 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -85,10 +85,15 @@ class IssueStore(Protocol): cursor: str | None = None, scope_type: IssueScope | None = None, metric: UsageAlertMetric | None = None, + scope_ids: tuple[str, ...] | None = None, ) -> tuple[IssueProjectionSignal, ...]: ... async def recover_issues(self, day: date) -> int: ... + async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: ... + + async def recover_issue_ids(self, ids: tuple[str, ...], observed_at: datetime) -> int: ... + async def projection_completed(self, day: date, version: str) -> bool: ... async def complete_projection(self, day: date, version: str) -> None: ... diff --git a/litellm/repositories/issue_families.py b/litellm/repositories/issue_families.py index 05b1e938206..230367421d6 100644 --- a/litellm/repositories/issue_families.py +++ b/litellm/repositories/issue_families.py @@ -132,8 +132,7 @@ async def update_family( WHERE NOT r.deleted AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz ORDER BY r.id FOR UPDATE OF r ), changed AS ( - UPDATE "LiteLLM_UsageAlertRule" r SET record=x.record,enabled=(x.record->>'enabled')::boolean, - updated_at=(x.record->>'updated_at')::timestamptz AT TIME ZONE 'UTC' + UPDATE "LiteLLM_UsageAlertRule" r SET record=x.record,enabled=(x.record->>'enabled')::boolean FROM incoming x WHERE r.id=x.record->>'id' AND NOT r.deleted AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz AND (SELECT count(*) FROM eligible)=(SELECT count(*) FROM incoming) RETURNING r.id diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index bec14f68417..415c14ba8a1 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -317,7 +317,14 @@ class IssuesRepository: cursor: str | None = None, scope_type: IssueScope | None = None, metric: UsageAlertMetric | None = None, + scope_ids: tuple[str, ...] | None = None, ) -> tuple[IssueProjectionSignal, ...]: + if not 1 <= limit <= 500: + raise ValueError("Read between one and 500 source signals") + if scope_ids == (): + return () + if scope_ids is not None and len(scope_ids) > 500: + raise ValueError("Filter at most 500 source scopes") return TypeAdapter(tuple[IssueProjectionSignal, ...]).validate_python( await self.database.query_raw( """SELECT a.record AS alert,r.record AS rule,a.evaluation_day @@ -328,33 +335,74 @@ class IssuesRepository: AND a.record->>'direction'=r.record->>'direction' AND (r.scope_id IS NULL OR a.scope_id=r.scope_id) AND ($2::text IS NULL OR a.id>$2) AND ($4::text IS NULL OR a.scope_type=$4) - AND ($5::text IS NULL OR a.metric=$5) ORDER BY a.id LIMIT $3""", + AND ($5::text IS NULL OR a.metric=$5) + AND ($6::text[] IS NULL OR a.scope_id=ANY($6::text[])) ORDER BY a.id LIMIT $3""", day.isoformat(), cursor, limit, scope_type, metric, + list(scope_ids) if scope_ids is not None else None, ) ) async def recover_issues(self, day: date) -> int: rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( await self.database.query_raw( - """WITH changed AS ( + """WITH healthy AS ( + SELECT i.id FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id + LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id + WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='usage_alert' + AND s.record->'usage_alert'->>'scope_type'=i.scope_type + AND s.record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM i.scope_id + AND ((i.family='spend' AND s.record->'usage_alert'->>'metric'='spend') + OR (i.family='latency' AND s.record->'usage_alert'->>'metric'='latency') + OR (i.family='traffic' AND s.record->'usage_alert'->>'metric'='requests') + OR (i.family='reliability' AND s.record->'usage_alert'->>'metric' IN ('errors','error_rate'))) + GROUP BY i.id HAVING bool_and(a.id IS NOT NULL AND NOT a.breached AND a.evaluation_day<=$1) + ), changed AS ( UPDATE "LiteLLM_Issue" i SET active=false, status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP - WHERE kind='performance' AND active - AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id AND s.source='usage_alert') - AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s - LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id - WHERE s.issue_id=i.id AND s.source='usage_alert' - AND (a.id IS NULL OR a.evaluation_day>$1 OR a.breached)) RETURNING i.id + WHERE id IN(SELECT id FROM healthy) AND active RETURNING id ) SELECT count(*)::int AS count FROM changed""", day.isoformat(), ) ) return rows[0].count + async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: + if not 1 <= limit <= 500: + raise ValueError("Read between one and 500 provider issues") + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw( + f"""SELECT {_RECORD} AS record {_JOIN} + WHERE i.kind='performance' AND i.scope_type='provider' AND i.active + AND ($2::text IS NULL OR i.id>$2) ORDER BY i.id LIMIT $1""", + limit, + cursor, + ) + ) + return tuple(row.record for row in rows) + + async def recover_issue_ids(self, ids: tuple[str, ...], observed_at: datetime) -> int: + if len(ids) > 500 or observed_at.tzinfo is None: + raise ValueError("Recover at most 500 issues with a timezone-aware observation") + if not ids: + return 0 + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + """WITH changed AS ( + UPDATE "LiteLLM_Issue" SET active=false, + status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP + WHERE id=ANY($1::text[]) AND kind='performance' AND active AND last_seen<=$2::timestamp(3) + RETURNING id + ) SELECT count(*)::int AS count FROM changed""", + list(ids), + _utc(observed_at), + ) + ) + return rows[0].count + async def projection_completed(self, day: date, version: str) -> bool: rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( await self.database.query_raw( @@ -377,7 +425,7 @@ class IssuesRepository: """WITH changed AS ( UPDATE "LiteLLM_Issue" SET active=false, status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP - WHERE kind='model_deprecation' AND active AND last_seen<$1::timestamp RETURNING id + WHERE kind='model_deprecation' AND active AND last_seen<$1::timestamp(3) RETURNING id ) SELECT count(*)::int AS count FROM changed""", _utc(observed_at), ) @@ -453,6 +501,10 @@ class IssuesRepository: JOIN "LiteLLM_UsageAlertRule" r ON r.id=b.rule_id JOIN "LiteLLM_UsageAlertIncident" a ON a.id=b.source_id AND a.rule_id=r.id WHERE b.delivery_id=d.id AND r.enabled AND NOT r.deleted AND a.breached AND a.status='open' + AND (r.record->'notification_channels') ? (d.record->>'channel') + AND (d.record->>'channel'<>'email' OR + (jsonb_array_length(d.record->'email_recipients')>0 + AND (d.record->'email_recipients') <@ (r.record->'email_recipients'))) AND (r.record->>'updated_at')::timestamptz=b.rule_revision::timestamptz AND NULLIF(a.evaluation_revision,'')::timestamptz=b.rule_revision::timestamptz AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric' diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index c173cef9e75..dfbb4ec80e0 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -1,25 +1,38 @@ +import asyncio import json +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager from datetime import date, datetime, timedelta, timezone from hashlib import sha256 -from typing import Final +from typing import Final, cast # noqa: TID251 # Generated Prisma query_raw has narrower overloads +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from uuid import uuid4 +import psycopg import pytest from integration._support.actors import ManagementDatabase from integration._support.client import Gateway +from prisma import Prisma +from prisma.types import DatasourceOverride +from psycopg import sql +from pydantic import BaseModel, TypeAdapter, ValidationError from litellm.repositories.issue_activity import IssueActivityRepository from litellm.repositories.issues import IssuesRepository -from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( IssueCohort, IssueComparison, + IssueDeliveryInput, + IssueFamilyPolicyUpdate, + IssueFamilyUpdate, IssueInput, IssueList, IssueModelPair, IssueSignalInput, IssueSignalList, ) +from litellm.types.proxy.usage_alert_defaults import default_usage_rule_id, default_usage_rules from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRuleInput @@ -54,19 +67,20 @@ def _issue(cohort: IssueCohort) -> IssueInput: ) +async def _query(db: ManagementDatabase, query: str, *args: object) -> object: + return await IssuesRepository.from_prisma(db.prisma).database.query_raw(query, *args) + + async def _remove_issues(db: ManagementDatabase, ids: tuple[str, ...]) -> None: - await db.prisma.writer_db.query_raw( + await _query( + db, 'DELETE FROM "LiteLLM_IssueDeliveryRule" WHERE delivery_id IN ' '(SELECT id FROM "LiteLLM_IssueDelivery" WHERE issue_id=ANY($1::text[])) RETURNING delivery_id', list(ids), ) - await db.prisma.writer_db.query_raw( - 'DELETE FROM "LiteLLM_IssueDelivery" WHERE issue_id=ANY($1::text[]) RETURNING id', list(ids) - ) - await db.prisma.writer_db.query_raw( - 'DELETE FROM "LiteLLM_IssueSignal" WHERE issue_id=ANY($1::text[]) RETURNING source_id', list(ids) - ) - await db.prisma.writer_db.query_raw('DELETE FROM "LiteLLM_Issue" WHERE id=ANY($1::text[]) RETURNING id', list(ids)) + await _query(db, 'DELETE FROM "LiteLLM_IssueDelivery" WHERE issue_id=ANY($1::text[]) RETURNING id', list(ids)) + await _query(db, 'DELETE FROM "LiteLLM_IssueSignal" WHERE issue_id=ANY($1::text[]) RETURNING source_id', list(ids)) + await _query(db, 'DELETE FROM "LiteLLM_Issue" WHERE id=ANY($1::text[]) RETURNING id', list(ids)) @pytest.mark.asyncio(loop_scope="session") @@ -139,15 +153,17 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa own: Final = gateway.request("GET", path, key=owner_key) assert own.status_code == 200 and own.json()["scope_label"] == "Example checkout" assert "99999" not in own.text and "9999" not in own.text - for offset in (0, 1, 2): - page: Final = gateway.request( + pages: Final = tuple( + gateway.request( "GET", "/issues", key=owner_key, params={"scope_type": "key", "scope_id": reporting_id, "limit": "1", "offset": str(offset)}, ) - assert page.status_code == 200 - parsed: Final = IssueList.model_validate(page.json()) + for offset in (0, 1, 2) + ) + assert all(page.status_code == 200 for page in pages) + for offset, parsed in enumerate(IssueList.model_validate(page.json()) for page in pages): assert parsed.total == 2 and len(parsed.data) == (1 if offset < 2 else 0) assert gateway.request("GET", "/issues", key=owner_key).status_code == 403 for suffix in ("", "/signals", "/activity", "/keys", "/users", "/impact"): @@ -185,9 +201,7 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa finally: await _remove_issues(db, (first.id, second.id, global_issue.id)) await alerts.delete_rule(rule.id) - await db.prisma.writer_db.query_raw( - 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id - ) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) @pytest.mark.asyncio(loop_scope="session") @@ -222,7 +236,8 @@ async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( ) ) ) - await db.prisma.writer_db.query_raw( + await _query( + db, """INSERT INTO "LiteLLM_DailyUserSpend" (id,user_id,date,api_key,model,custom_llm_provider,api_requests,spend,successful_requests,total_response_time_ms,timed_requests,updated_at) SELECT id,user_id,date,api_key,model,custom_llm_provider,api_requests,spend,successful_requests,total_response_time_ms,timed_requests,CURRENT_TIMESTAMP @@ -284,7 +299,350 @@ async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( assert len(exported) == 1 and exported[0].api_requests == 100 finally: await _remove_issues(db, (issue.id,)) - await db.prisma.writer_db.query_raw( + await _query( + db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id=ANY($1::text[]) RETURNING id', [entry["id"] for entry in entries], ) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + original: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + definition: Final = UsageAlertRuleInput( + name="Delivery fixture", + scope_type="user", + scope_id=original.cohort.scope_id, + metric="requests", + direction="increase", + notification_channels=("slack", "email"), + email_recipients=("alerts@example.com",), + ) + rule: Final = await alerts.save_rule(definition, "qa-admin") + assert rule is not None + observation: Final = UsageAlertObservation( + entity_id=rule.scope_id, day=original.observed_on, observed=420, baseline=120, threshold=360, breached=True + ) + delivery: Final = IssueDeliveryInput(id=str(uuid4()), issue_id=original.id, channel="slack") + invalid: Final = IssueDeliveryInput( + id=str(uuid4()), issue_id=original.id, channel="email", email_recipients=("outside@example.com",) + ) + next_issue_id: Final = str(uuid4()) + try: + source: Final = (await alerts.record_observations(rule, (observation,)))[0] + await _query( + db, + 'UPDATE "LiteLLM_UsageAlertIncident" SET evaluation_revision=$2 WHERE id=$1 RETURNING id', + source.id, + rule.updated_at.isoformat(), + ) + initial: Final = await store.upsert_issue(original) + assert tuple( + item.alert.id + for item in await store.active_signals( + original.observed_on, scope_type="user", metric="requests", scope_ids=(str(rule.scope_id),) + ) + ) == (source.id,) + assert await store.active_signals(original.observed_on, scope_ids=()) == () + assert await store.active_signals(original.observed_on, scope_ids=("other-example-user",)) == () + await store.add_signals( + initial.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),) + ) + await store.queue_delivery(delivery, rule.id, rule.updated_at.isoformat(), source.id) + await store.queue_delivery(invalid, rule.id, rule.updated_at.isoformat(), source.id) + pending: Final = await store.pending_delivery(delivery.id) + assert pending is not None and pending.alert.id == source.id and pending.issue.id == initial.id + assert await store.pending_delivery(invalid.id) is None + disabled: Final = await alerts.save_rule(definition.model_copy(update={"enabled": False}), "qa-admin", rule) + assert disabled is not None and await store.pending_delivery(delivery.id) is None + enabled: Final = await alerts.save_rule(definition, "qa-admin", disabled) + assert enabled is not None and await store.pending_delivery(delivery.id) is None + await _query( + db, + 'UPDATE "LiteLLM_UsageAlertIncident" SET evaluation_revision=$2 WHERE id=$1 RETURNING id', + source.id, + enabled.updated_at.isoformat(), + ) + await store.queue_delivery(delivery, enabled.id, enabled.updated_at.isoformat(), source.id) + assert await store.pending_delivery(delivery.id) is not None + await alerts.set_status(source.id, "dismissed") + assert await store.pending_delivery(delivery.id) is None + await alerts.set_status(source.id, "open") + assert await store.set_status(initial.id, "dismissed", "open") is not None + assert await store.pending_delivery(delivery.id) is None + later: Final = original.model_copy( + update={ + "observed_on": original.observed_on + timedelta(days=1), + "observed_at": original.observed_at + timedelta(minutes=1), + "title": "Later comparison", + "comparisons": (original.comparisons[0].model_copy(update={"observed_value": 999}),), + } + ) + continued: Final = await store.upsert_issue(later) + assert continued.status == "dismissed" and continued.comparisons == initial.comparisons + assert continued.title == initial.title and continued.observed_on == initial.observed_on + assert continued.last_seen > initial.last_seen + assert await store.set_status(initial.id, "open", "dismissed") is not None + await store.defer_delivery(delivery.id, "delivery_failed") + assert await store.pending_delivery(delivery.id) is None + await _query( + db, + 'UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP WHERE id=$1 RETURNING id', + delivery.id, + ) + retried: Final = await store.pending_delivery(delivery.id) + assert ( + retried is not None and retried.delivery.attempts == 1 and retried.delivery.last_error == "delivery_failed" + ) + await store.mark_delivery_sent(delivery.id) + await store.queue_delivery(delivery, enabled.id, enabled.updated_at.isoformat(), source.id) + assert await store.pending_delivery(delivery.id) is None + healthy: Final = observation.model_copy( + update={"day": observation.day + timedelta(days=1), "breached": False, "observed": 120} + ) + await alerts.record_observations(enabled, (healthy,)) + await store.recover_issues(observation.day + timedelta(days=2)) + recovered: Final = await store.get_issue(initial.id) + assert recovered is not None and not recovered.active and recovered.status == "resolved" + assert await store.find_active_issue(original.cohort_key) is None + repeated: Final = ( + await alerts.record_observations( + enabled, (observation.model_copy(update={"day": observation.day + timedelta(days=2)}),) + ) + )[0] + assert repeated.id != source.id + new_episode: Final = await store.upsert_issue( + original.model_copy( + update={ + "id": next_issue_id, + "observed_on": repeated.day, + "observed_at": later.observed_at + timedelta(minutes=1), + } + ) + ) + assert new_episode.active and new_episode.status == "open" and new_episode.id != initial.id + assert (await store.find_active_issue(original.cohort_key)) == new_episode + finally: + await _remove_issues(db, (original.id, next_issue_id)) + await alerts.delete_rule(rule.id) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) + + +class _Pid(BaseModel, frozen=True): + pid: int + + +class _Blocked(BaseModel, frozen=True): + blocked: bool + + +@asynccontextmanager +async def _family_database(db: ManagementDatabase) -> AsyncGenerator[Prisma]: + url: Final = db.gateway.database_url + assert url is not None + schema: Final = f"issues_{uuid4().hex}" + parsed: Final = urlsplit(url) + scoped: Final = urlunsplit(parsed._replace(query=urlencode({**dict(parse_qsl(parsed.query)), "schema": schema}))) + with psycopg.connect(url, autocommit=True) as setup: + setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + try: + setup.execute( + sql.SQL('CREATE TABLE {} (LIKE "LiteLLM_UsageAlertRule" INCLUDING ALL)').format( + sql.Identifier(schema, "LiteLLM_UsageAlertRule") + ) + ) + database: Final = Prisma(datasource=DatasourceOverride(url=scoped)) + await database.connect() + try: + yield database + finally: + await database.disconnect() + finally: + setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurrent_edits(db: ManagementDatabase) -> None: + async with _family_database(db) as database: + borrowed: Final = cast( # cast-ok: real generated Prisma client with narrower overloads + AlertDatabase, database + ) + alerts: Final = UsageAlertsRepository(borrowed) + store: Final = IssuesRepository(borrowed) + await alerts.ensure_default_rules(default_usage_rules()) + custom: Final = await alerts.save_rule( + UsageAlertRuleInput( + name="Custom spend", scope_type="key", scope_id="example-key", metric="spend", direction="increase" + ), + "qa-admin", + ) + assert custom is not None + removed_id: Final = default_usage_rule_id("model", "spend", "increase") + await alerts.delete_rule(removed_id) + result: Final = await store.update_family( + "spend", + IssueFamilyUpdate( + enabled=False, policies=(IssueFamilyPolicyUpdate(direction="increase", multiplier=5, min_value=25),) + ), + ) + assert result is not None and result.total_rules == 4 and result.enabled_rules == 0 + assert result.policies[0].multiplier == 5 and result.policies[0].min_value == 25 + assert await alerts.get_rule(removed_id) is None + assert await alerts.get_rule(custom.id) == custom + baseline: Final = await store.family_settings() + with pytest.raises(ValidationError, match="min_history_days"): + await store.update_family( + "spend", + IssueFamilyUpdate( + policies=(IssueFamilyPolicyUpdate(direction="increase", baseline_days=1, min_history_days=7),) + ), + ) + assert await store.family_settings() == baseline + ids: Final = tuple( + sorted(default_usage_rule_id(scope, "spend", "increase") for scope in ("deployment", "key", "team", "user")) + ) + async with asyncio.TaskGroup() as tasks: + async with database.tx(timeout=timedelta(seconds=30)) as transaction: + await transaction.query_raw('SELECT id FROM "LiteLLM_UsageAlertRule" WHERE id=$1 FOR UPDATE', ids[0]) + pid: Final = ( + TypeAdapter(tuple[_Pid, ...]) + .validate_python(await transaction.query_raw("SELECT pg_backend_pid() AS pid"))[0] + .pid + ) + update: Final = tasks.create_task(store.update_family("spend", IssueFamilyUpdate(enabled=True))) + async with asyncio.timeout(10): + while ( + not TypeAdapter(tuple[_Blocked, ...]) + .validate_python( + await database.query_raw( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE $1::int=ANY(pg_blocking_pids(pid))) AS blocked", + pid, + ) + )[0] + .blocked + ): + await asyncio.sleep(0.01) + await transaction.query_raw( + """UPDATE "LiteLLM_UsageAlertRule" SET record=record || + jsonb_build_object('name','Concurrent edit','updated_at',CURRENT_TIMESTAMP) + WHERE id=$1 RETURNING id""", + ids[0], + ) + assert update.result() is None + final: Final = await store.family_settings() + spend: Final = next(family for family in final.data if family.family == "spend") + assert spend.total_rules == 4 and spend.enabled_rules == 0 + edited: Final = await alerts.get_rule(ids[0]) + assert edited is not None and edited.name == "Concurrent edit" + assert await alerts.get_rule(custom.id) == custom + + +@pytest.mark.asyncio(loop_scope="session") +async def test_resource_recovery_ignores_broader_and_traffic_context(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + fixture: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + issue: Final = fixture.model_copy( + update={"family": "spend", "comparisons": (fixture.comparisons[0].model_copy(update={"metric": "spend"}),)} + ) + definitions: Final = ( + UsageAlertRuleInput( + name="Primary cost", scope_type="user", scope_id=issue.cohort.scope_id, metric="spend", direction="increase" + ), + UsageAlertRuleInput( + name="Traffic context", + scope_type="user", + scope_id=issue.cohort.scope_id, + metric="requests", + direction="increase", + ), + UsageAlertRuleInput(name="Broader cost", scope_type="deployment", metric="spend", direction="increase"), + ) + rules: Final = tuple([await alerts.save_rule(definition, "qa-admin") for definition in definitions]) + assert all(rule is not None for rule in rules) + try: + sources: Final = tuple( + [ + ( + await alerts.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=rule.scope_id, + day=issue.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + for rule in rules + if rule is not None + ] + ) + await store.upsert_issue(issue) + await store.add_signals( + issue.id, + tuple( + IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source) for source in sources + ), + ) + primary: Final = rules[0] + assert primary is not None + await alerts.record_observations( + primary, + ( + UsageAlertObservation( + entity_id=primary.scope_id, + day=issue.observed_on + timedelta(days=1), + observed=120, + baseline=120, + threshold=360, + breached=False, + ), + ), + ) + await store.recover_issues(issue.observed_on + timedelta(days=2)) + recovered: Final = await store.get_issue(issue.id) + assert recovered is not None and recovered.status == "resolved" and not recovered.active + supporting: Final = tuple([await alerts.get_alert(source.id) for source in sources[1:]]) + assert all(source is not None and source.status == "open" for source in supporting) + finally: + await _remove_issues(db, (issue.id,)) + for rule in rules: + if rule is not None: + await alerts.delete_rule(rule.id) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_provider_recovery_pages_saved_cohorts_and_rejects_stale_observations(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + prefix: Final = f"fixture-{uuid4()}" + initial: Final = _issue(IssueCohort(scope_type="provider", scope_id="example-provider")) + template: Final = initial.model_copy(update={"observed_at": initial.observed_at.replace(microsecond=123789)}) + inputs: Final = tuple( + template.model_copy(update={"id": prefix + suffix, "cohort_key": prefix + suffix}) + for suffix in ("-a", "-b", "-c") + ) + try: + await store.upsert_issues(inputs) + first: Final = await store.active_provider_issues(1, prefix) + second: Final = await store.active_provider_issues(1, first[0].id) + assert first[0].id == inputs[0].id and second[0].id == inputs[1].id + assert await store.set_status(inputs[0].id, "dismissed", "open") is not None + assert await store.recover_issue_ids((inputs[0].id,), template.observed_at - timedelta(seconds=1)) == 0 + assert await store.recover_issue_ids((inputs[0].id, inputs[1].id), template.observed_at) == 2 + dismissed: Final = await store.get_issue(inputs[0].id) + resolved: Final = await store.get_issue(inputs[1].id) + assert dismissed is not None and not dismissed.active and dismissed.status == "dismissed" + assert resolved is not None and not resolved.active and resolved.status == "resolved" + await store.recover_issues(template.observed_on + timedelta(days=1)) + remaining: Final = await store.active_provider_issues(1, prefix) + assert remaining[0].id == inputs[2].id + finally: + await _remove_issues(db, tuple(issue.id for issue in inputs)) From e6a9f11b49e6334171f223bfb4631fc885b3cdb7 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:34:25 +0800 Subject: [PATCH 14/38] fix: cast issue retention interval for Prisma --- litellm/repositories/issues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 415c14ba8a1..fda3bc33f89 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -551,7 +551,7 @@ class IssuesRepository: await self.database.query_raw( """WITH expired AS MATERIALIZED ( SELECT id FROM "LiteLLM_Issue" WHERE status<>'open' - AND updated_at$1) + AND updated_at$1::int) ORDER BY updated_at,id LIMIT $2 ), signals AS ( DELETE FROM "LiteLLM_IssueSignal" WHERE (issue_id,source,source_id) IN ( From 131c5856922bd41968b93215cd71c826c5d15085 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:37:38 +0800 Subject: [PATCH 15/38] fix: synchronize issue schemas and cover bounded cleanup --- .../litellm_proxy_extras/schema.prisma | 54 +++++++++++ litellm/proxy/schema.prisma | 54 +++++++++++ tests/integration/database/test_issues.py | 96 +++++++++++++++++++ 3 files changed, 204 insertions(+) diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 46ab28530a9..8a67d1b123a 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2132,11 +2132,13 @@ model LiteLLM_UsageAlertIncident { status String day String evaluation_day String + evaluation_revision String @default("") breached Boolean record Json notified_channels String[] @default([]) updated_at DateTime @default(now()) @@index([rule_id, scope_key, evaluation_day]) + @@index([evaluation_day, breached, id]) @@index([scope_type, scope_id, day]) @@index([status, day]) @@index([updated_at]) @@ -2149,3 +2151,55 @@ model LiteLLM_UsageAlertCheckpoint { expires_at DateTime @default(now()) @@index([expires_at]) } + +model LiteLLM_Issue { + id String @id + cohort_key String + kind String + family String + scope_type String + scope_id String? + status String + active Boolean @default(true) + ack_key String? + record Json + first_seen DateTime + last_seen DateTime + updated_at DateTime @default(now()) + + @@index([status, last_seen, id]) + @@index([kind, family, last_seen, id]) + @@index([scope_type, scope_id, last_seen, id]) + @@index([cohort_key, active]) +} + +model LiteLLM_IssueSignal { + issue_id String + source String + source_id String + record Json + + @@id([issue_id, source, source_id]) + @@index([source, source_id]) +} + +model LiteLLM_IssueDelivery { + id String @id + issue_id String + record Json + sent Boolean @default(false) + next_attempt DateTime @default(now()) + updated_at DateTime @default(now()) + + @@index([sent, next_attempt, id]) + @@index([issue_id]) +} + +model LiteLLM_IssueDeliveryRule { + delivery_id String + rule_id String + source_id String + rule_revision String + + @@id([delivery_id, rule_id, source_id]) +} diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 46ab28530a9..8a67d1b123a 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2132,11 +2132,13 @@ model LiteLLM_UsageAlertIncident { status String day String evaluation_day String + evaluation_revision String @default("") breached Boolean record Json notified_channels String[] @default([]) updated_at DateTime @default(now()) @@index([rule_id, scope_key, evaluation_day]) + @@index([evaluation_day, breached, id]) @@index([scope_type, scope_id, day]) @@index([status, day]) @@index([updated_at]) @@ -2149,3 +2151,55 @@ model LiteLLM_UsageAlertCheckpoint { expires_at DateTime @default(now()) @@index([expires_at]) } + +model LiteLLM_Issue { + id String @id + cohort_key String + kind String + family String + scope_type String + scope_id String? + status String + active Boolean @default(true) + ack_key String? + record Json + first_seen DateTime + last_seen DateTime + updated_at DateTime @default(now()) + + @@index([status, last_seen, id]) + @@index([kind, family, last_seen, id]) + @@index([scope_type, scope_id, last_seen, id]) + @@index([cohort_key, active]) +} + +model LiteLLM_IssueSignal { + issue_id String + source String + source_id String + record Json + + @@id([issue_id, source, source_id]) + @@index([source, source_id]) +} + +model LiteLLM_IssueDelivery { + id String @id + issue_id String + record Json + sent Boolean @default(false) + next_attempt DateTime @default(now()) + updated_at DateTime @default(now()) + + @@index([sent, next_attempt, id]) + @@index([issue_id]) +} + +model LiteLLM_IssueDeliveryRule { + delivery_id String + rule_id String + source_id String + rule_revision String + + @@id([delivery_id, rule_id, source_id]) +} diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index dfbb4ec80e0..4fa58c18adb 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -646,3 +646,99 @@ async def test_provider_recovery_pages_saved_cohorts_and_rejects_stale_observati assert remaining[0].id == inputs[2].id finally: await _remove_issues(db, tuple(issue.id for issue in inputs)) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_prune_removes_expired_children_in_bounded_passes( + db: ManagementDatabase, +) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + expired: Final = _issue( + IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}") + ) + open_issue: Final = _issue(expired.cohort) + recent: Final = _issue(expired.cohort) + rule: Final = await alerts.save_rule( + UsageAlertRuleInput( + name="Prune fixture", + scope_type="user", + scope_id=expired.cohort.scope_id, + metric="requests", + direction="increase", + ), + "qa-admin", + ) + assert rule is not None + deliveries: Final = tuple( + IssueDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") + for _ in range(2) + ) + try: + await store.upsert_issues((expired, open_issue, recent)) + source: Final = ( + await alerts.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=rule.scope_id, + day=expired.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + await store.add_signals( + expired.id, + ( + IssueSignalInput( + source="usage_alert", source_id=source.id, usage_alert=source + ), + ), + ) + for delivery in deliveries: + await store.queue_delivery( + delivery, rule.id, rule.updated_at.isoformat(), source.id + ) + assert await store.set_status(expired.id, "resolved", "open") is not None + assert await store.set_status(recent.id, "resolved", "open") is not None + await _query( + db, + "UPDATE \"LiteLLM_Issue\" SET updated_at=TIMESTAMP '2000-01-01' WHERE id=ANY($1::text[]) RETURNING id", + [expired.id, open_issue.id], + ) + await store.prune(retention_days=90, limit=1) + assert await store.get_issue(expired.id) is not None + for _ in range(7): + await store.prune(retention_days=90, limit=1) + assert await store.get_issue(expired.id) is None + assert await store.get_issue(open_issue.id) is not None + assert await store.get_issue(recent.id) is not None + assert (await store.list_signals(expired.id)).total == 0 + assert ( + await _query( + db, + 'SELECT id FROM "LiteLLM_IssueDelivery" WHERE issue_id=$1', + expired.id, + ) + == [] + ) + assert ( + await _query( + db, + 'SELECT delivery_id FROM "LiteLLM_IssueDeliveryRule" WHERE delivery_id=ANY($1::text[])', + [delivery.id for delivery in deliveries], + ) + == [] + ) + finally: + await _remove_issues(db, (expired.id, open_issue.id, recent.id)) + await alerts.delete_rule(rule.id) + await _query( + db, + 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', + rule.id, + ) From e98f2183cbf0395f8ee3efc873b8fdbb6bf031d8 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:40:36 +0800 Subject: [PATCH 16/38] chore: refresh activity latency OpenAPI snapshot --- litellm/proxy/_lazy_openapi_snapshot.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 6575552c6fd..a450688de51 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -4380,6 +4380,16 @@ "title": "Successful Requests", "type": "integer" }, + "timed_requests": { + "default": 0, + "title": "Timed Requests", + "type": "integer" + }, + "total_response_time_ms": { + "default": 0, + "title": "Total Response Time Ms", + "type": "integer" + }, "total_tokens": { "default": 0, "title": "Total Tokens", From 466305d294b910ce290ad161688fa33f2b53bd18 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:46:00 +0800 Subject: [PATCH 17/38] fix: pass standalone issue API lint checks --- litellm/proxy/proxy_server.py | 2 +- litellm/repositories/usage_alerts.py | 4 ---- litellm/types/proxy/usage_alert_defaults.py | 3 ++- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ad8a1bc67c0..4d5f8439f37 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -424,11 +424,11 @@ from litellm.messages.dispatch import messages_target from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import * from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling +from litellm.proxy.alerting_endpoints.issues import router as issues_router from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( router as model_deprecation_alerts_router, ) from litellm.proxy.alerting_endpoints.usage_alerts import router as usage_alerts_router -from litellm.proxy.alerting_endpoints.issues import router as issues_router from litellm.proxy.analytics_endpoints.analytics_endpoints import ( router as analytics_router, ) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 25d7a89d077..3ecfe60d679 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -92,10 +92,6 @@ class _ModelIdentity(BaseModel, frozen=True): model: str -class _Identifier(BaseModel, frozen=True): - id: str - - class _Pending(_Record, frozen=True): rule: Mapping[str, object] notified_channels: tuple[UsageAlertChannel, ...] diff --git a/litellm/types/proxy/usage_alert_defaults.py b/litellm/types/proxy/usage_alert_defaults.py index 7b54ac3eb45..887932a5194 100644 --- a/litellm/types/proxy/usage_alert_defaults.py +++ b/litellm/types/proxy/usage_alert_defaults.py @@ -67,7 +67,8 @@ def default_usage_rules( min_history_days=policy.min_history_days, notification_channels=( () - if legacy_user_spend_alerts and (scope, policy.metric, policy.direction) == ("user", "spend", "increase") + if legacy_user_spend_alerts + and (scope, policy.metric, policy.direction) == ("user", "spend", "increase") else channels ), ) From 1af30da74e080fcf1266ca52eaa335109f0e165f Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:56:34 +0800 Subject: [PATCH 18/38] fix: reject stale alert rule mutations --- litellm/models/usage_alerts.py | 2 +- .../proxy/alerting_endpoints/usage_alerts.py | 5 ++-- litellm/repositories/usage_alerts.py | 24 ++++++++++++--- .../integration/database/test_usage_alerts.py | 30 +++++++++++++++++++ 4 files changed, 54 insertions(+), 7 deletions(-) diff --git a/litellm/models/usage_alerts.py b/litellm/models/usage_alerts.py index 8858835eafc..bf90f6acb80 100644 --- a/litellm/models/usage_alerts.py +++ b/litellm/models/usage_alerts.py @@ -25,7 +25,7 @@ class UsageAlertStore(Protocol): self, data: UsageAlertRuleInput, created_by: str | None, existing: UsageAlertRule | None = None ) -> UsageAlertRule | None: ... - async def delete_rule(self, rule_id: str) -> None: ... + async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: ... async def list_rules( self, diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py index e6b1b22cc00..d571fc776d9 100644 --- a/litellm/proxy/alerting_endpoints/usage_alerts.py +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -188,7 +188,7 @@ async def update_usage_alert_rule( await require_scope(user, store, data.scope_type, data.scope_id, write=True) saved: Final = await store.save_rule(data, user.user_id, existing=rule) if saved is None: - raise HTTPException(status_code=404, detail="Alert rule not found") + raise HTTPException(status_code=409, detail="Alert rule changed while being updated. Refresh and try again") return saved @@ -199,7 +199,8 @@ async def delete_usage_alert_rule(rule_id: str, user: User, repository: Reposito if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") await require_scope(user, store, rule.scope_type, rule.scope_id, write=True) - await store.delete_rule(rule_id) + if not await store.delete_rule(rule_id, existing=rule): + raise HTTPException(status_code=409, detail="Alert rule changed while being deleted. Refresh and try again") return Response(status_code=204) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 3ecfe60d679..a7f65c535cd 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -223,19 +223,35 @@ class UsageAlertsRepository: rows: Final = await self.database.query_raw( """INSERT INTO "LiteLLM_UsageAlertRule" (id,scope_type,scope_id,enabled,record) VALUES ($1,$2,$3,$4,$5::jsonb) ON CONFLICT (id) DO UPDATE SET scope_type=EXCLUDED.scope_type,scope_id=EXCLUDED.scope_id,enabled=EXCLUDED.enabled,record=EXCLUDED.record - WHERE NOT "LiteLLM_UsageAlertRule".deleted RETURNING record""", + WHERE NOT "LiteLLM_UsageAlertRule".deleted + AND ("LiteLLM_UsageAlertRule".record->>'updated_at')::timestamptz=$6::text::timestamptz + AND "LiteLLM_UsageAlertRule".scope_type=$7 + AND "LiteLLM_UsageAlertRule".scope_id IS NOT DISTINCT FROM $8::text + RETURNING record""", rule.id, rule.scope_type, rule.scope_id, rule.enabled, rule.model_dump_json(), + existing.updated_at.isoformat() if existing else None, + existing.scope_type if existing else None, + existing.scope_id if existing else None, ) saved: Final = TypeAdapter(tuple[_Record, ...]).validate_python(rows) return _rule(saved[0].record) if saved else None - async def delete_rule(self, rule_id: str) -> None: - await self.database.query_raw( - 'UPDATE "LiteLLM_UsageAlertRule" SET deleted=true,enabled=false WHERE id=$1 RETURNING id', rule_id + async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: + return bool( + await self.database.query_raw( + """UPDATE "LiteLLM_UsageAlertRule" SET deleted=true,enabled=false + WHERE id=$1 AND NOT deleted AND ($2::text::timestamptz IS NULL OR ( + (record->>'updated_at')::timestamptz=$2::text::timestamptz + AND scope_type=$3 AND scope_id IS NOT DISTINCT FROM $4::text)) RETURNING id""", + rule_id, + existing.updated_at.isoformat() if existing else None, + existing.scope_type if existing else None, + existing.scope_id if existing else None, + ) ) async def list_rules( diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index 51978152ed2..c152d6c95f0 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -210,6 +210,36 @@ async def test_ongoing_episode_preserves_its_original_evidence_and_activity_link await repository.delete_rule(rule.id) +@pytest.mark.asyncio(loop_scope="session") +async def test_rule_scope_changes_reject_stale_updates_and_deletes(db: ManagementDatabase) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + definition: Final = UsageAlertRuleInput( + name="Concurrent scope edit", scope_type="user", scope_id="original-owner", metric="spend", direction="increase" + ) + original: Final = await repository.save_rule(definition, "qa-admin") + assert original is not None + try: + moved: Final = await repository.save_rule( + definition.model_copy(update={"scope_id": "different-owner"}), "qa-admin", existing=original + ) + assert moved is not None + assert await repository.save_rule(definition, "original-owner", existing=original) is None + assert not await repository.delete_rule(original.id, existing=original) + assert await repository.get_rule(original.id) == moved + current: Final = await repository.save_rule( + definition.model_copy(update={"scope_id": moved.scope_id, "name": "Current authorized edit"}), + "qa-admin", + existing=moved, + ) + assert current is not None + assert not await repository.delete_rule(original.id, existing=moved) + assert await repository.delete_rule(original.id, existing=current) + assert await repository.save_rule(definition, "qa-admin", existing=current) is None + assert await repository.get_rule(original.id) is None + finally: + await repository.delete_rule(original.id) + + @pytest.mark.asyncio(loop_scope="session") @pytest.mark.parametrize( "changed_field,new_value", From c1a198022059999040c4c1b79a0fc187cf36a06e Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 16:17:06 +0800 Subject: [PATCH 19/38] fix: enforce alert mutation and notification authority --- .../litellm_proxy_extras/schema.prisma | 1 - litellm/models/usage_alerts.py | 7 +- .../proxy/alerting_endpoints/usage_alerts.py | 50 +++++++++++- litellm/proxy/schema.prisma | 1 - litellm/repositories/usage_alerts.py | 27 +++++-- litellm/types/proxy/usage_alerts.py | 3 +- schema.prisma | 1 - .../integration/database/test_usage_alerts.py | 76 +++++++++++++++++++ .../alerting_endpoints/test_usage_alerts.py | 17 ++++- .../repositories/test_config_repository.py | 11 +-- tests/unit/types/proxy/test_usage_alerts.py | 3 + 11 files changed, 175 insertions(+), 22 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 8a67d1b123a..f76cbbc05e7 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2110,7 +2110,6 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } -// Daily alerts consume existing aggregate tables; no request-path writes. model LiteLLM_UsageAlertRule { id String @id scope_type String diff --git a/litellm/models/usage_alerts.py b/litellm/models/usage_alerts.py index bf90f6acb80..63e2e05b024 100644 --- a/litellm/models/usage_alerts.py +++ b/litellm/models/usage_alerts.py @@ -22,7 +22,12 @@ class UsageAlertStore(Protocol): async def get_rule(self, rule_id: str) -> UsageAlertRule | None: ... async def save_rule( - self, data: UsageAlertRuleInput, created_by: str | None, existing: UsageAlertRule | None = None + self, + data: UsageAlertRuleInput, + created_by: str | None, + existing: UsageAlertRule | None = None, + *, + overrides_defaults: bool = False, ) -> UsageAlertRule | None: ... async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: ... diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py index d571fc776d9..80d818f2dc7 100644 --- a/litellm/proxy/alerting_endpoints/usage_alerts.py +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -4,9 +4,11 @@ from datetime import date from typing import Annotated, Final, TypeAlias from fastapi import APIRouter, Depends, HTTPException, Query, Response +from pydantic import TypeAdapter from litellm.models.usage_alerts import UsageAlertStore -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.deps.repositories import UsageAlertsRepositoryDep from litellm.repositories.daily_activity_buckets import api_key_from_bucket @@ -44,6 +46,36 @@ Offset: TypeAlias = Annotated[int, Query(ge=0, le=100000)] ScopeId: TypeAlias = Annotated[str | None, Query(min_length=1, max_length=1024)] +def _allows_write(grant: str, route: str) -> bool: + if grant == "info_routes": + return False + paths: Final = ( + TypeAdapter(tuple[str, ...]).validate_python(LiteLLMRoutes[grant].value) + if grant in LiteLLMRoutes.__members__ + else (grant,) + ) + return RouteChecks.check_route_access(route, paths) + + +def require_write_access(user: UserAPIKeyAuth, route: str) -> None: + if not user.allowed_routes: + return + grants: Final = TypeAdapter(tuple[str, ...]).validate_python(user.allowed_routes) + if not any(_allows_write(grant, route) for grant in grants): + raise HTTPException(status_code=403, detail="This credential does not grant alert write access") + + +def require_notification_access( + user: UserAPIKeyAuth, data: UsageAlertRuleInput, existing: UsageAlertRule | None = None +) -> None: + if user.user_role == LitellmUserRoles.PROXY_ADMIN: + return + if data.notification_channels or data.email_recipients: + raise HTTPException(status_code=403, detail="Only proxy admins can configure notification delivery") + if existing is not None and (existing.notification_channels or existing.overrides_defaults): + raise HTTPException(status_code=403, detail="Only proxy admins can change operator-managed alert rules") + + async def require_scope( user: UserAPIKeyAuth, repository: UsageAlertStore, @@ -129,6 +161,7 @@ async def update_usage_alert_group( data: UsageAlertGroupUpdate, user: User, repository: Repository ) -> UsageAlertGroupUpdateResult: store: Final = require_repository(repository) + require_write_access(user, "/usage/alert-groups") await require_scope(user, store, data.scope_type, data.scope_id, write=True) return await store.set_group_status(data) @@ -150,8 +183,12 @@ async def list_usage_alert_models( async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, repository: Repository) -> UsageAlertRule: """Create a daily usage alert rule for a scope the caller can manage.""" store: Final = require_repository(repository) + require_write_access(user, "/usage/alert-rules") await require_scope(user, store, data.scope_type, data.scope_id, write=True) - saved: Final = await store.save_rule(data, user.user_id) + require_notification_access(user, data) + saved: Final = await store.save_rule( + data, user.user_id, overrides_defaults=user.user_role == LitellmUserRoles.PROXY_ADMIN + ) if saved is None: raise HTTPException(status_code=409, detail="Alert rule could not be saved") return saved @@ -172,6 +209,7 @@ async def update_usage_alert_rule( rule_id: str, data: UsageAlertRuleInput, user: User, repository: Repository ) -> UsageAlertRule: store: Final = require_repository(repository) + require_write_access(user, f"/usage/alert-rules/{rule_id}") rule: Final = await store.get_rule(rule_id) if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") @@ -186,7 +224,10 @@ async def update_usage_alert_rule( status_code=422, detail="Create a custom rule to change a built-in rule's scope, metric, or direction" ) await require_scope(user, store, data.scope_type, data.scope_id, write=True) - saved: Final = await store.save_rule(data, user.user_id, existing=rule) + require_notification_access(user, data, rule) + saved: Final = await store.save_rule( + data, user.user_id, existing=rule, overrides_defaults=user.user_role == LitellmUserRoles.PROXY_ADMIN + ) if saved is None: raise HTTPException(status_code=409, detail="Alert rule changed while being updated. Refresh and try again") return saved @@ -195,10 +236,12 @@ async def update_usage_alert_rule( @router.delete("/alert-rules/{rule_id}", status_code=204) async def delete_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> Response: store: Final = require_repository(repository) + require_write_access(user, f"/usage/alert-rules/{rule_id}") rule: Final = await store.get_rule(rule_id) if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") await require_scope(user, store, rule.scope_type, rule.scope_id, write=True) + require_notification_access(user, rule, rule) if not await store.delete_rule(rule_id, existing=rule): raise HTTPException(status_code=409, detail="Alert rule changed while being deleted. Refresh and try again") return Response(status_code=204) @@ -237,6 +280,7 @@ async def get_usage_alert(alert_id: str, user: User, repository: Repository) -> @router.patch("/alerts/{alert_id}", response_model=UsageAlert) async def update_usage_alert(alert_id: str, data: UsageAlertUpdate, user: User, repository: Repository) -> UsageAlert: store: Final = require_repository(repository) + require_write_access(user, f"/usage/alerts/{alert_id}") alert: Final = await store.get_alert(alert_id) if alert is None: raise HTTPException(status_code=404, detail="Usage alert not found") diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 8a67d1b123a..f76cbbc05e7 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2110,7 +2110,6 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } -// Daily alerts consume existing aggregate tables; no request-path writes. model LiteLLM_UsageAlertRule { id String @id scope_type String diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index a7f65c535cd..16fe6ef9421 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -157,6 +157,7 @@ def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now: rule.metric, rule.direction, item.day.isoformat(), + rule.updated_at.isoformat(), ] ), ) @@ -208,7 +209,12 @@ class UsageAlertsRepository: return _rule(rows[0].record) if rows else None async def save_rule( - self, data: UsageAlertRuleInput, created_by: str | None, existing: UsageAlertRule | None = None + self, + data: UsageAlertRuleInput, + created_by: str | None, + existing: UsageAlertRule | None = None, + *, + overrides_defaults: bool = False, ) -> UsageAlertRule | None: now: Final = datetime.now(timezone.utc) rule: Final = UsageAlertRule.model_validate( @@ -218,6 +224,7 @@ class UsageAlertsRepository: "created_by": existing.created_by if existing else created_by, "created_at": existing.created_at if existing else now, "updated_at": now, + "overrides_defaults": overrides_defaults, } ) rows: Final = await self.database.query_raw( @@ -232,7 +239,7 @@ class UsageAlertsRepository: rule.scope_type, rule.scope_id, rule.enabled, - rule.model_dump_json(), + json.dumps({**rule.model_dump(mode="json"), "overrides_defaults": overrides_defaults}), existing.updated_at.isoformat() if existing else None, existing.scope_type if existing else None, existing.scope_id if existing else None, @@ -652,7 +659,8 @@ class UsageAlertsRepository: """ WITH input AS (SELECT * FROM jsonb_to_recordset($2::jsonb) AS x(scope_key text, breached boolean, day text, record jsonb)), prior AS ( - SELECT x.*,p.id AS prior_id,p.evaluation_day,p.breached AS prior_breached,p.record AS prior_record + SELECT x.*,p.id AS prior_id,p.evaluation_day,p.evaluation_revision AS prior_revision, + p.breached AS prior_breached,p.record AS prior_record FROM input x LEFT JOIN LATERAL ( SELECT * FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=x.scope_key AND scope_type=x.record->>'scope_type' AND scope_id IS NOT DISTINCT FROM x.record->>'scope_id' @@ -662,18 +670,21 @@ class UsageAlertsRepository: WHERE EXISTS (SELECT 1 FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND enabled AND NOT deleted AND (record->>'updated_at')::timestamptz=$3::timestamptz) ), updated AS ( UPDATE "LiteLLM_UsageAlertIncident" i SET - evaluation_day=p.day, breached=p.breached, + evaluation_day=p.day, evaluation_revision=$3::text, breached=p.breached, record=i.record || jsonb_build_object( 'last_seen',CASE WHEN p.breached THEN p.record->'last_seen' ELSE i.record->'last_seen' END, 'status',CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END), status=CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END, updated_at=CURRENT_TIMESTAMP - FROM prior p WHERE i.id=p.prior_id AND p.evaluation_day>'id',$1,p.scope_key,p.record->>'scope_type',p.record->>'scope_id',p.record->>'metric','open',p.day,p.day,true,p.record - FROM prior p WHERE p.breached AND (p.prior_id IS NULL OR (NOT p.prior_breached AND p.evaluation_day>'id',$1,p.scope_key,p.record->>'scope_type',p.record->>'scope_id',p.record->>'metric','open',p.day,p.day,$3,true,p.record + FROM prior p WHERE p.breached AND (p.prior_id IS NULL OR (NOT p.prior_breached AND + (p.evaluation_day None: + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + admin: Final = scenario.user(user_role="proxy_admin") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + reader: Final = scenario.key(user_id=owner, key_type="default", allowed_routes=["info_routes"]) + admin_reader: Final = scenario.key(user_id=admin, key_type="default", allowed_routes=["info_routes"]) + config: Final = { + "name": "Read-only route grant", + "scope_type": "user", + "scope_id": owner, + "metric": "spend", + "direction": "increase", + } + created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + assert created.status_code == 201 + path: Final = f"/usage/alert-rules/{created.json()['id']}" + try: + assert gateway.request("GET", path, key=reader).status_code == 200 + assert gateway.request("POST", "/usage/alert-rules", config, key=reader).status_code == 403 + assert gateway.request("PUT", path, config, key=reader).status_code == 403 + assert gateway.request("DELETE", path, key=reader).status_code == 403 + assert ( + gateway.request("PATCH", "/issues/families/spend", {"enabled": False}, key=admin_reader).status_code + == 403 + ) + assert ( + gateway.request( + "PATCH", "/issues/unavailable", {"status": "resolved", "expected_status": "open"}, key=admin_reader + ).status_code + == 403 + ) + finally: + assert gateway.request("DELETE", path, key=owner_key).status_code == 204 + + +def test_only_admins_can_configure_operator_notifications(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + config: Final = { + "name": "Operator delivery", + "scope_type": "user", + "scope_id": owner, + "metric": "spend", + "direction": "increase", + } + for notification in ( + {"notification_channels": ["slack"]}, + {"notification_channels": ["email"], "email_recipients": ["ops@example.com"]}, + ): + assert ( + gateway.request("POST", "/usage/alert-rules", {**config, **notification}, key=owner_key).status_code + == 403 + ) + assert ( + gateway.request( + "POST", "/usage/alert-rules", {**config, "overrides_defaults": True}, key=owner_key + ).status_code + == 422 + ) + assert ( + gateway.request("POST", "/usage/alert-rules", {**config, "name": "Spend\r\nInjected: header"}).status_code + == 422 + ) + created: Final = gateway.request("POST", "/usage/alert-rules", {**config, "notification_channels": ["slack"]}) + assert created.status_code == 201 + assert "overrides_defaults" not in created.json() + path: Final = f"/usage/alert-rules/{created.json()['id']}" + try: + assert gateway.request("PUT", path, config, key=owner_key).status_code == 403 + assert gateway.request("DELETE", path, key=owner_key).status_code == 403 + finally: + assert gateway.request("DELETE", path).status_code == 204 + + @pytest.mark.asyncio(loop_scope="session") async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gateway, db: ManagementDatabase) -> None: """Populate an explicit storage fixture, then exercise the real incident HTTP API.""" diff --git a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py index e8221599f8b..9d8906eb4e0 100644 --- a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py +++ b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py @@ -6,7 +6,7 @@ import pytest from fastapi import HTTPException from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.alerting_endpoints.usage_alerts import require_scope +from litellm.proxy.alerting_endpoints.usage_alerts import require_scope, require_write_access from litellm.repositories.usage_alerts import UsageAlertsRepository, activity_path from litellm.types.proxy.auth.user_roles import LitellmUserRoles @@ -16,6 +16,21 @@ class NoDatabase: raise AssertionError("Unauthorized callers must not reach the database") +@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER]) +@pytest.mark.parametrize("key_type", [None, "default", "management"]) +def test_info_grants_do_not_authorize_mutations(role: LitellmUserRoles, key_type: str | None) -> None: + user: Final = UserAPIKeyAuth(user_role=role, key_type=key_type, allowed_routes=["info_routes", "openai_routes"]) + with pytest.raises(HTTPException) as error: + require_write_access(user, "/usage/alert-rules/example") + assert error.value.status_code == 403 + + +@pytest.mark.parametrize("grant", ["/usage/alert-rules/example", "/usage/*", "management_routes"]) +def test_explicit_management_grants_still_authorize_mutations(grant: str) -> None: + user: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, allowed_routes=["info_routes", grant]) + require_write_access(user, "/usage/alert-rules/example") + + @pytest.mark.asyncio @pytest.mark.parametrize( "role,key_type,write", diff --git a/tests/unit/repositories/test_config_repository.py b/tests/unit/repositories/test_config_repository.py index 0856aebf0ff..6c37a78b6ec 100644 --- a/tests/unit/repositories/test_config_repository.py +++ b/tests/unit/repositories/test_config_repository.py @@ -1,4 +1,5 @@ from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock import pytest @@ -7,16 +8,16 @@ from litellm.repositories.config_repository import ConfigRepository @pytest.mark.asyncio -async def test_batch_reads_decode_legacy_json_and_current_objects_from_writer(): - find_many = AsyncMock( +async def test_batch_reads_decode_legacy_json_and_current_objects_from_writer() -> None: + find_many: Final = AsyncMock( return_value=( SimpleNamespace(param_name="first", param_value='{"status":"resolved"}'), SimpleNamespace(param_name="second", param_value={"status": "dismissed"}), ) ) - prisma = SimpleNamespace(writer_db=SimpleNamespace(litellm_config=SimpleNamespace(find_many=find_many))) - repository = ConfigRepository(prisma, use_writer=True) - rows = await repository.get_params(("first", "second", "absent")) + prisma: Final = SimpleNamespace(writer_db=SimpleNamespace(litellm_config=SimpleNamespace(find_many=find_many))) + repository: Final = ConfigRepository(prisma, use_writer=True) + rows: Final = await repository.get_params(("first", "second", "absent")) assert {row.param_name: row.param_value for row in rows} == { "first": {"status": "resolved"}, "second": {"status": "dismissed"}, diff --git a/tests/unit/types/proxy/test_usage_alerts.py b/tests/unit/types/proxy/test_usage_alerts.py index 57d7e133438..3ee4618d619 100644 --- a/tests/unit/types/proxy/test_usage_alerts.py +++ b/tests/unit/types/proxy/test_usage_alerts.py @@ -27,6 +27,9 @@ RULE: Final = { {"notification_channels": ["email"]}, {"notification_channels": ["slack", "slack"]}, {"email_recipients": ["ops@example.com\r\nBcc: other@example.com"]}, + {"name": "Spend\r\nBcc: other@example.com"}, + {"name": "Spend\x00notification"}, + {"overrides_defaults": True}, ], ) def test_invalid_rules_cannot_be_stored(changes: dict[str, object]) -> None: From 2f898fa2de18e79a24837a2a3d33e58203b16ad2 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 16:27:10 +0800 Subject: [PATCH 20/38] fix(alerts): bound investigations and enforce team activity access --- docs/features/usage-alerts.md | 16 +- litellm/models/issues.py | 9 +- litellm/proxy/alerting_endpoints/issues.py | 75 ++++++- litellm/repositories/issue_activity.py | 28 +-- litellm/repositories/issues.py | 20 ++ litellm/types/proxy/issues.py | 11 + tests/integration/database/test_issues.py | 199 +++++++++++++++--- ui/litellm-dashboard/src/lib/http/schema.d.ts | 13 +- 8 files changed, 316 insertions(+), 55 deletions(-) diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index c14aa564221..4407b9a4e90 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -1,6 +1,6 @@ -# Issues and daily Performance alerts +# Alerts and daily Performance alerts -The Issues inbox combines model retirements with daily Performance alerts. Open an issue to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause. +The Alerts page combines model retirements with daily Performance alerts. Open an issue to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause. Performance comparisons cover completed UTC days. Retirement issues show recent model usage and the provider's retirement schedule, including today's usage when present. @@ -55,6 +55,8 @@ curl "$LITELLM_API_BASE/issues/$ISSUE_ID/signals?limit=50" \ Replace `issue-example` with an ID returned by `/issues`. Activity uses the issue's saved date range by default. Optional `start_date` and `end_date` are inclusive UTC dates and can span at most 366 days. `/keys` and `/users` provide paginated Activity rows with `limit` and `offset`. Their latency totals are `total_response_time_ms` and `timed_requests`; divide the former by the latter only when timed requests are present. +Alert Activity returns exact daily totals in `results` and exact range totals in `metadata`. It does not return the general Usage API's per-day dimension dictionaries. Key issues also return `model_breakdown`, with up to 20 models by range spend by default. Set `model_limit` between 1 and 100 to change that limit. `total_models` counts every matching model; `other` contains the combined metrics and `model_count` of all remaining models, or null when none remain. It is an aggregate, not a model identifier. Adding the displayed models and `other` recovers the full totals. Model identifiers longer than 1,024 characters are included in `other` without truncation. Non-key issues return null for `model_breakdown`. Activity queries use a bounded execution budget. A busy or over-budget query returns `503` with `Retry-After: 5`; retry later or choose a shorter date range. + `/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown. Supporting signals have their own saved comparisons and relationship explanations. Their baseline windows can differ, so do not derive a combined error rate or unit cost by adding their displayed values. Administrators and admin viewers can inspect broader supporting signals. A scoped reader receives only signals for the exact key, user, or team they can access. That restriction applies before the signal count and pagination; broader totals are not disclosed. @@ -72,6 +74,8 @@ curl -X PATCH "$LITELLM_API_BASE/issues/$ISSUE_ID" \ A stale status returns `409`; reload before retrying. Performance acknowledgements do not change other issues that share a supporting observation. Retirement acknowledgements use the same state as the existing model-deprecation API. An inactive retirement cannot be manually reopened. +The issue-detail response includes `can_update_status` and `can_view_activity` for the current reader. Clients should show status actions only when `can_update_status` is true and request Activity or resource breakdowns only when `can_view_activity` is true. These fields are null in responses where permission has not been evaluated, such as inbox lists. Every endpoint still checks authorization on each request. Team Activity requires team-admin access or the explicit `/team/daily/activity` permission; ordinary membership can read saved alert evidence but does not grant other members’ activity or key details. + ## Configure default families `GET /issues/families` returns the four Performance families, their enabled counts, and common policy values. A null common value means the underlying scopes have different settings. This is separate from custom rules and the model-retirement warning window. @@ -123,7 +127,7 @@ The response contains `updated_count`. Only incidents in that exact group that s ## Create a rule -This example alerts when a team's daily spend is more than three times its baseline and the absolute change is at least $10. Replace `team-example` with the team's ID. +This example alerts when a team's daily spend is at least three times its baseline and the absolute change is at least $10. Replace `team-example` with the team's ID. ```bash curl "$LITELLM_API_BASE/usage/alert-rules" \ @@ -144,7 +148,7 @@ curl "$LITELLM_API_BASE/usage/alert-rules" \ }' ``` -The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value below half the baseline. `min_value` is the minimum absolute difference in the metric's own units. +The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value at or below half the baseline. `min_value` is the minimum absolute difference in the metric's own units. | Metric | Unit | | --- | --- | @@ -177,6 +181,6 @@ Rules can specify `notification_channels: ["slack", "email"]` and `email_recipie Gateway administrators can manage every scope. Admin viewers can read every scope. Internal users can manage their own user and key scopes; team administrators can manage their team's scope and keys. Team members can read their team's incidents. Read-only users cannot change rules or incident state, and inference-only keys cannot access these endpoints. -Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` indicates that the gateway has no database connection. +Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` can indicate an unavailable database or exhausted analytics capacity; a `Retry-After` header indicates that the client should retry later. -The dashboard's Issues inbox is available to administrators and admin viewers. Other authorized roles can use the scoped API. +The dashboard's Alerts page is available to administrators and admin viewers. Other authorized roles can use the scoped API. diff --git a/litellm/models/issues.py b/litellm/models/issues.py index cd3f9767638..61d3f45f839 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -1,8 +1,10 @@ from datetime import date, datetime from typing import Literal, Protocol +from litellm.models.team import LiteLLM_TeamTable from litellm.types.proxy.issues import ( Issue, + IssueActivityResponse, IssueCohort, IssueDeliveryBinding, IssueDeliveryInput, @@ -24,7 +26,6 @@ from litellm.types.proxy.issues import ( from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, DailyActivityUserPageResponse, - SpendAnalyticsPaginatedResponse, ) from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus @@ -33,6 +34,8 @@ from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus class IssueStore(Protocol): async def get_issue(self, issue_id: str) -> Issue | None: ... + async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: ... + async def list_issues( self, kind: IssueKind | None = None, @@ -130,7 +133,9 @@ class IssueStore(Protocol): class IssueActivityStore(Protocol): - async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse: ... + async def daily_activity( + self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 + ) -> IssueActivityResponse: ... async def keys( self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int diff --git a/litellm/proxy/alerting_endpoints/issues.py b/litellm/proxy/alerting_endpoints/issues.py index cca5c730bb5..b87599a5d56 100644 --- a/litellm/proxy/alerting_endpoints/issues.py +++ b/litellm/proxy/alerting_endpoints/issues.py @@ -1,5 +1,7 @@ """Unified issue inbox and investigations backed by existing daily aggregates.""" +from collections.abc import Generator +from contextlib import contextmanager from datetime import date from typing import Final @@ -13,11 +15,16 @@ from litellm.proxy.alerting_endpoints.usage_alerts import ( User, require_repository, require_scope, + require_write_access, ) from litellm.proxy.deps.repositories import IssueActivityRepositoryDep, IssuesRepositoryDep, UsageAlertsRepositoryDep +from litellm.proxy.management.teams.authz import is_team_admin +from litellm.proxy.management_endpoints.common_utils import team_member_has_permission +from litellm.repositories.analytics_budget import AnalyticsCapacityError from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.issues import ( Issue, + IssueActivityResponse, IssueFamily, IssueFamilyList, IssueFamilySetting, @@ -32,7 +39,6 @@ from litellm.types.proxy.issues import ( from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, DailyActivityUserPageResponse, - SpendAnalyticsPaginatedResponse, ) from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertStatus @@ -52,6 +58,14 @@ def require_activity(repository: IssueActivityStore | None) -> IssueActivityStor return repository +@contextmanager +def activity_budget() -> Generator[None]: + try: + yield + except AnalyticsCapacityError as exc: + raise HTTPException(status_code=503, detail=str(exc), headers={"Retry-After": "5"}) from exc + + async def authorized_issue( issue_id: str, user: User, @@ -73,6 +87,25 @@ async def authorized_issue( return issue +async def can_view_activity(issue: Issue, user: User, repository: IssueStore) -> bool: + if issue.cohort.scope_type != "team" or user.user_role in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + return True + if user.user_id is None or issue.cohort.scope_id is None: + return False + team: Final = await repository.team_activity_access(issue.cohort.scope_id, user.user_id) + return team is not None and ( + is_team_admin(user, team) or team_member_has_permission(user, team, "/team/daily/activity") + ) + + +async def require_activity_access(issue: Issue, user: User, repository: IssuesRepositoryDep) -> None: + if not await can_view_activity(issue, user, require_issues(repository)): + raise HTTPException(status_code=403, detail="Full team Activity permission is required for this investigation") + + def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]: start: Final = start_date or issue.range_start end: Final = end_date or issue.range_end @@ -118,6 +151,7 @@ async def update_issue_family( repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, ) -> IssueFamilySetting: + require_write_access(user, f"/issues/families/{family}") await require_scope(user, require_repository(alerts), "deployment", None, write=True) try: result: Final = await require_issues(repository).update_family(family, data) @@ -137,7 +171,22 @@ async def get_issue( repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, ) -> Issue: - return await authorized_issue(issue_id, user, repository, alerts) + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + activity_allowed: Final = await can_view_activity(issue, user, require_issues(repository)) + try: + require_write_access(user, f"/issues/{issue_id}") + await require_scope( + user, + require_repository(alerts), + "model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type, + issue.cohort.scope_id, + write=True, + ) + except HTTPException as exc: + if exc.status_code != 403: + raise + return issue.model_copy(update={"can_update_status": False, "can_view_activity": activity_allowed}) + return issue.model_copy(update={"can_update_status": True, "can_view_activity": activity_allowed}) @router.patch("/{issue_id}", response_model=Issue) @@ -148,6 +197,7 @@ async def update_issue( repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, ) -> Issue: + require_write_access(user, f"/issues/{issue_id}") issue: Final = await authorized_issue(issue_id, user, repository, alerts, write=True) if issue.kind == "model_deprecation" and not issue.active and data.status == "open": raise HTTPException(status_code=409, detail="This retirement issue reopens when the model receives new traffic") @@ -181,7 +231,7 @@ async def list_issue_signals( return await require_issues(repository).list_signals(issue_id, limit, offset, visible_scope) -@router.get("/{issue_id}/activity", response_model=SpendAnalyticsPaginatedResponse) +@router.get("/{issue_id}/activity", response_model=IssueActivityResponse) async def get_issue_activity( issue_id: str, user: User, @@ -190,10 +240,13 @@ async def get_issue_activity( activity: IssueActivityRepositoryDep, start_date: date | None = None, end_date: date | None = None, -) -> SpendAnalyticsPaginatedResponse: + model_limit: Limit = 20, +) -> IssueActivityResponse: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).daily_activity(issue.cohort, start, end) + with activity_budget(): + return await require_activity(activity).daily_activity(issue.cohort, start, end, model_limit) @router.get("/{issue_id}/keys", response_model=DailyActivityKeyPageResponse) @@ -209,8 +262,10 @@ async def get_issue_keys( offset: Offset = 0, ) -> DailyActivityKeyPageResponse: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).keys(issue.cohort, start, end, limit, offset) + with activity_budget(): + return await require_activity(activity).keys(issue.cohort, start, end, limit, offset) @router.get("/{issue_id}/users", response_model=DailyActivityUserPageResponse) @@ -226,10 +281,12 @@ async def get_issue_users( offset: Offset = 0, ) -> DailyActivityUserPageResponse: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) if issue.cohort.scope_type == "team": raise HTTPException(status_code=422, detail="Historical team aggregates do not retain user membership") start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).users(issue.cohort, start, end, limit, offset) + with activity_budget(): + return await require_activity(activity).users(issue.cohort, start, end, limit, offset) @router.get("/{issue_id}/impact", response_model=IssueImpact) @@ -243,5 +300,7 @@ async def get_issue_impact( end_date: date | None = None, ) -> IssueImpact: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).impact(issue.cohort, start, end) + with activity_budget(): + return await require_activity(activity).impact(issue.cohort, start, end) diff --git a/litellm/repositories/issue_activity.py b/litellm/repositories/issue_activity.py index fed59a82cbc..6c85fa064b3 100644 --- a/litellm/repositories/issue_activity.py +++ b/litellm/repositories/issue_activity.py @@ -7,18 +7,22 @@ from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # generated Prism from pydantic import BaseModel, TypeAdapter from litellm.constants import PTU_SENTINEL_API_KEY +from litellm.repositories.analytics_budget import AnalyticsDatabase, BoundedAnalyticsDatabase +from litellm.repositories.bounded_daily_activity import BoundedDailyActivityRepository from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket -from litellm.repositories.daily_activity_repository import DailyActivityRepository +from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase, DailyActivityRepository from litellm.repositories.daily_activity_sql import PRISMA_TO_PG_TABLE, build_where_clause -from litellm.repositories.usage_alerts import AlertDatabase -from litellm.types.proxy.issues import IssueCohort, IssueImpact +from litellm.types.proxy.issues import ( + IssueActivityResponse, + IssueCohort, + IssueImpact, +) from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, DailyActivityUserPageResponse, KeyMetadata, KeySpendActivityRow, KeySpendMetrics, - SpendAnalyticsPaginatedResponse, UserActivityRow, ) from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable @@ -54,7 +58,7 @@ def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyAc class IssueActivityRepository: - def __init__(self, activity: DailyActivityRepository, database: AlertDatabase) -> None: + def __init__(self, activity: DailyActivityRepository, database: DailyActivityQueryDatabase) -> None: self.activity = activity self.database = database @@ -62,14 +66,14 @@ class IssueActivityRepository: def from_prisma(cls, prisma: "PrismaClient") -> "IssueActivityRepository": from litellm.proxy.management_endpoints.common_daily_activity import daily_activity_repository - database: Final = cast(AlertDatabase, prisma.db) # cast-ok: generated Prisma query_raw - return cls(daily_activity_repository(prisma), database) + database: Final = BoundedAnalyticsDatabase(cast(AnalyticsDatabase, prisma.db)) # cast-ok: generated Prisma tx + return cls(daily_activity_repository(prisma, query_database=database), database) - async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse: - from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated - - return await get_daily_activity_aggregated( - self.activity, issue_activity_scope(cohort, start, end), api_key_limit=20 + async def daily_activity( + self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 + ) -> IssueActivityResponse: + return await BoundedDailyActivityRepository(self.database).aggregate( + issue_activity_scope(cohort, start, end), model_limit=model_limit if cohort.scope_type == "key" else None ) async def keys( diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index fda3bc33f89..0d68a957ebd 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -10,6 +10,7 @@ from typing import ( from pydantic import BaseModel, TypeAdapter +from litellm.models.team import LiteLLM_TeamTable from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( @@ -165,6 +166,25 @@ class IssuesRepository: ) return (await self._labels((rows[0].record,)))[0] if rows else None + async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: + rows: Final = TypeAdapter(tuple[LiteLLM_TeamTable, ...]).validate_python( + await self.database.query_raw( + """SELECT t.team_id, + CASE WHEN m.role IS NULL THEN '[]'::jsonb + ELSE jsonb_build_array(jsonb_build_object('user_id',$2::text,'role',m.role)) END + AS members_with_roles, + CASE WHEN '/team/daily/activity'=ANY(t.team_member_permissions) + THEN ARRAY['/team/daily/activity'] ELSE ARRAY[]::text[] END AS team_member_permissions + FROM "LiteLLM_TeamTable" t LEFT JOIN LATERAL ( + SELECT member->>'role' AS role FROM jsonb_array_elements(t.members_with_roles) member + WHERE member->>'user_id'=$2 ORDER BY (member->>'role'='admin') DESC LIMIT 1 + ) m ON true WHERE t.team_id=$1""", + team_id, + user_id, + ) + ) + return rows[0] if rows else None + async def list_issues( self, kind: IssueKind | None = None, diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py index 1a4104726fb..42dd4355a53 100644 --- a/litellm/types/proxy/issues.py +++ b/litellm/types/proxy/issues.py @@ -3,6 +3,9 @@ from typing import Literal, Self, TypeAlias from pydantic import BaseModel, ConfigDict, Field, model_validator +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + BoundedDailyActivityResponse, +) from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import ( @@ -14,6 +17,8 @@ from litellm.types.proxy.usage_alerts import ( UsageAlertStatus, ) +IssueActivityResponse: TypeAlias = BoundedDailyActivityResponse + IssueKind: TypeAlias = Literal["performance", "model_deprecation"] IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] @@ -131,6 +136,12 @@ class Issue(IssueInput, frozen=True): updated_at: datetime scope_label: str | None = None active: bool = True + can_update_status: bool | None = Field( + default=None, description="Current reader's status permission, evaluated by the issue-detail endpoint" + ) + can_view_activity: bool | None = Field( + default=None, description="Current reader's permission to inspect Activity, keys, users and impact" + ) class IssueList(BaseModel, frozen=True): diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index 4fa58c18adb..d9631f3aee7 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Mapping from contextlib import asynccontextmanager from datetime import date, datetime, timedelta, timezone from hashlib import sha256 @@ -17,10 +17,12 @@ from prisma.types import DatasourceOverride from psycopg import sql from pydantic import BaseModel, TypeAdapter, ValidationError +from litellm.repositories.daily_activity_sql import build_bounded_daily_activity_sql from litellm.repositories.issue_activity import IssueActivityRepository from litellm.repositories.issues import IssuesRepository from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( + IssueActivityResponse, IssueCohort, IssueComparison, IssueDeliveryInput, @@ -92,6 +94,7 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa outsider: Final = scenario.user(user_role="internal_user") viewer: Final = scenario.user(user_role="proxy_admin_viewer") owner_key: Final = scenario.key(user_id=owner, key_type="management") + info_key: Final = scenario.key(user_id=owner, key_type="default", allowed_routes=["info_routes"]) outsider_key: Final = scenario.key(user_id=outsider, key_type="management") viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") inference_key: Final = scenario.key(user_id=owner, key_type="llm_api", key_alias="Example checkout") @@ -152,6 +155,16 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa path: Final = f"/issues/{first.id}" own: Final = gateway.request("GET", path, key=owner_key) assert own.status_code == 200 and own.json()["scope_label"] == "Example checkout" + assert own.json()["can_update_status"] is True + assert own.json()["can_view_activity"] is True + assert gateway.request("GET", path, key=viewer_key).json()["can_update_status"] is False + assert gateway.request("GET", path, key=info_key).json()["can_update_status"] is False + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=info_key + ).status_code + == 403 + ) assert "99999" not in own.text and "9999" not in own.text pages: Final = tuple( gateway.request( @@ -204,6 +217,159 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_detail_reports_team_status_permission(gateway: Gateway, db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + with gateway.scenario() as scenario: + admin: Final = scenario.user(user_role="internal_user") + member: Final = scenario.user(user_role="internal_user") + team: Final = scenario.team( + members_with_roles=[{"user_id": admin, "role": "admin"}, {"user_id": member, "role": "user"}] + ) + permitted_team: Final = scenario.team( + members_with_roles=[{"user_id": member, "role": "user"}], + team_member_permissions=["/team/daily/activity"], + ) + admin_key: Final = scenario.key(user_id=admin, key_type="management") + member_key: Final = scenario.key(user_id=member, key_type="management") + service_key: Final = scenario.key( + team_id=team, + user_id=None, + key_type="management", + metadata={"service_account_id": "example-activity-service"}, + ) + issue: Final = _issue(IssueCohort(scope_type="team", scope_id=team)) + permitted_issue: Final = _issue(IssueCohort(scope_type="team", scope_id=permitted_team)) + try: + await store.upsert_issues((issue, permitted_issue)) + path: Final = f"/issues/{issue.id}" + assert gateway.request("GET", path, key=admin_key).json()["can_update_status"] is True + assert gateway.request("GET", path, key=admin_key).json()["can_view_activity"] is True + assert gateway.request("GET", path, key=member_key).json()["can_update_status"] is False + assert gateway.request("GET", path, key=member_key).json()["can_view_activity"] is False + for suffix in ("/activity", "/keys", "/users", "/impact"): + assert gateway.request("GET", path + suffix, key=member_key).status_code == 403 + assert gateway.request("GET", path + suffix, key=service_key).status_code == 403 + for suffix in ("/activity", "/keys", "/impact"): + assert gateway.request("GET", path + suffix, key=admin_key).status_code == 200 + assert ( + gateway.request("GET", f"/issues/{permitted_issue.id}" + suffix, key=member_key).status_code == 200 + ) + assert ( + gateway.request("GET", f"/issues/{permitted_issue.id}", key=member_key).json()["can_view_activity"] + is True + ) + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=member_key + ).status_code + == 403 + ) + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=admin_key + ).status_code + == 200 + ) + finally: + await _remove_issues(db, (issue.id, permitted_issue.id)) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gateway, db: ManagementDatabase) -> None: + from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated + from litellm.repositories.issue_activity import issue_activity_scope + + repository: Final = IssueActivityRepository.from_prisma(db.prisma) + store: Final = IssuesRepository.from_prisma(db.prisma) + identity: Final = str(uuid4()) + key: Final = sha256(identity.encode()).hexdigest() + provider: Final = f"example-provider-{identity}" + day: Final = date.today() - timedelta(days=5) + model_count: Final = 137 + cohort: Final = IssueCohort(scope_type="key", scope_id=key) + issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day + timedelta(days=2)}) + await _query( + db, + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,model_group,custom_llm_provider,mcp_namespaced_tool_name,endpoint, + api_requests,spend,prompt_tokens,completion_tokens,successful_requests,failed_requests, + total_response_time_ms,timed_requests,updated_at) + SELECT $1||'-'||m||'-'||d,$1,($2::date+d::int)::text,$3, + CASE WHEN m=0 THEN 'Other' ELSE 'example-model-'||lpad(m::text,4,'0') END, + 'group-'||m,$4,'tool-'||m,'endpoint-'||m, + m+2,CASE WHEN m=0 AND d=0 THEN 10000 ELSE m+1 END,m+1,2,m+1,1,(m+1)*200,m+1,CURRENT_TIMESTAMP + FROM generate_series(0,$5::int-1) m CROSS JOIN generate_series(0,2) d RETURNING id""", + identity, + day.isoformat(), + key, + provider, + model_count, + ) + try: + await store.upsert_issue(issue) + scope: Final = issue_activity_scope(cohort, day, day + timedelta(days=2)) + original: Final = await get_daily_activity_aggregated(repository.activity, scope, api_key_limit=20) + response: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + assert response.status_code == 200 + bounded: Final = IssueActivityResponse.model_validate(response.json()) + assert bounded.metadata.model_dump(exclude={"api_key_limit"}) == original.metadata.model_dump( + exclude={"api_key_limit"} + ) + assert tuple((row.date, row.metrics) for row in bounded.results) == tuple( + (row.date, row.metrics) for row in original.results + ) + models: Final = bounded.model_breakdown + assert models is not None and models.total_models == model_count and len(models.models) == models.limit == 20 + assert models.models[0].model == "Other" and models.models[0].metrics.spend == 10002 + assert models.other is not None and models.other.model_count == model_count - models.limit + components: Final = (*tuple(row.metrics for row in models.models), models.other.metrics) + assert sum(row.spend for row in components) == bounded.metadata.total_spend + assert sum(row.api_requests for row in components) == bounded.metadata.total_api_requests + assert sum(row.total_response_time_ms for row in components) == bounded.metadata.total_response_time_ms + assert all(not row.breakdown.models and not row.breakdown.endpoints for row in bounded.results) + query: Final = build_bounded_daily_activity_sql(scope, model_limit=100) + raw: Final = TypeAdapter(tuple[object, ...]).validate_python( + await repository.database.query_raw(query.sql, *query.params) + ) + assert len(raw) == 3 + 1 + 100 + 1 + largest: Final = gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "100"}) + assert largest.status_code == 200 + large_models: Final = IssueActivityResponse.model_validate(largest.json()).model_breakdown + assert large_models is not None and len(large_models.models) == 100 + assert large_models.other is not None and large_models.other.model_count == model_count - 100 + assert gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "101"}).status_code == 422 + assert ( + gateway.request( + "GET", + f"/issues/{issue.id}/activity", + params={"start_date": day.isoformat(), "end_date": (day + timedelta(days=366)).isoformat()}, + ).status_code + == 422 + ) + non_key: Final = await repository.daily_activity( + IssueCohort(scope_type="provider", scope_id=provider), day, day + timedelta(days=2) + ) + assert non_key.model_breakdown is None and non_key.metadata == bounded.metadata + assert len(non_key.results) == 3 + schemas: Final = TypeAdapter(tuple[Mapping[str, str], ...]).validate_python( + await _query(db, "SELECT current_schema() AS name") + ) + assert gateway.database_url is not None + with psycopg.connect(gateway.database_url) as blocker: + blocker.execute( + sql.SQL("LOCK TABLE {} IN ACCESS EXCLUSIVE MODE").format( + sql.Identifier(schemas[0]["name"], "LiteLLM_DailyUserSpend") + ) + ) + busy: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + assert busy.status_code == 503 and busy.headers["Retry-After"] == "5" + assert gateway.request("GET", f"/issues/{issue.id}/activity").status_code == 200 + finally: + await _remove_issues(db, (issue.id,)) + await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1 RETURNING id', identity) + + @pytest.mark.asyncio(loop_scope="session") async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( gateway: Gateway, db: ManagementDatabase @@ -332,12 +498,6 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma next_issue_id: Final = str(uuid4()) try: source: Final = (await alerts.record_observations(rule, (observation,)))[0] - await _query( - db, - 'UPDATE "LiteLLM_UsageAlertIncident" SET evaluation_revision=$2 WHERE id=$1 RETURNING id', - source.id, - rule.updated_at.isoformat(), - ) initial: Final = await store.upsert_issue(original) assert tuple( item.alert.id @@ -359,12 +519,8 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma assert disabled is not None and await store.pending_delivery(delivery.id) is None enabled: Final = await alerts.save_rule(definition, "qa-admin", disabled) assert enabled is not None and await store.pending_delivery(delivery.id) is None - await _query( - db, - 'UPDATE "LiteLLM_UsageAlertIncident" SET evaluation_revision=$2 WHERE id=$1 RETURNING id', - source.id, - enabled.updated_at.isoformat(), - ) + reevaluated: Final = await alerts.record_observations(enabled, (observation,)) + assert len(reevaluated) == 1 and reevaluated[0].id == source.id await store.queue_delivery(delivery, enabled.id, enabled.updated_at.isoformat(), source.id) assert await store.pending_delivery(delivery.id) is not None await alerts.set_status(source.id, "dismissed") @@ -654,9 +810,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - expired: Final = _issue( - IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}") - ) + expired: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) open_issue: Final = _issue(expired.cohort) recent: Final = _issue(expired.cohort) rule: Final = await alerts.save_rule( @@ -671,8 +825,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) assert rule is not None deliveries: Final = tuple( - IssueDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") - for _ in range(2) + IssueDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") for _ in range(2) ) try: await store.upsert_issues((expired, open_issue, recent)) @@ -693,16 +846,10 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( )[0] await store.add_signals( expired.id, - ( - IssueSignalInput( - source="usage_alert", source_id=source.id, usage_alert=source - ), - ), + (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),), ) for delivery in deliveries: - await store.queue_delivery( - delivery, rule.id, rule.updated_at.isoformat(), source.id - ) + await store.queue_delivery(delivery, rule.id, rule.updated_at.isoformat(), source.id) assert await store.set_status(expired.id, "resolved", "open") is not None assert await store.set_status(recent.id, "resolved", "open") is not None await _query( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a0ee4b62418..8eb38be8bf9 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -34901,6 +34901,16 @@ export interface components { * @default true */ active: boolean; + /** + * Can Update Status + * @description Current reader's status permission, evaluated by the issue-detail endpoint + */ + can_update_status?: boolean | null; + /** + * Can View Activity + * @description Current reader's permission to inspect Activity, keys, users and impact + */ + can_view_activity?: boolean | null; cohort: components["schemas"]["IssueCohort"]; /** Cohort Key */ cohort_key: string; @@ -65677,6 +65687,7 @@ export interface operations { query?: { start_date?: string | null; end_date?: string | null; + model_limit?: number; }; header?: never; path: { @@ -65692,7 +65703,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + "application/json": components["schemas"]["BoundedDailyActivityResponse"]; }; }; /** @description Validation Error */ From 01e1198a88a773345fa6beaedd0178a6d0af40a8 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 16:33:27 +0800 Subject: [PATCH 21/38] fix(alerts): project bounded resource metadata --- docs/features/usage-alerts.md | 2 +- litellm/repositories/issue_activity.py | 72 +++++++++++++++++--- tests/integration/database/test_issues.py | 82 ++++++++++++++++++++++- 3 files changed, 146 insertions(+), 10 deletions(-) diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index 4407b9a4e90..f9c331b3566 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -57,7 +57,7 @@ Replace `issue-example` with an ID returned by `/issues`. Activity uses the issu Alert Activity returns exact daily totals in `results` and exact range totals in `metadata`. It does not return the general Usage API's per-day dimension dictionaries. Key issues also return `model_breakdown`, with up to 20 models by range spend by default. Set `model_limit` between 1 and 100 to change that limit. `total_models` counts every matching model; `other` contains the combined metrics and `model_count` of all remaining models, or null when none remain. It is an aggregate, not a model identifier. Adding the displayed models and `other` recovers the full totals. Model identifiers longer than 1,024 characters are included in `other` without truncation. Non-key issues return null for `model_breakdown`. Activity queries use a bounded execution budget. A busy or over-budget query returns `503` with `Retry-After: 5`; retry later or choose a shorter date range. -`/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown. +`/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Resource pages fetch labels only for the selected page, using the latest deleted-key label when needed. They do not load arbitrary key or user metadata. Display aliases are limited to 256 characters; email labels longer than 320 characters are omitted. Resource IDs remain unchanged. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown. Supporting signals have their own saved comparisons and relationship explanations. Their baseline windows can differ, so do not derive a combined error rate or unit cost by adding their displayed values. Administrators and admin viewers can inspect broader supporting signals. A scoped reader receives only signals for the exact key, user, or team they can access. That restriction applies before the signal count and pagination; broader totals are not disclosed. diff --git a/litellm/repositories/issue_activity.py b/litellm/repositories/issue_activity.py index 6c85fa064b3..a6b9bf96fe7 100644 --- a/litellm/repositories/issue_activity.py +++ b/litellm/repositories/issue_activity.py @@ -1,7 +1,9 @@ """Issue investigations reuse the same aggregate filters as entity Activity.""" +from collections.abc import Mapping from dataclasses import asdict, replace from datetime import date +from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # generated Prisma structural database interface from pydantic import BaseModel, TypeAdapter @@ -38,6 +40,17 @@ class _Counts(BaseModel, frozen=True): requests: int +class _KeyLabel(BaseModel, frozen=True): + api_key: str + metadata: KeyMetadata + + +class _UserLabel(BaseModel, frozen=True): + user_id: str + user_alias: str | None + user_email: str | None + + def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyActivityScope: if start > end or (end - start).days > 365: raise ValueError("Choose an ordered date range of at most 366 days") @@ -79,18 +92,16 @@ class IssueActivityRepository: async def keys( self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityKeyPageResponse: + if not 1 <= limit <= 100 or offset < 0: + raise ValueError("Read between one and 100 keys with a nonnegative offset") page: Final = await self.activity.key_page(issue_activity_scope(cohort, start, end), limit=limit, offset=offset) - metadata: Final = await self.activity.key_metadata( - frozenset(row.api_key for row in page.rows if row.api_key), None - ) + metadata: Final = await self._key_labels(tuple(row.api_key for row in page.rows if row.api_key)) return DailyActivityKeyPageResponse( api_keys=[ KeySpendActivityRow( api_key=api_key_bucket(row.api_key), metrics=KeySpendMetrics.model_validate(asdict(row)), - metadata=KeyMetadata.model_validate(asdict(metadata[row.api_key])) - if row.api_key in metadata - else KeyMetadata(), + metadata=metadata.get(row.api_key, KeyMetadata()), ) for row in page.rows ], @@ -102,19 +113,23 @@ class IssueActivityRepository: async def users( self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityUserPageResponse: + if not 1 <= limit <= 100 or offset < 0: + raise ValueError("Read between one and 100 users with a nonnegative offset") if cohort.scope_type == "team": raise ValueError("Historical team aggregates do not retain user membership") page: Final = await self.activity.user_page( issue_activity_scope(cohort, start, end), limit=limit, offset=offset ) - metadata: Final = await self.activity.user_metadata(frozenset(row.user_id for row in page.rows if row.user_id)) + metadata: Final = await self._user_labels(tuple(row.user_id for row in page.rows if row.user_id)) return DailyActivityUserPageResponse( users=tuple( UserActivityRow.model_validate( { **asdict(row), **( - asdict(metadata[row.user_id]) if row.user_id is not None and row.user_id in metadata else {} + metadata[row.user_id].model_dump() + if row.user_id is not None and row.user_id in metadata + else {} ), } ) @@ -125,6 +140,47 @@ class IssueActivityRepository: offset=offset, ) + async def _key_labels(self, ids: tuple[str, ...]) -> Mapping[str, KeyMetadata]: + if len(ids) > 100: + raise ValueError("Read at most 100 key labels") + if not ids: + return MappingProxyType({}) + rows: Final = TypeAdapter(tuple[_KeyLabel, ...]).validate_python( + await self.database.query_raw( + """SELECT w.api_key,jsonb_build_object( + 'key_alias',LEFT(COALESCE(k.key_alias,d.key_alias),256), + 'team_id',COALESCE(k.team_id,d.team_id),'user_id',COALESCE(k.user_id,d.user_id), + 'user_email',CASE WHEN length(u.user_email)<=320 THEN u.user_email END, + 'key_exists',CASE WHEN k.token IS NOT NULL THEN true WHEN d.id IS NOT NULL THEN false END + ) AS metadata + FROM unnest($1::text[]) w(api_key) + LEFT JOIN "LiteLLM_VerificationToken" k ON k.token=w.api_key + LEFT JOIN LATERAL ( + SELECT id,key_alias,team_id,user_id FROM "LiteLLM_DeletedVerificationToken" + WHERE token=w.api_key AND k.token IS NULL ORDER BY deleted_at DESC,id DESC LIMIT 1 + ) d ON true + LEFT JOIN "LiteLLM_UserTable" u ON u.user_id=COALESCE(k.user_id,d.user_id) + LIMIT 100""", + list(ids), + ) + ) + return MappingProxyType({row.api_key: row.metadata for row in rows}) + + async def _user_labels(self, ids: tuple[str, ...]) -> Mapping[str, _UserLabel]: + if len(ids) > 100: + raise ValueError("Read at most 100 user labels") + if not ids: + return MappingProxyType({}) + rows: Final = TypeAdapter(tuple[_UserLabel, ...]).validate_python( + await self.database.query_raw( + """SELECT user_id,LEFT(user_alias,256) AS user_alias, + CASE WHEN length(user_email)<=320 THEN user_email END AS user_email + FROM "LiteLLM_UserTable" WHERE user_id=ANY($1::text[]) LIMIT 100""", + list(ids), + ) + ) + return MappingProxyType({row.user_id: row for row in rows}) + async def _counts(self, scope: DailyActivityScope) -> _Counts: where, params = build_where_clause(scope) rows: Final = TypeAdapter(tuple[_Counts, ...]).validate_python( diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index d9631f3aee7..13bd87eb513 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import AsyncGenerator, Mapping +from collections.abc import AsyncGenerator, Mapping, Sequence from contextlib import asynccontextmanager from datetime import date, datetime, timedelta, timezone from hashlib import sha256 @@ -17,6 +17,7 @@ from prisma.types import DatasourceOverride from psycopg import sql from pydantic import BaseModel, TypeAdapter, ValidationError +from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase from litellm.repositories.daily_activity_sql import build_bounded_daily_activity_sql from litellm.repositories.issue_activity import IssueActivityRepository from litellm.repositories.issues import IssuesRepository @@ -889,3 +890,82 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id, ) + + +class _MeasuredActivityDatabase: + def __init__(self, database: DailyActivityQueryDatabase) -> None: + self.database = database + self.outputs: tuple[tuple[int, int], ...] = () + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]] | None: + result: Final = await self.database.query_raw(query, *args) + self.outputs = (*self.outputs, (len(result or ()), len(json.dumps(result, default=str)))) + return result + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_resource_metadata_bounds_labels_and_deleted_history( + gateway: Gateway, db: ManagementDatabase +) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + activity: Final = IssueActivityRepository.from_prisma(db.prisma) + measured: Final = _MeasuredActivityDatabase(activity.database) + repository: Final = IssueActivityRepository(activity.activity, measured) + identity: Final = uuid4().hex + reporting_key: Final = sha256(identity.encode()).hexdigest() + day: Final = date.today() - timedelta(days=3) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + issue: Final = _issue(IssueCohort(scope_type="user", scope_id=owner)).model_copy( + update={"range_start": day, "range_end": day} + ) + try: + await _query( + db, + """UPDATE "LiteLLM_UserTable" SET user_alias=repeat('u',1024), + user_email=repeat('m',400)||'@example.com' WHERE user_id=$1""", + owner, + ) + await _query( + db, + """INSERT INTO "LiteLLM_DeletedVerificationToken" + (id,token,key_alias,user_id,models,metadata,deleted_at) + SELECT $1::text||'-'||n,$2::text,'archived-'||n||'-'||repeat('x',1024),$3::text, + ARRAY[]::text[],jsonb_build_object('padding',repeat('x',65536)), + CURRENT_TIMESTAMP+n*INTERVAL '1 second' FROM generate_series(1,400) n""", + identity, + reporting_key, + owner, + ) + await _query( + db, + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,api_requests,successful_requests,updated_at) + VALUES($1,$2,$3,$4,'example-model',10,10,CURRENT_TIMESTAMP)""", + identity, + owner, + day.isoformat(), + reporting_key, + ) + await store.upsert_issue(issue) + keys: Final = await repository.keys(issue.cohort, day, day, 1, 0) + assert keys.total_api_keys == 1 and len(keys.api_keys) == 1 + key: Final = keys.api_keys[0] + assert key.api_key == reporting_key and key.metadata.key_exists is False + assert key.metadata.key_alias is not None and key.metadata.key_alias.startswith("archived-400-") + assert len(key.metadata.key_alias) == 256 and key.metadata.user_id == owner + assert key.metadata.user_email is None + assert measured.outputs[-1][0] == 1 and measured.outputs[-1][1] < 2048 + users: Final = await repository.users(issue.cohort, day, day, 1, 0) + assert users.total_users == 1 and len(users.users) == 1 + assert users.users[0].user_id == owner and users.users[0].user_alias == "u" * 256 + assert users.users[0].user_email is None + assert measured.outputs[-1][0] == 1 and measured.outputs[-1][1] < 2048 + for endpoint in ("keys", "users"): + response: Final = gateway.request("GET", f"/issues/{issue.id}/{endpoint}", params={"limit": "1"}) + assert response.status_code == 200 and len(response.content) < 2048 + assert "padding" not in response.text + finally: + await _remove_issues(db, (issue.id,)) + await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id=$1', identity) + await _query(db, 'DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', reporting_key) From 418685df5576ebf708251b45a7204123b7c861b8 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:11:24 +0800 Subject: [PATCH 22/38] fix(alerts): require current healthy evidence before recovering resource alerts --- litellm/models/issues.py | 2 +- litellm/repositories/issues.py | 40 ++++++++++-- tests/integration/database/test_issues.py | 80 +++++++++++++++++++++++ 3 files changed, 116 insertions(+), 6 deletions(-) diff --git a/litellm/models/issues.py b/litellm/models/issues.py index 61d3f45f839..6357072453a 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -91,7 +91,7 @@ class IssueStore(Protocol): scope_ids: tuple[str, ...] | None = None, ) -> tuple[IssueProjectionSignal, ...]: ... - async def recover_issues(self, day: date) -> int: ... + async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: ... async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: ... diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 0d68a957ebd..855b6962f61 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -32,7 +32,7 @@ from litellm.types.proxy.issues import ( IssueSignalInput, IssueSignalList, ) -from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS, PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus if TYPE_CHECKING: @@ -366,26 +366,56 @@ class IssuesRepository: ) ) - async def recover_issues(self, day: date) -> int: + async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: + if issue_ids == (): + return 0 + if issue_ids is not None and len(issue_ids) > 10: + raise ValueError("Recover at most 10 resource alerts") + selection: Final = "i.id=ANY($3::text[])" if issue_ids is not None else "$3::text[] IS NULL" rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( await self.database.query_raw( - """WITH healthy AS ( - SELECT i.id FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id + f"""WITH primary_signals AS ( + SELECT i.id,a.id AS source_id,a.breached,a.evaluation_day,a.evaluation_revision, + max(a.evaluation_day) OVER (PARTITION BY i.id) AS latest_observation_day, + r.id AS rule_id,r.record AS rule_record, + COALESCE(r.enabled AND NOT r.deleted + AND r.scope_type=a.scope_type AND r.record->>'metric'=a.metric + AND r.record->>'direction'=a.record->>'direction' + AND (r.scope_id IS NULL OR r.scope_id=a.scope_id) + AND (NOT(r.id=ANY($2::text[])) OR NOT EXISTS ( + SELECT 1 FROM "LiteLLM_UsageAlertRule" custom WHERE custom.enabled AND NOT custom.deleted + AND COALESCE((custom.record->>'overrides_defaults')::boolean,false) + AND NOT(custom.id=ANY($2::text[])) AND custom.scope_type=r.scope_type + AND custom.record->>'metric'=r.record->>'metric' + AND custom.record->>'direction'=r.record->>'direction' + AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id))),false) AS eligible + FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id + LEFT JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='usage_alert' + AND ({selection}) AND s.record->'usage_alert'->>'scope_type'=i.scope_type AND s.record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM i.scope_id AND ((i.family='spend' AND s.record->'usage_alert'->>'metric'='spend') OR (i.family='latency' AND s.record->'usage_alert'->>'metric'='latency') OR (i.family='traffic' AND s.record->'usage_alert'->>'metric'='requests') OR (i.family='reliability' AND s.record->'usage_alert'->>'metric' IN ('errors','error_rate'))) - GROUP BY i.id HAVING bool_and(a.id IS NOT NULL AND NOT a.breached AND a.evaluation_day<=$1) + ), observations AS ( + SELECT *,COALESCE(source_id IS NOT NULL AND NOT breached AND evaluation_day<=$1 + AND evaluation_day>=latest_observation_day + AND NULLIF(evaluation_revision,'')::timestamptz=(rule_record->>'updated_at')::timestamptz,false) AS healthy + FROM primary_signals + ), healthy AS ( + SELECT id FROM observations GROUP BY id + HAVING bool_or(eligible AND healthy) AND bool_and(NOT eligible OR healthy) ), changed AS ( UPDATE "LiteLLM_Issue" i SET active=false, status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP WHERE id IN(SELECT id FROM healthy) AND active RETURNING id ) SELECT count(*)::int AS count FROM changed""", day.isoformat(), + tuple(DEFAULT_USAGE_RULE_IDS), + issue_ids, ) ) return rows[0].count diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index 13bd87eb513..98530a6f3a9 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -969,3 +969,83 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( await _remove_issues(db, (issue.id,)) await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id=$1', identity) await _query(db, 'DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', reporting_key) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + definition: Final = UsageAlertRuleInput( + name="Primary traffic", + scope_type="user", + scope_id=issue.cohort.scope_id, + metric="requests", + direction="increase", + ) + first: Final = await alerts.save_rule(definition, "qa-admin") + second: Final = await alerts.save_rule(definition.model_copy(update={"name": "Another traffic policy"}), "qa-admin") + assert first is not None and second is not None + breach: Final = UsageAlertObservation( + entity_id=issue.cohort.scope_id, + day=issue.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ) + try: + sources: Final = tuple([(await alerts.record_observations(rule, (breach,)))[0] for rule in (first, second)]) + await store.upsert_issue(issue) + await store.add_signals( + issue.id, + tuple( + IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source) for source in sources + ), + ) + await alerts.record_observations( + first, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=1), + "observed": 120, + "breached": False, + } + ), + ), + ) + await alerts.record_observations( + second, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=2), + } + ), + ), + ) + assert await alerts.save_rule(definition.model_copy(update={"enabled": False}), "qa-admin", second) + await store.recover_issues(issue.observed_on + timedelta(days=3)) + unresolved: Final = await store.get_issue(issue.id) + assert unresolved is not None and unresolved.active + await alerts.record_observations( + first, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=3), + "observed": 120, + "breached": False, + } + ), + ), + ) + await store.recover_issues(issue.observed_on + timedelta(days=3)) + recovered: Final = await store.get_issue(issue.id) + assert recovered is not None and not recovered.active + finally: + await _remove_issues(db, (issue.id,)) + for rule in (first, second): + await alerts.delete_rule(rule.id) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) From d0c72a41befb46695b20dd1899f5469846313433 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:26:31 +0800 Subject: [PATCH 23/38] fix(alerts): validate write grants at the typed authorization boundary --- litellm/proxy/alerting_endpoints/usage_alerts.py | 10 +++++++--- .../unit/proxy/alerting_endpoints/test_usage_alerts.py | 6 ------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py index 80d818f2dc7..dfd50e6e720 100644 --- a/litellm/proxy/alerting_endpoints/usage_alerts.py +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -4,7 +4,7 @@ from datetime import date from typing import Annotated, Final, TypeAlias from fastapi import APIRouter, Depends, HTTPException, Query, Response -from pydantic import TypeAdapter +from pydantic import BaseModel, TypeAdapter from litellm.models.usage_alerts import UsageAlertStore from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth @@ -57,10 +57,14 @@ def _allows_write(grant: str, route: str) -> bool: return RouteChecks.check_route_access(route, paths) +class _WriteGrants(BaseModel, frozen=True): + allowed_routes: tuple[str, ...] | None = None + + def require_write_access(user: UserAPIKeyAuth, route: str) -> None: - if not user.allowed_routes: + grants: Final = _WriteGrants.model_validate(user, from_attributes=True).allowed_routes + if not grants: return - grants: Final = TypeAdapter(tuple[str, ...]).validate_python(user.allowed_routes) if not any(_allows_write(grant, route) for grant in grants): raise HTTPException(status_code=403, detail="This credential does not grant alert write access") diff --git a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py index 9d8906eb4e0..1270b6809a9 100644 --- a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py +++ b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py @@ -25,12 +25,6 @@ def test_info_grants_do_not_authorize_mutations(role: LitellmUserRoles, key_type assert error.value.status_code == 403 -@pytest.mark.parametrize("grant", ["/usage/alert-rules/example", "/usage/*", "management_routes"]) -def test_explicit_management_grants_still_authorize_mutations(grant: str) -> None: - user: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, allowed_routes=["info_routes", grant]) - require_write_access(user, "/usage/alert-rules/example") - - @pytest.mark.asyncio @pytest.mark.parametrize( "role,key_type,write", From acef1e3bc3301d65eda55a38f7eed6140bfb8661 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:23:07 +0800 Subject: [PATCH 24/38] feat(alerts): expose enabled family evaluation counts --- litellm/repositories/issue_families.py | 6 +- litellm/types/proxy/issues.py | 2 + .../unit/repositories/test_issue_families.py | 61 +++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 tests/unit/repositories/test_issue_families.py diff --git a/litellm/repositories/issue_families.py b/litellm/repositories/issue_families.py index 230367421d6..14716ce974a 100644 --- a/litellm/repositories/issue_families.py +++ b/litellm/repositories/issue_families.py @@ -1,6 +1,7 @@ """Bounded settings for the built-in performance policies.""" import json +from collections import Counter from datetime import datetime, timezone from itertools import chain, product from typing import Final @@ -21,7 +22,7 @@ from litellm.types.proxy.usage_alert_defaults import ( PerformanceIssueFamily, default_usage_rule_id, ) -from litellm.types.proxy.usage_alerts import UsageAlertRule +from litellm.types.proxy.usage_alerts import UsageAlertEvaluationCounts, UsageAlertRule class _RuleRow(BaseModel, frozen=True): @@ -70,6 +71,9 @@ def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) total_rules=len(family_rules), enabled_rules=sum(rule.enabled for rule in family_rules), policies=values, + evaluation_status_counts=UsageAlertEvaluationCounts.model_validate( + Counter(rule.evaluation_status for rule in family_rules if rule.enabled) + ), ) diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py index 42dd4355a53..e29f1d42768 100644 --- a/litellm/types/proxy/issues.py +++ b/litellm/types/proxy/issues.py @@ -12,6 +12,7 @@ from litellm.types.proxy.usage_alerts import ( UsageAlert, UsageAlertChannel, UsageAlertDirection, + UsageAlertEvaluationCounts, UsageAlertMetric, UsageAlertRule, UsageAlertStatus, @@ -243,6 +244,7 @@ class IssueFamilySetting(BaseModel, frozen=True): total_rules: int enabled_rules: int policies: tuple[IssueFamilyPolicy, ...] + evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts) class IssueFamilyList(BaseModel, frozen=True): diff --git a/tests/unit/repositories/test_issue_families.py b/tests/unit/repositories/test_issue_families.py new file mode 100644 index 00000000000..3647711c584 --- /dev/null +++ b/tests/unit/repositories/test_issue_families.py @@ -0,0 +1,61 @@ +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Final, get_args + +import pytest + +from litellm.repositories.issue_families import family_settings +from litellm.types.proxy.usage_alerts import ( + UsageAlertEvaluationStatus, + UsageAlertMetric, + UsageAlertRule, + UsageAlertScope, +) + + +@dataclass +class RuleRows: + rules: tuple[UsageAlertRule, ...] + calls: int = 0 + + async def query_raw(self, query: str, *args: object) -> object: + self.calls += 1 + return tuple({"record": rule.model_dump(mode="json")} for rule in self.rules) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", get_args(UsageAlertEvaluationStatus)) +async def test_family_counts_only_its_enabled_rules_without_an_extra_query(status: UsageAlertEvaluationStatus) -> None: + timestamp: Final = datetime(2026, 11, 5, tzinfo=timezone.utc) + inputs: Final[tuple[tuple[UsageAlertScope, UsageAlertMetric, bool], ...]] = ( + ("key", "spend", True), + ("team", "spend", True), + ("user", "spend", False), + ("model", "latency", True), + ) + rules: Final = tuple( + UsageAlertRule( + id=f"test-{scope}-{metric}", + name="Daily monitoring", + scope_type=scope, + metric=metric, + direction="increase", + enabled=enabled, + evaluation_status=status, + created_at=timestamp, + updated_at=timestamp, + created_by="test-admin", + ) + for scope, metric, enabled in inputs + ) + + database: Final = RuleRows(rules) + result: Final = next(setting for setting in (await family_settings(database)).data if setting.family == "spend") + + assert database.calls == 1 + assert result.enabled_rules == 2 + assert result.total_rules == 3 + assert result.evaluation_status_counts.model_dump() == { + **dict.fromkeys(get_args(UsageAlertEvaluationStatus), 0), + status: 2, + } From 17a5c99c54b77342fe9121be461e93ad2c1c7b6e Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:24:08 +0800 Subject: [PATCH 25/38] fix: expose pending alert projection in monitoring settings --- litellm/repositories/issue_families.py | 13 +++++++++- litellm/types/proxy/issues.py | 1 + tests/integration/database/test_issues.py | 29 +++++++++++++++++++---- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/litellm/repositories/issue_families.py b/litellm/repositories/issue_families.py index 14716ce974a..2c927fd1fb3 100644 --- a/litellm/repositories/issue_families.py +++ b/litellm/repositories/issue_families.py @@ -33,6 +33,10 @@ class _Count(BaseModel, frozen=True): count: int +class _ProjectionState(BaseModel, frozen=True): + pending: bool + + def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) -> IssueFamilySetting: policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family) groups: Final = tuple( @@ -84,11 +88,18 @@ async def family_settings(database: AlertDatabase) -> IssueFamilyList: list(DEFAULT_USAGE_RULE_IDS), ) ) + projection: Final = TypeAdapter(tuple[_ProjectionState, ...]).validate_python( + await database.query_raw( + """SELECT EXISTS(SELECT 1 FROM "LiteLLM_UsageAlertCheckpoint" + WHERE id LIKE 'performance-dirty:%' AND NOT completed LIMIT 1) AS pending""" + ) + ) return IssueFamilyList( data=tuple( _setting(family, tuple(row.record for row in rows)) for family in ("spend", "latency", "reliability", "traffic") - ) + ), + projection_pending=projection[0].pending, ) diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py index e29f1d42768..5bfd412e946 100644 --- a/litellm/types/proxy/issues.py +++ b/litellm/types/proxy/issues.py @@ -249,6 +249,7 @@ class IssueFamilySetting(BaseModel, frozen=True): class IssueFamilyList(BaseModel, frozen=True): data: tuple[IssueFamilySetting, ...] + projection_pending: bool = False class IssueFamilyPolicyUpdate(BaseModel, frozen=True): diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index 98530a6f3a9..128787f1524 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -605,14 +605,16 @@ async def _family_database(db: ManagementDatabase) -> AsyncGenerator[Prisma]: with psycopg.connect(url, autocommit=True) as setup: setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) try: - setup.execute( - sql.SQL('CREATE TABLE {} (LIKE "LiteLLM_UsageAlertRule" INCLUDING ALL)').format( - sql.Identifier(schema, "LiteLLM_UsageAlertRule") + for table in ("LiteLLM_UsageAlertRule", "LiteLLM_UsageAlertCheckpoint", "LiteLLM_Config"): + setup.execute( + sql.SQL("CREATE TABLE {} (LIKE {} INCLUDING ALL)").format( + sql.Identifier(schema, table), sql.Identifier(table) + ) ) - ) database: Final = Prisma(datasource=DatasourceOverride(url=scoped)) await database.connect() try: + assert await database.query_raw("SELECT current_schema() AS name") == [{"name": schema}] yield database finally: await database.disconnect() @@ -1049,3 +1051,22 @@ async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: M for rule in (first, second): await alerts.delete_rule(rule.id) await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_family_settings_expose_projection_backlog_without_delayed_rules(db: ManagementDatabase) -> None: + async with _family_database(db) as database: + borrowed: Final = cast(AlertDatabase, database) # cast-ok: real generated Prisma query delegate + store: Final = IssuesRepository(borrowed) + assert not (await store.family_settings()).projection_pending + await borrowed.query_raw( + """INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed,owner) + VALUES ('performance-dirty:2026-11-04',false,'fixture') RETURNING id""" + ) + pending: Final = await store.family_settings() + assert pending.projection_pending and all(item.total_rules == 0 for item in pending.data) + await borrowed.query_raw( + """UPDATE "LiteLLM_UsageAlertCheckpoint" SET completed=true + WHERE id='performance-dirty:2026-11-04' RETURNING id""" + ) + assert not (await store.family_settings()).projection_pending From 96580bb3cd4e3a05f261be0414159ace330ffd81 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:25:23 +0800 Subject: [PATCH 26/38] test: cover family counts with projection backlog reads --- tests/unit/repositories/test_issue_families.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/repositories/test_issue_families.py b/tests/unit/repositories/test_issue_families.py index 3647711c584..a22159e9b5a 100644 --- a/tests/unit/repositories/test_issue_families.py +++ b/tests/unit/repositories/test_issue_families.py @@ -20,6 +20,8 @@ class RuleRows: async def query_raw(self, query: str, *args: object) -> object: self.calls += 1 + if "SELECT EXISTS" in query: + return ({"pending": False},) return tuple({"record": rule.model_dump(mode="json")} for rule in self.rules) @@ -52,7 +54,7 @@ async def test_family_counts_only_its_enabled_rules_without_an_extra_query(statu database: Final = RuleRows(rules) result: Final = next(setting for setting in (await family_settings(database)).data if setting.family == "spend") - assert database.calls == 1 + assert database.calls == 2 # existing rule rows plus the separate projection-backlog read assert result.enabled_rules == 2 assert result.total_rules == 3 assert result.evaluation_status_counts.model_dump() == { From d690a76ca23037e5e81c71927ced44eedb2147ef Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:33:13 +0800 Subject: [PATCH 27/38] chore(api): regenerate alert monitoring status schemas --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8eb38be8bf9..9cce368bafb 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35054,6 +35054,11 @@ export interface components { IssueFamilyList: { /** Data */ data: components["schemas"]["IssueFamilySetting"][]; + /** + * Projection Pending + * @default false + */ + projection_pending: boolean; }; /** IssueFamilyPolicy */ IssueFamilyPolicy: { @@ -35107,6 +35112,7 @@ export interface components { enabled: boolean | null; /** Enabled Rules */ enabled_rules: number; + evaluation_status_counts?: components["schemas"]["UsageAlertEvaluationCounts"]; /** * Family * @enum {string} From 21ec701e1b0bd7c7206475cef75851d44cb0c9ab Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 09:16:27 +0800 Subject: [PATCH 28/38] fix: register alert relations for database tracing --- litellm/integrations/otel/model/spans.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index 0f98a8fce0e..5ca3d4e5d51 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -367,6 +367,13 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_SubscriptionAccountTable", "LiteLLM_LensSignalConfig", "LiteLLM_LensTraceSignal", + "LiteLLM_UsageAlertRule", + "LiteLLM_UsageAlertIncident", + "LiteLLM_UsageAlertCheckpoint", + "LiteLLM_Issue", + "LiteLLM_IssueSignal", + "LiteLLM_IssueDelivery", + "LiteLLM_IssueDeliveryRule", ) ) PRISMA_RELATIONS: Final[frozenset[str]] = _PRISMA_MODELS | _PRISMA_VIEWS From 380ad52f6f426762117ab71a5bf2970b899edcc5 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 14:24:41 +0800 Subject: [PATCH 29/38] fix: bound alert display labels before decoding --- litellm/repositories/usage_alerts.py | 4 ++-- .../integration/database/test_usage_alerts.py | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 16fe6ef9421..c1562bde50d 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -463,12 +463,12 @@ class UsageAlertsRepository: """WITH scopes AS ( SELECT DISTINCT scope_type,scope_id,key_id FROM jsonb_to_recordset($1::jsonb) AS x(scope_type text,scope_id text,key_id text) - ) SELECT s.scope_type,s.scope_id,CASE s.scope_type + ) SELECT s.scope_type,s.scope_id,LEFT(CASE s.scope_type WHEN 'key' THEN COALESCE(NULLIF(k.key_alias,''),NULLIF(k.key_name,'')) WHEN 'team' THEN NULLIF(t.team_alias,'') WHEN 'user' THEN COALESCE(NULLIF(u.user_alias,''),NULLIF(u.user_email,'')) WHEN 'model' THEN s.scope_id WHEN 'provider' THEN s.scope_id - WHEN 'deployment' THEN 'Gateway' END AS label + WHEN 'deployment' THEN 'Gateway' END,256) AS label FROM scopes s LEFT JOIN "LiteLLM_VerificationToken" k ON s.scope_type='key' AND k.token=s.key_id LEFT JOIN "LiteLLM_TeamTable" t ON s.scope_type='team' AND t.team_id=s.scope_id LEFT JOIN "LiteLLM_UserTable" u ON s.scope_type='user' AND u.user_id=s.scope_id""", diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index e5990d7eebe..10ddc8b2e14 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -18,6 +18,27 @@ from litellm.types.proxy.usage_alerts import ( ) +@pytest.mark.asyncio(loop_scope="session") +async def test_alert_resource_labels_bound_legacy_display_values(db: ManagementDatabase) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + identity: Final = "label-bound-" + uuid4().hex + scope: Final = UsageAlertRuleInput( + name="Display label", scope_type="user", scope_id=identity, metric="spend", direction="increase" + ) + await repository.database.query_raw( + 'INSERT INTO "LiteLLM_UserTable" (user_id,user_alias) VALUES ($1,repeat(chr(128640),65536))', identity + ) + try: + labels: Final = await repository.resource_labels((scope,)) + assert labels == {("user", identity): chr(128640) * 256} + stored: Final = await repository.database.query_raw( + 'SELECT length(user_alias) AS characters FROM "LiteLLM_UserTable" WHERE user_id=$1', identity + ) + assert stored[0]["characters"] == 65536 + finally: + await repository.database.query_raw('DELETE FROM "LiteLLM_UserTable" WHERE user_id=$1', identity) + + def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gateway: Gateway) -> None: with gateway.scenario() as scenario: owner: Final = scenario.user(user_role="internal_user") From 4af3d16b21753cba85199b130947bde12957d6e8 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 14:31:45 +0800 Subject: [PATCH 30/38] fix: bound legacy identifiers in alert resource pages --- .../repositories/daily_activity_repository.py | 12 ++++++--- litellm/repositories/daily_activity_sql.py | 16 +++++++++--- litellm/repositories/issue_activity.py | 11 +++++--- tests/integration/database/test_issues.py | 26 +++++++++++++++---- 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 0b285261658..e2e29b9b3ba 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -216,8 +216,10 @@ class DailyActivityRepository: rows: Final = _KEY_SPEND_ADAPTER.validate_python(await self._query(query)) return tuple(row.api_key for row in rows) - async def key_page(self, scope: DailyActivityScope, *, offset: int, limit: int) -> KeyPage: - query: Final = build_key_page_sql(scope, offset=offset, limit=limit) + async def key_page( + self, scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False + ) -> KeyPage: + query: Final = build_key_page_sql(scope, offset=offset, limit=limit, bounded_identifiers=bounded_identifiers) result: Final = await self._query(query) total_api_keys_value: Final = result[0].get("total_api_keys") if result else 0 total_api_keys: Final = ( @@ -226,8 +228,10 @@ class DailyActivityRepository: rows: Final = _KEY_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("api_key") is not None)) return KeyPage(rows=rows, total_api_keys=total_api_keys) - async def user_page(self, scope: DailyActivityScope, *, offset: int, limit: int) -> UserPage: - query: Final = build_user_page_sql(scope, offset=offset, limit=limit) + async def user_page( + self, scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False + ) -> UserPage: + query: Final = build_user_page_sql(scope, offset=offset, limit=limit, bounded_identifiers=bounded_identifiers) result: Final = await self._query(query) total_users_value: Final = result[0].get("total_users") if result else 0 total_users: Final = ( diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index da58389c09e..c00ce989a83 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -411,7 +411,9 @@ def _key_spend_select() -> str: COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests""" -def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery: +def build_key_page_sql( + scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False +) -> SqlQuery: if not 1 <= limit <= constants.USAGE_KEY_PAGE_MAX: raise ValueError(f"limit must be between 1 and {constants.USAGE_KEY_PAGE_MAX}") if offset < 0: @@ -420,11 +422,14 @@ def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> sentinel_param: Final = len(where_params) + 1 limit_param: Final = sentinel_param + 1 offset_param: Final = limit_param + 1 + identity_filter: Final = ( + "AND length(api_key)<=CASE WHEN api_key LIKE 'raw:%' THEN 1020 ELSE 1024 END" if bounded_identifiers else "" + ) sql: Final = f""" WITH ranked AS ( SELECT api_key,{_key_spend_select()}, SUM(spend::numeric) AS rank_spend FROM "{PRISMA_TO_PG_TABLE[scope.table]}" - WHERE {where_clause} AND api_key <> ${sentinel_param} + WHERE {where_clause} AND api_key <> ${sentinel_param} {identity_filter} GROUP BY api_key ) SELECT (SELECT COUNT(*) FROM ranked)::bigint AS total_api_keys, page.* @@ -451,7 +456,9 @@ def _user_spend_select() -> str: COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests""" -def build_user_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery: +def build_user_page_sql( + scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False +) -> SqlQuery: if not 1 <= limit <= constants.USAGE_USER_PAGE_MAX: raise ValueError(f"limit must be between 1 and {constants.USAGE_USER_PAGE_MAX}") if offset < 0: @@ -460,11 +467,12 @@ def build_user_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) - limit_param: Final = len(where_params) + 1 offset_param: Final = limit_param + 1 group_expr: Final = f"NULLIF(\"{scope.entity_id_field}\", '')" + identity_filter: Final = f"AND ({group_expr} IS NULL OR length({group_expr})<=1024)" if bounded_identifiers else "" sql: Final = f""" WITH ranked AS ( SELECT {group_expr} AS user_id,{_user_spend_select()}, SUM(spend::numeric) AS rank_spend FROM "{PRISMA_TO_PG_TABLE[scope.table]}" - WHERE {where_clause} + WHERE {where_clause} {identity_filter} GROUP BY {group_expr} ) SELECT (SELECT COUNT(*) FROM ranked)::bigint AS total_users, page.* diff --git a/litellm/repositories/issue_activity.py b/litellm/repositories/issue_activity.py index a6b9bf96fe7..c06cb51e9f6 100644 --- a/litellm/repositories/issue_activity.py +++ b/litellm/repositories/issue_activity.py @@ -94,7 +94,9 @@ class IssueActivityRepository: ) -> DailyActivityKeyPageResponse: if not 1 <= limit <= 100 or offset < 0: raise ValueError("Read between one and 100 keys with a nonnegative offset") - page: Final = await self.activity.key_page(issue_activity_scope(cohort, start, end), limit=limit, offset=offset) + page: Final = await self.activity.key_page( + issue_activity_scope(cohort, start, end), limit=limit, offset=offset, bounded_identifiers=True + ) metadata: Final = await self._key_labels(tuple(row.api_key for row in page.rows if row.api_key)) return DailyActivityKeyPageResponse( api_keys=[ @@ -118,7 +120,7 @@ class IssueActivityRepository: if cohort.scope_type == "team": raise ValueError("Historical team aggregates do not retain user membership") page: Final = await self.activity.user_page( - issue_activity_scope(cohort, start, end), limit=limit, offset=offset + issue_activity_scope(cohort, start, end), limit=limit, offset=offset, bounded_identifiers=True ) metadata: Final = await self._user_labels(tuple(row.user_id for row in page.rows if row.user_id)) return DailyActivityUserPageResponse( @@ -149,7 +151,10 @@ class IssueActivityRepository: await self.database.query_raw( """SELECT w.api_key,jsonb_build_object( 'key_alias',LEFT(COALESCE(k.key_alias,d.key_alias),256), - 'team_id',COALESCE(k.team_id,d.team_id),'user_id',COALESCE(k.user_id,d.user_id), + 'team_id',CASE WHEN length(COALESCE(k.team_id,d.team_id))<=1024 + THEN COALESCE(k.team_id,d.team_id) END, + 'user_id',CASE WHEN length(COALESCE(k.user_id,d.user_id))<=1024 + THEN COALESCE(k.user_id,d.user_id) END, 'user_email',CASE WHEN length(u.user_email)<=320 THEN u.user_email END, 'key_exists',CASE WHEN k.token IS NOT NULL THEN true WHEN d.id IS NOT NULL THEN false END ) AS metadata diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index 128787f1524..2409ab21a0f 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -17,6 +17,7 @@ from prisma.types import DatasourceOverride from psycopg import sql from pydantic import BaseModel, TypeAdapter, ValidationError +from litellm.proxy.management_endpoints.common_daily_activity import daily_activity_repository from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase from litellm.repositories.daily_activity_sql import build_bounded_daily_activity_sql from litellm.repositories.issue_activity import IssueActivityRepository @@ -912,13 +913,13 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( store: Final = IssuesRepository.from_prisma(db.prisma) activity: Final = IssueActivityRepository.from_prisma(db.prisma) measured: Final = _MeasuredActivityDatabase(activity.database) - repository: Final = IssueActivityRepository(activity.activity, measured) + repository: Final = IssueActivityRepository(daily_activity_repository(db.prisma, query_database=measured), measured) identity: Final = uuid4().hex reporting_key: Final = sha256(identity.encode()).hexdigest() day: Final = date.today() - timedelta(days=3) with gateway.scenario() as scenario: owner: Final = scenario.user(user_role="internal_user") - issue: Final = _issue(IssueCohort(scope_type="user", scope_id=owner)).model_copy( + issue: Final = _issue(IssueCohort(scope_type="provider", scope_id=identity)).model_copy( update={"range_start": day, "range_end": day} ) try: @@ -942,8 +943,10 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( await _query( db, """INSERT INTO "LiteLLM_DailyUserSpend" - (id,user_id,date,api_key,model,api_requests,successful_requests,updated_at) - VALUES($1,$2,$3,$4,'example-model',10,10,CURRENT_TIMESTAMP)""", + (id,user_id,date,api_key,model,custom_llm_provider,api_requests,successful_requests,updated_at) + VALUES($1,$2,$3,$4,'example-model',$1,10,10,CURRENT_TIMESTAMP), + ($1||'-wide',repeat(chr(128640),65536),$3,repeat(chr(128640),65536), + 'example-model',$1,20,20,CURRENT_TIMESTAMP)""", identity, owner, day.isoformat(), @@ -963,13 +966,26 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( assert users.users[0].user_id == owner and users.users[0].user_alias == "u" * 256 assert users.users[0].user_email is None assert measured.outputs[-1][0] == 1 and measured.outputs[-1][1] < 2048 + assert all(size < 2048 for _, size in measured.outputs) + totals: Final = await repository.daily_activity(issue.cohort, day, day) + assert totals.metadata.total_api_requests == 30 for endpoint in ("keys", "users"): response: Final = gateway.request("GET", f"/issues/{issue.id}/{endpoint}", params={"limit": "1"}) assert response.status_code == 200 and len(response.content) < 2048 assert "padding" not in response.text + await _query( + db, + """UPDATE "LiteLLM_DeletedVerificationToken" + SET user_id=repeat(chr(128640),65536),team_id=repeat(chr(128640),65536) + WHERE id=$1||'-400'""", + identity, + ) + legacy: Final = await repository.keys(issue.cohort, day, day, 1, 0) + assert legacy.api_keys[0].metadata.user_id is None and legacy.api_keys[0].metadata.team_id is None + assert all(size < 2048 for _, size in measured.outputs[-2:]) finally: await _remove_issues(db, (issue.id,)) - await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id=$1', identity) + await _query(db, "DELETE FROM \"LiteLLM_DailyUserSpend\" WHERE id IN ($1,$1||'-wide')", identity) await _query(db, 'DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', reporting_key) From 1f9969b8d9101d9dd1251b36db83d6e9594a5180 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 15:05:24 +0800 Subject: [PATCH 31/38] fix: check identifier byte size before character scans --- litellm/repositories/daily_activity_sql.py | 12 ++++++++++-- litellm/repositories/issue_activity.py | 16 ++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index c00ce989a83..25fb3304f4e 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -423,7 +423,10 @@ def build_key_page_sql( limit_param: Final = sentinel_param + 1 offset_param: Final = limit_param + 1 identity_filter: Final = ( - "AND length(api_key)<=CASE WHEN api_key LIKE 'raw:%' THEN 1020 ELSE 1024 END" if bounded_identifiers else "" + "AND CASE WHEN octet_length(api_key)<=4096 THEN " + "length(api_key)<=CASE WHEN api_key LIKE 'raw:%' THEN 1020 ELSE 1024 END ELSE false END" + if bounded_identifiers + else "" ) sql: Final = f""" WITH ranked AS ( @@ -467,7 +470,12 @@ def build_user_page_sql( limit_param: Final = len(where_params) + 1 offset_param: Final = limit_param + 1 group_expr: Final = f"NULLIF(\"{scope.entity_id_field}\", '')" - identity_filter: Final = f"AND ({group_expr} IS NULL OR length({group_expr})<=1024)" if bounded_identifiers else "" + identity_filter: Final = ( + f"AND ({group_expr} IS NULL OR CASE WHEN octet_length({group_expr})<=4096 " + f"THEN length({group_expr})<=1024 ELSE false END)" + if bounded_identifiers + else "" + ) sql: Final = f""" WITH ranked AS ( SELECT {group_expr} AS user_id,{_user_spend_select()}, SUM(spend::numeric) AS rank_spend diff --git a/litellm/repositories/issue_activity.py b/litellm/repositories/issue_activity.py index c06cb51e9f6..51ec62d69a8 100644 --- a/litellm/repositories/issue_activity.py +++ b/litellm/repositories/issue_activity.py @@ -151,11 +151,14 @@ class IssueActivityRepository: await self.database.query_raw( """SELECT w.api_key,jsonb_build_object( 'key_alias',LEFT(COALESCE(k.key_alias,d.key_alias),256), - 'team_id',CASE WHEN length(COALESCE(k.team_id,d.team_id))<=1024 - THEN COALESCE(k.team_id,d.team_id) END, - 'user_id',CASE WHEN length(COALESCE(k.user_id,d.user_id))<=1024 - THEN COALESCE(k.user_id,d.user_id) END, - 'user_email',CASE WHEN length(u.user_email)<=320 THEN u.user_email END, + 'team_id',CASE WHEN octet_length(COALESCE(k.team_id,d.team_id))<=4096 + THEN CASE WHEN length(COALESCE(k.team_id,d.team_id))<=1024 + THEN COALESCE(k.team_id,d.team_id) END END, + 'user_id',CASE WHEN octet_length(COALESCE(k.user_id,d.user_id))<=4096 + THEN CASE WHEN length(COALESCE(k.user_id,d.user_id))<=1024 + THEN COALESCE(k.user_id,d.user_id) END END, + 'user_email',CASE WHEN octet_length(u.user_email)<=1280 + THEN CASE WHEN length(u.user_email)<=320 THEN u.user_email END END, 'key_exists',CASE WHEN k.token IS NOT NULL THEN true WHEN d.id IS NOT NULL THEN false END ) AS metadata FROM unnest($1::text[]) w(api_key) @@ -179,7 +182,8 @@ class IssueActivityRepository: rows: Final = TypeAdapter(tuple[_UserLabel, ...]).validate_python( await self.database.query_raw( """SELECT user_id,LEFT(user_alias,256) AS user_alias, - CASE WHEN length(user_email)<=320 THEN user_email END AS user_email + CASE WHEN octet_length(user_email)<=1280 + THEN CASE WHEN length(user_email)<=320 THEN user_email END END AS user_email FROM "LiteLLM_UserTable" WHERE user_id=ANY($1::text[]) LIMIT 100""", list(ids), ) From 8c720dda733a4c0e0d4fe4b5ed9b1a6dabf3af65 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 16:02:05 +0800 Subject: [PATCH 32/38] docs(alerts): explain bounded resource identities --- docs/features/usage-alerts.md | 2 +- tests/integration/database/test_issues.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index f9c331b3566..af900e3ffdf 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -57,7 +57,7 @@ Replace `issue-example` with an ID returned by `/issues`. Activity uses the issu Alert Activity returns exact daily totals in `results` and exact range totals in `metadata`. It does not return the general Usage API's per-day dimension dictionaries. Key issues also return `model_breakdown`, with up to 20 models by range spend by default. Set `model_limit` between 1 and 100 to change that limit. `total_models` counts every matching model; `other` contains the combined metrics and `model_count` of all remaining models, or null when none remain. It is an aggregate, not a model identifier. Adding the displayed models and `other` recovers the full totals. Model identifiers longer than 1,024 characters are included in `other` without truncation. Non-key issues return null for `model_breakdown`. Activity queries use a bounded execution budget. A busy or over-budget query returns `503` with `Retry-After: 5`; retry later or choose a shorter date range. -`/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Resource pages fetch labels only for the selected page, using the latest deleted-key label when needed. They do not load arbitrary key or user metadata. Display aliases are limited to 256 characters; email labels longer than 320 characters are omitted. Resource IDs remain unchanged. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown. +`/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Resource pages fetch labels only for the selected page, using the latest deleted-key label when needed. They do not load arbitrary key or user metadata. Display aliases are limited to 256 characters; email labels longer than 320 characters are omitted. Resource pages omit IDs longer than 1,024 characters or 4,096 bytes instead of truncating them. Key IDs that need a `raw:` escape prefix reserve four characters within that limit. Oversized linked user or team IDs are omitted from resource metadata. Activity totals still include traffic from omitted resources. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown. Supporting signals have their own saved comparisons and relationship explanations. Their baseline windows can differ, so do not derive a combined error rate or unit cost by adding their displayed values. Administrators and admin viewers can inspect broader supporting signals. A scoped reader receives only signals for the exact key, user, or team they can access. That restriction applies before the signal count and pagination; broader totals are not disclosed. diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index 2409ab21a0f..f8c24d12c34 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -945,7 +945,7 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( """INSERT INTO "LiteLLM_DailyUserSpend" (id,user_id,date,api_key,model,custom_llm_provider,api_requests,successful_requests,updated_at) VALUES($1,$2,$3,$4,'example-model',$1,10,10,CURRENT_TIMESTAMP), - ($1||'-wide',repeat(chr(128640),65536),$3,repeat(chr(128640),65536), + ($1||'-wide',repeat(chr(128640),8192),$3,repeat(chr(128640),8192), 'example-model',$1,20,20,CURRENT_TIMESTAMP)""", identity, owner, @@ -976,7 +976,7 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( await _query( db, """UPDATE "LiteLLM_DeletedVerificationToken" - SET user_id=repeat(chr(128640),65536),team_id=repeat(chr(128640),65536) + SET user_id=repeat(chr(128640),8192),team_id=repeat(chr(128640),8192) WHERE id=$1||'-400'""", identity, ) From ae04c3f1617668993b7dd92c37575333fa85ec88 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 16:29:34 +0800 Subject: [PATCH 33/38] fix(alerts): honor custom overrides for queued notifications --- litellm/repositories/issues.py | 16 +++-- tests/integration/database/test_issues.py | 87 ++++++++++++++++++++++- 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 855b6962f61..ff4998a5a04 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -381,14 +381,14 @@ class IssuesRepository: COALESCE(r.enabled AND NOT r.deleted AND r.scope_type=a.scope_type AND r.record->>'metric'=a.metric AND r.record->>'direction'=a.record->>'direction' - AND (r.scope_id IS NULL OR r.scope_id=a.scope_id) - AND (NOT(r.id=ANY($2::text[])) OR NOT EXISTS ( + AND (r.scope_id IS NULL OR r.scope_id=a.scope_id),false) AS eligible, + COALESCE(r.id=ANY($2::text[]) AND EXISTS ( SELECT 1 FROM "LiteLLM_UsageAlertRule" custom WHERE custom.enabled AND NOT custom.deleted AND COALESCE((custom.record->>'overrides_defaults')::boolean,false) AND NOT(custom.id=ANY($2::text[])) AND custom.scope_type=r.scope_type AND custom.record->>'metric'=r.record->>'metric' AND custom.record->>'direction'=r.record->>'direction' - AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id))),false) AS eligible + AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id)),false) AS suppressed FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id LEFT JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id @@ -407,7 +407,7 @@ class IssuesRepository: FROM primary_signals ), healthy AS ( SELECT id FROM observations GROUP BY id - HAVING bool_or(eligible AND healthy) AND bool_and(NOT eligible OR healthy) + HAVING bool_or(eligible AND healthy) AND bool_and(NOT eligible OR suppressed OR healthy) ), changed AS ( UPDATE "LiteLLM_Issue" i SET active=false, status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP @@ -560,6 +560,13 @@ class IssuesRepository: AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric' AND a.record->>'direction'=r.record->>'direction' AND (r.scope_id IS NULL OR a.scope_id=r.scope_id) + AND (NOT(r.id=ANY($3::text[])) OR NOT EXISTS ( + SELECT 1 FROM "LiteLLM_UsageAlertRule" custom WHERE custom.enabled AND NOT custom.deleted + AND COALESCE((custom.record->>'overrides_defaults')::boolean,false) + AND NOT(custom.id=ANY($3::text[])) AND custom.scope_type=r.scope_type + AND custom.record->>'metric'=r.record->>'metric' + AND custom.record->>'direction'=r.record->>'direction' + AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id))) AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id AND s.source='usage_alert' AND s.source_id=a.id) ORDER BY b.rule_id,b.source_id LIMIT 1 @@ -569,6 +576,7 @@ class IssuesRepository: ORDER BY d.next_attempt,d.id LIMIT $1""", limit, delivery_id, + tuple(DEFAULT_USAGE_RULE_IDS), ) ) diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index f8c24d12c34..5ee34c8a3ab 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -37,7 +37,7 @@ from litellm.types.proxy.issues import ( IssueSignalList, ) from litellm.types.proxy.usage_alert_defaults import default_usage_rule_id, default_usage_rules -from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertObservation, UsageAlertRuleInput def _issue(cohort: IssueCohort) -> IssueInput: @@ -623,6 +623,91 @@ async def _family_database(db: ManagementDatabase) -> AsyncGenerator[Prisma]: setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) +@pytest.mark.asyncio(loop_scope="session") +async def test_late_default_override_stops_queued_delivery_and_recovers_on_fresh_evidence( + db: ManagementDatabase, +) -> None: + async with _family_database(db) as database: + borrowed: Final = cast(AlertDatabase, database) # cast-ok: real generated Prisma query delegate + for table in ( + "LiteLLM_UsageAlertIncident", + "LiteLLM_Issue", + "LiteLLM_IssueSignal", + "LiteLLM_IssueDelivery", + "LiteLLM_IssueDeliveryRule", + "LiteLLM_VerificationToken", + "LiteLLM_TeamTable", + "LiteLLM_UserTable", + ): + await borrowed.query_raw(f'CREATE TABLE "{table}" (LIKE public."{table}" INCLUDING ALL)') + alerts: Final = UsageAlertsRepository(borrowed) + store: Final = IssuesRepository(borrowed) + await alerts.ensure_default_rules(default_usage_rules(("slack",))) + rule: Final = await alerts.get_rule(default_usage_rule_id("key", "requests", "increase")) + assert rule is not None + issues: Final = tuple(_issue(IssueCohort(scope_type="key", scope_id=key)) for key in ("key-a", "key-b")) + observations: Final = tuple( + UsageAlertObservation( + entity_id=issue.cohort.scope_id, + day=issue.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ) + for issue in issues + ) + sources: Final = await alerts.record_observations(rule, observations) + await store.upsert_issues(issues) + for issue in issues: + source: UsageAlert = next(source for source in sources if source.scope_id == issue.cohort.scope_id) + await store.add_signals( + issue.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),) + ) + await store.queue_delivery( + IssueDeliveryInput(id=issue.id, issue_id=issue.id, channel="slack"), + rule.id, + rule.updated_at.isoformat(), + source.id, + ) + assert await store.pending_delivery(issue.id) is not None + definition: Final = UsageAlertRuleInput( + name="Project A threshold", + scope_type="key", + scope_id="key-a", + metric="requests", + direction="increase", + multiplier=100, + ) + ordinary: Final = await alerts.save_rule(definition, "example-user") + assert ordinary is not None and await store.pending_delivery(issues[0].id) is not None + assert await alerts.save_rule(definition, "qa-admin", ordinary, overrides_defaults=True) is not None + pending: Final = await store.pending_delivery(issues[0].id) + unaffected: Final = await store.pending_delivery(issues[1].id) + await store.recover_issues(issues[0].observed_on + timedelta(days=1), (issues[0].id,)) + awaiting_evidence: Final = await store.get_issue(issues[0].id) + assert awaiting_evidence is not None and awaiting_evidence.active + await alerts.record_observations( + rule, + ( + observations[0].model_copy( + update={"day": observations[0].day + timedelta(days=1), "observed": 120, "breached": False} + ), + ), + ) + await store.recover_issues(issues[0].observed_on + timedelta(days=2), (issues[0].id,)) + recovered: Final = await store.get_issue(issues[0].id) + other: Final = await store.get_issue(issues[1].id) + assert recovered is not None and other is not None + assert (pending is None, unaffected is not None, recovered.active, recovered.status, other.active) == ( + True, + True, + False, + "resolved", + True, + ), f"overridden delivery eligible={pending is not None}; recovered issue active={recovered.active}" + + @pytest.mark.asyncio(loop_scope="session") async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurrent_edits(db: ManagementDatabase) -> None: async with _family_database(db) as database: From 68a1ec4909e37926bf4457735941082d5f1b5445 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 16:46:24 +0800 Subject: [PATCH 34/38] fix(alerts): register backend routes and make migrations repeatable --- backend/routes/allowlist.py | 2 ++ .../migration.sql | 20 +++++++------- .../20261105200000_issues_inbox/migration.sql | 26 +++++++++---------- .../unit/proxy/alerting_endpoints/__init__.py | 0 4 files changed, 25 insertions(+), 23 deletions(-) create mode 100644 tests/unit/proxy/alerting_endpoints/__init__.py diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py index 2d3ccf40175..414c205b2b3 100644 --- a/backend/routes/allowlist.py +++ b/backend/routes/allowlist.py @@ -110,6 +110,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = ( "/v1/rust_control_plane/", # Alerting / email / IP allowlist "/alerting/", + "/issues/", "/email/", "/add/allowed_ip", "/delete/allowed_ip", @@ -153,6 +154,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset( { "/", "/routes", + "/issues", "/lens", "/openapi.json", "/docs", diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql index d68707b4395..99173d137ea 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql @@ -1,11 +1,11 @@ -CREATE TABLE "LiteLLM_UsageAlertRule" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertRule" ( "id" TEXT PRIMARY KEY, "scope_type" TEXT NOT NULL, "scope_id" TEXT, "enabled" BOOLEAN NOT NULL DEFAULT true, "record" JSONB NOT NULL, "deleted" BOOLEAN NOT NULL DEFAULT false ); -CREATE INDEX "LiteLLM_UsageAlertRule_enabled_deleted_id_idx" ON "LiteLLM_UsageAlertRule" ("enabled", "deleted", "id"); -CREATE INDEX "LiteLLM_UsageAlertRule_scope_type_scope_id_deleted_idx" ON "LiteLLM_UsageAlertRule" ("scope_type", "scope_id", "deleted"); -CREATE TABLE "LiteLLM_UsageAlertIncident" ( +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertRule_enabled_deleted_id_idx" ON "LiteLLM_UsageAlertRule" ("enabled", "deleted", "id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertRule_scope_type_scope_id_deleted_idx" ON "LiteLLM_UsageAlertRule" ("scope_type", "scope_id", "deleted"); +CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertIncident" ( "id" TEXT PRIMARY KEY, "rule_id" TEXT NOT NULL, "scope_key" TEXT NOT NULL, "scope_type" TEXT NOT NULL, "scope_id" TEXT, "metric" TEXT NOT NULL, "status" TEXT NOT NULL, "day" TEXT NOT NULL, "evaluation_day" TEXT NOT NULL, @@ -13,12 +13,12 @@ CREATE TABLE "LiteLLM_UsageAlertIncident" ( "notified_channels" TEXT[] DEFAULT ARRAY[]::TEXT[], "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP ); -CREATE INDEX "LiteLLM_UsageAlertIncident_rule_id_scope_key_evaluation_day_idx" ON "LiteLLM_UsageAlertIncident" ("rule_id", "scope_key", "evaluation_day"); -CREATE INDEX "LiteLLM_UsageAlertIncident_scope_type_scope_id_day_idx" ON "LiteLLM_UsageAlertIncident" ("scope_type", "scope_id", "day"); -CREATE INDEX "LiteLLM_UsageAlertIncident_status_day_idx" ON "LiteLLM_UsageAlertIncident" ("status", "day"); -CREATE INDEX "LiteLLM_UsageAlertIncident_updated_at_idx" ON "LiteLLM_UsageAlertIncident" ("updated_at"); -CREATE TABLE "LiteLLM_UsageAlertCheckpoint" ( +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_rule_id_scope_key_evaluation_day_idx" ON "LiteLLM_UsageAlertIncident" ("rule_id", "scope_key", "evaluation_day"); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_scope_type_scope_id_day_idx" ON "LiteLLM_UsageAlertIncident" ("scope_type", "scope_id", "day"); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_status_day_idx" ON "LiteLLM_UsageAlertIncident" ("status", "day"); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_updated_at_idx" ON "LiteLLM_UsageAlertIncident" ("updated_at"); +CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertCheckpoint" ( "id" TEXT PRIMARY KEY, "completed" BOOLEAN NOT NULL DEFAULT false, "owner" TEXT, "expires_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP ); -CREATE INDEX "LiteLLM_UsageAlertCheckpoint_expires_at_idx" ON "LiteLLM_UsageAlertCheckpoint" ("expires_at"); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertCheckpoint_expires_at_idx" ON "LiteLLM_UsageAlertCheckpoint" ("expires_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql index 3044320e8e8..f5bb7799d79 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql @@ -1,4 +1,4 @@ -CREATE TABLE "LiteLLM_Issue" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_Issue" ( "id" TEXT NOT NULL, "cohort_key" TEXT NOT NULL, "kind" TEXT NOT NULL, @@ -14,19 +14,19 @@ CREATE TABLE "LiteLLM_Issue" ( "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, CONSTRAINT "LiteLLM_Issue_pkey" PRIMARY KEY ("id") ); -CREATE INDEX "LiteLLM_Issue_status_last_seen_id_idx" ON "LiteLLM_Issue"("status", "last_seen", "id"); -CREATE INDEX "LiteLLM_Issue_kind_family_last_seen_id_idx" ON "LiteLLM_Issue"("kind", "family", "last_seen", "id"); -CREATE INDEX "LiteLLM_Issue_scope_type_scope_id_last_seen_id_idx" ON "LiteLLM_Issue"("scope_type", "scope_id", "last_seen", "id"); -CREATE INDEX "LiteLLM_Issue_cohort_key_active_idx" ON "LiteLLM_Issue"("cohort_key", "active"); -CREATE TABLE "LiteLLM_IssueSignal" ( +CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_status_last_seen_id_idx" ON "LiteLLM_Issue"("status", "last_seen", "id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_kind_family_last_seen_id_idx" ON "LiteLLM_Issue"("kind", "family", "last_seen", "id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_scope_type_scope_id_last_seen_id_idx" ON "LiteLLM_Issue"("scope_type", "scope_id", "last_seen", "id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_cohort_key_active_idx" ON "LiteLLM_Issue"("cohort_key", "active"); +CREATE TABLE IF NOT EXISTS "LiteLLM_IssueSignal" ( "issue_id" TEXT NOT NULL, "source" TEXT NOT NULL, "source_id" TEXT NOT NULL, "record" JSONB NOT NULL, CONSTRAINT "LiteLLM_IssueSignal_pkey" PRIMARY KEY ("issue_id", "source", "source_id") ); -CREATE INDEX "LiteLLM_IssueSignal_source_source_id_idx" ON "LiteLLM_IssueSignal"("source", "source_id"); -CREATE TABLE "LiteLLM_IssueDelivery" ( +CREATE INDEX IF NOT EXISTS "LiteLLM_IssueSignal_source_source_id_idx" ON "LiteLLM_IssueSignal"("source", "source_id"); +CREATE TABLE IF NOT EXISTS "LiteLLM_IssueDelivery" ( "id" TEXT NOT NULL, "issue_id" TEXT NOT NULL, "record" JSONB NOT NULL, @@ -35,15 +35,15 @@ CREATE TABLE "LiteLLM_IssueDelivery" ( "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, CONSTRAINT "LiteLLM_IssueDelivery_pkey" PRIMARY KEY ("id") ); -CREATE INDEX "LiteLLM_IssueDelivery_sent_next_attempt_id_idx" ON "LiteLLM_IssueDelivery"("sent", "next_attempt", "id"); -CREATE INDEX "LiteLLM_IssueDelivery_issue_id_idx" ON "LiteLLM_IssueDelivery"("issue_id"); -CREATE TABLE "LiteLLM_IssueDeliveryRule" ( +CREATE INDEX IF NOT EXISTS "LiteLLM_IssueDelivery_sent_next_attempt_id_idx" ON "LiteLLM_IssueDelivery"("sent", "next_attempt", "id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_IssueDelivery_issue_id_idx" ON "LiteLLM_IssueDelivery"("issue_id"); +CREATE TABLE IF NOT EXISTS "LiteLLM_IssueDeliveryRule" ( "delivery_id" TEXT NOT NULL, "rule_id" TEXT NOT NULL, "source_id" TEXT NOT NULL, "rule_revision" TEXT NOT NULL, CONSTRAINT "LiteLLM_IssueDeliveryRule_pkey" PRIMARY KEY ("delivery_id", "rule_id", "source_id") ); -ALTER TABLE "LiteLLM_UsageAlertIncident" ADD COLUMN "evaluation_revision" TEXT NOT NULL DEFAULT ''; -CREATE INDEX "LiteLLM_UsageAlertIncident_evaluation_day_breached_id_idx" +ALTER TABLE "LiteLLM_UsageAlertIncident" ADD COLUMN IF NOT EXISTS "evaluation_revision" TEXT NOT NULL DEFAULT ''; +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_evaluation_day_breached_id_idx" ON "LiteLLM_UsageAlertIncident"("evaluation_day", "breached", "id"); diff --git a/tests/unit/proxy/alerting_endpoints/__init__.py b/tests/unit/proxy/alerting_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 08760d243bff3fc69b88ebaf42e386ce8a58817f Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 16:50:07 +0800 Subject: [PATCH 35/38] fix(alerts): preserve scoped authorization and policy ownership --- litellm/models/issues.py | 3 + litellm/proxy/alerting_endpoints/issues.py | 37 +++- .../proxy/alerting_endpoints/usage_alerts.py | 4 +- litellm/repositories/issue_families.py | 2 + litellm/repositories/issues.py | 57 ++++- litellm/repositories/usage_alerts.py | 7 +- litellm/types/proxy/issues.py | 3 +- litellm/types/proxy/usage_alerts.py | 3 +- tests/integration/database/test_issues.py | 196 +++++++++++++++++- .../integration/database/test_usage_alerts.py | 23 ++ 10 files changed, 308 insertions(+), 27 deletions(-) diff --git a/litellm/models/issues.py b/litellm/models/issues.py index 6357072453a..22043c234d6 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -53,8 +53,11 @@ class IssueStore(Protocol): status: UsageAlertStatus, expected_status: UsageAlertStatus, actor: str | None = None, + authorized_scopes: tuple[IssueCohort, ...] | None = None, ) -> Issue | None: ... + async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]: ... + async def upsert_issue( self, data: IssueInput, diff --git a/litellm/proxy/alerting_endpoints/issues.py b/litellm/proxy/alerting_endpoints/issues.py index b87599a5d56..27b663abc18 100644 --- a/litellm/proxy/alerting_endpoints/issues.py +++ b/litellm/proxy/alerting_endpoints/issues.py @@ -25,6 +25,7 @@ from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.issues import ( Issue, IssueActivityResponse, + IssueCohort, IssueFamily, IssueFamilyList, IssueFamilySetting, @@ -106,6 +107,29 @@ async def require_activity_access(issue: Issue, user: User, repository: IssuesRe raise HTTPException(status_code=403, detail="Full team Activity permission is required for this investigation") +async def require_status_scopes( + issue: Issue, user: User, repository: IssueStore, alerts: UsageAlertsRepositoryDep +) -> tuple[IssueCohort, ...] | None: + store: Final = require_repository(alerts) + await require_scope( + user, + store, + "model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type, + issue.cohort.scope_id, + write=True, + ) + if user.user_role == LitellmUserRoles.PROXY_ADMIN: + return None + scopes: Final = await repository.status_scopes(issue.id) + if len(scopes) > 10: + raise HTTPException(status_code=403, detail="An administrator must update this alert's status") + for scope in scopes: + await require_scope( + user, store, "model" if scope.scope_type == "provider" else scope.scope_type, scope.scope_id, write=True + ) + return scopes + + def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]: start: Final = start_date or issue.range_start end: Final = end_date or issue.range_end @@ -175,13 +199,7 @@ async def get_issue( activity_allowed: Final = await can_view_activity(issue, user, require_issues(repository)) try: require_write_access(user, f"/issues/{issue_id}") - await require_scope( - user, - require_repository(alerts), - "model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type, - issue.cohort.scope_id, - write=True, - ) + await require_status_scopes(issue, user, require_issues(repository), alerts) except HTTPException as exc: if exc.status_code != 403: raise @@ -198,11 +216,12 @@ async def update_issue( alerts: UsageAlertsRepositoryDep, ) -> Issue: require_write_access(user, f"/issues/{issue_id}") - issue: Final = await authorized_issue(issue_id, user, repository, alerts, write=True) + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + scopes: Final = await require_status_scopes(issue, user, require_issues(repository), alerts) if issue.kind == "model_deprecation" and not issue.active and data.status == "open": raise HTTPException(status_code=409, detail="This retirement issue reopens when the model receives new traffic") changed: Final = await require_issues(repository).set_status( - issue_id, data.status, data.expected_status, user.user_id + issue_id, data.status, data.expected_status, user.user_id, authorized_scopes=scopes ) if changed is None: raise HTTPException(status_code=409, detail="Issue status changed; reload and retry") diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py index dfd50e6e720..c954c40cc48 100644 --- a/litellm/proxy/alerting_endpoints/usage_alerts.py +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -229,9 +229,7 @@ async def update_usage_alert_rule( ) await require_scope(user, store, data.scope_type, data.scope_id, write=True) require_notification_access(user, data, rule) - saved: Final = await store.save_rule( - data, user.user_id, existing=rule, overrides_defaults=user.user_role == LitellmUserRoles.PROXY_ADMIN - ) + saved: Final = await store.save_rule(data, user.user_id, existing=rule, overrides_defaults=rule.overrides_defaults) if saved is None: raise HTTPException(status_code=409, detail="Alert rule changed while being updated. Refresh and try again") return saved diff --git a/litellm/repositories/issue_families.py b/litellm/repositories/issue_families.py index 2c927fd1fb3..698a066db8b 100644 --- a/litellm/repositories/issue_families.py +++ b/litellm/repositories/issue_families.py @@ -132,6 +132,8 @@ async def update_family( **({"enabled": data.enabled} if data.enabled is not None else {}), **updates.get(row.record.direction, {}), "updated_at": timestamp, + "evaluation_status": "pending", + "last_evaluated_day": None, } ) for row in rows diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index ff4998a5a04..aead2ad0223 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -248,12 +248,20 @@ class IssuesRepository: status: UsageAlertStatus, expected_status: UsageAlertStatus, actor: str | None = None, + authorized_scopes: tuple[IssueCohort, ...] | None = None, ) -> Issue | None: + if authorized_scopes is not None and len(authorized_scopes) > 10: + raise ValueError("Authorize at most 10 alert source scopes") changed: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( await self.database.query_raw( f"""WITH candidate AS MATERIALIZED ( SELECT i.*,({_STATUS}) AS effective_status {_JOIN} WHERE i.id=$1 AND NOT(i.kind='model_deprecation' AND NOT i.active AND $2='open') + AND ($5::jsonb IS NULL OR NOT EXISTS ( + SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id AND NOT EXISTS ( + SELECT 1 FROM jsonb_to_recordset($5::jsonb) AS x(scope_type text,scope_id text) + WHERE s.source='usage_alert' AND x.scope_type=s.record->'usage_alert'->>'scope_type' + AND x.scope_id IS NOT DISTINCT FROM s.record->'usage_alert'->>'scope_id'))) ), ack AS ( INSERT INTO "LiteLLM_Config" AS prior (param_name,param_value) SELECT ack_key,jsonb_build_object('model',record->'deprecation'->>'usage_model', @@ -266,18 +274,34 @@ class IssuesRepository: WHERE EXISTS(SELECT 1 FROM candidate i WHERE i.ack_key=prior.param_name AND ({_PRIOR_STATUS})=$3) RETURNING param_name - ) UPDATE "LiteLLM_Issue" i SET status=$2,updated_at=CURRENT_TIMESTAMP - FROM candidate c WHERE i.id=c.id AND c.effective_status=$3 AND + ) UPDATE "LiteLLM_Issue" i SET status=$2, + updated_at=GREATEST(CURRENT_TIMESTAMP,i.updated_at+interval '1 millisecond') + FROM candidate c WHERE i.id=c.id AND (c.kind='model_deprecation' OR i.updated_at=c.updated_at) + AND c.effective_status=$3 AND ((c.kind<>'model_deprecation' AND i.status=$3) OR i.ack_key IN(SELECT param_name FROM ack)) RETURNING i.id""", issue_id, status, expected_status, actor, + json.dumps(tuple(scope.model_dump(mode="json") for scope in authorized_scopes)) + if authorized_scopes is not None + else None, ) ) return await self.get_issue(issue_id) if changed else None + async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]: + return TypeAdapter(tuple[IssueCohort, ...]).validate_python( + await self.database.query_raw( + """SELECT DISTINCT CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_type' + ELSE 'deployment' END AS scope_type, + CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_id' END AS scope_id + FROM "LiteLLM_IssueSignal" WHERE issue_id=$1 ORDER BY scope_type,scope_id LIMIT 11""", + issue_id, + ) + ) + async def add_signals(self, issue_id: str, signals: tuple[IssueSignalInput, ...]) -> None: await self.add_signal_batch( tuple(IssueSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in signals) @@ -289,11 +313,30 @@ class IssuesRepository: if len(signals) > 500: raise ValueError("Write at most 500 issue signals per batch") await self.database.query_raw( - """INSERT INTO "LiteLLM_IssueSignal" (issue_id,source,source_id,record) - SELECT DISTINCT ON(x->>'issue_id',x->>'source',x->>'source_id') - x->>'issue_id',x->>'source',x->>'source_id',x FROM jsonb_array_elements($1::jsonb) x - WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" i WHERE i.id=x->>'issue_id') - ON CONFLICT(issue_id,source,source_id) DO NOTHING RETURNING source_id""", + """WITH incoming AS MATERIALIZED ( + SELECT DISTINCT ON(x->>'issue_id',x->>'source',x->>'source_id') x AS record + FROM jsonb_array_elements($1::jsonb) x + ), parents AS MATERIALIZED ( + SELECT i.id FROM "LiteLLM_Issue" i WHERE i.id IN(SELECT record->>'issue_id' FROM incoming) + ORDER BY i.id FOR UPDATE + ), added AS ( + INSERT INTO "LiteLLM_IssueSignal" (issue_id,source,source_id,record) + SELECT x.record->>'issue_id',x.record->>'source',x.record->>'source_id',x.record + FROM incoming x JOIN parents p ON p.id=x.record->>'issue_id' + ON CONFLICT(issue_id,source,source_id) DO NOTHING RETURNING issue_id,source,record + ) UPDATE "LiteLLM_Issue" i SET + updated_at=GREATEST(CURRENT_TIMESTAMP,i.updated_at+interval '1 millisecond'), + status=CASE WHEN i.kind='performance' AND i.active AND i.status<>'open' AND EXISTS ( + SELECT 1 FROM added a WHERE a.issue_id=i.id AND a.source='usage_alert' + AND (a.record->'usage_alert'->>'scope_type'<>i.scope_type + OR a.record->'usage_alert'->>'scope_id' IS DISTINCT FROM i.scope_id) + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" prior + WHERE prior.issue_id=i.id AND prior.source='usage_alert' + AND prior.record->'usage_alert'->>'scope_type'=a.record->'usage_alert'->>'scope_type' + AND prior.record->'usage_alert'->>'scope_id' + IS NOT DISTINCT FROM a.record->'usage_alert'->>'scope_id') + ) THEN 'open' ELSE i.status END + WHERE i.id IN(SELECT issue_id FROM added) RETURNING i.id""", json.dumps(tuple(signal.model_dump(mode="json") for signal in signals)), ) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index c1562bde50d..1732eb86658 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -534,7 +534,12 @@ class UsageAlertsRepository: async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None: rows: Final = TypeAdapter(tuple[_Ownership, ...]).validate_python( await self.database.query_raw( - 'SELECT user_id,team_id FROM "LiteLLM_VerificationToken" WHERE token=$1', key_id + """SELECT user_id,team_id FROM "LiteLLM_VerificationToken" WHERE token=$1 + UNION ALL + (SELECT user_id,team_id FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1 + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_VerificationToken" WHERE token=$1) + ORDER BY deleted_at DESC,id DESC LIMIT 1)""", + key_id, ) ) return (rows[0].user_id, rows[0].team_id) if rows else None diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py index 5bfd412e946..4ad8f50cf42 100644 --- a/litellm/types/proxy/issues.py +++ b/litellm/types/proxy/issues.py @@ -1,7 +1,8 @@ from datetime import date, datetime -from typing import Literal, Self, TypeAlias +from typing import Literal, TypeAlias from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import Self from litellm.types.proxy.management_endpoints.common_daily_activity import ( BoundedDailyActivityResponse, diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index cc5e552ea01..4c65994befa 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -1,9 +1,10 @@ """Daily usage alert rules and incidents shared by the API and evaluator.""" from datetime import date, datetime -from typing import Literal, Self, TypeAlias +from typing import Literal, TypeAlias from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import Self UsageAlertScope: TypeAlias = Literal["deployment", "key", "user", "team", "model"] UsageAlertMetric: TypeAlias = Literal["spend", "requests", "latency", "errors", "error_rate"] diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index 5ee34c8a3ab..cde6cf84880 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -157,7 +157,7 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa path: Final = f"/issues/{first.id}" own: Final = gateway.request("GET", path, key=owner_key) assert own.status_code == 200 and own.json()["scope_label"] == "Example checkout" - assert own.json()["can_update_status"] is True + assert own.json()["can_update_status"] is False, "Broader source scopes require their own write permission" assert own.json()["can_view_activity"] is True assert gateway.request("GET", path, key=viewer_key).json()["can_update_status"] is False assert gateway.request("GET", path, key=info_key).json()["can_update_status"] is False @@ -204,15 +204,18 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa gateway.request( "PATCH", path, {"status": "dismissed", "expected_status": "open"}, key=owner_key ).status_code - == 200 + == 403 ) + assert gateway.request("PATCH", path, {"status": "dismissed", "expected_status": "open"}).status_code == 200 + assert gateway.request("PATCH", path, {"status": "resolved", "expected_status": "open"}).status_code == 409 + assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["status"] == "open" + assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["can_update_status"] is True assert ( gateway.request( - "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=owner_key + "PATCH", f"/issues/{second.id}", {"status": "dismissed", "expected_status": "open"}, key=owner_key ).status_code - == 409 + == 200 ) - assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["status"] == "open" finally: await _remove_issues(db, (first.id, second.id, global_issue.id)) await alerts.delete_rule(rule.id) @@ -784,6 +787,189 @@ async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurren assert await alerts.get_rule(custom.id) == custom +@pytest.mark.asyncio(loop_scope="session") +async def test_family_edit_requires_a_new_evaluation(db: ManagementDatabase) -> None: + async with _family_database(db) as database: + borrowed: Final = cast(AlertDatabase, database) # cast-ok: real generated Prisma query delegate + alerts: Final = UsageAlertsRepository(borrowed) + store: Final = IssuesRepository(borrowed) + await alerts.ensure_default_rules(default_usage_rules()) + rule: Final = await alerts.get_rule(default_usage_rule_id("key", "spend", "increase")) + assert rule is not None + await alerts.record_evaluation_status(rule, date.today() - timedelta(days=1), "complete") + result: Final = await store.update_family( + "spend", IssueFamilyUpdate(policies=(IssueFamilyPolicyUpdate(direction="increase", multiplier=5),)) + ) + saved: Final = await alerts.get_rule(rule.id) + assert result is not None and saved is not None + assert (result.evaluation_status_counts.complete, saved.evaluation_status, saved.last_evaluated_day) == ( + 0, + "pending", + None, + ), "Changed family policy must reset stale evaluation status" + + +@pytest.mark.asyncio(loop_scope="session") +async def test_deleted_key_owner_retains_historical_issue_access(gateway: Gateway, db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + outsider: Final = scenario.user(user_role="internal_user") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + outsider_key: Final = scenario.key(user_id=outsider, key_type="management") + reporting_id: Final = sha256(uuid4().bytes).hexdigest() + issue: Final = _issue(IssueCohort(scope_type="key", scope_id=reporting_id)) + await _query( + db, + 'INSERT INTO "LiteLLM_DeletedVerificationToken" (id,token,user_id,models) VALUES ($1,$1,$2,ARRAY[]::text[])', + reporting_id, + owner, + ) + try: + await store.upsert_issue(issue) + path: Final = f"/issues/{issue.id}" + detail: Final = gateway.request("GET", path, key=owner_key) + assert detail.status_code == 200, "Deleted key owner must retain historical issue access" + assert detail.json()["can_update_status"] is True + assert gateway.request("GET", path, key=outsider_key).status_code == 403 + assert ( + gateway.request( + "PATCH", path, {"status": "dismissed", "expected_status": "open"}, key=owner_key + ).status_code + == 200 + ) + await _query( + db, + 'INSERT INTO "LiteLLM_VerificationToken" (token,user_id,models) VALUES ($1,$2,ARRAY[]::text[])', + reporting_id, + outsider, + ) + assert gateway.request("GET", path, key=owner_key).status_code == 403 + assert gateway.request("GET", path, key=outsider_key).status_code == 200 + finally: + await _remove_issues(db, (issue.id,)) + await _query(db, 'DELETE FROM "LiteLLM_VerificationToken" WHERE token=$1', reporting_id) + await _query(db, 'DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', reporting_id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + rule: Final = await alerts.save_rule( + UsageAlertRuleInput( + name="Scope guard", + scope_type="user", + scope_id=issue.cohort.scope_id, + metric="requests", + direction="increase", + ), + "qa-admin", + ) + assert rule is not None + try: + source: Final = ( + await alerts.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=rule.scope_id, + day=issue.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + await store.upsert_issue(issue) + await store.add_signals( + issue.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),) + ) + authorized: Final = await store.status_scopes(issue.id) + assert authorized == (issue.cohort,) + assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=authorized) is not None + context: Final = source.model_copy(update={"id": str(uuid4()), "metric": "spend"}) + same_scope: Final = (IssueSignalInput(source="usage_alert", source_id=context.id, usage_alert=context),) + await store.add_signals(issue.id, same_scope) + unchanged: Final = await store.get_issue(issue.id) + assert unchanged is not None and unchanged.status == "dismissed" + await store.add_signals(issue.id, same_scope) + assert await store.get_issue(issue.id) == unchanged + broader: Final = source.model_copy(update={"id": str(uuid4()), "scope_type": "deployment", "scope_id": None}) + await store.add_signals( + issue.id, (IssueSignalInput(source="usage_alert", source_id=broader.id, usage_alert=broader),) + ) + assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=authorized) is None + current: Final = await store.get_issue(issue.id) + assert current is not None and current.status == "open" + all_scopes: Final = await store.status_scopes(issue.id) + assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=all_scopes) is not None + broader_context: Final = broader.model_copy(update={"id": str(uuid4()), "metric": "spend"}) + await store.add_signals( + issue.id, + (IssueSignalInput(source="usage_alert", source_id=broader_context.id, usage_alert=broader_context),), + ) + repeated_scope: Final = await store.get_issue(issue.id) + assert repeated_scope is not None and repeated_scope.status == "dismissed" + assert await store.set_status(issue.id, "open", "dismissed") is not None + racing: Final = source.model_copy(update={"id": str(uuid4()), "scope_type": "team", "scope_id": "racing-team"}) + async with asyncio.TaskGroup() as tasks: + async with db.prisma.writer_db.tx(timeout=timedelta(seconds=30)) as transaction: + await transaction.query_raw('SELECT id FROM "LiteLLM_Issue" WHERE id=$1 FOR UPDATE', issue.id) + pid: Final = ( + TypeAdapter(tuple[_Pid, ...]) + .validate_python(await transaction.query_raw("SELECT pg_backend_pid() AS pid"))[0] + .pid + ) + update: Final = tasks.create_task( + store.set_status(issue.id, "dismissed", "open", authorized_scopes=all_scopes) + ) + async with asyncio.timeout(10): + while ( + not TypeAdapter(tuple[_Blocked, ...]) + .validate_python( + await _query( + db, + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE $1::int=ANY(pg_blocking_pids(pid))) AS blocked", + pid, + ) + )[0] + .blocked + ): + await asyncio.sleep(0.01) + locked: Final = IssuesRepository( + cast(AlertDatabase, transaction) # cast-ok: generated Prisma transaction query delegate + ) + await locked.add_signals( + issue.id, (IssueSignalInput(source="usage_alert", source_id=racing.id, usage_alert=racing),) + ) + assert update.result() is None + raced: Final = await store.get_issue(issue.id) + assert raced is not None and raced.status == "open" + await store.add_signals( + issue.id, + tuple( + IssueSignalInput( + source="usage_alert", + source_id=f"scope-{index}", + usage_alert=source.model_copy(update={"id": f"scope-{index}", "scope_id": f"other-user-{index}"}), + ) + for index in range(20) + ), + ) + limited: Final = await store.status_scopes(issue.id) + assert len(limited) == 11 + with pytest.raises(ValueError, match="at most 10"): + await store.set_status(issue.id, "open", "dismissed", authorized_scopes=limited) + finally: + await _remove_issues(db, (issue.id,)) + await alerts.delete_rule(rule.id) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1', rule.id) + + @pytest.mark.asyncio(loop_scope="session") async def test_resource_recovery_ignores_broader_and_traffic_context(db: ManagementDatabase) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index 10ddc8b2e14..77362be35d0 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -86,6 +86,29 @@ def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gate assert gateway.request("GET", path, key=owner_key).status_code == 404 +def test_admin_edit_preserves_owner_rule_access(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + config: Final = { + "name": "Owner policy", + "scope_type": "user", + "scope_id": owner, + "metric": "spend", + "direction": "increase", + } + created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + assert created.status_code == 201, created.text + path: Final = f"/usage/alert-rules/{created.json()['id']}" + try: + edited: Final = gateway.request("PUT", path, {**config, "name": "Renamed by admin"}) + assert edited.status_code == 200, edited.text + owner_edit: Final = gateway.request("PUT", path, {**config, "multiplier": 5}, key=owner_key) + assert owner_edit.status_code == 200, "Admin edit must preserve the owner's rule access" + finally: + assert gateway.request("DELETE", path).status_code == 204 + + def test_team_admin_can_manage_team_alerts_but_member_can_only_read(gateway: Gateway) -> None: with gateway.scenario() as scenario: admin_id: Final = scenario.user(user_role="internal_user") From 4e59e493421d4fb43b7eb496f6f2698416884126 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 16:50:28 +0800 Subject: [PATCH 36/38] test(alerts): verify latency fields in activity responses --- .../test_daily_activity_routes.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index d658e59b76c..0b0dc0f02af 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -421,6 +421,8 @@ def _user_spend_row(user_id: str, rows: Sequence[_Activity]) -> UserSpendRow: api_requests=sum(row.api_requests for row in matching), successful_requests=sum(row.successful_requests for row in matching), failed_requests=sum(row.failed_requests for row in matching), + total_response_time_ms=sum(row.total_response_time_ms for row in matching), + timed_requests=sum(row.timed_requests for row in matching), ) @@ -445,6 +447,8 @@ def _key_spend_row(api_key: str, rows: Sequence[_Activity]) -> KeySpendRow: api_requests=sum(row.api_requests for row in matching), successful_requests=sum(row.successful_requests for row in matching), failed_requests=sum(row.failed_requests for row in matching), + total_response_time_ms=sum(row.total_response_time_ms for row in matching), + timed_requests=sum(row.timed_requests for row in matching), cache_read_input_tokens=sum(row.cache_read_input_tokens for row in matching), cache_creation_input_tokens=sum(row.cache_creation_input_tokens for row in matching), ) @@ -856,6 +860,8 @@ def test_key_page_routes_map_ranked_rows_and_totals( "api_requests": 1, "successful_requests": 1, "failed_requests": 0, + "total_response_time_ms": 100, + "timed_requests": 1, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 1, }, @@ -877,6 +883,8 @@ def test_key_page_routes_map_ranked_rows_and_totals( "api_requests": 2, "successful_requests": 2, "failed_requests": 0, + "total_response_time_ms": 200, + "timed_requests": 2, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 2, }, @@ -991,6 +999,8 @@ def test_user_page_route_ranks_users_by_spend_and_attaches_metadata( "api_requests": 1, "successful_requests": 1, "failed_requests": 0, + "total_response_time_ms": 100, + "timed_requests": 1, }, { "user_id": "user-a", @@ -1003,6 +1013,8 @@ def test_user_page_route_ranks_users_by_spend_and_attaches_metadata( "api_requests": 6, "successful_requests": 6, "failed_requests": 0, + "total_response_time_ms": 600, + "timed_requests": 6, }, ], "total_users": 2, @@ -1191,6 +1203,8 @@ def test_model_top_routes_rank_keys_and_include_metadata( api_requests=1, successful_requests=1, failed_requests=0, + total_response_time_ms=100, + timed_requests=1, cache_read_input_tokens=0, cache_creation_input_tokens=1, ) @@ -1486,6 +1500,8 @@ def test_user_cache_leakage_route_returns_cache_keys_and_metadata( api_requests=1, successful_requests=1, failed_requests=0, + total_response_time_ms=100, + timed_requests=1, cache_read_input_tokens=20, cache_creation_input_tokens=1, ) From 33226cc05f6eb3b25984c75211ff304a80355ab2 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 7 Nov 2026 11:55:26 +0800 Subject: [PATCH 37/38] fix(alerts): bound migration lock waits --- .../20261105000000_add_daily_usage_alerts/migration.sql | 2 ++ .../migrations/20261105200000_issues_inbox/migration.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql index 99173d137ea..f01406ae385 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql @@ -1,3 +1,5 @@ +SET LOCAL lock_timeout = '10s'; + CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertRule" ( "id" TEXT PRIMARY KEY, "scope_type" TEXT NOT NULL, "scope_id" TEXT, "enabled" BOOLEAN NOT NULL DEFAULT true, "record" JSONB NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql index f5bb7799d79..6afa8f18a0e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql @@ -1,3 +1,5 @@ +SET LOCAL lock_timeout = '10s'; + CREATE TABLE IF NOT EXISTS "LiteLLM_Issue" ( "id" TEXT NOT NULL, "cohort_key" TEXT NOT NULL, From be953d8abcfe3ba1498f2e8f73cd266c25131d3b Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 7 Nov 2026 11:55:04 +0800 Subject: [PATCH 38/38] fix(alerts): preserve incidents with simultaneous observations --- litellm/repositories/issues.py | 2 +- tests/integration/database/test_issues.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index aead2ad0223..7e4f2b436ec 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -487,7 +487,7 @@ class IssuesRepository: """WITH changed AS ( UPDATE "LiteLLM_Issue" SET active=false, status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP - WHERE id=ANY($1::text[]) AND kind='performance' AND active AND last_seen<=$2::timestamp(3) + WHERE id=ANY($1::text[]) AND kind='performance' AND active AND last_seen<$2::timestamp(3) RETURNING id ) SELECT count(*)::int AS count FROM changed""", list(ids), diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index cde6cf84880..e5457867b07 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -1067,7 +1067,15 @@ async def test_provider_recovery_pages_saved_cohorts_and_rejects_stale_observati assert first[0].id == inputs[0].id and second[0].id == inputs[1].id assert await store.set_status(inputs[0].id, "dismissed", "open") is not None assert await store.recover_issue_ids((inputs[0].id,), template.observed_at - timedelta(seconds=1)) == 0 - assert await store.recover_issue_ids((inputs[0].id, inputs[1].id), template.observed_at) == 2 + assert await store.recover_issue_ids((inputs[0].id, inputs[1].id), template.observed_at) == 0, ( + "A recovery at the last observed timestamp must not close the alert" + ) + assert ( + await store.recover_issue_ids( + (inputs[0].id, inputs[1].id), template.observed_at + timedelta(milliseconds=1) + ) + == 2 + ) dismissed: Final = await store.get_issue(inputs[0].id) resolved: Final = await store.get_issue(inputs[1].id) assert dismissed is not None and not dismissed.active and dismissed.status == "dismissed"