diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 914d984e541..e334ffee64a 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -551,6 +551,7 @@ jobs: tests/unit/proxy/management_helpers tests/unit/proxy/list_api tests/unit/proxy/analytics_endpoints + tests/unit/proxy/alerting_endpoints tests/unit/proxy/decisions_endpoints tests/unit/proxy/memory tests/unit/proxy/agent_endpoints diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py index 2d3ccf40175..d6a9bf20296 100644 --- a/backend/routes/allowlist.py +++ b/backend/routes/allowlist.py @@ -110,6 +110,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = ( "/v1/rust_control_plane/", # Alerting / email / IP allowlist "/alerting/", + "/alerts/", "/email/", "/add/allowed_ip", "/delete/allowed_ip", @@ -154,6 +155,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset( "/", "/routes", "/lens", + "/alerts", "/openapi.json", "/docs", "/docs/oauth2-redirect", diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261109013000_add_alerts/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261109013000_add_alerts/migration.sql new file mode 100644 index 00000000000..da623a0c6c6 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261109013000_add_alerts/migration.sql @@ -0,0 +1,22 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_Alert" ( + "id" BIGSERIAL PRIMARY KEY, + "detection_key" VARCHAR(64) NOT NULL, + "type" VARCHAR(32) NOT NULL, + "resource_type" VARCHAR(16) NOT NULL, + "resource_id" VARCHAR(1024) NOT NULL, + "status" VARCHAR(16) NOT NULL DEFAULT 'open', + "window_start" TIMESTAMP(3) NOT NULL, + "window_end" TIMESTAMP(3) NOT NULL, + "evidence" JSONB NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_by" VARCHAR(1024), + CHECK ("status" IN ('open', 'resolved', 'dismissed')), + CHECK ("window_end" > "window_start"), + CHECK (octet_length("evidence"::text) <= 8192) +); + +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_Alert_detection_key_key" ON "LiteLLM_Alert"("detection_key"); +CREATE INDEX IF NOT EXISTS "LiteLLM_Alert_status_id_idx" ON "LiteLLM_Alert"("status", "id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 4314a23e011..c02b3d19fa9 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2207,3 +2207,20 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +model LiteLLM_Alert { + id BigInt @id @default(autoincrement()) + detection_key String @unique @client.VarChar(64) + type String @client.VarChar(32) + resource_type String @client.VarChar(16) + resource_id String @client.VarChar(1024) + status String @default("open") @client.VarChar(16) + window_start DateTime + window_end DateTime + evidence Json + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + updated_by String? @client.VarChar(1024) + + @@index([status, id]) +} diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index ad275f64096..b8ebb2d6c6f 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -260,6 +260,7 @@ _PRISMA_VIEWS: Final[frozenset[str]] = frozenset( ) _PRISMA_MODELS: Final[frozenset[str]] = frozenset( ( + "LiteLLM_Alert", "LiteLLM_BudgetTable", "LiteLLM_CredentialsTable", "LiteLLM_ProxyModelTable", diff --git a/litellm/models/alert.py b/litellm/models/alert.py new file mode 100644 index 00000000000..c9a26c6aff9 --- /dev/null +++ b/litellm/models/alert.py @@ -0,0 +1,131 @@ +from datetime import date, datetime, timezone +from typing import Annotated, Final, Literal, Protocol, TypeAlias + +from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, field_validator, model_validator +from typing_extensions import Self + +AlertType: TypeAlias = Literal["spend", "traffic", "latency", "reliability", "model_deprecation"] +AlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"] +AlertResourceType: TypeAlias = Literal["gateway", "key", "team", "user", "model"] +MAX_ALERT_ID: Final = 2**63 - 1 + + +def _validate_alert_id(value: str) -> str: + if int(value) > MAX_ALERT_ID: + raise ValueError("Alert ID exceeds the supported range") + return value + + +AlertID: TypeAlias = Annotated[ + str, Field(pattern=r"^[1-9][0-9]{0,18}$", max_length=19), AfterValidator(_validate_alert_id) +] + + +class MetricEvidence(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + value: float = Field(ge=0, allow_inf_nan=False, description="Observed metric value in the alert window.") + baseline: float = Field(ge=0, allow_inf_nan=False, description="Comparison value in the same unit.") + unit: Literal["usd", "requests", "milliseconds", "ratio"] = Field(description="Unit for both values.") + sample_size: int = Field(ge=0, le=MAX_ALERT_ID, description="Number of requests in the alert window.") + + +class DeprecationEvidence(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + deprecation_date: date = Field(description="Date the affected model retires.") + replacement_model: str | None = Field( + default=None, min_length=1, max_length=1024, description="Suggested replacement." + ) + + +class AlertDetection(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + type: AlertType = Field(description="Built-in alert category.") + resource_type: AlertResourceType = Field(description="Kind of affected resource.") + resource_id: str = Field( + min_length=1, + max_length=1024, + description="Resource ID. Whole-gateway alerts use 'gateway'. Keys use their SHA-256 hash, never a secret.", + ) + window_start: AwareDatetime = Field( + description="Inclusive start of the observed window, normalized to UTC milliseconds." + ) + window_end: AwareDatetime = Field( + description="Exclusive end of the observed window, normalized to UTC milliseconds." + ) + evidence: MetricEvidence | DeprecationEvidence = Field(description="Small saved snapshot explaining the alert.") + + @field_validator("window_start", "window_end") + @classmethod + def normalize_window(cls, value: datetime) -> datetime: + return value.astimezone(timezone.utc).replace(microsecond=value.microsecond // 1000 * 1000) + + @model_validator(mode="after") + def validate_detection(self) -> Self: + if self.window_end <= self.window_start: + raise ValueError("window_end must be after window_start") + if self.resource_type == "gateway" and self.resource_id != "gateway": + raise ValueError("Whole-gateway alerts must use resource_id 'gateway'") + if self.resource_type == "key" and ( + len(self.resource_id) != 64 or any(c not in "0123456789abcdef" for c in self.resource_id) + ): + raise ValueError("Key resource IDs must be SHA-256 hashes") + if self.type == "model_deprecation": + if not isinstance(self.evidence, DeprecationEvidence) or self.resource_type != "model": + raise ValueError("Model deprecations require a model resource and deprecation evidence") + return self + expected_unit: Final = { + "spend": "usd", + "traffic": "requests", + "latency": "milliseconds", + "reliability": "ratio", + }[self.type] + if not isinstance(self.evidence, MetricEvidence) or self.evidence.unit != expected_unit: + raise ValueError("Evidence unit must match the alert type") + if self.type == "reliability" and (self.evidence.value > 1 or self.evidence.baseline > 1): + raise ValueError("Error ratios must be between zero and one") + return self + + def identity(self) -> tuple[str, ...]: + return ( + self.type, + self.resource_type, + self.resource_id, + self.window_start.astimezone(timezone.utc).isoformat(), + self.window_end.astimezone(timezone.utc).isoformat(), + ) + + +class Alert(AlertDetection): + id: AlertID = Field(description="Opaque alert ID, also used as a pagination cursor.") + status: AlertStatus = Field(description="Current review status.") + created_at: AwareDatetime = Field(description="When the alert was first saved.") + updated_at: AwareDatetime = Field(description="When its status last changed.") + updated_by: str | None = Field(default=None, max_length=1024, description="User who last changed the status.") + + +class AlertUpdate(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: AlertStatus = Field(description="Mark resolved, dismiss, or reopen the alert.") + + +class AlertPage(BaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[Alert, ...] = Field(description="Alerts ordered newest first.") + next_cursor: AlertID | None = Field( + default=None, description="Pass as cursor to fetch the next page, or null at the end." + ) + + +class AlertStore(Protocol): + async def record(self, detection: AlertDetection) -> Alert: ... + + async def list(self, status: AlertStatus, limit: int, cursor: int | None) -> AlertPage: ... + + async def get(self, alert_id: int) -> Alert | None: ... + + async def set_status(self, alert_id: int, status: AlertStatus, actor: str) -> Alert | None: ... diff --git a/litellm/proxy/alerting_endpoints/alerts.py b/litellm/proxy/alerting_endpoints/alerts.py new file mode 100644 index 00000000000..fa586ca7a25 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/alerts.py @@ -0,0 +1,56 @@ +from typing import Annotated, Final, TypeAlias + +from fastapi import APIRouter, Depends, HTTPException, Path, Query + +from litellm.models.alert import Alert, AlertID, AlertPage, AlertStatus, AlertStore, AlertUpdate +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.deps.repositories import AlertRepositoryDep +from litellm.types.proxy.auth.user_roles import LitellmUserRoles + +router: Final = APIRouter(tags=["alerts"]) +Auth: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] +AlertPath: TypeAlias = Annotated[AlertID, Path(description="Alert ID returned by GET /alerts.")] + + +def _authorized_store(user: UserAPIKeyAuth, repository: AlertStore | None, *, write: bool) -> AlertStore: + if user.user_role != LitellmUserRoles.PROXY_ADMIN and ( + write or user.user_role != LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ): + raise HTTPException(status_code=403, detail="Only gateway admins can manage alerts") + if repository is None: + raise HTTPException(status_code=503, detail="Alerts require a database connection") + return repository + + +def _found(alert: Alert | None) -> Alert: + if alert is None: + raise HTTPException(status_code=404, detail="Alert not found") + return alert + + +@router.get("/alerts", response_model=AlertPage, summary="List saved alerts") +async def list_alerts( + user: Auth, + repository: AlertRepositoryDep, + status: Annotated[AlertStatus, Query(description="Status to list.")] = "open", + limit: Annotated[int, Query(ge=1, le=100, description="Maximum alerts per page.")] = 25, + cursor: Annotated[AlertID | None, Query(description="next_cursor from the preceding page.")] = None, +) -> AlertPage: + """Read saved alerts, newest first. Gateway admins and view-only admins can read. No usage queries run.""" + store: Final = _authorized_store(user, repository, write=False) + return await store.list(status, limit, int(cursor) if cursor is not None else None) + + +@router.get("/alerts/{alert_id}", response_model=Alert, summary="Read an alert") +async def get_alert(alert_id: AlertPath, user: Auth, repository: AlertRepositoryDep) -> Alert: + """Return the saved evidence and status. Requires a gateway admin or view-only admin.""" + store: Final = _authorized_store(user, repository, write=False) + return _found(await store.get(int(alert_id))) + + +@router.patch("/alerts/{alert_id}", response_model=Alert, summary="Change an alert's status") +async def update_alert(alert_id: AlertPath, data: AlertUpdate, user: Auth, repository: AlertRepositoryDep) -> Alert: + """Resolve, dismiss, or reopen an alert. Only gateway admins can write. Returns the updated alert.""" + store: Final = _authorized_store(user, repository, write=True) + return _found(await store.set_status(int(alert_id), data.status, user.user_id or "proxy-admin")) diff --git a/litellm/proxy/deps/repositories.py b/litellm/proxy/deps/repositories.py index 503314640c2..6e54658d13d 100644 --- a/litellm/proxy/deps/repositories.py +++ b/litellm/proxy/deps/repositories.py @@ -2,6 +2,7 @@ from typing import Annotated, TypeAlias from fastapi import Depends +from litellm.models.alert import AlertStore from litellm.models.deployment_warning import DeploymentWarningIgnoreStore from litellm.models.fallback import RouterFallbackStore from litellm.models.gateway_activity import GatewayActivityStore @@ -9,6 +10,7 @@ from litellm.models.plugin_upload import PluginUploadStore from litellm.models.skill_submission import SkillSubmissionStore from litellm.models.user_banner import UserBannerStore from litellm.proxy.deps.database import DatabaseDep +from litellm.repositories.alert_repository import AlertRepository from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository from litellm.repositories.gateway_activity_repository import GatewayActivityRepository @@ -58,6 +60,13 @@ SkillSubmissionRepositoryDep: TypeAlias = Annotated[ ] +async def get_alert_repository(database: DatabaseDep) -> AlertStore | None: + return None if database is None else AlertRepository(database.prisma) + + +AlertRepositoryDep: TypeAlias = Annotated[AlertStore | None, Depends(get_alert_repository)] + + async def get_router_fallback_repository(database: DatabaseDep) -> RouterFallbackStore | None: return None if database is None else ConfigRepository(database.prisma, use_writer=True) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0c34f77890f..b1a746fbefd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -428,6 +428,7 @@ from litellm.messages.dispatch import messages_target from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import * from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling +from litellm.proxy.alerting_endpoints.alerts import router as alerts_router from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( router as model_deprecation_alerts_router, ) @@ -22591,6 +22592,7 @@ app.include_router(router_settings_router) app.include_router(fallback_management_router) app.include_router(cache_settings_router) app.include_router(model_deprecation_alerts_router) +app.include_router(alerts_router) app.include_router(coordination_redis_settings_router) app.include_router(user_agent_analytics_router) app.include_router(gateway_request_router) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 4314a23e011..c02b3d19fa9 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2207,3 +2207,20 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +model LiteLLM_Alert { + id BigInt @id @default(autoincrement()) + detection_key String @unique @client.VarChar(64) + type String @client.VarChar(32) + resource_type String @client.VarChar(16) + resource_id String @client.VarChar(1024) + status String @default("open") @client.VarChar(16) + window_start DateTime + window_end DateTime + evidence Json + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + updated_by String? @client.VarChar(1024) + + @@index([status, id]) +} diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py new file mode 100644 index 00000000000..1f02d3cdd81 --- /dev/null +++ b/litellm/repositories/alert_repository.py @@ -0,0 +1,63 @@ +import hashlib +import json +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # Prisma exposes generated table actions dynamically + +from litellm.models.alert import Alert, AlertDetection, AlertPage, AlertStatus +from litellm.repositories.prisma_protocols import PrismaRecord, TableActions + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +def _alert(row: PrismaRecord) -> Alert: + values: Final = row.dict() + return Alert.model_validate( + {**{key: value for key, value in values.items() if key != "detection_key"}, "id": str(values["id"])} + ) + + +class AlertRepository: + def __init__(self, prisma_client: "PrismaClient") -> None: + self._table = cast( # cast-ok: the generated Prisma client delegates typed table actions dynamically + TableActions[PrismaRecord], prisma_client.writer_db.litellm_alert + ) + + async def record(self, detection: AlertDetection) -> Alert: + from prisma import Json + from prisma.errors import UniqueViolationError + + key: Final = hashlib.sha256(json.dumps(detection.identity()).encode()).hexdigest() + try: + row: Final = await self._table.create( + data={ + **detection.model_dump(exclude={"evidence"}), + "detection_key": key, + "evidence": Json(detection.evidence.model_dump(mode="json")), + } + ) + return _alert(row) + except UniqueViolationError: + existing: Final = await self._table.find_unique(where={"detection_key": key}) + if existing is None: + raise + return _alert(existing) + + async def list(self, status: AlertStatus, limit: int, cursor: int | None) -> AlertPage: + size: Final = max(1, min(limit, 100)) + rows: Final = await self._table.find_many( + where={"status": status, **({"id": {"lt": cursor}} if cursor is not None else {})}, + order={"id": "desc"}, + take=size + 1, + ) + return AlertPage( + data=tuple(_alert(row) for row in rows[:size]), + next_cursor=str(rows[size - 1].dict()["id"]) if len(rows) > size else None, + ) + + async def get(self, alert_id: int) -> Alert | None: + row: Final = await self._table.find_unique(where={"id": alert_id}) + return None if row is None else _alert(row) + + async def set_status(self, alert_id: int, status: AlertStatus, actor: str) -> Alert | None: + row: Final = await self._table.update(where={"id": alert_id}, data={"status": status, "updated_by": actor}) + return None if row is None else _alert(row) diff --git a/schema.prisma b/schema.prisma index 4314a23e011..c02b3d19fa9 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2207,3 +2207,20 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +model LiteLLM_Alert { + id BigInt @id @default(autoincrement()) + detection_key String @unique @client.VarChar(64) + type String @client.VarChar(32) + resource_type String @client.VarChar(16) + resource_id String @client.VarChar(1024) + status String @default("open") @client.VarChar(16) + window_start DateTime + window_end DateTime + evidence Json + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + updated_by String? @client.VarChar(1024) + + @@index([status, id]) +} diff --git a/tests/integration/database/test_alerts.py b/tests/integration/database/test_alerts.py new file mode 100644 index 00000000000..b3054edba02 --- /dev/null +++ b/tests/integration/database/test_alerts.py @@ -0,0 +1,214 @@ +import asyncio +import os +import uuid +from collections.abc import AsyncIterator +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Final + +import pytest +import pytest_asyncio + +from litellm.models.alert import Alert, AlertDetection, AlertPage, AlertType +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.repositories.alert_repository import AlertRepository +from tests.integration._support.client import Gateway +from tests.integration._support.database import read_rows, write_rows + + +@dataclass(frozen=True, slots=True) +class Alerts: + repository: AlertRepository + resource_id: str + + def detection(self, kind: AlertType = "spend", day: int = 0) -> AlertDetection: + start: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + timedelta(days=day) + evidence: Final = ( + {"deprecation_date": "2026-03-01", "replacement_model": "example/replacement"} + if kind == "model_deprecation" + else { + "value": 0.5 if kind == "reliability" else 20, + "baseline": 0.1 if kind == "reliability" else 2, + "unit": {"spend": "usd", "traffic": "requests", "latency": "milliseconds", "reliability": "ratio"}[ + kind + ], + "sample_size": 100, + } + ) + return AlertDetection.model_validate( + { + "type": kind, + "resource_type": "model", + "resource_id": self.resource_id, + "window_start": start, + "window_end": start + timedelta(days=1), + "evidence": evidence, + } + ) + + +@pytest_asyncio.fixture(loop_scope="function") +async def alerts() -> AsyncIterator[Alerts]: + resource_id: Final = f"alerts-test-{uuid.uuid4().hex}" + client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache())) + await client.connect() + try: + yield Alerts(AlertRepository(client), resource_id) + finally: + try: + write_rows('DELETE FROM "LiteLLM_Alert" WHERE resource_id=%s', (resource_id,)) + finally: + await client.disconnect() + + +@pytest.mark.parametrize("kind", ("spend", "traffic", "latency", "reliability", "model_deprecation")) +@pytest.mark.asyncio +async def test_each_alert_type_round_trips_through_api_and_status_changes( + alerts: Alerts, gateway: Gateway, kind: AlertType +) -> None: + detection: Final = alerts.detection(kind) + saved: Final = await alerts.repository.record(detection) + response: Final = gateway.request("GET", f"/alerts/{saved.id}") + assert response.status_code == 200, response.text + assert Alert.model_validate_json(response.content) == saved + assert saved.status == "open" + assert saved.model_dump(include=set(AlertDetection.model_fields)) == detection.model_dump() + for status in ("resolved", "dismissed", "open"): + update: Final = gateway.request("PATCH", f"/alerts/{saved.id}", {"status": status}) + assert update.status_code == 200, update.text + changed: Final = Alert.model_validate_json(update.content) + assert changed.status == status + assert changed.updated_by + assert changed.evidence == saved.evidence + assert changed.created_at == saved.created_at + assert changed.updated_at >= saved.updated_at + assert await alerts.repository.get(int(saved.id)) == changed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ("spend", "traffic", "latency", "reliability")) +async def test_gateway_alert_round_trips_without_colliding_with_resource_alerts( + alerts: Alerts, gateway: Gateway, kind: AlertType +) -> None: + detection: Final = AlertDetection.model_validate( + {**alerts.detection(kind).model_dump(), "resource_type": "gateway", "resource_id": "gateway"} + ) + try: + saved: Final = await alerts.repository.record(detection) + scoped: Final = await alerts.repository.record(detection.model_copy(update={"resource_type": "model"})) + assert scoped.id != saved.id + detail: Final = gateway.request("GET", f"/alerts/{saved.id}") + assert detail.status_code == 200, detail.text + assert Alert.model_validate_json(detail.content) == saved + listed: Final = gateway.request("GET", "/alerts", params={"limit": "2"}) + assert listed.status_code == 200, listed.text + assert AlertPage.model_validate_json(listed.content).data == (scoped, saved) + update: Final = gateway.request("PATCH", f"/alerts/{saved.id}", {"status": "dismissed"}) + assert update.status_code == 200, update.text + dismissed: Final = Alert.model_validate_json(update.content) + assert dismissed.status == "dismissed" + assert await alerts.repository.record(detection) == dismissed + assert await alerts.repository.get(int(scoped.id)) == scoped + finally: + write_rows( + 'DELETE FROM "LiteLLM_Alert" WHERE resource_id=%s AND type=%s AND window_start=%s AND window_end=%s', + ("gateway", kind, detection.window_start, detection.window_end), + ) + + +@pytest.mark.asyncio +async def test_concurrent_detection_retries_preserve_dismissal_and_original_evidence(alerts: Alerts) -> None: + detection: Final = alerts.detection() + duplicates: Final = await asyncio.gather(*(alerts.repository.record(detection) for _ in range(10))) + assert len({alert.id for alert in duplicates}) == 1 + dismissed: Final = await alerts.repository.set_status(int(duplicates[0].id), "dismissed", "reviewer") + revised: Final = AlertDetection.model_validate( + {**detection.model_dump(), "evidence": {"value": 100, "baseline": 2, "unit": "usd", "sample_size": 100}} + ) + retried: Final = await alerts.repository.record(revised) + assert retried == dismissed + assert retried.evidence == detection.evidence + assert read_rows('SELECT count(*) AS count FROM "LiteLLM_Alert" WHERE resource_id=%s', (alerts.resource_id,)) == [ + {"count": 1} + ] + + +@pytest.mark.asyncio +async def test_keyset_pages_survive_status_changes_and_do_not_include_newer_alerts( + alerts: Alerts, gateway: Gateway +) -> None: + oldest: Final = await alerts.repository.record(alerts.detection(day=0)) + middle: Final = await alerts.repository.record(alerts.detection(day=1)) + newest: Final = await alerts.repository.record(alerts.detection(day=2)) + first: Final = gateway.request("GET", "/alerts", params={"limit": "2"}) + assert first.status_code == 200, first.text + page: Final = AlertPage.model_validate_json(first.content) + assert tuple(alert.id for alert in page.data) == (newest.id, middle.id) + assert page.next_cursor == middle.id + await alerts.repository.set_status(int(middle.id), "resolved", "reviewer") + late: Final = await alerts.repository.record(alerts.detection(day=3)) + second: Final = gateway.request("GET", "/alerts", params={"limit": "2", "cursor": middle.id}) + assert second.status_code == 200, second.text + remainder: Final = AlertPage.model_validate_json(second.content) + assert oldest.id in tuple(alert.id for alert in remainder.data) + assert not {newest.id, middle.id, late.id}.intersection(alert.id for alert in remainder.data) + resolved: Final = gateway.request("GET", "/alerts", params={"status": "resolved", "limit": "1"}) + assert resolved.status_code == 200, resolved.text + assert AlertPage.model_validate_json(resolved.content).data[0].id == middle.id + + +@pytest.mark.asyncio +async def test_repository_caps_page_size_and_follows_to_end(alerts: Alerts) -> None: + for day in range(102): + await alerts.repository.record(alerts.detection(day=day)) + page: Final = await alerts.repository.list("open", 100_000, None) + assert len(page.data) == 100 + assert page.next_cursor == page.data[-1].id + next_page: Final = await alerts.repository.list("open", 100, int(page.next_cursor)) + ours: Final = tuple(alert.id for alert in next_page.data if alert.resource_id == alerts.resource_id) + assert len(ours) == 2 + assert not set(ours).intersection(alert.id for alert in page.data) + empty: Final = await alerts.repository.list("dismissed", 25, None) + assert all(alert.status == "dismissed" for alert in empty.data) + + +@pytest.mark.asyncio +async def test_live_auth_denies_nonadmins_and_viewers_cannot_write(alerts: Alerts, gateway: Gateway) -> None: + saved: Final = await alerts.repository.record(alerts.detection()) + with gateway.scenario() as scenario: + for role in ("internal_user", "proxy_admin_viewer"): + user: Final = scenario.user(user_role=role) + key: Final = scenario.key(user_id=user) + for path in ("/alerts", f"/alerts/{saved.id}"): + response: Final = gateway.request("GET", path, key=key) + assert response.status_code in ((200,) if role == "proxy_admin_viewer" else (401, 403)), response.text + write: Final = gateway.request("PATCH", f"/alerts/{saved.id}", {"status": "dismissed"}, key=key) + assert write.status_code in (401, 403), write.text + assert await alerts.repository.get(int(saved.id)) == saved + + +@pytest.mark.asyncio +async def test_unknown_alert_returns_none_for_read_and_write(alerts: Alerts) -> None: + assert await alerts.repository.get(2**63 - 1) is None + assert await alerts.repository.set_status(2**63 - 1, "dismissed", "reviewer") is None + + +@pytest.mark.asyncio +async def test_windows_round_trip_at_database_precision_and_timezone(alerts: Alerts) -> None: + start: Final = datetime(2026, 1, 1, microsecond=123456, tzinfo=timezone.utc) + detection: Final = AlertDetection.model_validate( + {**alerts.detection().model_dump(), "window_start": start, "window_end": start + timedelta(seconds=1)} + ) + saved: Final = await alerts.repository.record(detection) + assert saved.window_start == start.replace(microsecond=123000) + assert saved.window_end == saved.window_start + timedelta(seconds=1) + offset: Final = timezone(timedelta(hours=8)) + shifted: Final = AlertDetection.model_validate( + { + **detection.model_dump(), + "window_start": start.astimezone(offset), + "window_end": (start + timedelta(seconds=1)).astimezone(offset), + } + ) + assert await alerts.repository.record(shifted) == saved diff --git a/tests/unit/models/test_alert.py b/tests/unit/models/test_alert.py new file mode 100644 index 00000000000..cf915c530b8 --- /dev/null +++ b/tests/unit/models/test_alert.py @@ -0,0 +1,68 @@ +from collections.abc import Mapping +from datetime import timedelta, timezone +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.models.alert import AlertDetection + +DETECTION: Final = { + "type": "spend", + "resource_type": "team", + "resource_id": "example-team", + "window_start": "2026-01-08T00:00:00Z", + "window_end": "2026-01-09T00:00:00Z", + "evidence": {"value": 20, "baseline": 2, "unit": "usd", "sample_size": 50}, +} + + +@pytest.mark.parametrize( + "change", + ( + {"resource_id": "x" * 1025}, + {"resource_id": ""}, + {"resource_type": "key", "resource_id": "sk-example-not-a-real-key"}, + {"resource_type": "gateway", "resource_id": "another-gateway"}, + { + "resource_type": "gateway", + "resource_id": "gateway", + "type": "model_deprecation", + "evidence": {"deprecation_date": "2026-02-01"}, + }, + {"window_start": "2026-01-08T00:00:00"}, + {"window_start": "2026-01-08T00:00:00.123456Z", "window_end": "2026-01-08T00:00:00.123457Z"}, + {"window_end": "2026-01-08T00:00:00Z"}, + {"window_end": "2026-01-07T00:00:00Z"}, + {"type": "custom"}, + {"type": "latency"}, + {"type": "model_deprecation"}, + {"evidence": {"value": float("inf"), "baseline": 2, "unit": "usd", "sample_size": 1}}, + {"evidence": {"value": 2, "baseline": float("nan"), "unit": "usd", "sample_size": 1}}, + {"evidence": {"value": -1, "baseline": 2, "unit": "usd", "sample_size": 1}}, + {"evidence": {"value": 2, "baseline": 2, "unit": "usd", "sample_size": -1}}, + {"evidence": {"value": 2, "baseline": 2, "unit": "usd", "sample_size": 1, "unbounded": "extra"}}, + {"type": "reliability", "evidence": {"value": 1.1, "baseline": 0, "unit": "ratio", "sample_size": 1}}, + {"type": "reliability", "evidence": {"value": 0, "baseline": 1.1, "unit": "ratio", "sample_size": 1}}, + { + "resource_type": "model", + "type": "model_deprecation", + "evidence": {"deprecation_date": "2026-02-01", "replacement_model": "x" * 1025}, + }, + ), +) +def test_invalid_or_unbounded_detection_is_rejected(change: Mapping[str, object]) -> None: + with pytest.raises(ValidationError): + AlertDetection.model_validate({**DETECTION, **change}) + + +def test_identity_uses_instants_not_timezone_or_evidence() -> None: + original: Final = AlertDetection.model_validate(DETECTION) + revised: Final = original.model_copy( + update={ + "window_start": original.window_start.astimezone(timezone(timedelta(hours=8))), + "window_end": original.window_end.astimezone(timezone(timedelta(hours=8))), + "evidence": original.evidence.model_copy(update={"value": 30}), + } + ) + assert original.identity() == revised.identity() diff --git a/tests/unit/proxy/alerting_endpoints/__init__.py b/tests/unit/proxy/alerting_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/alerting_endpoints/test_alerts.py b/tests/unit/proxy/alerting_endpoints/test_alerts.py new file mode 100644 index 00000000000..15dfad975fc --- /dev/null +++ b/tests/unit/proxy/alerting_endpoints/test_alerts.py @@ -0,0 +1,161 @@ +from collections.abc import Iterator +from datetime import datetime, timezone +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pydantic import BaseModel, ConfigDict + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.alerting_endpoints.alerts import router +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.deps.database import get_database +from litellm.repositories.database import Database +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from tests._support.native_database import native_database + + +class StoredAlert(BaseModel): + model_config = ConfigDict(frozen=True) + id: int = 5 + detection_key: str = "digest" + type: str = "spend" + resource_type: str = "team" + resource_id: str = "example-team" + status: str = "open" + window_start: datetime = datetime(2026, 1, 8, tzinfo=timezone.utc) + window_end: datetime = datetime(2026, 1, 9, tzinfo=timezone.utc) + evidence: object = {"value": 20, "baseline": 2, "unit": "usd", "sample_size": 50} + created_at: datetime = datetime(2026, 1, 9, tzinfo=timezone.utc) + updated_at: datetime = datetime(2026, 1, 9, tzinfo=timezone.utc) + updated_by: str | None = None + + +@pytest.fixture +def table() -> MagicMock: + table: Final = MagicMock() + table.find_many = AsyncMock(return_value=[StoredAlert()]) + table.find_unique = AsyncMock(return_value=None) + table.update = AsyncMock(return_value=None) + return table + + +@pytest.fixture +def app(table: MagicMock) -> FastAPI: + app: Final = FastAPI() + app.include_router(router) + prisma: Final = MagicMock() + prisma.writer_db.litellm_alert = table + app.dependency_overrides[get_database] = lambda: Database(prisma, native_database()) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN + ) + return app + + +@pytest.fixture +def client(app: FastAPI) -> Iterator[TestClient]: + with TestClient(app) as client: + yield client + + +@pytest.mark.parametrize( + "role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.TEAM, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) +) +@pytest.mark.parametrize("method,path", (("GET", "/alerts"), ("GET", "/alerts/5"), ("PATCH", "/alerts/5"))) +def test_permissions_precede_database_access( + app: FastAPI, table: MagicMock, role: LitellmUserRoles, method: str, path: str +) -> None: + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role) + app.dependency_overrides[get_database] = lambda: None + with TestClient(app) as client: + response: Final = client.request(method, path, json={"status": "dismissed"} if method == "PATCH" else None) + assert response.status_code == (503 if role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY and method == "GET" else 403) + table.find_many.assert_not_awaited() + table.find_unique.assert_not_awaited() + table.update.assert_not_awaited() + + +@pytest.mark.parametrize( + "query", ("limit=0", "limit=101", "cursor=0", "cursor=9223372036854775808", "cursor=oops", "status=invalid") +) +def test_invalid_page_never_reads_database(client: TestClient, table: MagicMock, query: str) -> None: + assert client.get(f"/alerts?{query}").status_code == 422 + table.find_many.assert_not_awaited() + + +@pytest.mark.parametrize("alert_id", ("0", "-1", "01", "1.0", "oops", "9223372036854775808", "1" * 20)) +@pytest.mark.parametrize("method", ("GET", "PATCH")) +def test_invalid_id_never_accesses_database(client: TestClient, table: MagicMock, alert_id: str, method: str) -> None: + response: Final = client.request( + method, f"/alerts/{alert_id}", json={"status": "resolved"} if method == "PATCH" else None + ) + assert response.status_code == 422 + table.find_unique.assert_not_awaited() + table.update.assert_not_awaited() + + +@pytest.mark.parametrize("alert_id", (9007199254740993, 9223372036854775807)) +def test_large_ids_remain_exact_in_reads_updates_and_pagination( + client: TestClient, table: MagicMock, alert_id: int +) -> None: + row: Final = StoredAlert(id=alert_id) + table.find_unique.return_value = row + table.update.return_value = row.model_copy(update={"status": "resolved", "updated_by": "admin-user"}) + table.find_many.return_value = [row, StoredAlert(id=alert_id - 1)] + detail: Final = client.get(f"/alerts/{alert_id}") + assert detail.status_code == 200, detail.text + assert detail.json()["id"] == str(alert_id) + table.find_unique.assert_awaited_once_with(where={"id": alert_id}) + updated: Final = client.patch(f"/alerts/{alert_id}", json={"status": "resolved"}) + assert updated.status_code == 200, updated.text + assert updated.json()["id"] == str(alert_id) + assert updated.json()["status"] == "resolved" + table.update.assert_awaited_once_with( + where={"id": alert_id}, data={"status": "resolved", "updated_by": "admin-user"} + ) + page: Final = client.get("/alerts?limit=1") + assert page.status_code == 200, page.text + assert page.json()["next_cursor"] == str(alert_id) + following: Final = client.get("/alerts", params={"cursor": page.json()["next_cursor"], "limit": 1}) + assert following.status_code == 200, following.text + table.find_many.assert_awaited_with(where={"status": "open", "id": {"lt": alert_id}}, order={"id": "desc"}, take=2) + + +def test_openapi_keeps_ids_and_cursors_as_strings(client: TestClient) -> None: + spec: Final = client.get("/openapi.json").json() + paths: Final = spec["paths"] + for method in ("get", "patch"): + parameter: Final = next(p for p in paths["/alerts/{alert_id}"][method]["parameters"] if p["name"] == "alert_id") + assert parameter["schema"]["type"] == "string" + cursor: Final = next(p for p in paths["/alerts"]["get"]["parameters"] if p["name"] == "cursor") + assert {s["type"] for s in cursor["schema"]["anyOf"]} == {"string", "null"} + schemas: Final = spec["components"]["schemas"] + assert schemas["Alert"]["properties"]["id"]["type"] == "string" + assert {s["type"] for s in schemas["AlertPage"]["properties"]["next_cursor"]["anyOf"]} == {"string", "null"} + + +def test_viewer_reads_saved_alert_with_string_id(app: FastAPI, table: MagicMock) -> None: + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ) + with TestClient(app) as client: + response: Final = client.get("/alerts?status=open&limit=1") + assert response.status_code == 200, response.text + expected: Final = StoredAlert().model_dump(mode="json", exclude={"detection_key"}) + assert response.json() == {"data": [{**expected, "id": "5"}], "next_cursor": None} + + +@pytest.mark.parametrize("method", ("GET", "PATCH")) +def test_missing_alert_returns_not_found(client: TestClient, method: str) -> None: + response: Final = client.request(method, "/alerts/5", json={"status": "resolved"} if method == "PATCH" else None) + assert response.status_code == 404 + assert response.json() == {"detail": "Alert not found"} + + +@pytest.mark.parametrize("body", ({"status": "unknown"}, {"status": "open", "evidence": {}}, {})) +def test_status_update_rejects_other_changes(client: TestClient, table: MagicMock, body: object) -> None: + assert client.patch("/alerts/5", json=body).status_code == 422 + table.update.assert_not_awaited() diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a446cf87857..3d680bf333d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1002,6 +1002,50 @@ export interface paths { patch?: never; trace?: never; }; + "/alerts": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List saved alerts + * @description Read saved alerts, newest first. Gateway admins and view-only admins can read. No usage queries run. + */ + get: operations["list_alerts_alerts_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/{alert_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Read an alert + * @description Return the saved evidence and status. Requires a gateway admin or view-only admin. + */ + get: operations["get_alert_alerts__alert_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** + * Change an alert's status + * @description Resolve, dismiss, or reopen an alert. Only gateway admins can write. Returns the updated alert. + */ + patch: operations["update_alert_alerts__alert_id__patch"]; + trace?: never; + }; "/anthropic/{endpoint}": { parameters: { query?: never; @@ -27106,12 +27150,99 @@ export interface components { /** Url */ url: string; }; + /** Alert */ + Alert: { + /** + * Created At + * Format: date-time + * @description When the alert was first saved. + */ + created_at: string; + /** + * Evidence + * @description Small saved snapshot explaining the alert. + */ + evidence: components["schemas"]["MetricEvidence"] | components["schemas"]["DeprecationEvidence"]; + /** + * Id + * @description Opaque alert ID, also used as a pagination cursor. + */ + id: string; + /** + * Resource Id + * @description Resource ID. Whole-gateway alerts use 'gateway'. Keys use their SHA-256 hash, never a secret. + */ + resource_id: string; + /** + * Resource Type + * @description Kind of affected resource. + * @enum {string} + */ + resource_type: "gateway" | "key" | "team" | "user" | "model"; + /** + * Status + * @description Current review status. + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + /** + * Type + * @description Built-in alert category. + * @enum {string} + */ + type: "spend" | "traffic" | "latency" | "reliability" | "model_deprecation"; + /** + * Updated At + * Format: date-time + * @description When its status last changed. + */ + updated_at: string; + /** + * Updated By + * @description User who last changed the status. + */ + updated_by?: string | null; + /** + * Window End + * Format: date-time + * @description Exclusive end of the observed window, normalized to UTC milliseconds. + */ + window_end: string; + /** + * Window Start + * Format: date-time + * @description Inclusive start of the observed window, normalized to UTC milliseconds. + */ + window_start: string; + }; + /** AlertPage */ + AlertPage: { + /** + * Data + * @description Alerts ordered newest first. + */ + data: components["schemas"]["Alert"][]; + /** + * Next Cursor + * @description Pass as cursor to fetch the next page, or null at the end. + */ + next_cursor?: string | null; + }; /** * AlertType * @description Enum for alert types and management event types * @enum {string} */ AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted" | "mcp_tool_description_blocked" | "mcp_pinned_tools_changed"; + /** AlertUpdate */ + AlertUpdate: { + /** + * Status + * @description Mark resolved, dismiss, or reopen the alert. + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + }; /** AllowedVectorStoreIndexItem */ AllowedVectorStoreIndexItem: { /** Index Name */ @@ -33055,6 +33186,20 @@ export interface components { model: string; status: components["schemas"]["DeprecationAlertStatus"]; }; + /** DeprecationEvidence */ + DeprecationEvidence: { + /** + * Deprecation Date + * Format: date + * @description Date the affected model retires. + */ + deprecation_date: string; + /** + * Replacement Model + * @description Suggested replacement. + */ + replacement_model?: string | null; + }; /** DeprecationTagUsage */ DeprecationTagUsage: { /** Api Key */ @@ -39714,6 +39859,30 @@ export interface components { } & { [key: string]: unknown; }; + /** MetricEvidence */ + MetricEvidence: { + /** + * Baseline + * @description Comparison value in the same unit. + */ + baseline: number; + /** + * Sample Size + * @description Number of requests in the alert window. + */ + sample_size: number; + /** + * Unit + * @description Unit for both values. + * @enum {string} + */ + unit: "usd" | "requests" | "milliseconds" | "ratio"; + /** + * Value + * @description Observed metric value in the alert window. + */ + value: number; + }; /** MetricWithMetadata */ MetricWithMetadata: { /** Api Key Breakdown */ @@ -54531,6 +54700,110 @@ export interface operations { }; }; }; + list_alerts_alerts_get: { + parameters: { + query?: { + /** @description Status to list. */ + status?: "open" | "resolved" | "dismissed"; + /** @description Maximum alerts per page. */ + limit?: number; + /** @description next_cursor from the preceding page. */ + cursor?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertPage"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_alert_alerts__alert_id__get: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Alert ID returned by GET /alerts. */ + alert_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Alert"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_alert_alerts__alert_id__patch: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Alert ID returned by GET /alerts. */ + alert_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AlertUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Alert"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; anthropic_proxy_route_anthropic__endpoint__get: { parameters: { query?: never;