From baf5718b47fc5a4fee308cf6dfa9ae72c14f4b37 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 12 Nov 2026 10:23:03 +0800 Subject: [PATCH] feat(alerts): enable monitoring by default with a five-alert free allowance (#51123) * feat(alerts): add default-on monitoring and a five-alert free allowance * fix(alerts): handle monitoring races and retry-safe migration * fix(ui): match monitoring upgrade link to status --- .../migration.sql | 5 + .../litellm_proxy_extras/schema.prisma | 2 + litellm/models/performance_alert.py | 29 +++- litellm/proxy/_types.py | 5 +- litellm/proxy/alerting_endpoints/alerts.py | 31 +++- litellm/proxy/deps/repositories.py | 21 +++ litellm/proxy/proxy_server.py | 6 +- litellm/proxy/schema.prisma | 2 + .../spend_tracking/performance_alerts.py | 4 +- litellm/repositories/alert_repository.py | 12 +- .../performance_alert_repository.py | 85 ++++++++++- schema.prisma | 2 + tests/integration/database/test_alerts.py | 4 +- .../database/test_performance_alerts.py | 141 +++++++++++++++++- .../proxy/alerting_endpoints/test_alerts.py | 46 ++++++ .../test_performance_alert_repository.py | 70 ++++++++- .../Alerts/Alerts.integration.test.tsx | 120 ++++++++++++++- .../src/components/Alerts/Alerts.tsx | 19 +-- .../Alerts/PerformanceMonitoring.tsx | 99 ++++++++++++ ui/litellm-dashboard/src/i18n/locales/en.json | 12 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 112 +++++++++++++- 21 files changed, 789 insertions(+), 38 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261111000000_add_performance_alert_access/migration.sql create mode 100644 ui/litellm-dashboard/src/components/Alerts/PerformanceMonitoring.tsx diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261111000000_add_performance_alert_access/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261111000000_add_performance_alert_access/migration.sql new file mode 100644 index 00000000000..c878ea297dc --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261111000000_add_performance_alert_access/migration.sql @@ -0,0 +1,5 @@ +SET LOCAL lock_timeout = '10s'; + +ALTER TABLE "LiteLLM_PerformanceAlertState" + ADD COLUMN IF NOT EXISTS "enabled" BOOLEAN NOT NULL DEFAULT true, + ADD COLUMN IF NOT EXISTS "alerts_used" INTEGER; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 70d0536cbed..498f86cc40f 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2229,6 +2229,8 @@ model LiteLLM_Alert { } model LiteLLM_PerformanceAlertState { + enabled Boolean @default(true) + alerts_used Int? id Int @id owner String @client.VarChar(32) next_run_at DateTime @client.Timestamptz(3) diff --git a/litellm/models/performance_alert.py b/litellm/models/performance_alert.py index 37497b3742e..c07397b0ae1 100644 --- a/litellm/models/performance_alert.py +++ b/litellm/models/performance_alert.py @@ -10,6 +10,7 @@ from litellm.models.alert import MAX_ALERT_ID, AlertDetection, AlertType, Metric PerformanceScope: TypeAlias = Literal["gateway", "model", "team", "user", "key"] SCOPES: Final[tuple[PerformanceScope, ...]] = ("gateway", "model", "team", "user", "key") BASELINE_DAYS: Final = 7 +FREE_ALERT_LIMIT: Final = 5 MIN_ACTIVE_DAYS: Final = 4 SPEND_MULTIPLIER: Final = 3 @@ -83,11 +84,35 @@ class ResourceUnreadableError(Exception): Any other database error ends the run so the next run resumes from the same place.""" +class AlertMonitoring(BaseModel): + model_config = ConfigDict(frozen=True) + + enabled: bool = Field(description="Whether performance monitoring is enabled. Defaults to true.") + alerts_used: int = Field(ge=0, le=FREE_ALERT_LIMIT, description="Lifetime free allowance consumed, capped at five.") + alert_limit: int | None = Field(description="Free allowance, or null for Enterprise.") + disabled_by_config: bool = Field(description="Whether performance_alerts_enabled is false in proxy settings.") + + @property + def active(self) -> bool: + return self.enabled and (self.alert_limit is None or self.alerts_used < self.alert_limit) + + +class AlertMonitoringUpdate(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + enabled: bool = Field(strict=True, description="Enable or disable performance monitoring for this gateway.") + + +class AlertMonitoringStore(Protocol): + async def monitoring(self) -> AlertMonitoring: ... + async def set_enabled(self, enabled: bool) -> AlertMonitoring: ... + + class PerformanceAlertStore(Protocol): async def claim(self, interval: timedelta) -> Claim | None: ... async def key_index_ready(self) -> bool: ... - async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState: ... - async def skip_next(self, owner: str, state: SweepState) -> SweepState: ... + async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState | None: ... + async def skip_next(self, owner: str, state: SweepState) -> SweepState | None: ... def target_day(now: datetime) -> date: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ee99895e209..d4169d3ae04 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2968,10 +2968,11 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): ), ) performance_alerts_enabled: bool = Field( - False, + True, description=( "Check yesterday's usage for the gateway, models, teams, users and keys against the week before, " - "and save unusual spend, traffic, response time and failure rate changes as alerts. Off by default." + "and save unusual spend, traffic, response time and failure rate changes as alerts. On by default. " + "Set false to disable monitoring. Free gateways stop after five saved performance alerts." ), ) spend_capture_rate_check: SpendCaptureRateCheckSettings | None = Field( diff --git a/litellm/proxy/alerting_endpoints/alerts.py b/litellm/proxy/alerting_endpoints/alerts.py index 3adcab90ec8..a6ed06c3b96 100644 --- a/litellm/proxy/alerting_endpoints/alerts.py +++ b/litellm/proxy/alerting_endpoints/alerts.py @@ -12,9 +12,10 @@ from litellm.models.alert import ( AlertView, AlertViewPage, ) +from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.deps.repositories import AlertRepositoryDep +from litellm.proxy.deps.repositories import AlertMonitoringRepositoryDep, AlertRepositoryDep from litellm.types.proxy.auth.user_roles import LitellmUserRoles router: Final = APIRouter(tags=["alerts"]) @@ -22,11 +23,15 @@ Auth: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] AlertPath: TypeAlias = Annotated[AlertID, Path(description="Alert ID returned by GET /alerts.")] -def _authorized_store(user: UserAPIKeyAuth, repository: AlertStore | None, *, write: bool) -> AlertStore: +def _authorize(user: UserAPIKeyAuth, *, write: bool) -> None: if user.user_role != LitellmUserRoles.PROXY_ADMIN and ( write or user.user_role != LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY ): raise HTTPException(status_code=403, detail="Only gateway admins can manage alerts") + + +def _authorized_store(user: UserAPIKeyAuth, repository: AlertStore | None, *, write: bool) -> AlertStore: + _authorize(user, write=write) if repository is None: raise HTTPException(status_code=503, detail="Alerts require a database connection") return repository @@ -55,6 +60,28 @@ async def list_alerts( return AlertViewPage(data=await store.enrich(page.data), next_cursor=page.next_cursor) +@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read performance monitoring settings") +async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -> AlertMonitoring: + """Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin.""" + _authorize(user, write=False) + if repository is None: + raise HTTPException(status_code=503, detail="Alerts require a database connection") + return await repository.monitoring() + + +@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable performance monitoring") +async def update_monitoring( + data: AlertMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep +) -> AlertMonitoring: + """Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts.""" + _authorize(user, write=True) + if repository is None: + raise HTTPException(status_code=503, detail="Alerts require a database connection") + if data.enabled and (await repository.monitoring()).disabled_by_config: + raise HTTPException(status_code=409, detail="Performance monitoring is disabled in proxy settings") + return await repository.set_enabled(data.enabled) + + @router.get("/alerts/{alert_id}", response_model=AlertView, summary="Read an alert") async def get_alert(alert_id: AlertPath, user: Auth, repository: AlertRepositoryDep) -> AlertView: """Return the saved evidence and status. Requires a gateway admin or view-only admin.""" diff --git a/litellm/proxy/deps/repositories.py b/litellm/proxy/deps/repositories.py index 6e54658d13d..b75c82f7889 100644 --- a/litellm/proxy/deps/repositories.py +++ b/litellm/proxy/deps/repositories.py @@ -6,6 +6,7 @@ from litellm.models.alert import AlertStore from litellm.models.deployment_warning import DeploymentWarningIgnoreStore from litellm.models.fallback import RouterFallbackStore from litellm.models.gateway_activity import GatewayActivityStore +from litellm.models.performance_alert import AlertMonitoringStore from litellm.models.plugin_upload import PluginUploadStore from litellm.models.skill_submission import SkillSubmissionStore from litellm.models.user_banner import UserBannerStore @@ -14,6 +15,7 @@ from litellm.repositories.alert_repository import AlertRepository from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository from litellm.repositories.gateway_activity_repository import GatewayActivityRepository +from litellm.repositories.performance_alert_repository import PerformanceAlertRepository from litellm.repositories.plugin_upload_repository import PluginUploadRepository from litellm.repositories.skill_submission_repository import SkillSubmissionRepository from litellm.repositories.user_banner_repository import UserBannerRepository @@ -67,6 +69,25 @@ async def get_alert_repository(database: DatabaseDep) -> AlertStore | None: AlertRepositoryDep: TypeAlias = Annotated[AlertStore | None, Depends(get_alert_repository)] +async def get_alert_monitoring_repository(database: DatabaseDep) -> AlertMonitoringStore | None: + from litellm.proxy.proxy_server import general_settings_view, premium_user + + return ( + None + if database is None + else PerformanceAlertRepository( + database.prisma, + enterprise=premium_user is True, + configured=general_settings_view().get("performance_alerts_enabled", True) is True, + ) + ) + + +AlertMonitoringRepositoryDep: TypeAlias = Annotated[ + AlertMonitoringStore | None, Depends(get_alert_monitoring_repository) +] + + async def get_router_fallback_repository(database: DatabaseDep) -> RouterFallbackStore | None: return None if database is None else ConfigRepository(database.prisma, use_writer=True) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 014d39455de..6382c1a393e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11922,11 +11922,9 @@ class ProxyStartupEvent: from litellm.proxy.spend_tracking.performance_alerts import RUN_INTERVAL, run_performance_alerts from litellm.repositories.performance_alert_repository import PerformanceAlertRepository - performance_alert_repository: Final = PerformanceAlertRepository(prisma_client) - async def run_performance_alerts_job() -> None: - if _current_general_settings().get("performance_alerts_enabled") is True: - await run_performance_alerts(performance_alert_repository) + if _current_general_settings().get("performance_alerts_enabled", True) is True: + await run_performance_alerts(PerformanceAlertRepository(prisma_client, enterprise=premium_user is True)) scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # APScheduler does not provide typed job methods run_performance_alerts_job, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 70d0536cbed..498f86cc40f 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2229,6 +2229,8 @@ model LiteLLM_Alert { } model LiteLLM_PerformanceAlertState { + enabled Boolean @default(true) + alerts_used Int? id Int @id owner String @client.VarChar(32) next_run_at DateTime @client.Timestamptz(3) diff --git a/litellm/proxy/spend_tracking/performance_alerts.py b/litellm/proxy/spend_tracking/performance_alerts.py index bb9a47d02c5..b5249a9e8b8 100644 --- a/litellm/proxy/spend_tracking/performance_alerts.py +++ b/litellm/proxy/spend_tracking/performance_alerts.py @@ -22,7 +22,7 @@ PAGE_SIZE: Final = 25 PAUSE_SECONDS: Final = 0.1 -async def _step(store: PerformanceAlertStore, owner: str, state: SweepState) -> SweepState: +async def _step(store: PerformanceAlertStore, owner: str, state: SweepState) -> SweepState | None: """One page, falling back to a single resource and then stepping over it, so one bad resource cannot hold up the rest of the sweep.""" try: @@ -78,6 +78,8 @@ async def run_performance_alerts( verbose_proxy_logger.info("Performance alerts waiting for %s before checking keys", KEY_DATE_INDEX) break state = await _step(store, claim.owner, state) + if state is None: + return await sleep(PAUSE_SECONDS) if state.completed: verbose_proxy_logger.info( diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py index 82048d53d83..eccdf16263d 100644 --- a/litellm/repositories/alert_repository.py +++ b/litellm/repositories/alert_repository.py @@ -116,16 +116,21 @@ def _detection_key(detection: AlertDetection) -> str: return hashlib.sha256(json.dumps(detection.identity()).encode()).hexdigest() -async def record_alerts(transaction: AlertTransaction, detections: Sequence[AlertDetection]) -> None: +async def record_alerts( + transaction: AlertTransaction, detections: Sequence[AlertDetection], limit: int | None = None +) -> int: if not detections: - return - await transaction.execute_raw( + return 0 + return await transaction.execute_raw( """INSERT INTO "LiteLLM_Alert" (detection_key, type, resource_type, resource_id, window_start, window_end, evidence, created_at, updated_at) SELECT detection_key, type, resource_type, resource_id, window_start, window_end, evidence, now() AT TIME ZONE 'UTC', now() AT TIME ZONE 'UTC' FROM jsonb_to_recordset($1::jsonb) AS rows(detection_key text, type text, resource_type text, resource_id text, window_start timestamp, window_end timestamp, evidence jsonb) + WHERE NOT EXISTS (SELECT 1 FROM "LiteLLM_Alert" a WHERE a.detection_key = rows.detection_key) + ORDER BY resource_type, resource_id, type + LIMIT $2 ON CONFLICT (detection_key) DO NOTHING""", json.dumps( tuple( @@ -133,6 +138,7 @@ async def record_alerts(transaction: AlertTransaction, detections: Sequence[Aler for detection in detections ) ), + limit, ) diff --git a/litellm/repositories/performance_alert_repository.py b/litellm/repositories/performance_alert_repository.py index 6686d158ff7..5902a02dc0a 100644 --- a/litellm/repositories/performance_alert_repository.py +++ b/litellm/repositories/performance_alert_repository.py @@ -11,6 +11,8 @@ from typing_extensions import LiteralString from litellm.models.performance_alert import ( BASELINE_DAYS, + FREE_ALERT_LIMIT, + AlertMonitoring, Claim, PerformanceScope, ResourceUnreadableError, @@ -40,6 +42,13 @@ class _Database(Protocol): def tx(self, *, timeout: timedelta) -> AbstractAsyncContextManager[_Transaction]: ... +class _MonitoringRow(BaseModel): + model_config = ConfigDict(frozen=True) + + enabled: bool + alerts_used: int | None + + class _StateRow(BaseModel): model_config = ConfigDict(frozen=True) @@ -164,10 +173,55 @@ _INDEX_READY: Final = """SELECT 1 AS resource_id FROM pg_index WHERE indexrelid class PerformanceAlertRepository: - def __init__(self, prisma_client: "PrismaClient") -> None: + def __init__(self, prisma_client: "PrismaClient", *, enterprise: bool = False, configured: bool = True) -> None: self._database: Final = cast( # cast-ok: generated Prisma transactions expose the bounded SQL surface _Database, prisma_client.writer_db ) + self._enterprise: Final = enterprise + self._configured: Final = configured + + async def _monitoring(self, transaction: _Transaction, *, lock: bool = False) -> AlertMonitoring: + query: Final = 'SELECT enabled, alerts_used FROM "LiteLLM_PerformanceAlertState" WHERE id = 1' + rows: Final = await transaction.query_raw(query + (" FOR UPDATE" if lock else "")) + if not rows: + await transaction.execute_raw( + """INSERT INTO "LiteLLM_PerformanceAlertState" (id, owner, next_run_at, updated_at) + VALUES (1, '', now(), now() AT TIME ZONE 'UTC') ON CONFLICT (id) DO NOTHING""" + ) + return await self._monitoring(transaction, lock=lock) + row: Final = _MonitoringRow.model_validate(rows[0]) + if row.alerts_used is None: + await transaction.execute_raw( + """UPDATE "LiteLLM_PerformanceAlertState" SET alerts_used = ( + SELECT COUNT(*) FROM (SELECT 1 FROM "LiteLLM_Alert" + WHERE type != 'model_deprecation' LIMIT $1) existing + ) WHERE id = 1 AND alerts_used IS NULL""", + FREE_ALERT_LIMIT, + ) + return await self._monitoring(transaction, lock=lock) + return AlertMonitoring( + enabled=self._configured and row.enabled, + alerts_used=row.alerts_used, + alert_limit=None if self._enterprise else FREE_ALERT_LIMIT, + disabled_by_config=not self._configured, + ) + + async def monitoring(self) -> AlertMonitoring: + async with self._transaction() as transaction: + return await self._monitoring(transaction) + + async def set_enabled(self, enabled: bool) -> AlertMonitoring: + async with self._database.tx(timeout=timedelta(seconds=10)) as transaction: + await transaction.query_raw( + "SELECT set_config('statement_timeout', '6s', true), set_config('lock_timeout', '5s', true)" + ) + await self._monitoring(transaction, lock=True) + await transaction.execute_raw( + """UPDATE "LiteLLM_PerformanceAlertState" SET enabled = $1, + updated_at = now() AT TIME ZONE 'UTC' WHERE id = 1""", + enabled, + ) + return await self._monitoring(transaction) @asynccontextmanager async def _transaction(self) -> AsyncGenerator[_Transaction]: @@ -181,6 +235,8 @@ class PerformanceAlertRepository: interval.""" owner: Final = uuid.uuid4().hex async with self._transaction() as transaction: + if not (await self._monitoring(transaction)).active: + return None rows: Final = _STATE_ROWS.validate_python( await transaction.query_raw(_CLAIM, owner, f"{int(interval.total_seconds())} seconds") ) @@ -195,7 +251,7 @@ class PerformanceAlertRepository: async with self._transaction() as transaction: return bool(await transaction.query_raw(_INDEX_READY, f'"{KEY_DATE_INDEX}"')) - async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState: + async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState | None: """Evaluate the next `limit` resources and save their alerts together with the new position.""" from prisma.errors import RawQueryError @@ -208,21 +264,38 @@ class PerformanceAlertRepository: raise ResourceUnreadableError() from exc raise - async def _evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState: + async def _evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState | None: async with self._transaction() as transaction: + monitoring: Final = await self._monitoring(transaction, lock=True) + if not monitoring.active: + return None windows: Final = _WINDOWS.validate_python( await transaction.query_raw(_page_sql(state.scope, state.wrapped), *self._arguments(state, limit)) ) - await record_alerts( - transaction, tuple(chain.from_iterable(detect(state.scope, window, state.day) for window in windows)) + saved: Final = await record_alerts( + transaction, + tuple(chain.from_iterable(detect(state.scope, window, state.day) for window in windows)), + None if self._enterprise else FREE_ALERT_LIMIT - monitoring.alerts_used, ) + if saved: + await transaction.execute_raw( + """UPDATE "LiteLLM_PerformanceAlertState" + SET alerts_used = LEAST($1, alerts_used + $2) WHERE id = 1""", + FREE_ALERT_LIMIT, + saved, + ) + if not self._enterprise and monitoring.alerts_used + saved >= FREE_ALERT_LIMIT: + await self._save(transaction, owner, state) + return None advanced: Final = advance(state, tuple(window.resource_id for window in windows), limit) await self._save(transaction, owner, advanced) return advanced - async def skip_next(self, owner: str, state: SweepState) -> SweepState: + async def skip_next(self, owner: str, state: SweepState) -> SweepState | None: """Step over the next resource without evaluating it, for one that cannot be read within the limits.""" async with self._transaction() as transaction: + if not (await self._monitoring(transaction, lock=True)).active: + return None rows: Final = ( (_IdRow(resource_id="gateway"),) if state.scope == "gateway" diff --git a/schema.prisma b/schema.prisma index 70d0536cbed..498f86cc40f 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2229,6 +2229,8 @@ model LiteLLM_Alert { } model LiteLLM_PerformanceAlertState { + enabled Boolean @default(true) + alerts_used Int? id Int @id owner String @client.VarChar(32) next_run_at DateTime @client.Timestamptz(3) diff --git a/tests/integration/database/test_alerts.py b/tests/integration/database/test_alerts.py index dc62886b75b..aaf36dd7ef2 100644 --- a/tests/integration/database/test_alerts.py +++ b/tests/integration/database/test_alerts.py @@ -201,11 +201,13 @@ async def test_live_auth_denies_nonadmins_and_viewers_cannot_write(alerts: Alert for role in ("internal_user", "proxy_admin_viewer"): user: Final = scenario.user(user_role=role) key: Final = scenario.key(user_id=user) - for path in ("/alerts", f"/alerts/{saved.id}"): + for path in ("/alerts", f"/alerts/{saved.id}", "/alerts/monitoring"): response: Final = gateway.request("GET", path, key=key) assert response.status_code in ((200,) if role == "proxy_admin_viewer" else (401, 403)), response.text write: Final = gateway.request("PATCH", f"/alerts/{saved.id}", {"status": "dismissed"}, key=key) assert write.status_code in (401, 403), write.text + monitoring: Final = gateway.request("PATCH", "/alerts/monitoring", {"enabled": False}, key=key) + assert monitoring.status_code in (401, 403), monitoring.text assert await alerts.repository.get(int(saved.id)) == saved diff --git a/tests/integration/database/test_performance_alerts.py b/tests/integration/database/test_performance_alerts.py index 735f135065c..64a18e09fda 100644 --- a/tests/integration/database/test_performance_alerts.py +++ b/tests/integration/database/test_performance_alerts.py @@ -13,11 +13,14 @@ import pytest_asyncio from litellm_proxy_extras.request_log_indexes import ensure_request_log_indexes from psycopg import sql +from litellm.models.alert import AlertDetection, DeprecationEvidence +from litellm.models.performance_alert import FREE_ALERT_LIMIT, SweepOwnershipLostError, SweepState from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.spend_tracking.performance_alerts import run_performance_alerts from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.alert_repository import AlertRepository from litellm.repositories.performance_alert_repository import PerformanceAlertRepository +from tests.integration._support.client import Gateway from tests.integration._support.database import read_rows, write_rows DAY: Final = date(2026, 1, 8) @@ -87,6 +90,7 @@ class Fixture: prefix: str repository: PerformanceAlertRepository alerts: AlertRepository + client: PrismaClient def key(self, name: str) -> str: return hashlib.sha256(f"{self.prefix}-{name}".encode()).hexdigest() @@ -118,7 +122,7 @@ async def fixture() -> AsyncIterator[Fixture]: client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache())) await client.connect() write_rows('DELETE FROM "LiteLLM_PerformanceAlertState"', ()) - built: Final = Fixture(prefix, PerformanceAlertRepository(client), AlertRepository(client)) + built: Final = Fixture(prefix, PerformanceAlertRepository(client, enterprise=True), AlertRepository(client), client) try: yield built finally: @@ -215,3 +219,138 @@ async def test_a_timer_firing_just_before_the_run_is_due_still_takes_it( ("earlier-worker", f"{seconds_until_due} seconds"), ) assert (await fixture.repository.claim(timedelta(minutes=5)) is not None) is claimed + + +@pytest.mark.asyncio +async def test_free_allowance_is_atomic_and_enterprise_resumes_the_unfinished_page(fixture: Fixture) -> None: + free: Final = PerformanceAlertRepository(fixture.client) + for index in range(FREE_ALERT_LIMIT + 3): + _seed("team", f"{fixture.prefix}-{index}", Day(spend=10), Day(spend=100)) + claim: Final = await free.claim(timedelta(0)) + assert claim is not None + state: Final = SweepState(day=DAY, scope="team", cursor=fixture.prefix) + await asyncio.gather( + free.evaluate_page(claim.owner, state, 4), + free.evaluate_page(claim.owner, state.model_copy(update={"cursor": f"{fixture.prefix}-3"}), 4), + ) + assert len(fixture.saved()) == FREE_ALERT_LIMIT + assert (await free.monitoring()).alerts_used == FREE_ALERT_LIMIT + assert await free.claim(timedelta(0)) is None + assert await free.evaluate_page(claim.owner, state, 25) is None + upgraded: Final = await fixture.repository.claim(timedelta(0)) + assert upgraded is not None and upgraded.state is not None + assert not upgraded.state.completed + await fixture.repository.evaluate_page(upgraded.owner, upgraded.state, 25) + assert len(fixture.saved()) == FREE_ALERT_LIMIT + 3 + assert (await fixture.repository.monitoring()).alert_limit is None + assert await free.claim(timedelta(0)) is None + + +@pytest.mark.asyncio +async def test_retries_status_changes_deletion_and_reconnect_do_not_restore_allowance(fixture: Fixture) -> None: + free: Final = PerformanceAlertRepository(fixture.client) + _seed("team", f"{fixture.prefix}-a", Day(spend=10), Day(spend=100)) + claim: Final = await free.claim(timedelta(0)) + assert claim is not None + state: Final = SweepState(day=DAY, scope="team", cursor=fixture.prefix) + for _ in range(2): + await free.evaluate_page(claim.owner, state, 25) + assert (await free.monitoring()).alerts_used == 1 + (saved,) = read_rows('SELECT id FROM "LiteLLM_Alert" WHERE resource_id = %s', (f"{fixture.prefix}-a",)) + for status in ("resolved", "dismissed", "open"): + response: Final = await fixture.alerts.set_status(int(str(saved["id"])), status, "admin") + assert response is not None and response.status == status + write_rows('DELETE FROM "LiteLLM_Alert" WHERE id = %s', (saved["id"],)) + await free.set_enabled(False) + await free.set_enabled(True) + client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache())) + await client.connect() + try: + restarted: Final = PerformanceAlertRepository(client) + assert (await restarted.monitoring()).alerts_used == 1 + finally: + await client.disconnect() + + +@pytest.mark.asyncio +async def test_opt_out_is_shared_and_config_false_cannot_be_overridden(fixture: Fixture, gateway: Gateway) -> None: + response: Final = gateway.request("GET", "/alerts/monitoring") + assert response.status_code == 200, response.text + assert response.json() == {"enabled": True, "alerts_used": 0, "alert_limit": 5, "disabled_by_config": False} + claim: Final = await fixture.repository.claim(timedelta(0)) + assert claim is not None + disabled: Final = gateway.request("PATCH", "/alerts/monitoring", {"enabled": False}) + assert disabled.status_code == 200, disabled.text + assert disabled.json()["enabled"] is False + assert await fixture.repository.claim(timedelta(0)) is None + assert await fixture.repository.evaluate_page(claim.owner, SweepState(day=DAY), 25) is None + enabled: Final = gateway.request("PATCH", "/alerts/monitoring", {"enabled": True}) + assert enabled.status_code == 200, enabled.text + assert (await fixture.repository.monitoring()).enabled + configured_off: Final = PerformanceAlertRepository(fixture.client, configured=False) + assert not (await configured_off.set_enabled(True)).enabled + assert await configured_off.claim(timedelta(0)) is None + + +@pytest.mark.asyncio +async def test_admin_can_opt_out_while_a_detection_batch_holds_the_state_lock( + fixture: Fixture, + gateway: Gateway, +) -> None: + await fixture.repository.monitoring() + with psycopg.connect(os.environ["DATABASE_URL"]) as detector: + detector.execute('SELECT id FROM "LiteLLM_PerformanceAlertState" WHERE id = 1 FOR UPDATE') + request: Final = asyncio.create_task( + asyncio.to_thread(gateway.request, "PATCH", "/alerts/monitoring", {"enabled": False}) + ) + try: + async with asyncio.timeout(3): + while not request.done(): + waiting: Final = read_rows( + "SELECT pid FROM pg_stat_activity WHERE %s::int = ANY(pg_blocking_pids(pid)) " + "AND clock_timestamp() - query_start > interval '200 milliseconds'", + (str(detector.info.backend_pid),), + ) + if waiting: + break + await asyncio.sleep(0.01) + finally: + detector.rollback() + response: Final = await asyncio.wait_for(request, timeout=5) + assert response.status_code == 200, response.text + assert response.json()["enabled"] is False + assert not (await fixture.repository.monitoring()).enabled + assert await fixture.repository.claim(timedelta(0)) is None + + +@pytest.mark.asyncio +async def test_existing_performance_alerts_consume_allowance_but_deprecations_do_not(fixture: Fixture) -> None: + start: Final = datetime(2026, 1, 8, tzinfo=timezone.utc) + await fixture.alerts.record( + AlertDetection( + type="model_deprecation", + resource_type="model", + resource_id=fixture.prefix, + window_start=start, + window_end=start + timedelta(days=1), + evidence=DeprecationEvidence(deprecation_date=DAY + timedelta(days=90)), + ) + ) + assert (await fixture.repository.monitoring()).alerts_used == 0 + _seed("team", f"{fixture.prefix}-a", Day(spend=10), Day(spend=100)) + await fixture.sweep() + write_rows('UPDATE "LiteLLM_PerformanceAlertState" SET alerts_used = NULL', ()) + free: Final = PerformanceAlertRepository(fixture.client) + assert (await free.monitoring()).alerts_used == 1 + + +@pytest.mark.asyncio +async def test_losing_ownership_rolls_back_alerts_and_allowance(fixture: Fixture) -> None: + free: Final = PerformanceAlertRepository(fixture.client) + for index in range(FREE_ALERT_LIMIT): + _seed("team", f"{fixture.prefix}-{index}", Day(spend=10), Day(spend=100)) + assert await free.claim(timedelta(0)) is not None + with pytest.raises(SweepOwnershipLostError): + await free.evaluate_page("old-owner", SweepState(day=DAY, scope="team", cursor=fixture.prefix), 25) + assert fixture.saved() == {} + assert (await free.monitoring()).alerts_used == 0 diff --git a/tests/unit/proxy/alerting_endpoints/test_alerts.py b/tests/unit/proxy/alerting_endpoints/test_alerts.py index bb651ef3807..97d13c176ff 100644 --- a/tests/unit/proxy/alerting_endpoints/test_alerts.py +++ b/tests/unit/proxy/alerting_endpoints/test_alerts.py @@ -12,7 +12,9 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.alerting_endpoints.alerts import router from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.deps.database import get_database +from litellm.proxy.deps.repositories import get_alert_monitoring_repository from litellm.repositories.database import Database +from litellm.repositories.performance_alert_repository import PerformanceAlertRepository from litellm.types.proxy.auth.user_roles import LitellmUserRoles from tests._support.native_database import native_database @@ -93,6 +95,50 @@ def test_invalid_page_never_reads_database(client: TestClient, table: MagicMock, table.find_many.assert_not_awaited() +@pytest.mark.parametrize( + "role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, LitellmUserRoles.PROXY_ADMIN) +) +@pytest.mark.parametrize("method", ("GET", "PATCH")) +def test_monitoring_authorization_precedes_missing_database(app: FastAPI, role: LitellmUserRoles, method: str) -> None: + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role) + app.dependency_overrides[get_database] = lambda: None + with TestClient(app) as client: + response: Final = client.request( + method, "/alerts/monitoring", json={"enabled": False} if method == "PATCH" else None + ) + allowed: Final = role == LitellmUserRoles.PROXY_ADMIN or ( + role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY and method == "GET" + ) + assert response.status_code == (503 if allowed else 403), response.text + + +@pytest.mark.parametrize( + "body", ({"enabled": True, "alerts_used": 0}, {"enabled": True, "alert_limit": None}, {"enabled": "false"}, {}) +) +def test_monitoring_only_accepts_a_boolean_opt_out( + client: TestClient, history_query: AsyncMock, body: dict[str, object] +) -> None: + response: Final = client.patch("/alerts/monitoring", json=body) + assert response.status_code == 422, response.text + history_query.assert_not_awaited() + + +def test_config_opt_out_cannot_be_enabled_from_the_api(app: FastAPI) -> None: + prisma: Final = MagicMock() + transaction: Final = prisma.writer_db.tx.return_value.__aenter__.return_value + transaction.query_raw = AsyncMock(return_value=[{"enabled": True, "alerts_used": 0}]) + transaction.execute_raw = AsyncMock() + app.dependency_overrides[get_alert_monitoring_repository] = lambda: PerformanceAlertRepository( + prisma, configured=False + ) + with TestClient(app) as client: + status: Final = client.get("/alerts/monitoring") + response: Final = client.patch("/alerts/monitoring", json={"enabled": True}) + assert status.json() == {"enabled": False, "alerts_used": 0, "alert_limit": 5, "disabled_by_config": True} + assert response.status_code == 409, response.text + transaction.execute_raw.assert_not_awaited() + + @pytest.mark.parametrize("alert_id", ("0", "-1", "01", "1.0", "oops", "9223372036854775808", "1" * 20)) @pytest.mark.parametrize("method", ("GET", "PATCH")) def test_invalid_id_never_accesses_database(client: TestClient, table: MagicMock, alert_id: str, method: str) -> None: diff --git a/tests/unit/repositories/test_performance_alert_repository.py b/tests/unit/repositories/test_performance_alert_repository.py index 2ea3bc98e0f..6210ff5c9d1 100644 --- a/tests/unit/repositories/test_performance_alert_repository.py +++ b/tests/unit/repositories/test_performance_alert_repository.py @@ -1,5 +1,5 @@ from collections.abc import Mapping, Sequence -from datetime import date, timedelta +from datetime import date, datetime, timedelta, timezone from typing import Final from unittest.mock import ANY, AsyncMock, MagicMock @@ -7,6 +7,7 @@ import pytest from pydantic import TypeAdapter from litellm.models.performance_alert import ResourceUnreadableError, SweepOwnershipLostError, SweepState +from litellm.proxy.spend_tracking.performance_alerts import run_performance_alerts from litellm.repositories.performance_alert_repository import PerformanceAlertRepository @@ -38,13 +39,17 @@ async def test_only_a_statement_timeout_marks_a_resource_unreadable(message: str def _repository( rows: Sequence[Mapping[str, object]] | Exception, + *, + enabled: bool = True, + used: int = 0, + enterprise: bool = False, ) -> tuple[PerformanceAlertRepository, AsyncMock, AsyncMock]: - query: Final = AsyncMock(side_effect=((), rows)) + query: Final = AsyncMock(side_effect=((), ({"enabled": enabled, "alerts_used": used},), rows)) execute: Final = AsyncMock(return_value=1) transaction: Final = MagicMock(query_raw=query, execute_raw=execute) context: Final = MagicMock(__aenter__=AsyncMock(return_value=transaction), __aexit__=AsyncMock(return_value=False)) client: Final = MagicMock(writer_db=MagicMock(tx=MagicMock(return_value=context))) - return PerformanceAlertRepository(client), query, execute + return PerformanceAlertRepository(client, enterprise=enterprise), query, execute @pytest.mark.asyncio @@ -129,7 +134,7 @@ async def test_a_page_saves_the_detected_evidence_and_its_resume_position(spend_ @pytest.mark.asyncio async def test_lost_ownership_aborts_the_page_transaction(spend_window: Mapping[str, object]) -> None: repository, _query, execute = _repository((spend_window,)) - execute.side_effect = (1, 0) + execute.side_effect = (1, 1, 0) with pytest.raises(SweepOwnershipLostError): await repository.evaluate_page("previous-worker", SweepState(day=date(2026, 1, 8), scope="model"), 1) @@ -157,3 +162,60 @@ async def test_skipping_an_unreadable_resource_saves_progress_without_counting_i execute.assert_awaited_once_with( ANY, "worker-a", "2026-01-08", "team", "team-b", wrapped, 0, 1, False, "team", "team-z" ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("enabled", "used"), ((True, 5), (False, 0))) +async def test_stopped_monitoring_never_reads_usage_or_claims_work(enabled: bool, used: int) -> None: + repository, query, execute = _repository((), enabled=enabled, used=used) + await run_performance_alerts(repository, now=lambda: datetime(2026, 1, 9, tzinfo=timezone.utc)) + assert query.await_count == 2 + execute.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enterprise", (False, True)) +async def test_first_read_initializes_the_allowance_once(enterprise: bool) -> None: + repository, query, execute = _repository((), enterprise=enterprise) + saved: Final = ({"enabled": True, "alerts_used": 3},) + query.side_effect = ((), (), ({"enabled": True, "alerts_used": None},), saved, (), saved) + first: Final = await repository.monitoring() + assert first.alerts_used == 3 + assert first.alert_limit == (None if enterprise else 5) + assert first.active + assert await repository.monitoring() == first + assert execute.await_count == 2 + execute.assert_awaited_with(ANY, 5) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enabled", (False, True)) +async def test_admin_toggle_persists_the_requested_value(enabled: bool) -> None: + repository, query, execute = _repository(()) + query.side_effect = ((), ({"enabled": not enabled, "alerts_used": 2},), ({"enabled": enabled, "alerts_used": 2},)) + result: Final = await repository.set_enabled(enabled) + assert result.enabled is enabled + assert result.active is enabled + assert result.alerts_used == 2 + execute.assert_awaited_once_with(ANY, enabled) + + +@pytest.mark.asyncio +async def test_last_free_alert_keeps_the_page_available_for_an_enterprise_upgrade( + spend_window: Mapping[str, object], +) -> None: + repository, _query, execute = _repository((spend_window,), used=4) + state: Final = SweepState(day=date(2026, 1, 8), scope="model") + assert await repository.evaluate_page("worker-a", state, 1) is None + execute.assert_any_await(ANY, ANY, 1) + execute.assert_any_await(ANY, 5, 1) + execute.assert_awaited_with(ANY, "worker-a", "2026-01-08", "model", "", False, 0, 0, False, None, "") + + +@pytest.mark.asyncio +async def test_enterprise_keeps_saving_alerts_after_the_free_allowance(spend_window: Mapping[str, object]) -> None: + repository, _query, execute = _repository((spend_window,), used=5, enterprise=True) + state: Final = SweepState(day=date(2026, 1, 8), scope="model") + result: Final = await repository.evaluate_page("worker-a", state, 1) + assert result == state.model_copy(update={"cursor": "model-a", "checked": 1}) + execute.assert_any_await(ANY, ANY, None) diff --git a/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx b/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx index 2b7c7c0d93b..073d22e52f5 100644 --- a/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import userEvent from "@testing-library/user-event"; -import { renderWithProviders, screen, testQueryClient, within } from "../../../tests/test-utils"; +import { act, renderWithProviders, screen, testQueryClient, waitFor, within } from "../../../tests/test-utils"; import Alerts from "./Alerts"; import type { SavedAlert } from "./alertPresentation"; @@ -27,6 +27,8 @@ const calls: { path: string; query: URLSearchParams; method: string; body: unkno let status = "open"; let rejectWrite = false; let rejectHistory = false; +let monitoring = { enabled: true, alerts_used: 0, alert_limit: 5 as number | null, disabled_by_config: false }; +let rejectMonitoring = false; function response(data: unknown, code = 200) { return new Response(JSON.stringify(data), { status: code, headers: { "Content-Type": "application/json" } }); } @@ -38,6 +40,8 @@ describe("Alerts", () => { status = "open"; rejectWrite = false; rejectHistory = false; + rejectMonitoring = false; + monitoring = { enabled: true, alerts_used: 0, alert_limit: 5, disabled_by_config: false }; role = "proxy_admin"; document.cookie = `token=${btoa(JSON.stringify({ alg: "none" }))}.${btoa(JSON.stringify({ key: "example-session", exp: 4102444800 }))}.example; path=/`; vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { @@ -49,6 +53,14 @@ describe("Alerts", () => { if (url.pathname === "/litellm/.well-known/litellm-ui-config") return response({}); if (url.pathname === "/management/v1/users/me") return response({ user_id: "example-admin", user_role: role }); if (url.pathname === "/model/deprecations") return response({ deprecated: [], imminent: [] }); + if (url.pathname === "/alerts/monitoring") { + if (rejectMonitoring) return response({ detail: "Unavailable" }, 503); + if (request.method === "PATCH") { + if (rejectWrite) return response({ detail: "Unavailable" }, 503); + monitoring = { ...monitoring, enabled: body.enabled }; + } + return response(monitoring); + } if (request.method === "PATCH") { if (rejectWrite) return response({ detail: "Unavailable" }, 503); status = body.status; @@ -149,4 +161,110 @@ describe("Alerts", () => { expect(screen.queryByRole("button", { name: "Dismiss" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Mark resolved" })).not.toBeInTheDocument(); }); + it("defaults on and persists opting out through the monitoring API", async () => { + const user = userEvent.setup(); + renderWithProviders(); + const toggle = await screen.findByRole("switch", { name: "Performance monitoring" }); + expect(toggle).toBeChecked(); + expect(screen.getByText("0 of 5 free alerts used.")).toBeInTheDocument(); + await user.click(toggle); + expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument(); + expect(toggle).not.toBeChecked(); + expect(calls.find((call) => call.method === "PATCH")).toMatchObject({ + path: "/alerts/monitoring", + body: { enabled: false }, + }); + expect(screen.getByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument(); + await user.click(toggle); + expect(await screen.findByText("0 of 5 free alerts used.")).toBeInTheDocument(); + }); + + it("explains exhaustion without blocking saved alerts or model deprecations", async () => { + monitoring.alerts_used = 5; + const user = userEvent.setup(); + renderWithProviders(); + expect( + await screen.findByText("You’ve used all 5 free alerts. New performance alerts are paused."), + ).toBeInTheDocument(); + expect(screen.getByRole("link", { name: "Upgrade to Enterprise" })).toHaveAttribute( + "href", + "https://www.litellm.ai/enterprise", + ); + expect(screen.getByRole("button", { name: "Mark resolved" })).toBeEnabled(); + expect(screen.getByRole("button", { name: "Set up email reminders" })).toBeInTheDocument(); + await user.click(screen.getByRole("switch", { name: "Performance monitoring" })); + expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument(); + expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument(); + }); + + it("keeps the saved toggle value when an older refresh finishes later", async () => { + const user = userEvent.setup(); + renderWithProviders(); + const toggle = await screen.findByRole("switch", { name: "Performance monitoring" }); + const originalFetch = vi.mocked(globalThis.fetch).getMockImplementation()!; + const staleResponse = response(monitoring); + const pendingRefresh = Promise.withResolvers(); + vi.mocked(globalThis.fetch).mockImplementation((input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + if (new URL(request.url).pathname === "/alerts/monitoring" && request.method === "GET") { + return pendingRefresh.promise; + } + return originalFetch(input, init); + }); + await user.click(screen.getByRole("button", { name: "Refresh" })); + await user.click(toggle); + expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument(); + await act(async () => { + pendingRefresh.resolve(staleResponse); + await pendingRefresh.promise; + }); + await waitFor(() => expect(testQueryClient.isFetching({ queryKey: ["alert-monitoring"] })).toBe(0)); + expect(toggle).not.toBeChecked(); + expect(screen.getByText("Off. Saved alerts remain available.")).toBeInTheDocument(); + }); + + it("shows Enterprise monitoring without a free limit", async () => { + monitoring.alert_limit = null; + monitoring.alerts_used = 5; + renderWithProviders(); + expect(await screen.findByText("On with Enterprise.")).toBeInTheDocument(); + expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument(); + }); + + it("respects config opt-out and view-only permissions", async () => { + monitoring.disabled_by_config = true; + monitoring.alerts_used = 5; + monitoring.enabled = false; + renderWithProviders(); + expect(await screen.findByText("Disabled in proxy settings.")).toBeInTheDocument(); + expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument(); + expect(screen.getByRole("switch", { name: "Performance monitoring" })).toHaveAttribute("aria-disabled", "true"); + }); + + it("lets viewers see monitoring status without a toggle", async () => { + role = "proxy_admin_viewer"; + renderWithProviders(); + expect(await screen.findByText("0 of 5 free alerts used.")).toBeInTheDocument(); + expect(screen.queryByRole("switch")).not.toBeInTheDocument(); + }); + + it("keeps the saved preference when a monitoring change fails", async () => { + rejectWrite = true; + const user = userEvent.setup(); + renderWithProviders(); + await user.click(await screen.findByRole("switch", { name: "Performance monitoring" })); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not save monitoring settings"); + expect(screen.getByRole("switch", { name: "Performance monitoring" })).toBeChecked(); + }); + + it("can retry monitoring settings without hiding the inbox", async () => { + rejectMonitoring = true; + const user = userEvent.setup(); + renderWithProviders(); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not load monitoring settings"); + expect(screen.getByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument(); + rejectMonitoring = false; + await user.click(screen.getByRole("button", { name: "Retry" })); + expect(await screen.findByRole("switch", { name: "Performance monitoring" })).toBeChecked(); + }); }); diff --git a/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx b/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx index 2f9d502741b..3a4d8127bea 100644 --- a/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx +++ b/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx @@ -3,6 +3,7 @@ import { parseAsBoolean, parseAsString, parseAsStringEnum, parseAsStringLiteral, useQueryStates } from "nuqs"; import { RefreshCw } from "lucide-react"; import { useTranslation } from "react-i18next"; +import { useQueryClient } from "@tanstack/react-query"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { Page, PageTabs, PageTabsList, PageTabsTrigger } from "@/components/shared/Page"; import { Button } from "@/components/ui/button"; @@ -17,6 +18,7 @@ import { ALERT_TYPES, AlertFilter, AlertItem, TYPE_LABEL_KEYS, presentAlert } fr import AlertActions from "./AlertActions"; import AlertChart from "./AlertChart"; import AlertDetail from "./AlertDetail"; +import PerformanceMonitoring from "./PerformanceMonitoring"; const parsers = { alert: parseAsString, @@ -97,6 +99,7 @@ function SelectedAlert({ auth, onBack }: { auth: ReturnType }) { const { t } = useTranslation(); + const client = useQueryClient(); const [params, setParams] = useQueryStates(parsers); const scope = JSON.stringify([auth.userId, auth.userRole]); const showDeprecations = params.type === "all" || params.type === "model-deprecations"; @@ -113,9 +116,15 @@ function AlertsInbox({ auth }: { auth: ReturnType }) { const items = [...retirementItems, ...performanceItems]; const loading = list.isLoading || deprecations.isLoading; const hasError = (showPerformance && list.isError) || (showDeprecations && deprecations.isError); + function refresh() { + void client.invalidateQueries({ queryKey: ["alert-monitoring"] }); + if (showPerformance) void list.refetch(); + if (showDeprecations) void deprecations.refetch(); + } return ( <>

{t("alerts.unified.title")}

+ void setParams({ state: value as AlertStatus })} @@ -142,15 +151,7 @@ function AlertsInbox({ auth }: { auth: ReturnType }) { ))} - diff --git a/ui/litellm-dashboard/src/components/Alerts/PerformanceMonitoring.tsx b/ui/litellm-dashboard/src/components/Alerts/PerformanceMonitoring.tsx new file mode 100644 index 00000000000..1839bf00abd --- /dev/null +++ b/ui/litellm-dashboard/src/components/Alerts/PerformanceMonitoring.tsx @@ -0,0 +1,99 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/components/ui/button"; +import { Switch } from "@/components/ui/switch"; +import { backgroundFetchClient } from "@/lib/http/api"; +import { getProxyBaseUrl } from "@/components/networking"; +import type { components } from "@/lib/http/schema"; + +function statusMessage(data: components["schemas"]["AlertMonitoring"]) { + if (data.disabled_by_config) return "alerts.monitoring.disabledByConfig"; + if (!data.enabled) return "alerts.monitoring.off"; + if (data.alert_limit === null) return "alerts.monitoring.enterprise"; + if (data.alerts_used >= data.alert_limit) return "alerts.monitoring.exhausted"; + return "alerts.monitoring.allowance"; +} + +export default function PerformanceMonitoring({ scope, readOnly }: { scope: string; readOnly: boolean }) { + const { t } = useTranslation(); + const client = useQueryClient(); + const queryKey = ["alert-monitoring", getProxyBaseUrl(), scope]; + const queryOptions = { + queryKey, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const { data } = await backgroundFetchClient.GET("/alerts/monitoring", { signal }); + if (!data) throw new Error("Could not load monitoring settings"); + return data; + }, + staleTime: 60_000, + retry: false, + }; + const query = useQuery(queryOptions); + const mutation = useMutation({ + mutationFn: async (enabled: boolean) => { + const { data } = await backgroundFetchClient.PATCH("/alerts/monitoring", { body: { enabled } }); + if (!data) throw new Error("Could not save monitoring settings"); + return data; + }, + onSuccess: async (data) => { + await client.cancelQueries({ queryKey }); + client.setQueryData(queryKey, data); + }, + }); + if (query.isError) + return ( +
+

{t("alerts.monitoring.loadError")}

+ +
+ ); + if (!query.data) + return ( +

+ {t("alerts.monitoring.loading")} +

+ ); + const data = query.data; + const messageKey = statusMessage(data); + const message = t(messageKey, { used: data.alerts_used, limit: data.alert_limit }); + return ( +
+
+
+ + {!readOnly && ( + mutation.mutate(enabled)} + aria-describedby="performance-monitoring-status" + /> + )} +
+

+ {message} + {messageKey === "alerts.monitoring.exhausted" && ( + + {t("alerts.monitoring.upgrade")} + + )} +

+
+ {mutation.isError && ( +

+ {t("alerts.monitoring.saveError")} +

+ )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index 06db34266bd..626b5f64350 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -5998,6 +5998,18 @@ "showMore": "Show {{hidden}} more usage tags", "showLess": "Show less", "more": "+{{hidden}} more" + }, + "monitoring": { + "title": "Performance monitoring", + "loading": "Loading monitoring settings…", + "allowance": "{{used}} of {{limit}} free alerts used.", + "enterprise": "On with Enterprise.", + "off": "Off. Saved alerts remain available.", + "exhausted": "You’ve used all 5 free alerts. New performance alerts are paused.", + "disabledByConfig": "Disabled in proxy settings.", + "upgrade": "Upgrade to Enterprise", + "loadError": "Could not load monitoring settings.", + "saveError": "Could not save monitoring settings. Try again." } }, "caching": { diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 4eb22db44c9..0825dbbd473 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1022,6 +1022,30 @@ export interface paths { patch?: never; trace?: never; }; + "/alerts/monitoring": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Read performance monitoring settings + * @description Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin. + */ + get: operations["get_monitoring_alerts_monitoring_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** + * Enable or disable performance monitoring + * @description Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts. + */ + patch: operations["update_monitoring_alerts_monitoring_patch"]; + trace?: never; + }; "/alerts/{alert_id}": { parameters: { query?: never; @@ -27229,6 +27253,37 @@ export interface components { */ value: number; }; + /** AlertMonitoring */ + AlertMonitoring: { + /** + * Alert Limit + * @description Free allowance, or null for Enterprise. + */ + alert_limit: number | null; + /** + * Alerts Used + * @description Lifetime free allowance consumed, capped at five. + */ + alerts_used: number; + /** + * Disabled By Config + * @description Whether performance_alerts_enabled is false in proxy settings. + */ + disabled_by_config: boolean; + /** + * Enabled + * @description Whether performance monitoring is enabled. Defaults to true. + */ + enabled: boolean; + }; + /** AlertMonitoringUpdate */ + AlertMonitoringUpdate: { + /** + * Enabled + * @description Enable or disable performance monitoring for this gateway. + */ + enabled: boolean; + }; /** * AlertType * @description Enum for alert types and management event types @@ -31596,8 +31651,8 @@ export interface components { password_policy_require_uppercase?: boolean | null; /** * Performance Alerts Enabled - * @description Check yesterday's usage for the gateway, models, teams, users and keys against the week before, and save unusual spend, traffic, response time and failure rate changes as alerts. Off by default. - * @default false + * @description Check yesterday's usage for the gateway, models, teams, users and keys against the week before, and save unusual spend, traffic, response time and failure rate changes as alerts. On by default. Set false to disable monitoring. Free gateways stop after five saved performance alerts. + * @default true */ performance_alerts_enabled: boolean; /** @@ -54833,6 +54888,59 @@ export interface operations { }; }; }; + get_monitoring_alerts_monitoring_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertMonitoring"]; + }; + }; + }; + }; + update_monitoring_alerts_monitoring_patch: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AlertMonitoringUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertMonitoring"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_alert_alerts__alert_id__get: { parameters: { query?: never;