mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
feat(alerts): enable monitoring by default with a five-alert free allowance (#51123)
* feat(alerts): add default-on monitoring and a five-alert free allowance * fix(alerts): handle monitoring races and retry-safe migration * fix(ui): match monitoring upgrade link to status
This commit is contained in:
moe-berri
GitHub
parent
16a36c66b4
commit
baf5718b47
21 changed files with 789 additions and 38 deletions
|
|
@ -0,0 +1,5 @@
|
|||
SET LOCAL lock_timeout = '10s';
|
||||
|
||||
ALTER TABLE "LiteLLM_PerformanceAlertState"
|
||||
ADD COLUMN IF NOT EXISTS "enabled" BOOLEAN NOT NULL DEFAULT true,
|
||||
ADD COLUMN IF NOT EXISTS "alerts_used" INTEGER;
|
||||
|
|
@ -2229,6 +2229,8 @@ model LiteLLM_Alert {
|
|||
}
|
||||
|
||||
model LiteLLM_PerformanceAlertState {
|
||||
enabled Boolean @default(true)
|
||||
alerts_used Int?
|
||||
id Int @id
|
||||
owner String @client.VarChar(32)
|
||||
next_run_at DateTime @client.Timestamptz(3)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from litellm.models.alert import MAX_ALERT_ID, AlertDetection, AlertType, Metric
|
|||
PerformanceScope: TypeAlias = Literal["gateway", "model", "team", "user", "key"]
|
||||
SCOPES: Final[tuple[PerformanceScope, ...]] = ("gateway", "model", "team", "user", "key")
|
||||
BASELINE_DAYS: Final = 7
|
||||
FREE_ALERT_LIMIT: Final = 5
|
||||
|
||||
MIN_ACTIVE_DAYS: Final = 4
|
||||
SPEND_MULTIPLIER: Final = 3
|
||||
|
|
@ -83,11 +84,35 @@ class ResourceUnreadableError(Exception):
|
|||
Any other database error ends the run so the next run resumes from the same place."""
|
||||
|
||||
|
||||
class AlertMonitoring(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
enabled: bool = Field(description="Whether performance monitoring is enabled. Defaults to true.")
|
||||
alerts_used: int = Field(ge=0, le=FREE_ALERT_LIMIT, description="Lifetime free allowance consumed, capped at five.")
|
||||
alert_limit: int | None = Field(description="Free allowance, or null for Enterprise.")
|
||||
disabled_by_config: bool = Field(description="Whether performance_alerts_enabled is false in proxy settings.")
|
||||
|
||||
@property
|
||||
def active(self) -> bool:
|
||||
return self.enabled and (self.alert_limit is None or self.alerts_used < self.alert_limit)
|
||||
|
||||
|
||||
class AlertMonitoringUpdate(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
enabled: bool = Field(strict=True, description="Enable or disable performance monitoring for this gateway.")
|
||||
|
||||
|
||||
class AlertMonitoringStore(Protocol):
|
||||
async def monitoring(self) -> AlertMonitoring: ...
|
||||
async def set_enabled(self, enabled: bool) -> AlertMonitoring: ...
|
||||
|
||||
|
||||
class PerformanceAlertStore(Protocol):
|
||||
async def claim(self, interval: timedelta) -> Claim | None: ...
|
||||
async def key_index_ready(self) -> bool: ...
|
||||
async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState: ...
|
||||
async def skip_next(self, owner: str, state: SweepState) -> SweepState: ...
|
||||
async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState | None: ...
|
||||
async def skip_next(self, owner: str, state: SweepState) -> SweepState | None: ...
|
||||
|
||||
|
||||
def target_day(now: datetime) -> date:
|
||||
|
|
|
|||
|
|
@ -2968,10 +2968,11 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
),
|
||||
)
|
||||
performance_alerts_enabled: bool = Field(
|
||||
False,
|
||||
True,
|
||||
description=(
|
||||
"Check yesterday's usage for the gateway, models, teams, users and keys against the week before, "
|
||||
"and save unusual spend, traffic, response time and failure rate changes as alerts. Off by default."
|
||||
"and save unusual spend, traffic, response time and failure rate changes as alerts. On by default. "
|
||||
"Set false to disable monitoring. Free gateways stop after five saved performance alerts."
|
||||
),
|
||||
)
|
||||
spend_capture_rate_check: SpendCaptureRateCheckSettings | None = Field(
|
||||
|
|
|
|||
|
|
@ -12,9 +12,10 @@ from litellm.models.alert import (
|
|||
AlertView,
|
||||
AlertViewPage,
|
||||
)
|
||||
from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.deps.repositories import AlertRepositoryDep
|
||||
from litellm.proxy.deps.repositories import AlertMonitoringRepositoryDep, AlertRepositoryDep
|
||||
from litellm.types.proxy.auth.user_roles import LitellmUserRoles
|
||||
|
||||
router: Final = APIRouter(tags=["alerts"])
|
||||
|
|
@ -22,11 +23,15 @@ Auth: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)]
|
|||
AlertPath: TypeAlias = Annotated[AlertID, Path(description="Alert ID returned by GET /alerts.")]
|
||||
|
||||
|
||||
def _authorized_store(user: UserAPIKeyAuth, repository: AlertStore | None, *, write: bool) -> AlertStore:
|
||||
def _authorize(user: UserAPIKeyAuth, *, write: bool) -> None:
|
||||
if user.user_role != LitellmUserRoles.PROXY_ADMIN and (
|
||||
write or user.user_role != LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY
|
||||
):
|
||||
raise HTTPException(status_code=403, detail="Only gateway admins can manage alerts")
|
||||
|
||||
|
||||
def _authorized_store(user: UserAPIKeyAuth, repository: AlertStore | None, *, write: bool) -> AlertStore:
|
||||
_authorize(user, write=write)
|
||||
if repository is None:
|
||||
raise HTTPException(status_code=503, detail="Alerts require a database connection")
|
||||
return repository
|
||||
|
|
@ -55,6 +60,28 @@ async def list_alerts(
|
|||
return AlertViewPage(data=await store.enrich(page.data), next_cursor=page.next_cursor)
|
||||
|
||||
|
||||
@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read performance monitoring settings")
|
||||
async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -> AlertMonitoring:
|
||||
"""Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin."""
|
||||
_authorize(user, write=False)
|
||||
if repository is None:
|
||||
raise HTTPException(status_code=503, detail="Alerts require a database connection")
|
||||
return await repository.monitoring()
|
||||
|
||||
|
||||
@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable performance monitoring")
|
||||
async def update_monitoring(
|
||||
data: AlertMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep
|
||||
) -> AlertMonitoring:
|
||||
"""Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts."""
|
||||
_authorize(user, write=True)
|
||||
if repository is None:
|
||||
raise HTTPException(status_code=503, detail="Alerts require a database connection")
|
||||
if data.enabled and (await repository.monitoring()).disabled_by_config:
|
||||
raise HTTPException(status_code=409, detail="Performance monitoring is disabled in proxy settings")
|
||||
return await repository.set_enabled(data.enabled)
|
||||
|
||||
|
||||
@router.get("/alerts/{alert_id}", response_model=AlertView, summary="Read an alert")
|
||||
async def get_alert(alert_id: AlertPath, user: Auth, repository: AlertRepositoryDep) -> AlertView:
|
||||
"""Return the saved evidence and status. Requires a gateway admin or view-only admin."""
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from litellm.models.alert import AlertStore
|
|||
from litellm.models.deployment_warning import DeploymentWarningIgnoreStore
|
||||
from litellm.models.fallback import RouterFallbackStore
|
||||
from litellm.models.gateway_activity import GatewayActivityStore
|
||||
from litellm.models.performance_alert import AlertMonitoringStore
|
||||
from litellm.models.plugin_upload import PluginUploadStore
|
||||
from litellm.models.skill_submission import SkillSubmissionStore
|
||||
from litellm.models.user_banner import UserBannerStore
|
||||
|
|
@ -14,6 +15,7 @@ from litellm.repositories.alert_repository import AlertRepository
|
|||
from litellm.repositories.config_repository import ConfigRepository
|
||||
from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository
|
||||
from litellm.repositories.gateway_activity_repository import GatewayActivityRepository
|
||||
from litellm.repositories.performance_alert_repository import PerformanceAlertRepository
|
||||
from litellm.repositories.plugin_upload_repository import PluginUploadRepository
|
||||
from litellm.repositories.skill_submission_repository import SkillSubmissionRepository
|
||||
from litellm.repositories.user_banner_repository import UserBannerRepository
|
||||
|
|
@ -67,6 +69,25 @@ async def get_alert_repository(database: DatabaseDep) -> AlertStore | None:
|
|||
AlertRepositoryDep: TypeAlias = Annotated[AlertStore | None, Depends(get_alert_repository)]
|
||||
|
||||
|
||||
async def get_alert_monitoring_repository(database: DatabaseDep) -> AlertMonitoringStore | None:
|
||||
from litellm.proxy.proxy_server import general_settings_view, premium_user
|
||||
|
||||
return (
|
||||
None
|
||||
if database is None
|
||||
else PerformanceAlertRepository(
|
||||
database.prisma,
|
||||
enterprise=premium_user is True,
|
||||
configured=general_settings_view().get("performance_alerts_enabled", True) is True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
AlertMonitoringRepositoryDep: TypeAlias = Annotated[
|
||||
AlertMonitoringStore | None, Depends(get_alert_monitoring_repository)
|
||||
]
|
||||
|
||||
|
||||
async def get_router_fallback_repository(database: DatabaseDep) -> RouterFallbackStore | None:
|
||||
return None if database is None else ConfigRepository(database.prisma, use_writer=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -11922,11 +11922,9 @@ class ProxyStartupEvent:
|
|||
from litellm.proxy.spend_tracking.performance_alerts import RUN_INTERVAL, run_performance_alerts
|
||||
from litellm.repositories.performance_alert_repository import PerformanceAlertRepository
|
||||
|
||||
performance_alert_repository: Final = PerformanceAlertRepository(prisma_client)
|
||||
|
||||
async def run_performance_alerts_job() -> None:
|
||||
if _current_general_settings().get("performance_alerts_enabled") is True:
|
||||
await run_performance_alerts(performance_alert_repository)
|
||||
if _current_general_settings().get("performance_alerts_enabled", True) is True:
|
||||
await run_performance_alerts(PerformanceAlertRepository(prisma_client, enterprise=premium_user is True))
|
||||
|
||||
scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # APScheduler does not provide typed job methods
|
||||
run_performance_alerts_job,
|
||||
|
|
|
|||
|
|
@ -2229,6 +2229,8 @@ model LiteLLM_Alert {
|
|||
}
|
||||
|
||||
model LiteLLM_PerformanceAlertState {
|
||||
enabled Boolean @default(true)
|
||||
alerts_used Int?
|
||||
id Int @id
|
||||
owner String @client.VarChar(32)
|
||||
next_run_at DateTime @client.Timestamptz(3)
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ PAGE_SIZE: Final = 25
|
|||
PAUSE_SECONDS: Final = 0.1
|
||||
|
||||
|
||||
async def _step(store: PerformanceAlertStore, owner: str, state: SweepState) -> SweepState:
|
||||
async def _step(store: PerformanceAlertStore, owner: str, state: SweepState) -> SweepState | None:
|
||||
"""One page, falling back to a single resource and then stepping over it, so one bad resource
|
||||
cannot hold up the rest of the sweep."""
|
||||
try:
|
||||
|
|
@ -78,6 +78,8 @@ async def run_performance_alerts(
|
|||
verbose_proxy_logger.info("Performance alerts waiting for %s before checking keys", KEY_DATE_INDEX)
|
||||
break
|
||||
state = await _step(store, claim.owner, state)
|
||||
if state is None:
|
||||
return
|
||||
await sleep(PAUSE_SECONDS)
|
||||
if state.completed:
|
||||
verbose_proxy_logger.info(
|
||||
|
|
|
|||
|
|
@ -116,16 +116,21 @@ def _detection_key(detection: AlertDetection) -> str:
|
|||
return hashlib.sha256(json.dumps(detection.identity()).encode()).hexdigest()
|
||||
|
||||
|
||||
async def record_alerts(transaction: AlertTransaction, detections: Sequence[AlertDetection]) -> None:
|
||||
async def record_alerts(
|
||||
transaction: AlertTransaction, detections: Sequence[AlertDetection], limit: int | None = None
|
||||
) -> int:
|
||||
if not detections:
|
||||
return
|
||||
await transaction.execute_raw(
|
||||
return 0
|
||||
return await transaction.execute_raw(
|
||||
"""INSERT INTO "LiteLLM_Alert"
|
||||
(detection_key, type, resource_type, resource_id, window_start, window_end, evidence, created_at, updated_at)
|
||||
SELECT detection_key, type, resource_type, resource_id, window_start, window_end, evidence,
|
||||
now() AT TIME ZONE 'UTC', now() AT TIME ZONE 'UTC'
|
||||
FROM jsonb_to_recordset($1::jsonb) AS rows(detection_key text, type text, resource_type text,
|
||||
resource_id text, window_start timestamp, window_end timestamp, evidence jsonb)
|
||||
WHERE NOT EXISTS (SELECT 1 FROM "LiteLLM_Alert" a WHERE a.detection_key = rows.detection_key)
|
||||
ORDER BY resource_type, resource_id, type
|
||||
LIMIT $2
|
||||
ON CONFLICT (detection_key) DO NOTHING""",
|
||||
json.dumps(
|
||||
tuple(
|
||||
|
|
@ -133,6 +138,7 @@ async def record_alerts(transaction: AlertTransaction, detections: Sequence[Aler
|
|||
for detection in detections
|
||||
)
|
||||
),
|
||||
limit,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ from typing_extensions import LiteralString
|
|||
|
||||
from litellm.models.performance_alert import (
|
||||
BASELINE_DAYS,
|
||||
FREE_ALERT_LIMIT,
|
||||
AlertMonitoring,
|
||||
Claim,
|
||||
PerformanceScope,
|
||||
ResourceUnreadableError,
|
||||
|
|
@ -40,6 +42,13 @@ class _Database(Protocol):
|
|||
def tx(self, *, timeout: timedelta) -> AbstractAsyncContextManager[_Transaction]: ...
|
||||
|
||||
|
||||
class _MonitoringRow(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
enabled: bool
|
||||
alerts_used: int | None
|
||||
|
||||
|
||||
class _StateRow(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
|
@ -164,10 +173,55 @@ _INDEX_READY: Final = """SELECT 1 AS resource_id FROM pg_index WHERE indexrelid
|
|||
|
||||
|
||||
class PerformanceAlertRepository:
|
||||
def __init__(self, prisma_client: "PrismaClient") -> None:
|
||||
def __init__(self, prisma_client: "PrismaClient", *, enterprise: bool = False, configured: bool = True) -> None:
|
||||
self._database: Final = cast( # cast-ok: generated Prisma transactions expose the bounded SQL surface
|
||||
_Database, prisma_client.writer_db
|
||||
)
|
||||
self._enterprise: Final = enterprise
|
||||
self._configured: Final = configured
|
||||
|
||||
async def _monitoring(self, transaction: _Transaction, *, lock: bool = False) -> AlertMonitoring:
|
||||
query: Final = 'SELECT enabled, alerts_used FROM "LiteLLM_PerformanceAlertState" WHERE id = 1'
|
||||
rows: Final = await transaction.query_raw(query + (" FOR UPDATE" if lock else ""))
|
||||
if not rows:
|
||||
await transaction.execute_raw(
|
||||
"""INSERT INTO "LiteLLM_PerformanceAlertState" (id, owner, next_run_at, updated_at)
|
||||
VALUES (1, '', now(), now() AT TIME ZONE 'UTC') ON CONFLICT (id) DO NOTHING"""
|
||||
)
|
||||
return await self._monitoring(transaction, lock=lock)
|
||||
row: Final = _MonitoringRow.model_validate(rows[0])
|
||||
if row.alerts_used is None:
|
||||
await transaction.execute_raw(
|
||||
"""UPDATE "LiteLLM_PerformanceAlertState" SET alerts_used = (
|
||||
SELECT COUNT(*) FROM (SELECT 1 FROM "LiteLLM_Alert"
|
||||
WHERE type != 'model_deprecation' LIMIT $1) existing
|
||||
) WHERE id = 1 AND alerts_used IS NULL""",
|
||||
FREE_ALERT_LIMIT,
|
||||
)
|
||||
return await self._monitoring(transaction, lock=lock)
|
||||
return AlertMonitoring(
|
||||
enabled=self._configured and row.enabled,
|
||||
alerts_used=row.alerts_used,
|
||||
alert_limit=None if self._enterprise else FREE_ALERT_LIMIT,
|
||||
disabled_by_config=not self._configured,
|
||||
)
|
||||
|
||||
async def monitoring(self) -> AlertMonitoring:
|
||||
async with self._transaction() as transaction:
|
||||
return await self._monitoring(transaction)
|
||||
|
||||
async def set_enabled(self, enabled: bool) -> AlertMonitoring:
|
||||
async with self._database.tx(timeout=timedelta(seconds=10)) as transaction:
|
||||
await transaction.query_raw(
|
||||
"SELECT set_config('statement_timeout', '6s', true), set_config('lock_timeout', '5s', true)"
|
||||
)
|
||||
await self._monitoring(transaction, lock=True)
|
||||
await transaction.execute_raw(
|
||||
"""UPDATE "LiteLLM_PerformanceAlertState" SET enabled = $1,
|
||||
updated_at = now() AT TIME ZONE 'UTC' WHERE id = 1""",
|
||||
enabled,
|
||||
)
|
||||
return await self._monitoring(transaction)
|
||||
|
||||
@asynccontextmanager
|
||||
async def _transaction(self) -> AsyncGenerator[_Transaction]:
|
||||
|
|
@ -181,6 +235,8 @@ class PerformanceAlertRepository:
|
|||
interval."""
|
||||
owner: Final = uuid.uuid4().hex
|
||||
async with self._transaction() as transaction:
|
||||
if not (await self._monitoring(transaction)).active:
|
||||
return None
|
||||
rows: Final = _STATE_ROWS.validate_python(
|
||||
await transaction.query_raw(_CLAIM, owner, f"{int(interval.total_seconds())} seconds")
|
||||
)
|
||||
|
|
@ -195,7 +251,7 @@ class PerformanceAlertRepository:
|
|||
async with self._transaction() as transaction:
|
||||
return bool(await transaction.query_raw(_INDEX_READY, f'"{KEY_DATE_INDEX}"'))
|
||||
|
||||
async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState:
|
||||
async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState | None:
|
||||
"""Evaluate the next `limit` resources and save their alerts together with the new position."""
|
||||
from prisma.errors import RawQueryError
|
||||
|
||||
|
|
@ -208,21 +264,38 @@ class PerformanceAlertRepository:
|
|||
raise ResourceUnreadableError() from exc
|
||||
raise
|
||||
|
||||
async def _evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState:
|
||||
async def _evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState | None:
|
||||
async with self._transaction() as transaction:
|
||||
monitoring: Final = await self._monitoring(transaction, lock=True)
|
||||
if not monitoring.active:
|
||||
return None
|
||||
windows: Final = _WINDOWS.validate_python(
|
||||
await transaction.query_raw(_page_sql(state.scope, state.wrapped), *self._arguments(state, limit))
|
||||
)
|
||||
await record_alerts(
|
||||
transaction, tuple(chain.from_iterable(detect(state.scope, window, state.day) for window in windows))
|
||||
saved: Final = await record_alerts(
|
||||
transaction,
|
||||
tuple(chain.from_iterable(detect(state.scope, window, state.day) for window in windows)),
|
||||
None if self._enterprise else FREE_ALERT_LIMIT - monitoring.alerts_used,
|
||||
)
|
||||
if saved:
|
||||
await transaction.execute_raw(
|
||||
"""UPDATE "LiteLLM_PerformanceAlertState"
|
||||
SET alerts_used = LEAST($1, alerts_used + $2) WHERE id = 1""",
|
||||
FREE_ALERT_LIMIT,
|
||||
saved,
|
||||
)
|
||||
if not self._enterprise and monitoring.alerts_used + saved >= FREE_ALERT_LIMIT:
|
||||
await self._save(transaction, owner, state)
|
||||
return None
|
||||
advanced: Final = advance(state, tuple(window.resource_id for window in windows), limit)
|
||||
await self._save(transaction, owner, advanced)
|
||||
return advanced
|
||||
|
||||
async def skip_next(self, owner: str, state: SweepState) -> SweepState:
|
||||
async def skip_next(self, owner: str, state: SweepState) -> SweepState | None:
|
||||
"""Step over the next resource without evaluating it, for one that cannot be read within the limits."""
|
||||
async with self._transaction() as transaction:
|
||||
if not (await self._monitoring(transaction, lock=True)).active:
|
||||
return None
|
||||
rows: Final = (
|
||||
(_IdRow(resource_id="gateway"),)
|
||||
if state.scope == "gateway"
|
||||
|
|
|
|||
|
|
@ -2229,6 +2229,8 @@ model LiteLLM_Alert {
|
|||
}
|
||||
|
||||
model LiteLLM_PerformanceAlertState {
|
||||
enabled Boolean @default(true)
|
||||
alerts_used Int?
|
||||
id Int @id
|
||||
owner String @client.VarChar(32)
|
||||
next_run_at DateTime @client.Timestamptz(3)
|
||||
|
|
|
|||
|
|
@ -201,11 +201,13 @@ async def test_live_auth_denies_nonadmins_and_viewers_cannot_write(alerts: Alert
|
|||
for role in ("internal_user", "proxy_admin_viewer"):
|
||||
user: Final = scenario.user(user_role=role)
|
||||
key: Final = scenario.key(user_id=user)
|
||||
for path in ("/alerts", f"/alerts/{saved.id}"):
|
||||
for path in ("/alerts", f"/alerts/{saved.id}", "/alerts/monitoring"):
|
||||
response: Final = gateway.request("GET", path, key=key)
|
||||
assert response.status_code in ((200,) if role == "proxy_admin_viewer" else (401, 403)), response.text
|
||||
write: Final = gateway.request("PATCH", f"/alerts/{saved.id}", {"status": "dismissed"}, key=key)
|
||||
assert write.status_code in (401, 403), write.text
|
||||
monitoring: Final = gateway.request("PATCH", "/alerts/monitoring", {"enabled": False}, key=key)
|
||||
assert monitoring.status_code in (401, 403), monitoring.text
|
||||
assert await alerts.repository.get(int(saved.id)) == saved
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,11 +13,14 @@ import pytest_asyncio
|
|||
from litellm_proxy_extras.request_log_indexes import ensure_request_log_indexes
|
||||
from psycopg import sql
|
||||
|
||||
from litellm.models.alert import AlertDetection, DeprecationEvidence
|
||||
from litellm.models.performance_alert import FREE_ALERT_LIMIT, SweepOwnershipLostError, SweepState
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.spend_tracking.performance_alerts import run_performance_alerts
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.repositories.alert_repository import AlertRepository
|
||||
from litellm.repositories.performance_alert_repository import PerformanceAlertRepository
|
||||
from tests.integration._support.client import Gateway
|
||||
from tests.integration._support.database import read_rows, write_rows
|
||||
|
||||
DAY: Final = date(2026, 1, 8)
|
||||
|
|
@ -87,6 +90,7 @@ class Fixture:
|
|||
prefix: str
|
||||
repository: PerformanceAlertRepository
|
||||
alerts: AlertRepository
|
||||
client: PrismaClient
|
||||
|
||||
def key(self, name: str) -> str:
|
||||
return hashlib.sha256(f"{self.prefix}-{name}".encode()).hexdigest()
|
||||
|
|
@ -118,7 +122,7 @@ async def fixture() -> AsyncIterator[Fixture]:
|
|||
client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache()))
|
||||
await client.connect()
|
||||
write_rows('DELETE FROM "LiteLLM_PerformanceAlertState"', ())
|
||||
built: Final = Fixture(prefix, PerformanceAlertRepository(client), AlertRepository(client))
|
||||
built: Final = Fixture(prefix, PerformanceAlertRepository(client, enterprise=True), AlertRepository(client), client)
|
||||
try:
|
||||
yield built
|
||||
finally:
|
||||
|
|
@ -215,3 +219,138 @@ async def test_a_timer_firing_just_before_the_run_is_due_still_takes_it(
|
|||
("earlier-worker", f"{seconds_until_due} seconds"),
|
||||
)
|
||||
assert (await fixture.repository.claim(timedelta(minutes=5)) is not None) is claimed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_free_allowance_is_atomic_and_enterprise_resumes_the_unfinished_page(fixture: Fixture) -> None:
|
||||
free: Final = PerformanceAlertRepository(fixture.client)
|
||||
for index in range(FREE_ALERT_LIMIT + 3):
|
||||
_seed("team", f"{fixture.prefix}-{index}", Day(spend=10), Day(spend=100))
|
||||
claim: Final = await free.claim(timedelta(0))
|
||||
assert claim is not None
|
||||
state: Final = SweepState(day=DAY, scope="team", cursor=fixture.prefix)
|
||||
await asyncio.gather(
|
||||
free.evaluate_page(claim.owner, state, 4),
|
||||
free.evaluate_page(claim.owner, state.model_copy(update={"cursor": f"{fixture.prefix}-3"}), 4),
|
||||
)
|
||||
assert len(fixture.saved()) == FREE_ALERT_LIMIT
|
||||
assert (await free.monitoring()).alerts_used == FREE_ALERT_LIMIT
|
||||
assert await free.claim(timedelta(0)) is None
|
||||
assert await free.evaluate_page(claim.owner, state, 25) is None
|
||||
upgraded: Final = await fixture.repository.claim(timedelta(0))
|
||||
assert upgraded is not None and upgraded.state is not None
|
||||
assert not upgraded.state.completed
|
||||
await fixture.repository.evaluate_page(upgraded.owner, upgraded.state, 25)
|
||||
assert len(fixture.saved()) == FREE_ALERT_LIMIT + 3
|
||||
assert (await fixture.repository.monitoring()).alert_limit is None
|
||||
assert await free.claim(timedelta(0)) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retries_status_changes_deletion_and_reconnect_do_not_restore_allowance(fixture: Fixture) -> None:
|
||||
free: Final = PerformanceAlertRepository(fixture.client)
|
||||
_seed("team", f"{fixture.prefix}-a", Day(spend=10), Day(spend=100))
|
||||
claim: Final = await free.claim(timedelta(0))
|
||||
assert claim is not None
|
||||
state: Final = SweepState(day=DAY, scope="team", cursor=fixture.prefix)
|
||||
for _ in range(2):
|
||||
await free.evaluate_page(claim.owner, state, 25)
|
||||
assert (await free.monitoring()).alerts_used == 1
|
||||
(saved,) = read_rows('SELECT id FROM "LiteLLM_Alert" WHERE resource_id = %s', (f"{fixture.prefix}-a",))
|
||||
for status in ("resolved", "dismissed", "open"):
|
||||
response: Final = await fixture.alerts.set_status(int(str(saved["id"])), status, "admin")
|
||||
assert response is not None and response.status == status
|
||||
write_rows('DELETE FROM "LiteLLM_Alert" WHERE id = %s', (saved["id"],))
|
||||
await free.set_enabled(False)
|
||||
await free.set_enabled(True)
|
||||
client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache()))
|
||||
await client.connect()
|
||||
try:
|
||||
restarted: Final = PerformanceAlertRepository(client)
|
||||
assert (await restarted.monitoring()).alerts_used == 1
|
||||
finally:
|
||||
await client.disconnect()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_opt_out_is_shared_and_config_false_cannot_be_overridden(fixture: Fixture, gateway: Gateway) -> None:
|
||||
response: Final = gateway.request("GET", "/alerts/monitoring")
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"enabled": True, "alerts_used": 0, "alert_limit": 5, "disabled_by_config": False}
|
||||
claim: Final = await fixture.repository.claim(timedelta(0))
|
||||
assert claim is not None
|
||||
disabled: Final = gateway.request("PATCH", "/alerts/monitoring", {"enabled": False})
|
||||
assert disabled.status_code == 200, disabled.text
|
||||
assert disabled.json()["enabled"] is False
|
||||
assert await fixture.repository.claim(timedelta(0)) is None
|
||||
assert await fixture.repository.evaluate_page(claim.owner, SweepState(day=DAY), 25) is None
|
||||
enabled: Final = gateway.request("PATCH", "/alerts/monitoring", {"enabled": True})
|
||||
assert enabled.status_code == 200, enabled.text
|
||||
assert (await fixture.repository.monitoring()).enabled
|
||||
configured_off: Final = PerformanceAlertRepository(fixture.client, configured=False)
|
||||
assert not (await configured_off.set_enabled(True)).enabled
|
||||
assert await configured_off.claim(timedelta(0)) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_can_opt_out_while_a_detection_batch_holds_the_state_lock(
|
||||
fixture: Fixture,
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
await fixture.repository.monitoring()
|
||||
with psycopg.connect(os.environ["DATABASE_URL"]) as detector:
|
||||
detector.execute('SELECT id FROM "LiteLLM_PerformanceAlertState" WHERE id = 1 FOR UPDATE')
|
||||
request: Final = asyncio.create_task(
|
||||
asyncio.to_thread(gateway.request, "PATCH", "/alerts/monitoring", {"enabled": False})
|
||||
)
|
||||
try:
|
||||
async with asyncio.timeout(3):
|
||||
while not request.done():
|
||||
waiting: Final = read_rows(
|
||||
"SELECT pid FROM pg_stat_activity WHERE %s::int = ANY(pg_blocking_pids(pid)) "
|
||||
"AND clock_timestamp() - query_start > interval '200 milliseconds'",
|
||||
(str(detector.info.backend_pid),),
|
||||
)
|
||||
if waiting:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
finally:
|
||||
detector.rollback()
|
||||
response: Final = await asyncio.wait_for(request, timeout=5)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["enabled"] is False
|
||||
assert not (await fixture.repository.monitoring()).enabled
|
||||
assert await fixture.repository.claim(timedelta(0)) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_existing_performance_alerts_consume_allowance_but_deprecations_do_not(fixture: Fixture) -> None:
|
||||
start: Final = datetime(2026, 1, 8, tzinfo=timezone.utc)
|
||||
await fixture.alerts.record(
|
||||
AlertDetection(
|
||||
type="model_deprecation",
|
||||
resource_type="model",
|
||||
resource_id=fixture.prefix,
|
||||
window_start=start,
|
||||
window_end=start + timedelta(days=1),
|
||||
evidence=DeprecationEvidence(deprecation_date=DAY + timedelta(days=90)),
|
||||
)
|
||||
)
|
||||
assert (await fixture.repository.monitoring()).alerts_used == 0
|
||||
_seed("team", f"{fixture.prefix}-a", Day(spend=10), Day(spend=100))
|
||||
await fixture.sweep()
|
||||
write_rows('UPDATE "LiteLLM_PerformanceAlertState" SET alerts_used = NULL', ())
|
||||
free: Final = PerformanceAlertRepository(fixture.client)
|
||||
assert (await free.monitoring()).alerts_used == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_losing_ownership_rolls_back_alerts_and_allowance(fixture: Fixture) -> None:
|
||||
free: Final = PerformanceAlertRepository(fixture.client)
|
||||
for index in range(FREE_ALERT_LIMIT):
|
||||
_seed("team", f"{fixture.prefix}-{index}", Day(spend=10), Day(spend=100))
|
||||
assert await free.claim(timedelta(0)) is not None
|
||||
with pytest.raises(SweepOwnershipLostError):
|
||||
await free.evaluate_page("old-owner", SweepState(day=DAY, scope="team", cursor=fixture.prefix), 25)
|
||||
assert fixture.saved() == {}
|
||||
assert (await free.monitoring()).alerts_used == 0
|
||||
|
|
|
|||
|
|
@ -12,7 +12,9 @@ from litellm.proxy._types import UserAPIKeyAuth
|
|||
from litellm.proxy.alerting_endpoints.alerts import router
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.deps.database import get_database
|
||||
from litellm.proxy.deps.repositories import get_alert_monitoring_repository
|
||||
from litellm.repositories.database import Database
|
||||
from litellm.repositories.performance_alert_repository import PerformanceAlertRepository
|
||||
from litellm.types.proxy.auth.user_roles import LitellmUserRoles
|
||||
from tests._support.native_database import native_database
|
||||
|
||||
|
|
@ -93,6 +95,50 @@ def test_invalid_page_never_reads_database(client: TestClient, table: MagicMock,
|
|||
table.find_many.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, LitellmUserRoles.PROXY_ADMIN)
|
||||
)
|
||||
@pytest.mark.parametrize("method", ("GET", "PATCH"))
|
||||
def test_monitoring_authorization_precedes_missing_database(app: FastAPI, role: LitellmUserRoles, method: str) -> None:
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role)
|
||||
app.dependency_overrides[get_database] = lambda: None
|
||||
with TestClient(app) as client:
|
||||
response: Final = client.request(
|
||||
method, "/alerts/monitoring", json={"enabled": False} if method == "PATCH" else None
|
||||
)
|
||||
allowed: Final = role == LitellmUserRoles.PROXY_ADMIN or (
|
||||
role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY and method == "GET"
|
||||
)
|
||||
assert response.status_code == (503 if allowed else 403), response.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body", ({"enabled": True, "alerts_used": 0}, {"enabled": True, "alert_limit": None}, {"enabled": "false"}, {})
|
||||
)
|
||||
def test_monitoring_only_accepts_a_boolean_opt_out(
|
||||
client: TestClient, history_query: AsyncMock, body: dict[str, object]
|
||||
) -> None:
|
||||
response: Final = client.patch("/alerts/monitoring", json=body)
|
||||
assert response.status_code == 422, response.text
|
||||
history_query.assert_not_awaited()
|
||||
|
||||
|
||||
def test_config_opt_out_cannot_be_enabled_from_the_api(app: FastAPI) -> None:
|
||||
prisma: Final = MagicMock()
|
||||
transaction: Final = prisma.writer_db.tx.return_value.__aenter__.return_value
|
||||
transaction.query_raw = AsyncMock(return_value=[{"enabled": True, "alerts_used": 0}])
|
||||
transaction.execute_raw = AsyncMock()
|
||||
app.dependency_overrides[get_alert_monitoring_repository] = lambda: PerformanceAlertRepository(
|
||||
prisma, configured=False
|
||||
)
|
||||
with TestClient(app) as client:
|
||||
status: Final = client.get("/alerts/monitoring")
|
||||
response: Final = client.patch("/alerts/monitoring", json={"enabled": True})
|
||||
assert status.json() == {"enabled": False, "alerts_used": 0, "alert_limit": 5, "disabled_by_config": True}
|
||||
assert response.status_code == 409, response.text
|
||||
transaction.execute_raw.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alert_id", ("0", "-1", "01", "1.0", "oops", "9223372036854775808", "1" * 20))
|
||||
@pytest.mark.parametrize("method", ("GET", "PATCH"))
|
||||
def test_invalid_id_never_accesses_database(client: TestClient, table: MagicMock, alert_id: str, method: str) -> None:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from datetime import date, timedelta
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import Final
|
||||
from unittest.mock import ANY, AsyncMock, MagicMock
|
||||
|
||||
|
|
@ -7,6 +7,7 @@ import pytest
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.models.performance_alert import ResourceUnreadableError, SweepOwnershipLostError, SweepState
|
||||
from litellm.proxy.spend_tracking.performance_alerts import run_performance_alerts
|
||||
from litellm.repositories.performance_alert_repository import PerformanceAlertRepository
|
||||
|
||||
|
||||
|
|
@ -38,13 +39,17 @@ async def test_only_a_statement_timeout_marks_a_resource_unreadable(message: str
|
|||
|
||||
def _repository(
|
||||
rows: Sequence[Mapping[str, object]] | Exception,
|
||||
*,
|
||||
enabled: bool = True,
|
||||
used: int = 0,
|
||||
enterprise: bool = False,
|
||||
) -> tuple[PerformanceAlertRepository, AsyncMock, AsyncMock]:
|
||||
query: Final = AsyncMock(side_effect=((), rows))
|
||||
query: Final = AsyncMock(side_effect=((), ({"enabled": enabled, "alerts_used": used},), rows))
|
||||
execute: Final = AsyncMock(return_value=1)
|
||||
transaction: Final = MagicMock(query_raw=query, execute_raw=execute)
|
||||
context: Final = MagicMock(__aenter__=AsyncMock(return_value=transaction), __aexit__=AsyncMock(return_value=False))
|
||||
client: Final = MagicMock(writer_db=MagicMock(tx=MagicMock(return_value=context)))
|
||||
return PerformanceAlertRepository(client), query, execute
|
||||
return PerformanceAlertRepository(client, enterprise=enterprise), query, execute
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -129,7 +134,7 @@ async def test_a_page_saves_the_detected_evidence_and_its_resume_position(spend_
|
|||
@pytest.mark.asyncio
|
||||
async def test_lost_ownership_aborts_the_page_transaction(spend_window: Mapping[str, object]) -> None:
|
||||
repository, _query, execute = _repository((spend_window,))
|
||||
execute.side_effect = (1, 0)
|
||||
execute.side_effect = (1, 1, 0)
|
||||
with pytest.raises(SweepOwnershipLostError):
|
||||
await repository.evaluate_page("previous-worker", SweepState(day=date(2026, 1, 8), scope="model"), 1)
|
||||
|
||||
|
|
@ -157,3 +162,60 @@ async def test_skipping_an_unreadable_resource_saves_progress_without_counting_i
|
|||
execute.assert_awaited_once_with(
|
||||
ANY, "worker-a", "2026-01-08", "team", "team-b", wrapped, 0, 1, False, "team", "team-z"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("enabled", "used"), ((True, 5), (False, 0)))
|
||||
async def test_stopped_monitoring_never_reads_usage_or_claims_work(enabled: bool, used: int) -> None:
|
||||
repository, query, execute = _repository((), enabled=enabled, used=used)
|
||||
await run_performance_alerts(repository, now=lambda: datetime(2026, 1, 9, tzinfo=timezone.utc))
|
||||
assert query.await_count == 2
|
||||
execute.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("enterprise", (False, True))
|
||||
async def test_first_read_initializes_the_allowance_once(enterprise: bool) -> None:
|
||||
repository, query, execute = _repository((), enterprise=enterprise)
|
||||
saved: Final = ({"enabled": True, "alerts_used": 3},)
|
||||
query.side_effect = ((), (), ({"enabled": True, "alerts_used": None},), saved, (), saved)
|
||||
first: Final = await repository.monitoring()
|
||||
assert first.alerts_used == 3
|
||||
assert first.alert_limit == (None if enterprise else 5)
|
||||
assert first.active
|
||||
assert await repository.monitoring() == first
|
||||
assert execute.await_count == 2
|
||||
execute.assert_awaited_with(ANY, 5)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("enabled", (False, True))
|
||||
async def test_admin_toggle_persists_the_requested_value(enabled: bool) -> None:
|
||||
repository, query, execute = _repository(())
|
||||
query.side_effect = ((), ({"enabled": not enabled, "alerts_used": 2},), ({"enabled": enabled, "alerts_used": 2},))
|
||||
result: Final = await repository.set_enabled(enabled)
|
||||
assert result.enabled is enabled
|
||||
assert result.active is enabled
|
||||
assert result.alerts_used == 2
|
||||
execute.assert_awaited_once_with(ANY, enabled)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_last_free_alert_keeps_the_page_available_for_an_enterprise_upgrade(
|
||||
spend_window: Mapping[str, object],
|
||||
) -> None:
|
||||
repository, _query, execute = _repository((spend_window,), used=4)
|
||||
state: Final = SweepState(day=date(2026, 1, 8), scope="model")
|
||||
assert await repository.evaluate_page("worker-a", state, 1) is None
|
||||
execute.assert_any_await(ANY, ANY, 1)
|
||||
execute.assert_any_await(ANY, 5, 1)
|
||||
execute.assert_awaited_with(ANY, "worker-a", "2026-01-08", "model", "", False, 0, 0, False, None, "")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enterprise_keeps_saving_alerts_after_the_free_allowance(spend_window: Mapping[str, object]) -> None:
|
||||
repository, _query, execute = _repository((spend_window,), used=5, enterprise=True)
|
||||
state: Final = SweepState(day=date(2026, 1, 8), scope="model")
|
||||
result: Final = await repository.evaluate_page("worker-a", state, 1)
|
||||
assert result == state.model_copy(update={"cursor": "model-a", "checked": 1})
|
||||
execute.assert_any_await(ANY, ANY, None)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { renderWithProviders, screen, testQueryClient, within } from "../../../tests/test-utils";
|
||||
import { act, renderWithProviders, screen, testQueryClient, waitFor, within } from "../../../tests/test-utils";
|
||||
import Alerts from "./Alerts";
|
||||
import type { SavedAlert } from "./alertPresentation";
|
||||
|
||||
|
|
@ -27,6 +27,8 @@ const calls: { path: string; query: URLSearchParams; method: string; body: unkno
|
|||
let status = "open";
|
||||
let rejectWrite = false;
|
||||
let rejectHistory = false;
|
||||
let monitoring = { enabled: true, alerts_used: 0, alert_limit: 5 as number | null, disabled_by_config: false };
|
||||
let rejectMonitoring = false;
|
||||
function response(data: unknown, code = 200) {
|
||||
return new Response(JSON.stringify(data), { status: code, headers: { "Content-Type": "application/json" } });
|
||||
}
|
||||
|
|
@ -38,6 +40,8 @@ describe("Alerts", () => {
|
|||
status = "open";
|
||||
rejectWrite = false;
|
||||
rejectHistory = false;
|
||||
rejectMonitoring = false;
|
||||
monitoring = { enabled: true, alerts_used: 0, alert_limit: 5, disabled_by_config: false };
|
||||
role = "proxy_admin";
|
||||
document.cookie = `token=${btoa(JSON.stringify({ alg: "none" }))}.${btoa(JSON.stringify({ key: "example-session", exp: 4102444800 }))}.example; path=/`;
|
||||
vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
|
||||
|
|
@ -49,6 +53,14 @@ describe("Alerts", () => {
|
|||
if (url.pathname === "/litellm/.well-known/litellm-ui-config") return response({});
|
||||
if (url.pathname === "/management/v1/users/me") return response({ user_id: "example-admin", user_role: role });
|
||||
if (url.pathname === "/model/deprecations") return response({ deprecated: [], imminent: [] });
|
||||
if (url.pathname === "/alerts/monitoring") {
|
||||
if (rejectMonitoring) return response({ detail: "Unavailable" }, 503);
|
||||
if (request.method === "PATCH") {
|
||||
if (rejectWrite) return response({ detail: "Unavailable" }, 503);
|
||||
monitoring = { ...monitoring, enabled: body.enabled };
|
||||
}
|
||||
return response(monitoring);
|
||||
}
|
||||
if (request.method === "PATCH") {
|
||||
if (rejectWrite) return response({ detail: "Unavailable" }, 503);
|
||||
status = body.status;
|
||||
|
|
@ -149,4 +161,110 @@ describe("Alerts", () => {
|
|||
expect(screen.queryByRole("button", { name: "Dismiss" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Mark resolved" })).not.toBeInTheDocument();
|
||||
});
|
||||
it("defaults on and persists opting out through the monitoring API", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
const toggle = await screen.findByRole("switch", { name: "Performance monitoring" });
|
||||
expect(toggle).toBeChecked();
|
||||
expect(screen.getByText("0 of 5 free alerts used.")).toBeInTheDocument();
|
||||
await user.click(toggle);
|
||||
expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument();
|
||||
expect(toggle).not.toBeChecked();
|
||||
expect(calls.find((call) => call.method === "PATCH")).toMatchObject({
|
||||
path: "/alerts/monitoring",
|
||||
body: { enabled: false },
|
||||
});
|
||||
expect(screen.getByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument();
|
||||
await user.click(toggle);
|
||||
expect(await screen.findByText("0 of 5 free alerts used.")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("explains exhaustion without blocking saved alerts or model deprecations", async () => {
|
||||
monitoring.alerts_used = 5;
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
expect(
|
||||
await screen.findByText("You’ve used all 5 free alerts. New performance alerts are paused."),
|
||||
).toBeInTheDocument();
|
||||
expect(screen.getByRole("link", { name: "Upgrade to Enterprise" })).toHaveAttribute(
|
||||
"href",
|
||||
"https://www.litellm.ai/enterprise",
|
||||
);
|
||||
expect(screen.getByRole("button", { name: "Mark resolved" })).toBeEnabled();
|
||||
expect(screen.getByRole("button", { name: "Set up email reminders" })).toBeInTheDocument();
|
||||
await user.click(screen.getByRole("switch", { name: "Performance monitoring" }));
|
||||
expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps the saved toggle value when an older refresh finishes later", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
const toggle = await screen.findByRole("switch", { name: "Performance monitoring" });
|
||||
const originalFetch = vi.mocked(globalThis.fetch).getMockImplementation()!;
|
||||
const staleResponse = response(monitoring);
|
||||
const pendingRefresh = Promise.withResolvers<Response>();
|
||||
vi.mocked(globalThis.fetch).mockImplementation((input, init) => {
|
||||
const request = input instanceof Request ? input : new Request(input, init);
|
||||
if (new URL(request.url).pathname === "/alerts/monitoring" && request.method === "GET") {
|
||||
return pendingRefresh.promise;
|
||||
}
|
||||
return originalFetch(input, init);
|
||||
});
|
||||
await user.click(screen.getByRole("button", { name: "Refresh" }));
|
||||
await user.click(toggle);
|
||||
expect(await screen.findByText("Off. Saved alerts remain available.")).toBeInTheDocument();
|
||||
await act(async () => {
|
||||
pendingRefresh.resolve(staleResponse);
|
||||
await pendingRefresh.promise;
|
||||
});
|
||||
await waitFor(() => expect(testQueryClient.isFetching({ queryKey: ["alert-monitoring"] })).toBe(0));
|
||||
expect(toggle).not.toBeChecked();
|
||||
expect(screen.getByText("Off. Saved alerts remain available.")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows Enterprise monitoring without a free limit", async () => {
|
||||
monitoring.alert_limit = null;
|
||||
monitoring.alerts_used = 5;
|
||||
renderWithProviders(<Alerts />);
|
||||
expect(await screen.findByText("On with Enterprise.")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("respects config opt-out and view-only permissions", async () => {
|
||||
monitoring.disabled_by_config = true;
|
||||
monitoring.alerts_used = 5;
|
||||
monitoring.enabled = false;
|
||||
renderWithProviders(<Alerts />);
|
||||
expect(await screen.findByText("Disabled in proxy settings.")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("link", { name: "Upgrade to Enterprise" })).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("switch", { name: "Performance monitoring" })).toHaveAttribute("aria-disabled", "true");
|
||||
});
|
||||
|
||||
it("lets viewers see monitoring status without a toggle", async () => {
|
||||
role = "proxy_admin_viewer";
|
||||
renderWithProviders(<Alerts />);
|
||||
expect(await screen.findByText("0 of 5 free alerts used.")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("switch")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps the saved preference when a monitoring change fails", async () => {
|
||||
rejectWrite = true;
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
await user.click(await screen.findByRole("switch", { name: "Performance monitoring" }));
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("Could not save monitoring settings");
|
||||
expect(screen.getByRole("switch", { name: "Performance monitoring" })).toBeChecked();
|
||||
});
|
||||
|
||||
it("can retry monitoring settings without hiding the inbox", async () => {
|
||||
rejectMonitoring = true;
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Alerts />);
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("Could not load monitoring settings");
|
||||
expect(screen.getByRole("button", { name: /Spend on Research agent/ })).toBeInTheDocument();
|
||||
rejectMonitoring = false;
|
||||
await user.click(screen.getByRole("button", { name: "Retry" }));
|
||||
expect(await screen.findByRole("switch", { name: "Performance monitoring" })).toBeChecked();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
import { parseAsBoolean, parseAsString, parseAsStringEnum, parseAsStringLiteral, useQueryStates } from "nuqs";
|
||||
import { RefreshCw } from "lucide-react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { Page, PageTabs, PageTabsList, PageTabsTrigger } from "@/components/shared/Page";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
|
@ -17,6 +18,7 @@ import { ALERT_TYPES, AlertFilter, AlertItem, TYPE_LABEL_KEYS, presentAlert } fr
|
|||
import AlertActions from "./AlertActions";
|
||||
import AlertChart from "./AlertChart";
|
||||
import AlertDetail from "./AlertDetail";
|
||||
import PerformanceMonitoring from "./PerformanceMonitoring";
|
||||
|
||||
const parsers = {
|
||||
alert: parseAsString,
|
||||
|
|
@ -97,6 +99,7 @@ function SelectedAlert({ auth, onBack }: { auth: ReturnType<typeof useAuthorized
|
|||
|
||||
function AlertsInbox({ auth }: { auth: ReturnType<typeof useAuthorized> }) {
|
||||
const { t } = useTranslation();
|
||||
const client = useQueryClient();
|
||||
const [params, setParams] = useQueryStates(parsers);
|
||||
const scope = JSON.stringify([auth.userId, auth.userRole]);
|
||||
const showDeprecations = params.type === "all" || params.type === "model-deprecations";
|
||||
|
|
@ -113,9 +116,15 @@ function AlertsInbox({ auth }: { auth: ReturnType<typeof useAuthorized> }) {
|
|||
const items = [...retirementItems, ...performanceItems];
|
||||
const loading = list.isLoading || deprecations.isLoading;
|
||||
const hasError = (showPerformance && list.isError) || (showDeprecations && deprecations.isError);
|
||||
function refresh() {
|
||||
void client.invalidateQueries({ queryKey: ["alert-monitoring"] });
|
||||
if (showPerformance) void list.refetch();
|
||||
if (showDeprecations) void deprecations.refetch();
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<h1 className="text-2xl font-semibold">{t("alerts.unified.title")}</h1>
|
||||
<PerformanceMonitoring scope={scope} readOnly={Boolean(auth.isViewOnly)} />
|
||||
<PageTabs
|
||||
value={params.state}
|
||||
onValueChange={(value) => void setParams({ state: value as AlertStatus })}
|
||||
|
|
@ -142,15 +151,7 @@ function AlertsInbox({ auth }: { auth: ReturnType<typeof useAuthorized> }) {
|
|||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={list.isFetching || deprecations.isFetching}
|
||||
onClick={() => {
|
||||
if (showPerformance) void list.refetch();
|
||||
if (showDeprecations) void deprecations.refetch();
|
||||
}}
|
||||
>
|
||||
<Button variant="outline" size="sm" disabled={list.isFetching || deprecations.isFetching} onClick={refresh}>
|
||||
<RefreshCw className="size-3.5" />
|
||||
{t("alerts.unified.refresh")}
|
||||
</Button>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { backgroundFetchClient } from "@/lib/http/api";
|
||||
import { getProxyBaseUrl } from "@/components/networking";
|
||||
import type { components } from "@/lib/http/schema";
|
||||
|
||||
function statusMessage(data: components["schemas"]["AlertMonitoring"]) {
|
||||
if (data.disabled_by_config) return "alerts.monitoring.disabledByConfig";
|
||||
if (!data.enabled) return "alerts.monitoring.off";
|
||||
if (data.alert_limit === null) return "alerts.monitoring.enterprise";
|
||||
if (data.alerts_used >= data.alert_limit) return "alerts.monitoring.exhausted";
|
||||
return "alerts.monitoring.allowance";
|
||||
}
|
||||
|
||||
export default function PerformanceMonitoring({ scope, readOnly }: { scope: string; readOnly: boolean }) {
|
||||
const { t } = useTranslation();
|
||||
const client = useQueryClient();
|
||||
const queryKey = ["alert-monitoring", getProxyBaseUrl(), scope];
|
||||
const queryOptions = {
|
||||
queryKey,
|
||||
queryFn: async ({ signal }: { signal: AbortSignal }) => {
|
||||
const { data } = await backgroundFetchClient.GET("/alerts/monitoring", { signal });
|
||||
if (!data) throw new Error("Could not load monitoring settings");
|
||||
return data;
|
||||
},
|
||||
staleTime: 60_000,
|
||||
retry: false,
|
||||
};
|
||||
const query = useQuery(queryOptions);
|
||||
const mutation = useMutation({
|
||||
mutationFn: async (enabled: boolean) => {
|
||||
const { data } = await backgroundFetchClient.PATCH("/alerts/monitoring", { body: { enabled } });
|
||||
if (!data) throw new Error("Could not save monitoring settings");
|
||||
return data;
|
||||
},
|
||||
onSuccess: async (data) => {
|
||||
await client.cancelQueries({ queryKey });
|
||||
client.setQueryData(queryKey, data);
|
||||
},
|
||||
});
|
||||
if (query.isError)
|
||||
return (
|
||||
<div role="alert" className="flex flex-wrap items-center gap-2 text-sm">
|
||||
<p>{t("alerts.monitoring.loadError")}</p>
|
||||
<Button variant="outline" size="sm" onClick={() => void query.refetch()}>
|
||||
{t("alerts.panel.retry")}
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
if (!query.data)
|
||||
return (
|
||||
<p role="status" className="text-sm text-muted-foreground">
|
||||
{t("alerts.monitoring.loading")}
|
||||
</p>
|
||||
);
|
||||
const data = query.data;
|
||||
const messageKey = statusMessage(data);
|
||||
const message = t(messageKey, { used: data.alerts_used, limit: data.alert_limit });
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<div className="space-y-2 text-sm">
|
||||
<div className="flex items-center gap-3">
|
||||
<label htmlFor="performance-monitoring" className="font-medium">
|
||||
{t("alerts.monitoring.title")}
|
||||
</label>
|
||||
{!readOnly && (
|
||||
<Switch
|
||||
id="performance-monitoring"
|
||||
checked={data.enabled}
|
||||
disabled={mutation.isPending || data.disabled_by_config}
|
||||
onCheckedChange={(enabled) => mutation.mutate(enabled)}
|
||||
aria-describedby="performance-monitoring-status"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
<p id="performance-monitoring-status" className="text-muted-foreground" role="status">
|
||||
{message}
|
||||
{messageKey === "alerts.monitoring.exhausted" && (
|
||||
<a
|
||||
className="ml-2 text-foreground underline underline-offset-4"
|
||||
href="https://www.litellm.ai/enterprise"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
{t("alerts.monitoring.upgrade")}
|
||||
</a>
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
{mutation.isError && (
|
||||
<p role="alert" className="text-sm text-destructive">
|
||||
{t("alerts.monitoring.saveError")}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -5998,6 +5998,18 @@
|
|||
"showMore": "Show {{hidden}} more usage tags",
|
||||
"showLess": "Show less",
|
||||
"more": "+{{hidden}} more"
|
||||
},
|
||||
"monitoring": {
|
||||
"title": "Performance monitoring",
|
||||
"loading": "Loading monitoring settings…",
|
||||
"allowance": "{{used}} of {{limit}} free alerts used.",
|
||||
"enterprise": "On with Enterprise.",
|
||||
"off": "Off. Saved alerts remain available.",
|
||||
"exhausted": "You’ve used all 5 free alerts. New performance alerts are paused.",
|
||||
"disabledByConfig": "Disabled in proxy settings.",
|
||||
"upgrade": "Upgrade to Enterprise",
|
||||
"loadError": "Could not load monitoring settings.",
|
||||
"saveError": "Could not save monitoring settings. Try again."
|
||||
}
|
||||
},
|
||||
"caching": {
|
||||
|
|
|
|||
112
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
112
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -1022,6 +1022,30 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/alerts/monitoring": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* Read performance monitoring settings
|
||||
* @description Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin.
|
||||
*/
|
||||
get: operations["get_monitoring_alerts_monitoring_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
/**
|
||||
* Enable or disable performance monitoring
|
||||
* @description Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts.
|
||||
*/
|
||||
patch: operations["update_monitoring_alerts_monitoring_patch"];
|
||||
trace?: never;
|
||||
};
|
||||
"/alerts/{alert_id}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -27229,6 +27253,37 @@ export interface components {
|
|||
*/
|
||||
value: number;
|
||||
};
|
||||
/** AlertMonitoring */
|
||||
AlertMonitoring: {
|
||||
/**
|
||||
* Alert Limit
|
||||
* @description Free allowance, or null for Enterprise.
|
||||
*/
|
||||
alert_limit: number | null;
|
||||
/**
|
||||
* Alerts Used
|
||||
* @description Lifetime free allowance consumed, capped at five.
|
||||
*/
|
||||
alerts_used: number;
|
||||
/**
|
||||
* Disabled By Config
|
||||
* @description Whether performance_alerts_enabled is false in proxy settings.
|
||||
*/
|
||||
disabled_by_config: boolean;
|
||||
/**
|
||||
* Enabled
|
||||
* @description Whether performance monitoring is enabled. Defaults to true.
|
||||
*/
|
||||
enabled: boolean;
|
||||
};
|
||||
/** AlertMonitoringUpdate */
|
||||
AlertMonitoringUpdate: {
|
||||
/**
|
||||
* Enabled
|
||||
* @description Enable or disable performance monitoring for this gateway.
|
||||
*/
|
||||
enabled: boolean;
|
||||
};
|
||||
/**
|
||||
* AlertType
|
||||
* @description Enum for alert types and management event types
|
||||
|
|
@ -31596,8 +31651,8 @@ export interface components {
|
|||
password_policy_require_uppercase?: boolean | null;
|
||||
/**
|
||||
* Performance Alerts Enabled
|
||||
* @description Check yesterday's usage for the gateway, models, teams, users and keys against the week before, and save unusual spend, traffic, response time and failure rate changes as alerts. Off by default.
|
||||
* @default false
|
||||
* @description Check yesterday's usage for the gateway, models, teams, users and keys against the week before, and save unusual spend, traffic, response time and failure rate changes as alerts. On by default. Set false to disable monitoring. Free gateways stop after five saved performance alerts.
|
||||
* @default true
|
||||
*/
|
||||
performance_alerts_enabled: boolean;
|
||||
/**
|
||||
|
|
@ -54833,6 +54888,59 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
get_monitoring_alerts_monitoring_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["AlertMonitoring"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
update_monitoring_alerts_monitoring_patch: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["AlertMonitoringUpdate"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["AlertMonitoring"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_alert_alerts__alert_id__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue