feat: add unified issue storage and investigation APIs

This commit is contained in:
moe-berri 2026-11-05 14:50:43 +08:00 • committed by ishaan-j-berri
parent f334a32717
commit 5ab3a0a739
21 changed files with 2537 additions and 25 deletions

View file

@ -0,0 +1,49 @@
CREATE TABLE "LiteLLM_Issue" (
"id" TEXT NOT NULL,
"cohort_key" TEXT NOT NULL,
"kind" TEXT NOT NULL,
"family" TEXT NOT NULL,
"scope_type" TEXT NOT NULL,
"scope_id" TEXT,
"status" TEXT NOT NULL,
"active" BOOLEAN NOT NULL DEFAULT true,
"ack_key" TEXT,
"record" JSONB NOT NULL,
"first_seen" TIMESTAMP(3) NOT NULL,
"last_seen" TIMESTAMP(3) NOT NULL,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "LiteLLM_Issue_pkey" PRIMARY KEY ("id")
);
CREATE INDEX "LiteLLM_Issue_status_last_seen_id_idx" ON "LiteLLM_Issue"("status", "last_seen", "id");
CREATE INDEX "LiteLLM_Issue_kind_family_last_seen_id_idx" ON "LiteLLM_Issue"("kind", "family", "last_seen", "id");
CREATE INDEX "LiteLLM_Issue_scope_type_scope_id_last_seen_id_idx" ON "LiteLLM_Issue"("scope_type", "scope_id", "last_seen", "id");
CREATE INDEX "LiteLLM_Issue_cohort_key_active_idx" ON "LiteLLM_Issue"("cohort_key", "active");
CREATE TABLE "LiteLLM_IssueSignal" (
"issue_id" TEXT NOT NULL,
"source" TEXT NOT NULL,
"source_id" TEXT NOT NULL,
"record" JSONB NOT NULL,
CONSTRAINT "LiteLLM_IssueSignal_pkey" PRIMARY KEY ("issue_id", "source", "source_id")
);
CREATE INDEX "LiteLLM_IssueSignal_source_source_id_idx" ON "LiteLLM_IssueSignal"("source", "source_id");
CREATE TABLE "LiteLLM_IssueDelivery" (
"id" TEXT NOT NULL,
"issue_id" TEXT NOT NULL,
"record" JSONB NOT NULL,
"sent" BOOLEAN NOT NULL DEFAULT false,
"next_attempt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "LiteLLM_IssueDelivery_pkey" PRIMARY KEY ("id")
);
CREATE INDEX "LiteLLM_IssueDelivery_sent_next_attempt_id_idx" ON "LiteLLM_IssueDelivery"("sent", "next_attempt", "id");
CREATE INDEX "LiteLLM_IssueDelivery_issue_id_idx" ON "LiteLLM_IssueDelivery"("issue_id");
CREATE TABLE "LiteLLM_IssueDeliveryRule" (
"delivery_id" TEXT NOT NULL,
"rule_id" TEXT NOT NULL,
"source_id" TEXT NOT NULL,
"rule_revision" TEXT NOT NULL,
CONSTRAINT "LiteLLM_IssueDeliveryRule_pkey" PRIMARY KEY ("delivery_id", "rule_id", "source_id")
);
ALTER TABLE "LiteLLM_UsageAlertIncident" ADD COLUMN "evaluation_revision" TEXT NOT NULL DEFAULT '';
CREATE INDEX "LiteLLM_UsageAlertIncident_evaluation_day_breached_id_idx"
ON "LiteLLM_UsageAlertIncident"("evaluation_day", "breached", "id");

134
litellm/models/issues.py Normal file
View file

@ -0,0 +1,134 @@
from datetime import date, datetime
from typing import Literal, Protocol
from litellm.types.proxy.issues import (
Issue,
IssueCohort,
IssueDeliveryBinding,
IssueDeliveryInput,
IssueFamily,
IssueFamilyList,
IssueFamilySetting,
IssueFamilyUpdate,
IssueImpact,
IssueInput,
IssueKind,
IssueList,
IssuePendingDelivery,
IssueProjectionSignal,
IssueScope,
IssueSignal,
IssueSignalInput,
IssueSignalList,
)
from litellm.types.proxy.management_endpoints.common_daily_activity import (
DailyActivityKeyPageResponse,
DailyActivityUserPageResponse,
SpendAnalyticsPaginatedResponse,
)
from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily
from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus
class IssueStore(Protocol):
async def get_issue(self, issue_id: str) -> Issue | None: ...
async def list_issues(
self,
kind: IssueKind | None = None,
family: IssueFamily | None = None,
status: UsageAlertStatus | None = None,
scope_type: IssueScope | None = None,
scope_id: str | None = None,
limit: int = 50,
offset: int = 0,
) -> IssueList: ...
async def set_status(
self,
issue_id: str,
status: UsageAlertStatus,
expected_status: UsageAlertStatus,
actor: str | None = None,
) -> Issue | None: ...
async def upsert_issue(
self,
data: IssueInput,
reopen: bool = False,
status: UsageAlertStatus | None = None,
) -> Issue: ...
async def upsert_issues(
self,
inputs: tuple[IssueInput, ...],
reopen: bool = False,
status: UsageAlertStatus | None = None,
) -> tuple[Issue, ...]: ...
async def find_active_issue(self, cohort_key: str) -> Issue | None: ...
async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: ...
async def add_signals(self, issue_id: str, signals: tuple[IssueSignalInput, ...]) -> None: ...
async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: ...
async def list_signals(self, issue_id: str, limit: int = 50, offset: int = 0) -> IssueSignalList: ...
async def active_signals(
self,
day: date,
limit: int = 500,
cursor: str | None = None,
scope_type: IssueScope | None = None,
metric: UsageAlertMetric | None = None,
) -> tuple[IssueProjectionSignal, ...]: ...
async def recover_issues(self, day: date) -> int: ...
async def projection_completed(self, day: date, version: str) -> bool: ...
async def complete_projection(self, day: date, version: str) -> None: ...
async def finish_deprecation_projection(self, observed_at: datetime) -> int: ...
async def family_settings(self) -> IssueFamilyList: ...
async def update_family(
self, family: PerformanceIssueFamily, data: IssueFamilyUpdate
) -> IssueFamilySetting | None: ...
async def queue_delivery(
self,
data: IssueDeliveryInput,
rule_id: str,
rule_revision: str,
source_id: str,
) -> None: ...
async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: ...
async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: ...
async def mark_delivery_sent(self, delivery_id: str) -> None: ...
async def defer_delivery(
self, delivery_id: str, error: Literal["delivery_failed", "transport_not_configured"]
) -> None: ...
async def prune(self, retention_days: int = 90, limit: int = 500) -> None: ...
class IssueActivityStore(Protocol):
async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse: ...
async def keys(
self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
) -> DailyActivityKeyPageResponse: ...
async def users(
self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
) -> DailyActivityUserPageResponse: ...
async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: ...

View file

@ -585,6 +585,15 @@ class LiteLLMRoutes(enum.Enum):
"/usage/alert-rules/{rule_id}",
"/usage/alerts",
"/usage/alerts/{alert_id}",
"/issues",
"/issues/families",
"/issues/families/{family}",
"/issues/{issue_id}",
"/issues/{issue_id}/signals",
"/issues/{issue_id}/activity",
"/issues/{issue_id}/keys",
"/issues/{issue_id}/users",
"/issues/{issue_id}/impact",
"/key/info",
"/key/health",
"/team/info",
@ -658,6 +667,15 @@ class LiteLLMRoutes(enum.Enum):
"/usage/alert-rules/{rule_id}",
"/usage/alerts",
"/usage/alerts/{alert_id}",
"/issues",
"/issues/families",
"/issues/families/{family}",
"/issues/{issue_id}",
"/issues/{issue_id}/signals",
"/issues/{issue_id}/activity",
"/issues/{issue_id}/keys",
"/issues/{issue_id}/users",
"/issues/{issue_id}/impact",
# user
"/user/new",
"/management/v1/users/bulk",
@ -874,6 +892,15 @@ class LiteLLMRoutes(enum.Enum):
"/usage/alert-rules/{rule_id}",
"/usage/alerts",
"/usage/alerts/{alert_id}",
"/issues",
"/issues/families",
"/issues/families/{family}",
"/issues/{issue_id}",
"/issues/{issue_id}/signals",
"/issues/{issue_id}/activity",
"/issues/{issue_id}/keys",
"/issues/{issue_id}/users",
"/issues/{issue_id}/impact",
)
self_managed_routes = [

View file

@ -0,0 +1,235 @@
"""Unified issue inbox and investigations backed by existing daily aggregates."""
from datetime import date
from typing import Final
from fastapi import APIRouter, HTTPException
from litellm.models.issues import IssueActivityStore, IssueStore
from litellm.proxy.alerting_endpoints.usage_alerts import (
Limit,
Offset,
ScopeId,
User,
require_repository,
require_scope,
)
from litellm.proxy.deps.repositories import IssueActivityRepositoryDep, IssuesRepositoryDep, UsageAlertsRepositoryDep
from litellm.types.proxy.issues import (
Issue,
IssueFamily,
IssueFamilyList,
IssueFamilySetting,
IssueFamilyUpdate,
IssueImpact,
IssueKind,
IssueList,
IssueScope,
IssueSignalList,
IssueUpdate,
)
from litellm.types.proxy.management_endpoints.common_daily_activity import (
DailyActivityKeyPageResponse,
DailyActivityUserPageResponse,
SpendAnalyticsPaginatedResponse,
)
from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily
from litellm.types.proxy.usage_alerts import UsageAlertStatus
router: Final = APIRouter(prefix="/issues", tags=["issues"])
def require_issues(repository: IssueStore | None) -> IssueStore:
if repository is None:
raise HTTPException(status_code=503, detail="Issues require a database connection")
return repository
def require_activity(repository: IssueActivityStore | None) -> IssueActivityStore:
if repository is None:
raise HTTPException(status_code=503, detail="Issue Activity requires a database connection")
return repository
async def authorized_issue(
issue_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
*,
write: bool = False,
) -> Issue:
issue: Final = await require_issues(repository).get_issue(issue_id)
if issue is None:
raise HTTPException(status_code=404, detail="Issue not found")
await require_scope(
user,
require_repository(alerts),
"model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type,
issue.cohort.scope_id,
write=write,
)
return issue
def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]:
start: Final = start_date or issue.range_start
end: Final = end_date or issue.range_end
if start > end or (end - start).days > 365:
raise HTTPException(status_code=422, detail="Choose an ordered date range of at most 366 days")
return start, end
@router.get("", response_model=IssueList)
async def list_issues(
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
kind: IssueKind | None = None,
family: IssueFamily | None = None,
status: UsageAlertStatus | None = None,
scope_type: IssueScope | None = None,
scope_id: ScopeId = None,
limit: Limit = 50,
offset: Offset = 0,
) -> IssueList:
await require_scope(
user, require_repository(alerts), "model" if scope_type == "provider" else scope_type, scope_id, write=False
)
return await require_issues(repository).list_issues(kind, family, status, scope_type, scope_id, limit, offset)
@router.get("/families", response_model=IssueFamilyList)
async def list_issue_families(
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
) -> IssueFamilyList:
await require_scope(user, require_repository(alerts), "deployment", None, write=False)
return await require_issues(repository).family_settings()
@router.patch("/families/{family}", response_model=IssueFamilySetting)
async def update_issue_family(
family: PerformanceIssueFamily,
data: IssueFamilyUpdate,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
) -> IssueFamilySetting:
await require_scope(user, require_repository(alerts), "deployment", None, write=True)
try:
result: Final = await require_issues(repository).update_family(family, data)
except ValueError as exc:
raise HTTPException(
status_code=422, detail="Invalid family policy; check direction and history bounds"
) from exc
if result is None:
raise HTTPException(status_code=409, detail="Built-in rules changed or are unavailable; reload and retry")
return result
@router.get("/{issue_id}", response_model=Issue)
async def get_issue(
issue_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
) -> Issue:
return await authorized_issue(issue_id, user, repository, alerts)
@router.patch("/{issue_id}", response_model=Issue)
async def update_issue(
issue_id: str,
data: IssueUpdate,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
) -> Issue:
await authorized_issue(issue_id, user, repository, alerts, write=True)
changed: Final = await require_issues(repository).set_status(
issue_id, data.status, data.expected_status, user.user_id
)
if changed is None:
raise HTTPException(status_code=409, detail="Issue status changed; reload and retry")
return changed
@router.get("/{issue_id}/signals", response_model=IssueSignalList)
async def list_issue_signals(
issue_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
limit: Limit = 50,
offset: Offset = 0,
) -> IssueSignalList:
await authorized_issue(issue_id, user, repository, alerts)
return await require_issues(repository).list_signals(issue_id, limit, offset)
@router.get("/{issue_id}/activity", response_model=SpendAnalyticsPaginatedResponse)
async def get_issue_activity(
issue_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
activity: IssueActivityRepositoryDep,
start_date: date | None = None,
end_date: date | None = None,
) -> SpendAnalyticsPaginatedResponse:
issue: Final = await authorized_issue(issue_id, user, repository, alerts)
start, end = activity_range(issue, start_date, end_date)
return await require_activity(activity).daily_activity(issue.cohort, start, end)
@router.get("/{issue_id}/keys", response_model=DailyActivityKeyPageResponse)
async def get_issue_keys(
issue_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
activity: IssueActivityRepositoryDep,
start_date: date | None = None,
end_date: date | None = None,
limit: Limit = 20,
offset: Offset = 0,
) -> DailyActivityKeyPageResponse:
issue: Final = await authorized_issue(issue_id, user, repository, alerts)
start, end = activity_range(issue, start_date, end_date)
return await require_activity(activity).keys(issue.cohort, start, end, limit, offset)
@router.get("/{issue_id}/users", response_model=DailyActivityUserPageResponse)
async def get_issue_users(
issue_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
activity: IssueActivityRepositoryDep,
start_date: date | None = None,
end_date: date | None = None,
limit: Limit = 20,
offset: Offset = 0,
) -> DailyActivityUserPageResponse:
issue: Final = await authorized_issue(issue_id, user, repository, alerts)
if issue.cohort.scope_type == "team":
raise HTTPException(status_code=422, detail="Historical team aggregates do not retain user membership")
start, end = activity_range(issue, start_date, end_date)
return await require_activity(activity).users(issue.cohort, start, end, limit, offset)
@router.get("/{issue_id}/impact", response_model=IssueImpact)
async def get_issue_impact(
issue_id: str,
user: User,
repository: IssuesRepositoryDep,
alerts: UsageAlertsRepositoryDep,
activity: IssueActivityRepositoryDep,
start_date: date | None = None,
end_date: date | None = None,
) -> IssueImpact:
issue: Final = await authorized_issue(issue_id, user, repository, alerts)
start, end = activity_range(issue, start_date, end_date)
return await require_activity(activity).impact(issue.cohort, start, end)

View file

@ -4,12 +4,15 @@ from fastapi import Depends
from litellm.models.deployment_warning import DeploymentWarningIgnoreStore
from litellm.models.gateway_activity import GatewayActivityStore
from litellm.models.issues import IssueActivityStore, IssueStore
from litellm.models.plugin_upload import PluginUploadStore
from litellm.models.usage_alerts import UsageAlertStore
from litellm.models.user_banner import UserBannerStore
from litellm.proxy.deps.database import DatabaseDep
from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository
from litellm.repositories.gateway_activity_repository import GatewayActivityRepository
from litellm.repositories.issue_activity import IssueActivityRepository
from litellm.repositories.issues import IssuesRepository
from litellm.repositories.plugin_upload_repository import PluginUploadRepository
from litellm.repositories.usage_alerts import UsageAlertsRepository
from litellm.repositories.user_banner_repository import UserBannerRepository
@ -52,3 +55,17 @@ async def get_usage_alerts_repository(database: DatabaseDep) -> UsageAlertStore
UsageAlertsRepositoryDep: TypeAlias = Annotated[UsageAlertStore | None, Depends(get_usage_alerts_repository)]
async def get_issues_repository(database: DatabaseDep) -> IssueStore | None:
return None if database is None else IssuesRepository.from_prisma(database.prisma)
IssuesRepositoryDep: TypeAlias = Annotated[IssueStore | None, Depends(get_issues_repository)]
async def get_issue_activity_repository(database: DatabaseDep) -> IssueActivityStore | None:
return None if database is None else IssueActivityRepository.from_prisma(database.prisma)
IssueActivityRepositoryDep: TypeAlias = Annotated[IssueActivityStore | None, Depends(get_issue_activity_repository)]

View file

@ -1128,7 +1128,7 @@ async def get_daily_activity_aggregated(
scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit
),
_guardrail_spend_or_empty(repository, scope)
if include_guardrail_breakdown and not scope.model
if include_guardrail_breakdown and not scope.model and scope.provider is None and not scope.model_pairs
else asyncio.sleep(0, result=()),
)
records: Final = aggregated_rows.grouping_rows

View file

@ -136,6 +136,8 @@ def _key_activity_row(row: KeySpendRow, metadata: Mapping[str, KeyMetadataRow])
failed_requests=row.failed_requests,
cache_read_input_tokens=row.cache_read_input_tokens,
cache_creation_input_tokens=row.cache_creation_input_tokens,
total_response_time_ms=row.total_response_time_ms,
timed_requests=row.timed_requests,
),
metadata=_key_metadata(row.api_key, metadata),
)
@ -169,6 +171,8 @@ def _user_activity_row(row: UserSpendRow, metadata: Mapping[str, UserMetadataRow
api_requests=row.api_requests,
successful_requests=row.successful_requests,
failed_requests=row.failed_requests,
total_response_time_ms=row.total_response_time_ms,
timed_requests=row.timed_requests,
)

View file

@ -428,6 +428,7 @@ from litellm.proxy.alerting_endpoints.model_deprecation_alerts import (
router as model_deprecation_alerts_router,
)
from litellm.proxy.alerting_endpoints.usage_alerts import router as usage_alerts_router
from litellm.proxy.alerting_endpoints.issues import router as issues_router
from litellm.proxy.analytics_endpoints.analytics_endpoints import (
router as analytics_router,
)
@ -22440,6 +22441,7 @@ app.include_router(fallback_management_router)
app.include_router(cache_settings_router)
app.include_router(model_deprecation_alerts_router)
app.include_router(usage_alerts_router)
app.include_router(issues_router)
app.include_router(coordination_redis_settings_router)
app.include_router(user_agent_analytics_router)
app.include_router(gateway_request_router)

View file

@ -324,6 +324,8 @@ class DailyActivityRepository:
)
async def daily_rows(self, scope: DailyActivityScope, *, page: int, page_size: int) -> DailyRowsPage:
if scope.provider is not None or scope.model_pairs:
raise ValueError("Provider cohorts require the aggregated Activity API")
table: Final = _daily_rows_table(self._prisma_client, scope.table)
adjusted_start, adjusted_end = adjust_dates_for_timezone(
scope.start_date,

View file

@ -1,3 +1,4 @@
import json
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
@ -75,6 +76,8 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu
exclusion_index: Final = entity_index + int(has_entity_array)
model_index: Final = exclusion_index + int(bool(scope.exclude_entity_ids))
api_keys_index: Final = model_index + int(bool(scope.model))
provider_index: Final = api_keys_index + int(bool(scope.api_keys))
pairs_index: Final = provider_index + int(scope.provider is not None)
conditions: Final = (
f"date >= ${start_index}",
f"date <= ${start_index + 1}",
@ -85,6 +88,21 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu
if has_entity_array
else ()
),
*(
(f"COALESCE(NULLIF(custom_llm_provider, ''), 'unknown') = ${provider_index}",)
if scope.provider is not None
else ()
),
*(
(
f"EXISTS (SELECT 1 FROM jsonb_to_recordset(${pairs_index}::jsonb) "
f"AS cohort(match_provider text, match_model text) WHERE "
f"cohort.match_provider=COALESCE(NULLIF(custom_llm_provider, ''), 'unknown') "
f"AND cohort.match_model={_MODEL_EXPR})",
)
if scope.model_pairs
else ()
),
*(
(
f'("{scope.entity_id_field}" IS NULL '
@ -113,6 +131,16 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu
*((list(scope.exclude_entity_ids),) if scope.exclude_entity_ids else ()),
*((scope.model,) if scope.model else ()),
*((list(scope.api_keys),) if scope.api_keys else ()),
*((scope.provider,) if scope.provider is not None else ()),
*(
(
json.dumps(
tuple({"match_provider": provider, "match_model": model} for provider, model in scope.model_pairs)
),
)
if scope.model_pairs
else ()
),
)
return " AND ".join(conditions), params
@ -301,7 +329,9 @@ def _key_spend_select() -> str:
COALESCE(SUM(successful_requests), 0)::bigint AS successful_requests,
COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests,
COALESCE(SUM(cache_read_input_tokens), 0)::bigint AS cache_read_input_tokens,
COALESCE(SUM(cache_creation_input_tokens), 0)::bigint AS cache_creation_input_tokens"""
COALESCE(SUM(cache_creation_input_tokens), 0)::bigint AS cache_creation_input_tokens,
COALESCE(SUM(total_response_time_ms), 0)::bigint AS total_response_time_ms,
COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests"""
def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery:
@ -339,7 +369,9 @@ def _user_spend_select() -> str:
(COALESCE(SUM(prompt_tokens), 0) + COALESCE(SUM(completion_tokens), 0))::bigint AS total_tokens,
COALESCE(SUM(api_requests), 0)::bigint AS api_requests,
COALESCE(SUM(successful_requests), 0)::bigint AS successful_requests,
COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests"""
COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests,
COALESCE(SUM(total_response_time_ms), 0)::bigint AS total_response_time_ms,
COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests"""
def build_user_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery:

View file

@ -0,0 +1,154 @@
"""Issue investigations reuse the same aggregate filters as entity Activity."""
from dataclasses import asdict, replace
from datetime import date
from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # generated Prisma structural database interface
from pydantic import BaseModel, TypeAdapter
from litellm.constants import PTU_SENTINEL_API_KEY
from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket
from litellm.repositories.daily_activity_repository import DailyActivityRepository
from litellm.repositories.daily_activity_sql import PRISMA_TO_PG_TABLE, build_where_clause
from litellm.repositories.usage_alerts import AlertDatabase
from litellm.types.proxy.issues import IssueCohort, IssueImpact
from litellm.types.proxy.management_endpoints.common_daily_activity import (
DailyActivityKeyPageResponse,
DailyActivityUserPageResponse,
KeyMetadata,
KeySpendActivityRow,
KeySpendMetrics,
SpendAnalyticsPaginatedResponse,
UserActivityRow,
)
from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
class _Counts(BaseModel, frozen=True):
keys: int
entities: int
models: int
requests: int
def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyActivityScope:
if start > end or (end - start).days > 365:
raise ValueError("Choose an ordered date range of at most 366 days")
return DailyActivityScope(
table=DailyActivityTable.TEAM if cohort.scope_type == "team" else DailyActivityTable.USER,
entity_id_field="team_id" if cohort.scope_type == "team" else "user_id",
entity_ids=(cohort.scope_id,) if cohort.scope_type in ("team", "user") and cohort.scope_id else None,
exclude_entity_ids=(),
api_keys=(api_key_from_bucket(cohort.scope_id),) if cohort.scope_type == "key" and cohort.scope_id else None,
start_date=start.isoformat(),
end_date=end.isoformat(),
model=cohort.scope_id if cohort.scope_type == "model" else None,
timezone_offset_minutes=0,
include_current_utc_day=True,
provider=cohort.scope_id if cohort.scope_type == "provider" else None,
model_pairs=tuple((pair.provider, pair.model) for pair in cohort.model_pairs),
)
class IssueActivityRepository:
def __init__(self, activity: DailyActivityRepository, database: AlertDatabase) -> None:
self.activity = activity
self.database = database
@classmethod
def from_prisma(cls, prisma: "PrismaClient") -> "IssueActivityRepository":
from litellm.proxy.management_endpoints.common_daily_activity import daily_activity_repository
database: Final = cast(AlertDatabase, prisma.db) # cast-ok: generated Prisma query_raw
return cls(daily_activity_repository(prisma), database)
async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse:
from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated
return await get_daily_activity_aggregated(
self.activity, issue_activity_scope(cohort, start, end), api_key_limit=20
)
async def keys(
self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
) -> DailyActivityKeyPageResponse:
page: Final = await self.activity.key_page(issue_activity_scope(cohort, start, end), limit=limit, offset=offset)
metadata: Final = await self.activity.key_metadata(
frozenset(row.api_key for row in page.rows if row.api_key), None
)
return DailyActivityKeyPageResponse(
api_keys=[
KeySpendActivityRow(
api_key=api_key_bucket(row.api_key),
metrics=KeySpendMetrics.model_validate(asdict(row)),
metadata=KeyMetadata.model_validate(asdict(metadata[row.api_key]))
if row.api_key in metadata
else KeyMetadata(),
)
for row in page.rows
],
total_api_keys=page.total_api_keys,
limit=limit,
offset=offset,
)
async def users(
self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
) -> DailyActivityUserPageResponse:
if cohort.scope_type == "team":
raise ValueError("Historical team aggregates do not retain user membership")
page: Final = await self.activity.user_page(
issue_activity_scope(cohort, start, end), limit=limit, offset=offset
)
metadata: Final = await self.activity.user_metadata(frozenset(row.user_id for row in page.rows if row.user_id))
return DailyActivityUserPageResponse(
users=tuple(
UserActivityRow.model_validate(
{
**asdict(row),
**(
asdict(metadata[row.user_id]) if row.user_id is not None and row.user_id in metadata else {}
),
}
)
for row in page.rows
),
total_users=page.total_users,
limit=limit,
offset=offset,
)
async def _counts(self, scope: DailyActivityScope) -> _Counts:
where, params = build_where_clause(scope)
rows: Final = TypeAdapter(tuple[_Counts, ...]).validate_python(
await self.database.query_raw(
f'''SELECT COUNT(DISTINCT NULLIF(api_key,'')) FILTER(WHERE api_key<>${len(params) + 1})::int AS keys,
COUNT(DISTINCT NULLIF("{scope.entity_id_field}",''))
FILTER(WHERE "{scope.entity_id_field}" NOT IN ('default_user_id','litellm-dashboard'))::int AS entities,
COUNT(DISTINCT NULLIF(model,''))::int AS models,
COALESCE(SUM(api_requests),0)::bigint AS requests
FROM "{PRISMA_TO_PG_TABLE[scope.table]}" WHERE {where}''',
*params,
PTU_SENTINEL_API_KEY,
)
)
return rows[0]
async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact:
scope: Final = issue_activity_scope(cohort, start, end)
counts: Final = await self._counts(scope)
teams: Final = (
await self._counts(replace(scope, table=DailyActivityTable.TEAM, entity_id_field="team_id"))
if cohort.scope_type not in ("user", "team")
else None
)
return IssueImpact(
keys=counts.keys,
models=counts.models,
requests=counts.requests,
users=counts.entities if cohort.scope_type != "team" else None,
teams=counts.entities if cohort.scope_type == "team" else teams.entities if teams else None,
)

View file

@ -0,0 +1,149 @@
"""Bounded settings for the built-in performance policies."""
import json
from datetime import datetime, timezone
from itertools import chain, product
from typing import Final
from pydantic import BaseModel, TypeAdapter
from litellm.repositories.usage_alerts import AlertDatabase
from litellm.types.proxy.issues import (
IssueFamilyList,
IssueFamilyPolicy,
IssueFamilySetting,
IssueFamilyUpdate,
)
from litellm.types.proxy.usage_alert_defaults import (
DEFAULT_USAGE_POLICIES,
DEFAULT_USAGE_RULE_IDS,
USAGE_ALERT_SCOPES,
PerformanceIssueFamily,
default_usage_rule_id,
)
from litellm.types.proxy.usage_alerts import UsageAlertRule
class _RuleRow(BaseModel, frozen=True):
record: UsageAlertRule
class _Count(BaseModel, frozen=True):
count: int
def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) -> IssueFamilySetting:
policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family)
groups: Final = tuple(
(policy, tuple(rule for rule in rules if rule.metric == policy.metric and rule.direction == policy.direction))
for policy in policies
)
values: Final = tuple(
IssueFamilyPolicy(
direction=policy.direction,
enabled=group[0].enabled if group and len({rule.enabled for rule in group}) == 1 else None,
multiplier=group[0].multiplier if group and len({rule.multiplier for rule in group}) == 1 else None,
min_value=group[0].min_value if group and len({rule.min_value for rule in group}) == 1 else None,
min_requests=group[0].min_requests if group and len({rule.min_requests for rule in group}) == 1 else None,
baseline_days=group[0].baseline_days
if group and len({rule.baseline_days for rule in group}) == 1
else None,
min_history_days=group[0].min_history_days
if group and len({rule.min_history_days for rule in group}) == 1
else None,
mixed=len(
{
(r.enabled, r.multiplier, r.min_value, r.min_requests, r.baseline_days, r.min_history_days)
for r in group
}
)
> 1,
total_rules=len(group),
enabled_rules=sum(rule.enabled for rule in group),
)
for policy, group in groups
)
family_rules: Final = tuple(chain.from_iterable(group for _, group in groups))
return IssueFamilySetting(
family=family,
enabled=family_rules[0].enabled if family_rules and len({r.enabled for r in family_rules}) == 1 else None,
total_rules=len(family_rules),
enabled_rules=sum(rule.enabled for rule in family_rules),
policies=values,
)
async def family_settings(database: AlertDatabase) -> IssueFamilyList:
rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python(
await database.query_raw(
'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted',
list(DEFAULT_USAGE_RULE_IDS),
)
)
return IssueFamilyList(
data=tuple(
_setting(family, tuple(row.record for row in rows))
for family in ("spend", "latency", "reliability", "traffic")
)
)
async def update_family(
database: AlertDatabase,
family: PerformanceIssueFamily,
data: IssueFamilyUpdate,
) -> IssueFamilySetting | None:
policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family)
if any(update.direction not in {policy.direction for policy in policies} for update in data.policies):
raise ValueError("This family does not support that direction")
ids: Final = tuple(
default_usage_rule_id(scope, policy.metric, policy.direction)
for policy, scope in product(policies, USAGE_ALERT_SCOPES)
)
rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python(
await database.query_raw(
'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted',
list(ids),
)
)
timestamp: Final = datetime.now(timezone.utc).isoformat()
updates: Final = {
policy.direction: policy.model_dump(exclude_none=True, exclude={"direction"}) for policy in data.policies
}
records: Final = tuple(
UsageAlertRule.model_validate(
{
**row.record.model_dump(),
**({"enabled": data.enabled} if data.enabled is not None else {}),
**updates.get(row.record.direction, {}),
"updated_at": timestamp,
}
)
for row in rows
)
if not records:
return None
changed: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
await database.query_raw(
"""WITH incoming AS MATERIALIZED (
SELECT x->'record' AS record,x->>'revision' AS revision FROM jsonb_array_elements($1::jsonb) x
), eligible AS MATERIALIZED (
SELECT r.id FROM "LiteLLM_UsageAlertRule" r JOIN incoming x ON r.id=x.record->>'id'
WHERE NOT r.deleted AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz
ORDER BY r.id FOR UPDATE OF r
), changed AS (
UPDATE "LiteLLM_UsageAlertRule" r SET record=x.record,enabled=(x.record->>'enabled')::boolean,
updated_at=(x.record->>'updated_at')::timestamptz AT TIME ZONE 'UTC'
FROM incoming x WHERE r.id=x.record->>'id' AND NOT r.deleted
AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz
AND (SELECT count(*) FROM eligible)=(SELECT count(*) FROM incoming) RETURNING r.id
) SELECT count(*)::int AS count FROM changed""",
json.dumps(
tuple(
{"record": record.model_dump(mode="json"), "revision": row.record.updated_at.isoformat()}
for record, row in zip(records, rows, strict=True)
)
),
)
)
return _setting(family, records) if changed[0].count == len(records) else None

View file

@ -0,0 +1,504 @@
import json
from collections.abc import Mapping
from datetime import date, datetime, timezone
from typing import (
TYPE_CHECKING,
Final,
Literal,
cast, # noqa: TID251 # generated Prisma client implements query_raw
)
from pydantic import BaseModel, TypeAdapter
from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key
from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository
from litellm.types.proxy.issues import (
Issue,
IssueDeliveryBinding,
IssueDeliveryInput,
IssueFamily,
IssueFamilyList,
IssueFamilySetting,
IssueFamilyUpdate,
IssueInput,
IssueKind,
IssueList,
IssuePendingDelivery,
IssueProjectionSignal,
IssueScope,
IssueSignal,
IssueSignalInput,
IssueSignalList,
)
from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily
from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
class _IssueRow(BaseModel, frozen=True):
record: Issue
class _SignalRow(BaseModel, frozen=True):
record: IssueSignal
class _Count(BaseModel, frozen=True):
count: int
class _Identifier(BaseModel, frozen=True):
id: str
_ACK: Final = "CASE WHEN jsonb_typeof(c.param_value)='string' THEN (c.param_value#>>'{}')::jsonb ELSE c.param_value END"
_STATUS: Final = f"""CASE WHEN i.kind <> 'model_deprecation' THEN i.status
WHEN NOT i.active THEN CASE WHEN ({_ACK})->>'status'='dismissed' THEN 'dismissed' ELSE 'resolved' END
WHEN COALESCE(({_ACK})->>'status','open')='open'
OR ({_ACK})->>'last_seen_date' IS NULL
OR i.record->'deprecation'->>'last_used_date' > ({_ACK})->>'last_seen_date'
OR (i.record->'deprecation'->>'last_used_date' = ({_ACK})->>'last_seen_date'
AND (i.record->'deprecation'->>'last_day_requests')::bigint
> COALESCE((({_ACK})->>'last_seen_requests')::bigint,0)) THEN 'open'
ELSE ({_ACK})->>'status' END"""
_JOIN: Final = 'FROM "LiteLLM_Issue" i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key'
_RECORD: Final = f"""i.record || jsonb_build_object('status',({_STATUS}),'active',i.active,
'first_seen',i.first_seen AT TIME ZONE 'UTC','last_seen',i.last_seen AT TIME ZONE 'UTC',
'updated_at',i.updated_at AT TIME ZONE 'UTC') || CASE WHEN i.kind='model_deprecation'
THEN jsonb_build_object('deprecation',i.record->'deprecation' ||
jsonb_build_object('state',COALESCE(({_ACK}),i.record->'deprecation'->'state')))
ELSE '{{}}'::jsonb END"""
def _utc(value: datetime) -> str:
return value.astimezone(timezone.utc).replace(tzinfo=None).isoformat()
def _issue_payload(data: IssueInput, status: UsageAlertStatus | None) -> Mapping[str, object]:
initial: Final = Issue.model_validate(
{
**data.model_dump(),
"status": status or "open",
"first_seen": data.observed_at,
"last_seen": data.observed_at,
"updated_at": data.observed_at,
}
)
return {
"record": initial.model_dump(mode="json"),
"ack_key": deprecation_alert_state_key(data.deprecation.usage_model, data.deprecation.model.deprecation_date)
if data.deprecation
else None,
}
class IssuesRepository:
def __init__(self, database: AlertDatabase) -> None:
self.database = database
@classmethod
def from_prisma(cls, prisma: "PrismaClient") -> "IssuesRepository":
return cls(cast(AlertDatabase, prisma.writer_db)) # cast-ok: generated Prisma delegate implements query_raw
async def upsert_issue(
self, data: IssueInput, reopen: bool = False, status: UsageAlertStatus | None = None
) -> Issue:
rows: Final = await self.upsert_issues((data,), reopen=reopen, status=status)
return rows[0]
async def upsert_issues(
self,
inputs: tuple[IssueInput, ...],
reopen: bool = False,
status: UsageAlertStatus | None = None,
) -> tuple[Issue, ...]:
if not inputs:
return ()
if len(inputs) > 500 or len(frozenset(item.id for item in inputs)) != len(inputs):
raise ValueError("Write at most 500 distinct issues per batch")
payload: Final = tuple(_issue_payload(item, status) for item in inputs)
rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
await self.database.query_raw(
f"""WITH incoming AS (
SELECT x->'record' AS record,x->>'ack_key' AS ack_key FROM jsonb_array_elements($1::jsonb) x
), changed AS (
INSERT INTO "LiteLLM_Issue" AS old
(id,cohort_key,kind,family,scope_type,scope_id,status,ack_key,record,first_seen,last_seen,updated_at)
SELECT record->>'id',record->>'cohort_key',record->>'kind',record->>'family',
record->'cohort'->>'scope_type',record->'cohort'->>'scope_id',record->>'status',ack_key,record,
(record->>'observed_at')::timestamptz AT TIME ZONE 'UTC',
(record->>'observed_at')::timestamptz AT TIME ZONE 'UTC',
(record->>'observed_at')::timestamptz AT TIME ZONE 'UTC' FROM incoming
ON CONFLICT(id) DO UPDATE SET
record=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen
THEN EXCLUDED.record ELSE old.record END,
last_seen=GREATEST(old.last_seen,EXCLUDED.last_seen),
active=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen
THEN true ELSE old.active END,
status=CASE WHEN $2 AND old.active AND EXCLUDED.last_seen > old.updated_at
THEN 'open' ELSE old.status END,
updated_at=GREATEST(old.updated_at,EXCLUDED.last_seen)
WHERE old.kind=EXCLUDED.kind AND old.cohort_key=EXCLUDED.cohort_key
AND old.scope_type=EXCLUDED.scope_type AND old.scope_id IS NOT DISTINCT FROM EXCLUDED.scope_id
RETURNING *
) SELECT {_RECORD} AS record FROM changed i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key""",
json.dumps(payload),
reopen,
)
)
if len(rows) != len(inputs):
raise ValueError("An issue ID already belongs to another cohort")
return tuple(row.record for row in rows)
async def get_issue(self, issue_id: str) -> Issue | None:
rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
await self.database.query_raw(f"SELECT {_RECORD} AS record {_JOIN} WHERE i.id=$1", issue_id)
)
return (await self._labels((rows[0].record,)))[0] if rows else None
async def list_issues(
self,
kind: IssueKind | None = None,
family: IssueFamily | None = None,
status: UsageAlertStatus | None = None,
scope_type: IssueScope | None = None,
scope_id: str | None = None,
limit: int = 50,
offset: int = 0,
) -> IssueList:
where: Final = f"""($1::text IS NULL OR i.kind=$1) AND ($2::text IS NULL OR i.family=$2)
AND ($3::text IS NULL OR ({_STATUS})=$3) AND ($4::text IS NULL OR i.scope_type=$4)
AND ($5::text IS NULL OR i.scope_id=$5)"""
args: Final = (kind, family, status, scope_type, scope_id)
count: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
await self.database.query_raw(f"SELECT count(*)::int AS count {_JOIN} WHERE {where}", *args)
)
rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
await self.database.query_raw(
f"SELECT {_RECORD} AS record {_JOIN} WHERE {where} ORDER BY i.last_seen DESC,i.id LIMIT $6 OFFSET $7",
*args,
limit,
offset,
)
)
return IssueList(
data=await self._labels(tuple(row.record for row in rows)), total=count[0].count, limit=limit, offset=offset
)
async def _labels(self, issues: tuple[Issue, ...]) -> tuple[Issue, ...]:
if not issues:
return ()
labels: Final = await UsageAlertsRepository(self.database).resource_labels(
tuple(issue.cohort for issue in issues)
)
return tuple(
issue.model_copy(update={"scope_label": labels.get((issue.cohort.scope_type, issue.cohort.scope_id))})
for issue in issues
)
async def find_active_issue(self, cohort_key: str) -> Issue | None:
rows: Final = await self.find_active_issues((cohort_key,))
return rows[0] if rows else None
async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]:
if len(cohort_keys) > 500:
raise ValueError("Read at most 500 issue cohorts per batch")
rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
await self.database.query_raw(
f"SELECT DISTINCT ON(i.cohort_key) {_RECORD} AS record {_JOIN} "
"WHERE i.cohort_key=ANY($1::text[]) AND i.active "
"ORDER BY i.cohort_key,i.first_seen DESC,i.id",
list(cohort_keys),
)
)
return tuple(row.record for row in rows)
async def set_status(
self,
issue_id: str,
status: UsageAlertStatus,
expected_status: UsageAlertStatus,
actor: str | None = None,
) -> Issue | None:
changed: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python(
await self.database.query_raw(
f"""WITH candidate AS MATERIALIZED (
SELECT i.*,({_STATUS}) AS effective_status {_JOIN} WHERE i.id=$1
), ack AS (
INSERT INTO "LiteLLM_Config" AS prior (param_name,param_value)
SELECT ack_key,jsonb_build_object('model',record->'deprecation'->>'usage_model',
'deprecation_date',record->'deprecation'->'model'->>'deprecation_date',
'status',$2::text,'last_seen_date',record->'deprecation'->>'last_used_date',
'last_seen_requests',record->'deprecation'->'last_day_requests',
'updated_at',CURRENT_TIMESTAMP,'updated_by',$4::text)
FROM candidate WHERE kind='model_deprecation' AND effective_status=$3
ON CONFLICT(param_name) DO UPDATE SET param_value=EXCLUDED.param_value
WHERE (CASE WHEN COALESCE((CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'status','open')='open'
OR (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' IS NULL
OR EXCLUDED.param_value->>'last_seen_date' > (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date'
OR (EXCLUDED.param_value->>'last_seen_date'=(CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date'
AND (EXCLUDED.param_value->>'last_seen_requests')::bigint
> COALESCE(((CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_requests')::bigint,0))
THEN 'open' ELSE (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'status' END)=$3
RETURNING param_name
) UPDATE "LiteLLM_Issue" i SET status=$2,updated_at=CURRENT_TIMESTAMP
FROM candidate c WHERE i.id=c.id AND c.effective_status=$3 AND
((c.kind<>'model_deprecation' AND i.status=$3) OR i.ack_key IN(SELECT param_name FROM ack))
RETURNING i.id""",
issue_id,
status,
expected_status,
actor,
)
)
return await self.get_issue(issue_id) if changed else None
async def add_signals(self, issue_id: str, signals: tuple[IssueSignalInput, ...]) -> None:
await self.add_signal_batch(
tuple(IssueSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in signals)
)
async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None:
if not signals:
return
if len(signals) > 500:
raise ValueError("Write at most 500 issue signals per batch")
await self.database.query_raw(
"""INSERT INTO "LiteLLM_IssueSignal" (issue_id,source,source_id,record)
SELECT DISTINCT ON(x->>'issue_id',x->>'source',x->>'source_id')
x->>'issue_id',x->>'source',x->>'source_id',x FROM jsonb_array_elements($1::jsonb) x
WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" i WHERE i.id=x->>'issue_id'
AND (i.scope_type NOT IN ('key','team','user') OR
(x->'usage_alert'->>'scope_type'=i.scope_type
AND x->'usage_alert'->>'scope_id'=i.scope_id)))
ON CONFLICT(issue_id,source,source_id) DO NOTHING RETURNING source_id""",
json.dumps(tuple(signal.model_dump(mode="json") for signal in signals)),
)
async def list_signals(self, issue_id: str, limit: int = 50, offset: int = 0) -> IssueSignalList:
counts: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
await self.database.query_raw(
'SELECT count(*)::int AS count FROM "LiteLLM_IssueSignal" WHERE issue_id=$1',
issue_id,
)
)
rows: Final = TypeAdapter(tuple[_SignalRow, ...]).validate_python(
await self.database.query_raw(
'SELECT record FROM "LiteLLM_IssueSignal" WHERE issue_id=$1 ORDER BY source,source_id LIMIT $2 OFFSET $3',
issue_id,
limit,
offset,
)
)
return IssueSignalList(
data=tuple(row.record for row in rows), total=counts[0].count, limit=limit, offset=offset
)
async def active_signals(
self,
day: date,
limit: int = 500,
cursor: str | None = None,
scope_type: IssueScope | None = None,
metric: UsageAlertMetric | None = None,
) -> tuple[IssueProjectionSignal, ...]:
return TypeAdapter(tuple[IssueProjectionSignal, ...]).validate_python(
await self.database.query_raw(
"""SELECT a.record AS alert,r.record AS rule,a.evaluation_day
FROM "LiteLLM_UsageAlertIncident" a JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id
WHERE a.evaluation_day=$1 AND a.breached AND r.enabled AND NOT r.deleted
AND NULLIF(a.evaluation_revision,'')::timestamptz=(r.record->>'updated_at')::timestamptz
AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric'
AND a.record->>'direction'=r.record->>'direction'
AND (r.scope_id IS NULL OR a.scope_id=r.scope_id)
AND ($2::text IS NULL OR a.id>$2) AND ($4::text IS NULL OR a.scope_type=$4)
AND ($5::text IS NULL OR a.metric=$5) ORDER BY a.id LIMIT $3""",
day.isoformat(),
cursor,
limit,
scope_type,
metric,
)
)
async def recover_issues(self, day: date) -> int:
rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
await self.database.query_raw(
"""WITH changed AS (
UPDATE "LiteLLM_Issue" i SET active=false,
status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP
WHERE kind='performance' AND active
AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id AND s.source='usage_alert')
AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s
LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id
WHERE s.issue_id=i.id AND s.source='usage_alert'
AND (a.id IS NULL OR a.evaluation_day>$1 OR a.breached)) RETURNING i.id
) SELECT count(*)::int AS count FROM changed""",
day.isoformat(),
)
)
return rows[0].count
async def projection_completed(self, day: date, version: str) -> bool:
rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python(
await self.database.query_raw(
'SELECT id FROM "LiteLLM_UsageAlertCheckpoint" WHERE id=$1 AND completed',
f"issues:{version}:{day.isoformat()}",
)
)
return bool(rows)
async def complete_projection(self, day: date, version: str) -> None:
await self.database.query_raw(
"""INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed,expires_at) VALUES($1,true,CURRENT_TIMESTAMP)
ON CONFLICT(id) DO UPDATE SET completed=true,expires_at=CURRENT_TIMESTAMP RETURNING id""",
f"issues:{version}:{day.isoformat()}",
)
async def finish_deprecation_projection(self, observed_at: datetime) -> int:
rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
await self.database.query_raw(
"""WITH changed AS (
UPDATE "LiteLLM_Issue" SET active=false,
status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP
WHERE kind='model_deprecation' AND active AND last_seen<$1::timestamp RETURNING id
) SELECT count(*)::int AS count FROM changed""",
_utc(observed_at),
)
)
return rows[0].count
async def family_settings(self) -> "IssueFamilyList":
from litellm.repositories.issue_families import family_settings
return await family_settings(self.database)
async def update_family(
self, family: "PerformanceIssueFamily", data: "IssueFamilyUpdate"
) -> "IssueFamilySetting | None":
from litellm.repositories.issue_families import update_family
return await update_family(self.database, family, data)
async def queue_delivery(
self,
data: IssueDeliveryInput,
rule_id: str,
rule_revision: str,
source_id: str,
) -> None:
await self.queue_deliveries(
(IssueDeliveryBinding(delivery=data, rule_id=rule_id, rule_revision=rule_revision, source_id=source_id),)
)
async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None:
if not bindings:
return
if len(bindings) > 500:
raise ValueError("Queue at most 500 delivery bindings per batch")
await self.database.query_raw(
"""WITH incoming AS MATERIALIZED (
SELECT x->'delivery' AS delivery,x->>'rule_id' AS rule_id,
x->>'rule_revision' AS revision,x->>'source_id' AS source_id
FROM jsonb_array_elements($1::jsonb) x
WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" WHERE id=x->'delivery'->>'issue_id')
), saved AS (
INSERT INTO "LiteLLM_IssueDelivery" (id,issue_id,record)
SELECT DISTINCT ON(delivery->>'id') delivery->>'id',delivery->>'issue_id',delivery
FROM incoming ON CONFLICT(id) DO NOTHING RETURNING id
) INSERT INTO "LiteLLM_IssueDeliveryRule" (delivery_id,rule_id,source_id,rule_revision)
SELECT DISTINCT ON(delivery->>'id',rule_id,source_id) delivery->>'id',rule_id,source_id,revision
FROM incoming WHERE delivery->>'id' IN(SELECT id FROM saved)
OR EXISTS(SELECT 1 FROM "LiteLLM_IssueDelivery" d WHERE d.id=delivery->>'id'
AND d.issue_id=delivery->>'issue_id')
ON CONFLICT(delivery_id,rule_id,source_id) DO UPDATE SET rule_revision=EXCLUDED.rule_revision
RETURNING delivery_id""",
json.dumps(tuple(binding.model_dump(mode="json") for binding in bindings)),
)
async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]:
if not 1 <= limit <= 500:
raise ValueError("Read between one and 500 deliveries")
return TypeAdapter(tuple[IssuePendingDelivery, ...]).validate_python(
await self.database.query_raw(
f"""SELECT d.record || jsonb_build_object('next_attempt',d.next_attempt AT TIME ZONE 'UTC') AS delivery,
{_RECORD} AS issue,bound.rule,bound.alert
FROM "LiteLLM_IssueDelivery" d JOIN "LiteLLM_Issue" i ON i.id=d.issue_id
LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key
JOIN LATERAL (
SELECT r.record AS rule,a.record AS alert FROM "LiteLLM_IssueDeliveryRule" b
JOIN "LiteLLM_UsageAlertRule" r ON r.id=b.rule_id
JOIN "LiteLLM_UsageAlertIncident" a ON a.id=b.source_id AND a.rule_id=r.id
WHERE b.delivery_id=d.id AND r.enabled AND NOT r.deleted AND a.breached AND a.status='open'
AND (r.record->>'updated_at')::timestamptz=b.rule_revision::timestamptz
AND NULLIF(a.evaluation_revision,'')::timestamptz=b.rule_revision::timestamptz
AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric'
AND a.record->>'direction'=r.record->>'direction'
AND (r.scope_id IS NULL OR a.scope_id=r.scope_id)
AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id
AND s.source='usage_alert' AND s.source_id=a.id)
ORDER BY b.rule_id,b.source_id LIMIT 1
) bound ON true
WHERE NOT d.sent AND d.next_attempt<=CURRENT_TIMESTAMP AND i.active AND ({_STATUS})='open'
ORDER BY d.next_attempt,d.id LIMIT $1""",
limit,
)
)
async def mark_delivery_sent(self, delivery_id: str) -> None:
await self.database.query_raw(
"""UPDATE "LiteLLM_IssueDelivery" SET sent=true,updated_at=CURRENT_TIMESTAMP,
record=record || jsonb_build_object('last_error',NULL,'attempts',
COALESCE((record->>'attempts')::int,0)+1)
WHERE id=$1 AND NOT sent RETURNING id""",
delivery_id,
)
async def defer_delivery(
self,
delivery_id: str,
error: Literal["delivery_failed", "transport_not_configured"],
) -> None:
await self.database.query_raw(
"""UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP+interval '1 hour',
updated_at=CURRENT_TIMESTAMP,record=record || jsonb_build_object('last_error',$2::text,
'attempts',COALESCE((record->>'attempts')::int,0)+1)
WHERE id=$1 AND NOT sent RETURNING id""",
delivery_id,
error,
)
async def prune(self, retention_days: int = 90, limit: int = 500) -> None:
if not 1 <= limit <= 500 or retention_days < 1:
raise ValueError("Retention and cleanup batch must be bounded positive values")
await self.database.query_raw(
"""WITH expired AS MATERIALIZED (
SELECT id FROM "LiteLLM_Issue" WHERE status<>'open'
AND updated_at<CURRENT_TIMESTAMP-make_interval(days=>$1)
ORDER BY updated_at,id LIMIT $2
), signals AS (
DELETE FROM "LiteLLM_IssueSignal" WHERE (issue_id,source,source_id) IN (
SELECT issue_id,source,source_id FROM "LiteLLM_IssueSignal"
WHERE issue_id IN(SELECT id FROM expired) ORDER BY issue_id,source,source_id LIMIT $2)
RETURNING issue_id
), bindings AS (
DELETE FROM "LiteLLM_IssueDeliveryRule" WHERE (delivery_id,rule_id,source_id) IN (
SELECT b.delivery_id,b.rule_id,b.source_id FROM "LiteLLM_IssueDeliveryRule" b
JOIN "LiteLLM_IssueDelivery" d ON d.id=b.delivery_id
WHERE d.issue_id IN(SELECT id FROM expired) ORDER BY b.delivery_id,b.rule_id,b.source_id LIMIT $2)
RETURNING delivery_id
), deliveries AS (
DELETE FROM "LiteLLM_IssueDelivery" d WHERE d.id IN (
SELECT id FROM "LiteLLM_IssueDelivery" d WHERE issue_id IN(SELECT id FROM expired)
AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueDeliveryRule" b WHERE b.delivery_id=d.id)
ORDER BY id LIMIT $2) RETURNING issue_id
), removed AS (
DELETE FROM "LiteLLM_Issue" i WHERE id IN(SELECT id FROM expired)
AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id)
AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueDelivery" d WHERE d.issue_id=i.id) RETURNING id
) SELECT count(*)::int AS count FROM removed""",
retention_days,
limit,
)

View file

@ -3,7 +3,6 @@
import json
from collections.abc import Mapping
from datetime import date, datetime, timedelta, timezone
from itertools import product
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -17,6 +16,7 @@ from uuid import NAMESPACE_URL, uuid4, uuid5
from pydantic import BaseModel, TypeAdapter
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS
from litellm.types.proxy.usage_alerts import (
UsageAlert,
UsageAlertChannel,
@ -49,6 +49,14 @@ class AlertDatabase(Protocol):
async def query_raw(self, query: str, *args: object) -> object: ...
class ResourceLabelScope(Protocol):
@property
def scope_type(self) -> str: ...
@property
def scope_id(self) -> str | None: ...
class _Record(BaseModel, frozen=True):
record: Mapping[str, object]
@ -75,7 +83,7 @@ class _GroupPage(BaseModel, frozen=True):
class _ResourceLabel(BaseModel, frozen=True):
scope_type: UsageAlertScope
scope_type: str
scope_id: str | None
label: str | None
@ -98,14 +106,7 @@ class _Ownership(BaseModel, frozen=True):
team_id: str | None
_DEFAULT_RULE_IDS: Final = frozenset(
str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{scope}:{metric}:{direction}"))
for scope, metric, direction in product(
("deployment", "key", "user", "team", "model"),
("spend", "requests", "latency", "errors"),
("increase", "decrease"),
)
)
_DEFAULT_RULE_IDS: Final = DEFAULT_USAGE_RULE_IDS
def _rule(record: Mapping[str, object]) -> UsageAlertRule:
@ -405,8 +406,8 @@ class UsageAlertsRepository:
)
)
page: Final = rows[0]
labels: Final[Mapping[tuple[UsageAlertScope, str | None], str | None]] = (
await self._group_labels(page.data) if page.data else {}
labels: Final[Mapping[tuple[str, str | None], str | None]] = (
await self.resource_labels(page.data) if page.data else {}
)
return UsageAlertGroupList(
data=tuple(
@ -434,9 +435,9 @@ class UsageAlertsRepository:
offset=offset,
)
async def _group_labels(
self, groups: tuple[_GroupRow, ...]
) -> Mapping[tuple[UsageAlertScope, str | None], str | None]:
async def resource_labels(
self, groups: tuple[ResourceLabelScope, ...]
) -> Mapping[tuple[str, str | None], str | None]:
rows: Final = TypeAdapter(tuple[_ResourceLabel, ...]).validate_python(
await self.database.query_raw(
"""WITH scopes AS (
@ -446,7 +447,8 @@ class UsageAlertsRepository:
WHEN 'key' THEN COALESCE(NULLIF(k.key_alias,''),NULLIF(k.key_name,''))
WHEN 'team' THEN NULLIF(t.team_alias,'')
WHEN 'user' THEN COALESCE(NULLIF(u.user_alias,''),NULLIF(u.user_email,''))
WHEN 'model' THEN s.scope_id WHEN 'deployment' THEN 'Gateway' END AS label
WHEN 'model' THEN s.scope_id WHEN 'provider' THEN s.scope_id
WHEN 'deployment' THEN 'Gateway' END AS label
FROM scopes s LEFT JOIN "LiteLLM_VerificationToken" k ON s.scope_type='key' AND k.token=s.key_id
LEFT JOIN "LiteLLM_TeamTable" t ON s.scope_type='team' AND t.team_id=s.scope_id
LEFT JOIN "LiteLLM_UserTable" u ON s.scope_type='user' AND u.user_id=s.scope_id""",

View file

@ -0,0 +1,260 @@
from datetime import date, datetime
from typing import Literal, Self, TypeAlias
from pydantic import BaseModel, ConfigDict, Field, model_validator
from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo
from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily
from litellm.types.proxy.usage_alerts import (
UsageAlert,
UsageAlertChannel,
UsageAlertDirection,
UsageAlertMetric,
UsageAlertRule,
UsageAlertStatus,
)
IssueKind: TypeAlias = Literal["performance", "model_deprecation"]
IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"]
IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"]
IssueGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"]
class IssueModelPair(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
provider: str = Field(min_length=1, max_length=256)
model: str = Field(min_length=1, max_length=1024)
class IssueCohort(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
scope_type: IssueScope
scope_id: str | None = Field(default=None, min_length=1, max_length=1024)
model_pairs: tuple[IssueModelPair, ...] = Field(default=(), max_length=100)
@model_validator(mode="after")
def validate_scope(self) -> Self:
if (self.scope_type == "deployment") != (self.scope_id is None):
raise ValueError("Only a deployment cohort omits scope_id")
if self.scope_type == "key" and self.scope_id and self.scope_id.removeprefix("raw:").startswith("sk-"):
raise ValueError("Use a reporting key ID, never a secret key")
if len(frozenset((pair.provider, pair.model) for pair in self.model_pairs)) != len(self.model_pairs):
raise ValueError("model_pairs must be unique")
return self
class IssueComparison(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False)
metric: UsageAlertMetric
direction: UsageAlertDirection
observed_value: float = Field(ge=0)
baseline_value: float = Field(ge=0)
threshold: float
baseline_start: date
baseline_end: date
baseline_sample_days: int | None = Field(default=None, ge=1, le=30)
@model_validator(mode="after")
def validate_comparison(self) -> Self:
if self.baseline_start > self.baseline_end:
raise ValueError("baseline_start must not follow baseline_end")
if self.metric == "error_rate" and (self.observed_value > 100 or self.baseline_value > 100):
raise ValueError("error_rate uses percentages from 0 to 100")
return self
class IssueImpact(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
keys: int | None = Field(default=None, ge=0)
users: int | None = Field(default=None, ge=0)
teams: int | None = Field(default=None, ge=0)
models: int | None = Field(default=None, ge=0)
requests: int | None = Field(default=None, ge=0)
measured_keys: int | None = Field(default=None, ge=0)
measured_models: int | None = Field(default=None, ge=0)
class IssueDeprecation(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
model: ModelDeprecationInfo
usage_model: str = Field(min_length=1, max_length=1024)
requests: int = Field(ge=0)
last_used_date: date
last_day_requests: int = Field(ge=0)
state: DeprecationAlertState
class IssueInput(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
id: str = Field(min_length=1, max_length=200)
cohort_key: str = Field(min_length=1, max_length=200)
kind: IssueKind
family: IssueFamily
title: str = Field(min_length=1, max_length=2048)
cohort: IssueCohort
observed_on: date
observed_at: datetime
range_start: date
range_end: date
grouping_reason: IssueGroupingReason
explanation: str = Field(default="", max_length=2000)
comparisons: tuple[IssueComparison, ...] = Field(default=(), max_length=10)
impact: IssueImpact = Field(default_factory=IssueImpact)
deprecation: IssueDeprecation | None = None
@model_validator(mode="after")
def validate_issue(self) -> Self:
if self.range_start > self.range_end:
raise ValueError("range_start must not follow range_end")
if (self.kind == "model_deprecation") != (self.deprecation is not None):
raise ValueError("Model deprecation issues require deprecation details")
if (self.kind == "model_deprecation") != (self.family == "retirement"):
raise ValueError("Only model deprecation issues use the retirement family")
if self.observed_at.tzinfo is None:
raise ValueError("observed_at must include a timezone")
return self
class Issue(IssueInput, frozen=True):
status: UsageAlertStatus
first_seen: datetime
last_seen: datetime
updated_at: datetime
scope_label: str | None = None
active: bool = True
class IssueList(BaseModel, frozen=True):
data: tuple[Issue, ...]
total: int
limit: int
offset: int
class IssueUpdate(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
status: UsageAlertStatus
expected_status: UsageAlertStatus
class IssueSignalInput(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
source: Literal["usage_alert", "model_deprecation"]
source_id: str = Field(min_length=1, max_length=200)
usage_alert: UsageAlert | None = None
deprecation: IssueDeprecation | None = None
@model_validator(mode="after")
def validate_signal(self) -> Self:
if self.source == "usage_alert":
if self.usage_alert is None or self.deprecation is not None or self.usage_alert.id != self.source_id:
raise ValueError("Usage signals require their matching source alert")
elif self.deprecation is None or self.usage_alert is not None:
raise ValueError("Deprecation signals require deprecation details")
return self
class IssueSignal(IssueSignalInput, frozen=True):
issue_id: str
class IssueSignalList(BaseModel, frozen=True):
data: tuple[IssueSignal, ...]
total: int
limit: int
offset: int
class IssueProjectionSignal(BaseModel, frozen=True):
alert: UsageAlert
rule: UsageAlertRule
evaluation_day: date
class IssueDeliveryInput(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
id: str = Field(min_length=1, max_length=200)
issue_id: str = Field(min_length=1, max_length=200)
channel: UsageAlertChannel
email_recipients: tuple[str, ...] = Field(default=(), max_length=20)
class IssueDeliveryBinding(BaseModel, frozen=True):
delivery: IssueDeliveryInput
rule_id: str
rule_revision: str
source_id: str
class IssueDelivery(IssueDeliveryInput, frozen=True):
attempts: int = 0
next_attempt: datetime | None = None
last_error: Literal["delivery_failed", "transport_not_configured"] | None = None
class IssuePendingDelivery(BaseModel, frozen=True):
delivery: IssueDelivery
issue: Issue
rule: UsageAlertRule
alert: UsageAlert
class IssueFamilyPolicy(BaseModel, frozen=True):
direction: UsageAlertDirection
enabled: bool | None
multiplier: float | None
min_value: float | None
min_requests: int | None
baseline_days: int | None
min_history_days: int | None
mixed: bool
total_rules: int
enabled_rules: int
class IssueFamilySetting(BaseModel, frozen=True):
family: PerformanceIssueFamily
enabled: bool | None
total_rules: int
enabled_rules: int
policies: tuple[IssueFamilyPolicy, ...]
class IssueFamilyList(BaseModel, frozen=True):
data: tuple[IssueFamilySetting, ...]
class IssueFamilyPolicyUpdate(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
direction: UsageAlertDirection
enabled: bool | None = None
multiplier: float | None = Field(default=None, gt=1, le=1000)
min_value: float | None = Field(default=None, ge=0)
min_requests: int | None = Field(default=None, ge=0)
baseline_days: int | None = Field(default=None, ge=1, le=30)
min_history_days: int | None = Field(default=None, ge=1, le=30)
class IssueFamilyUpdate(BaseModel, frozen=True):
model_config = ConfigDict(frozen=True, extra="forbid")
enabled: bool | None = None
policies: tuple[IssueFamilyPolicyUpdate, ...] = Field(default=(), max_length=2)
@model_validator(mode="after")
def validate_update(self) -> Self:
if len(frozenset(policy.direction for policy in self.policies)) != len(self.policies):
raise ValueError("Each direction can occur once")
if self.enabled is None and not self.policies:
raise ValueError("Specify enabled or a policy update")
return self

View file

@ -182,6 +182,8 @@ class KeySpendMetrics(LiteLLMBaseModel):
failed_requests: int = 0
cache_read_input_tokens: int = 0
cache_creation_input_tokens: int = 0
total_response_time_ms: int = 0
timed_requests: int = 0
class KeySpendActivityRow(LiteLLMBaseModel):
@ -214,6 +216,8 @@ class UserActivityRow(LiteLLMBaseModel):
api_requests: int = 0
successful_requests: int = 0
failed_requests: int = 0
total_response_time_ms: int = 0
timed_requests: int = 0
class DailyActivityUserPageResponse(LiteLLMBaseModel):

View file

@ -0,0 +1,75 @@
from dataclasses import dataclass
from itertools import product
from typing import Final, Literal, TypeAlias
from uuid import NAMESPACE_URL, uuid5
from litellm.types.proxy.usage_alerts import (
UsageAlertChannel,
UsageAlertDirection,
UsageAlertMetric,
UsageAlertRule,
UsageAlertRuleInput,
UsageAlertScope,
)
PerformanceIssueFamily: TypeAlias = Literal["spend", "latency", "reliability", "traffic"]
USAGE_ALERT_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("deployment", "key", "user", "team", "model")
@dataclass(frozen=True, slots=True)
class DefaultUsagePolicy:
family: PerformanceIssueFamily
metric: UsageAlertMetric
direction: UsageAlertDirection
multiplier: float
min_value: float
min_requests: int = 0
min_history_days: int = 3
DEFAULT_USAGE_POLICIES: Final = (
DefaultUsagePolicy("spend", "spend", "increase", 3, 10),
DefaultUsagePolicy("latency", "latency", "increase", 3, 500, 100),
DefaultUsagePolicy("reliability", "error_rate", "increase", 3, 2, 500),
DefaultUsagePolicy("traffic", "requests", "increase", 10, 1000),
DefaultUsagePolicy("traffic", "requests", "decrease", 5, 0, 500, 7),
)
def default_usage_rule_id(scope: UsageAlertScope, metric: UsageAlertMetric, direction: UsageAlertDirection) -> str:
return str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{scope}:{metric}:{direction}"))
DEFAULT_USAGE_RULE_IDS: Final = frozenset(
default_usage_rule_id(scope, policy.metric, policy.direction)
for scope, policy in product(USAGE_ALERT_SCOPES, DEFAULT_USAGE_POLICIES)
)
def is_default_usage_rule(rule: UsageAlertRule) -> bool:
return rule.id in DEFAULT_USAGE_RULE_IDS and rule.id == default_usage_rule_id(
rule.scope_type, rule.metric, rule.direction
)
def default_usage_rules(
channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False
) -> tuple[UsageAlertRuleInput, ...]:
return tuple(
UsageAlertRuleInput(
name=f"Daily {scope} {policy.metric.replace('_', ' ')} {policy.direction}",
scope_type=scope,
metric=policy.metric,
direction=policy.direction,
multiplier=policy.multiplier,
min_value=policy.min_value,
min_requests=policy.min_requests,
min_history_days=policy.min_history_days,
notification_channels=(
()
if legacy_user_spend_alerts and (scope, policy.metric, policy.direction) == ("user", "spend", "increase")
else channels
),
)
for scope, policy in product(USAGE_ALERT_SCOPES, DEFAULT_USAGE_POLICIES)
)

View file

@ -6,7 +6,7 @@ from typing import Literal, Self, TypeAlias
from pydantic import BaseModel, ConfigDict, Field, model_validator
UsageAlertScope: TypeAlias = Literal["deployment", "key", "user", "team", "model"]
UsageAlertMetric: TypeAlias = Literal["spend", "requests", "latency", "errors"]
UsageAlertMetric: TypeAlias = Literal["spend", "requests", "latency", "errors", "error_rate"]
UsageAlertDirection: TypeAlias = Literal["increase", "decrease"]
UsageAlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"]
UsageAlertChannel: TypeAlias = Literal["email", "slack"]

View file

@ -39,6 +39,8 @@ class DailyActivityScope:
model: str | None
timezone_offset_minutes: int | None
include_current_utc_day: bool = False
provider: str | None = None
model_pairs: tuple[tuple[str, str], ...] = ()
def __post_init__(self) -> None:
if self.entity_id_field not in _ENTITY_FIELDS[self.table]:
@ -57,6 +59,8 @@ class KeySpendRow:
failed_requests: int
cache_read_input_tokens: int
cache_creation_input_tokens: int
total_response_time_ms: int = 0
timed_requests: int = 0
@dataclass(frozen=True, slots=True)
@ -75,6 +79,8 @@ class UserSpendRow:
api_requests: int
successful_requests: int
failed_requests: int
total_response_time_ms: int = 0
timed_requests: int = 0
@dataclass(frozen=True, slots=True)

View file

@ -2132,11 +2132,13 @@ model LiteLLM_UsageAlertIncident {
status String
day String
evaluation_day String
evaluation_revision String @default("")
breached Boolean
record Json
notified_channels String[] @default([])
updated_at DateTime @default(now())
@@index([rule_id, scope_key, evaluation_day])
@@index([evaluation_day, breached, id])
@@index([scope_type, scope_id, day])
@@index([status, day])
@@index([updated_at])
@ -2149,3 +2151,55 @@ model LiteLLM_UsageAlertCheckpoint {
expires_at DateTime @default(now())
@@index([expires_at])
}
model LiteLLM_Issue {
id String @id
cohort_key String
kind String
family String
scope_type String
scope_id String?
status String
active Boolean @default(true)
ack_key String?
record Json
first_seen DateTime
last_seen DateTime
updated_at DateTime @default(now())
@@index([status, last_seen, id])
@@index([kind, family, last_seen, id])
@@index([scope_type, scope_id, last_seen, id])
@@index([cohort_key, active])
}
model LiteLLM_IssueSignal {
issue_id String
source String
source_id String
record Json
@@id([issue_id, source, source_id])
@@index([source, source_id])
}
model LiteLLM_IssueDelivery {
id String @id
issue_id String
record Json
sent Boolean @default(false)
next_attempt DateTime @default(now())
updated_at DateTime @default(now())
@@index([sent, next_attempt, id])
@@index([issue_id])
}
model LiteLLM_IssueDeliveryRule {
delivery_id String
rule_id String
source_id String
rule_revision String
@@id([delivery_id, rule_id, source_id])
}

View file

@ -8343,6 +8343,160 @@ export interface paths {
patch?: never;
trace?: never;
};
"/issues": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** List Issues */
get: operations["list_issues_issues_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/issues/families": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** List Issue Families */
get: operations["list_issue_families_issues_families_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/issues/families/{family}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
/** Update Issue Family */
patch: operations["update_issue_family_issues_families__family__patch"];
trace?: never;
};
"/issues/{issue_id}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Get Issue */
get: operations["get_issue_issues__issue_id__get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
/** Update Issue */
patch: operations["update_issue_issues__issue_id__patch"];
trace?: never;
};
"/issues/{issue_id}/activity": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Get Issue Activity */
get: operations["get_issue_activity_issues__issue_id__activity_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/issues/{issue_id}/impact": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Get Issue Impact */
get: operations["get_issue_impact_issues__issue_id__impact_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/issues/{issue_id}/keys": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Get Issue Keys */
get: operations["get_issue_keys_issues__issue_id__keys_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/issues/{issue_id}/signals": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** List Issue Signals */
get: operations["list_issue_signals_issues__issue_id__signals_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/issues/{issue_id}/users": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Get Issue Users */
get: operations["get_issue_users_issues__issue_id__users_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/jwt/key/mapping/delete": {
parameters: {
query?: never;
@ -34710,6 +34864,302 @@ export interface components {
/** Is Accepted */
is_accepted: boolean;
};
/** Issue */
Issue: {
/**
* Active
* @default true
*/
active: boolean;
cohort: components["schemas"]["IssueCohort"];
/** Cohort Key */
cohort_key: string;
/**
* Comparisons
* @default []
*/
comparisons: components["schemas"]["IssueComparison"][];
deprecation?: components["schemas"]["IssueDeprecation"] | null;
/**
* Explanation
* @default
*/
explanation: string;
/**
* Family
* @enum {string}
*/
family: "retirement" | "latency" | "reliability" | "spend" | "traffic";
/**
* First Seen
* Format: date-time
*/
first_seen: string;
/**
* Grouping Reason
* @enum {string}
*/
grouping_reason: "same_resource" | "dominant_key" | "provider_cohort" | "model_deprecation";
/** Id */
id: string;
impact?: components["schemas"]["IssueImpact"];
/**
* Kind
* @enum {string}
*/
kind: "performance" | "model_deprecation";
/**
* Last Seen
* Format: date-time
*/
last_seen: string;
/**
* Observed At
* Format: date-time
*/
observed_at: string;
/**
* Observed On
* Format: date
*/
observed_on: string;
/**
* Range End
* Format: date
*/
range_end: string;
/**
* Range Start
* Format: date
*/
range_start: string;
/** Scope Label */
scope_label?: string | null;
/**
* Status
* @enum {string}
*/
status: "open" | "resolved" | "dismissed";
/** Title */
title: string;
/**
* Updated At
* Format: date-time
*/
updated_at: string;
};
/** IssueCohort */
IssueCohort: {
/**
* Model Pairs
* @default []
*/
model_pairs: components["schemas"]["IssueModelPair"][];
/** Scope Id */
scope_id?: string | null;
/**
* Scope Type
* @enum {string}
*/
scope_type: "deployment" | "key" | "user" | "team" | "model" | "provider";
};
/** IssueComparison */
IssueComparison: {
/**
* Baseline End
* Format: date
*/
baseline_end: string;
/** Baseline Sample Days */
baseline_sample_days?: number | null;
/**
* Baseline Start
* Format: date
*/
baseline_start: string;
/** Baseline Value */
baseline_value: number;
/**
* Direction
* @enum {string}
*/
direction: "increase" | "decrease";
/**
* Metric
* @enum {string}
*/
metric: "spend" | "requests" | "latency" | "errors" | "error_rate";
/** Observed Value */
observed_value: number;
/** Threshold */
threshold: number;
};
/** IssueDeprecation */
IssueDeprecation: {
/** Last Day Requests */
last_day_requests: number;
/**
* Last Used Date
* Format: date
*/
last_used_date: string;
model: components["schemas"]["ModelDeprecationInfo"];
/** Requests */
requests: number;
state: components["schemas"]["DeprecationAlertState"];
/** Usage Model */
usage_model: string;
};
/** IssueFamilyList */
IssueFamilyList: {
/** Data */
data: components["schemas"]["IssueFamilySetting"][];
};
/** IssueFamilyPolicy */
IssueFamilyPolicy: {
/** Baseline Days */
baseline_days: number | null;
/**
* Direction
* @enum {string}
*/
direction: "increase" | "decrease";
/** Enabled */
enabled: boolean | null;
/** Enabled Rules */
enabled_rules: number;
/** Min History Days */
min_history_days: number | null;
/** Min Requests */
min_requests: number | null;
/** Min Value */
min_value: number | null;
/** Mixed */
mixed: boolean;
/** Multiplier */
multiplier: number | null;
/** Total Rules */
total_rules: number;
};
/** IssueFamilyPolicyUpdate */
IssueFamilyPolicyUpdate: {
/** Baseline Days */
baseline_days?: number | null;
/**
* Direction
* @enum {string}
*/
direction: "increase" | "decrease";
/** Enabled */
enabled?: boolean | null;
/** Min History Days */
min_history_days?: number | null;
/** Min Requests */
min_requests?: number | null;
/** Min Value */
min_value?: number | null;
/** Multiplier */
multiplier?: number | null;
};
/** IssueFamilySetting */
IssueFamilySetting: {
/** Enabled */
enabled: boolean | null;
/** Enabled Rules */
enabled_rules: number;
/**
* Family
* @enum {string}
*/
family: "spend" | "latency" | "reliability" | "traffic";
/** Policies */
policies: components["schemas"]["IssueFamilyPolicy"][];
/** Total Rules */
total_rules: number;
};
/** IssueFamilyUpdate */
IssueFamilyUpdate: {
/** Enabled */
enabled?: boolean | null;
/**
* Policies
* @default []
*/
policies: components["schemas"]["IssueFamilyPolicyUpdate"][];
};
/** IssueImpact */
IssueImpact: {
/** Keys */
keys?: number | null;
/** Measured Keys */
measured_keys?: number | null;
/** Measured Models */
measured_models?: number | null;
/** Models */
models?: number | null;
/** Requests */
requests?: number | null;
/** Teams */
teams?: number | null;
/** Users */
users?: number | null;
};
/** IssueList */
IssueList: {
/** Data */
data: components["schemas"]["Issue"][];
/** Limit */
limit: number;
/** Offset */
offset: number;
/** Total */
total: number;
};
/** IssueModelPair */
IssueModelPair: {
/** Model */
model: string;
/** Provider */
provider: string;
};
/** IssueSignal */
IssueSignal: {
deprecation?: components["schemas"]["IssueDeprecation"] | null;
/** Issue Id */
issue_id: string;
/**
* Source
* @enum {string}
*/
source: "usage_alert" | "model_deprecation";
/** Source Id */
source_id: string;
usage_alert?: components["schemas"]["UsageAlert"] | null;
};
/** IssueSignalList */
IssueSignalList: {
/** Data */
data: components["schemas"]["IssueSignal"][];
/** Limit */
limit: number;
/** Offset */
offset: number;
/** Total */
total: number;
};
/** IssueUpdate */
IssueUpdate: {
/**
* Expected Status
* @enum {string}
*/
expected_status: "open" | "resolved" | "dismissed";
/**
* Status
* @enum {string}
*/
status: "open" | "resolved" | "dismissed";
};
/**
* ItemReference
* @description An internal identifier for an item to reference.
@ -34868,6 +35318,16 @@ export interface components {
* @default 0
*/
successful_requests: number;
/**
* Timed Requests
* @default 0
*/
timed_requests: number;
/**
* Total Response Time Ms
* @default 0
*/
total_response_time_ms: number;
/**
* Total Tokens
* @default 0
@ -50485,7 +50945,7 @@ export interface components {
* Metric
* @enum {string}
*/
metric: "spend" | "requests" | "latency" | "errors";
metric: "spend" | "requests" | "latency" | "errors" | "error_rate";
/** Observed Value */
observed_value: number;
/** Rule Id */
@ -50588,7 +51048,7 @@ export interface components {
* Metric
* @enum {string}
*/
metric: "spend" | "requests" | "latency" | "errors";
metric: "spend" | "requests" | "latency" | "errors" | "error_rate";
representative: components["schemas"]["UsageAlert"];
};
/** UsageAlertGroupUpdate */
@ -50706,7 +51166,7 @@ export interface components {
* Metric
* @enum {string}
*/
metric: "spend" | "requests" | "latency" | "errors";
metric: "spend" | "requests" | "latency" | "errors" | "error_rate";
/**
* Min History Days
* @default 3
@ -50774,7 +51234,7 @@ export interface components {
* Metric
* @enum {string}
*/
metric: "spend" | "requests" | "latency" | "errors";
metric: "spend" | "requests" | "latency" | "errors" | "error_rate";
/**
* Min History Days
* @default 3
@ -51349,6 +51809,16 @@ export interface components {
* @default 0
*/
successful_requests: number;
/**
* Timed Requests
* @default 0
*/
timed_requests: number;
/**
* Total Response Time Ms
* @default 0
*/
total_response_time_ms: number;
/**
* Total Tokens
* @default 0
@ -64994,6 +65464,338 @@ export interface operations {
};
};
};
list_issues_issues_get: {
parameters: {
query?: {
kind?: ("performance" | "model_deprecation") | null;
family?: ("retirement" | "latency" | "reliability" | "spend" | "traffic") | null;
status?: ("open" | "resolved" | "dismissed") | null;
scope_type?: ("deployment" | "key" | "user" | "team" | "model" | "provider") | null;
scope_id?: string | null;
limit?: number;
offset?: number;
};
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["IssueList"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
list_issue_families_issues_families_get: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["IssueFamilyList"];
};
};
};
};
update_issue_family_issues_families__family__patch: {
parameters: {
query?: never;
header?: never;
path: {
family: "spend" | "latency" | "reliability" | "traffic";
};
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["IssueFamilyUpdate"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["IssueFamilySetting"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_issue_issues__issue_id__get: {
parameters: {
query?: never;
header?: never;
path: {
issue_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Issue"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
update_issue_issues__issue_id__patch: {
parameters: {
query?: never;
header?: never;
path: {
issue_id: string;
};
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["IssueUpdate"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Issue"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_issue_activity_issues__issue_id__activity_get: {
parameters: {
query?: {
start_date?: string | null;
end_date?: string | null;
};
header?: never;
path: {
issue_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_issue_impact_issues__issue_id__impact_get: {
parameters: {
query?: {
start_date?: string | null;
end_date?: string | null;
};
header?: never;
path: {
issue_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["IssueImpact"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_issue_keys_issues__issue_id__keys_get: {
parameters: {
query?: {
start_date?: string | null;
end_date?: string | null;
limit?: number;
offset?: number;
};
header?: never;
path: {
issue_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["DailyActivityKeyPageResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
list_issue_signals_issues__issue_id__signals_get: {
parameters: {
query?: {
limit?: number;
offset?: number;
};
header?: never;
path: {
issue_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["IssueSignalList"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_issue_users_issues__issue_id__users_get: {
parameters: {
query?: {
start_date?: string | null;
end_date?: string | null;
limit?: number;
offset?: number;
};
header?: never;
path: {
issue_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["DailyActivityUserPageResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
delete_jwt_key_mapping_jwt_key_mapping_delete_post: {
parameters: {
query?: never;
@ -79013,7 +79815,7 @@ export interface operations {
query?: {
scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null;
scope_id?: string | null;
metric?: ("spend" | "requests" | "latency" | "errors") | null;
metric?: ("spend" | "requests" | "latency" | "errors" | "error_rate") | null;
status?: ("open" | "resolved" | "dismissed") | null;
start_date?: string | null;
end_date?: string | null;