From 5ab3a0a7399b44f2c5c5fa6d5932ced536a90caf Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 14:50:43 +0800 Subject: [PATCH] feat: add unified issue storage and investigation APIs --- .../20261105200000_issues_inbox/migration.sql | 49 ++ litellm/models/issues.py | 134 +++ litellm/proxy/_types.py | 27 + litellm/proxy/alerting_endpoints/issues.py | 235 +++++ litellm/proxy/deps/repositories.py | 17 + .../common_daily_activity.py | 2 +- .../daily_activity_routes.py | 4 + litellm/proxy/proxy_server.py | 2 + .../repositories/daily_activity_repository.py | 2 + litellm/repositories/daily_activity_sql.py | 36 +- litellm/repositories/issue_activity.py | 154 ++++ litellm/repositories/issue_families.py | 149 ++++ litellm/repositories/issues.py | 504 +++++++++++ litellm/repositories/usage_alerts.py | 34 +- litellm/types/proxy/issues.py | 260 ++++++ .../common_daily_activity.py | 4 + litellm/types/proxy/usage_alert_defaults.py | 75 ++ litellm/types/proxy/usage_alerts.py | 2 +- litellm/types/repositories/daily_activity.py | 6 + schema.prisma | 54 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 812 +++++++++++++++++- 21 files changed, 2537 insertions(+), 25 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql create mode 100644 litellm/models/issues.py create mode 100644 litellm/proxy/alerting_endpoints/issues.py create mode 100644 litellm/repositories/issue_activity.py create mode 100644 litellm/repositories/issue_families.py create mode 100644 litellm/repositories/issues.py create mode 100644 litellm/types/proxy/issues.py create mode 100644 litellm/types/proxy/usage_alert_defaults.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql new file mode 100644 index 00000000000..3044320e8e8 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql @@ -0,0 +1,49 @@ +CREATE TABLE "LiteLLM_Issue" ( + "id" TEXT NOT NULL, + "cohort_key" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "family" TEXT NOT NULL, + "scope_type" TEXT NOT NULL, + "scope_id" TEXT, + "status" TEXT NOT NULL, + "active" BOOLEAN NOT NULL DEFAULT true, + "ack_key" TEXT, + "record" JSONB NOT NULL, + "first_seen" TIMESTAMP(3) NOT NULL, + "last_seen" TIMESTAMP(3) NOT NULL, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "LiteLLM_Issue_pkey" PRIMARY KEY ("id") +); +CREATE INDEX "LiteLLM_Issue_status_last_seen_id_idx" ON "LiteLLM_Issue"("status", "last_seen", "id"); +CREATE INDEX "LiteLLM_Issue_kind_family_last_seen_id_idx" ON "LiteLLM_Issue"("kind", "family", "last_seen", "id"); +CREATE INDEX "LiteLLM_Issue_scope_type_scope_id_last_seen_id_idx" ON "LiteLLM_Issue"("scope_type", "scope_id", "last_seen", "id"); +CREATE INDEX "LiteLLM_Issue_cohort_key_active_idx" ON "LiteLLM_Issue"("cohort_key", "active"); +CREATE TABLE "LiteLLM_IssueSignal" ( + "issue_id" TEXT NOT NULL, + "source" TEXT NOT NULL, + "source_id" TEXT NOT NULL, + "record" JSONB NOT NULL, + CONSTRAINT "LiteLLM_IssueSignal_pkey" PRIMARY KEY ("issue_id", "source", "source_id") +); +CREATE INDEX "LiteLLM_IssueSignal_source_source_id_idx" ON "LiteLLM_IssueSignal"("source", "source_id"); +CREATE TABLE "LiteLLM_IssueDelivery" ( + "id" TEXT NOT NULL, + "issue_id" TEXT NOT NULL, + "record" JSONB NOT NULL, + "sent" BOOLEAN NOT NULL DEFAULT false, + "next_attempt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "LiteLLM_IssueDelivery_pkey" PRIMARY KEY ("id") +); +CREATE INDEX "LiteLLM_IssueDelivery_sent_next_attempt_id_idx" ON "LiteLLM_IssueDelivery"("sent", "next_attempt", "id"); +CREATE INDEX "LiteLLM_IssueDelivery_issue_id_idx" ON "LiteLLM_IssueDelivery"("issue_id"); +CREATE TABLE "LiteLLM_IssueDeliveryRule" ( + "delivery_id" TEXT NOT NULL, + "rule_id" TEXT NOT NULL, + "source_id" TEXT NOT NULL, + "rule_revision" TEXT NOT NULL, + CONSTRAINT "LiteLLM_IssueDeliveryRule_pkey" PRIMARY KEY ("delivery_id", "rule_id", "source_id") +); +ALTER TABLE "LiteLLM_UsageAlertIncident" ADD COLUMN "evaluation_revision" TEXT NOT NULL DEFAULT ''; +CREATE INDEX "LiteLLM_UsageAlertIncident_evaluation_day_breached_id_idx" + ON "LiteLLM_UsageAlertIncident"("evaluation_day", "breached", "id"); diff --git a/litellm/models/issues.py b/litellm/models/issues.py new file mode 100644 index 00000000000..7d0692d26db --- /dev/null +++ b/litellm/models/issues.py @@ -0,0 +1,134 @@ +from datetime import date, datetime +from typing import Literal, Protocol + +from litellm.types.proxy.issues import ( + Issue, + IssueCohort, + IssueDeliveryBinding, + IssueDeliveryInput, + IssueFamily, + IssueFamilyList, + IssueFamilySetting, + IssueFamilyUpdate, + IssueImpact, + IssueInput, + IssueKind, + IssueList, + IssuePendingDelivery, + IssueProjectionSignal, + IssueScope, + IssueSignal, + IssueSignalInput, + IssueSignalList, +) +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailyActivityKeyPageResponse, + DailyActivityUserPageResponse, + SpendAnalyticsPaginatedResponse, +) +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus + + +class IssueStore(Protocol): + async def get_issue(self, issue_id: str) -> Issue | None: ... + + async def list_issues( + self, + kind: IssueKind | None = None, + family: IssueFamily | None = None, + status: UsageAlertStatus | None = None, + scope_type: IssueScope | None = None, + scope_id: str | None = None, + limit: int = 50, + offset: int = 0, + ) -> IssueList: ... + + async def set_status( + self, + issue_id: str, + status: UsageAlertStatus, + expected_status: UsageAlertStatus, + actor: str | None = None, + ) -> Issue | None: ... + + async def upsert_issue( + self, + data: IssueInput, + reopen: bool = False, + status: UsageAlertStatus | None = None, + ) -> Issue: ... + + async def upsert_issues( + self, + inputs: tuple[IssueInput, ...], + reopen: bool = False, + status: UsageAlertStatus | None = None, + ) -> tuple[Issue, ...]: ... + + async def find_active_issue(self, cohort_key: str) -> Issue | None: ... + + async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: ... + + async def add_signals(self, issue_id: str, signals: tuple[IssueSignalInput, ...]) -> None: ... + + async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: ... + + async def list_signals(self, issue_id: str, limit: int = 50, offset: int = 0) -> IssueSignalList: ... + + async def active_signals( + self, + day: date, + limit: int = 500, + cursor: str | None = None, + scope_type: IssueScope | None = None, + metric: UsageAlertMetric | None = None, + ) -> tuple[IssueProjectionSignal, ...]: ... + + async def recover_issues(self, day: date) -> int: ... + + async def projection_completed(self, day: date, version: str) -> bool: ... + + async def complete_projection(self, day: date, version: str) -> None: ... + + async def finish_deprecation_projection(self, observed_at: datetime) -> int: ... + + async def family_settings(self) -> IssueFamilyList: ... + + async def update_family( + self, family: PerformanceIssueFamily, data: IssueFamilyUpdate + ) -> IssueFamilySetting | None: ... + + async def queue_delivery( + self, + data: IssueDeliveryInput, + rule_id: str, + rule_revision: str, + source_id: str, + ) -> None: ... + + async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: ... + + async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: ... + + async def mark_delivery_sent(self, delivery_id: str) -> None: ... + + async def defer_delivery( + self, delivery_id: str, error: Literal["delivery_failed", "transport_not_configured"] + ) -> None: ... + + async def prune(self, retention_days: int = 90, limit: int = 500) -> None: ... + + +class IssueActivityStore(Protocol): + async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse: ... + + async def keys( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityKeyPageResponse: ... + + async def users( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityUserPageResponse: ... + + async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: ... diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7254a0fb6b7..633768fa9c3 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -585,6 +585,15 @@ class LiteLLMRoutes(enum.Enum): "/usage/alert-rules/{rule_id}", "/usage/alerts", "/usage/alerts/{alert_id}", + "/issues", + "/issues/families", + "/issues/families/{family}", + "/issues/{issue_id}", + "/issues/{issue_id}/signals", + "/issues/{issue_id}/activity", + "/issues/{issue_id}/keys", + "/issues/{issue_id}/users", + "/issues/{issue_id}/impact", "/key/info", "/key/health", "/team/info", @@ -658,6 +667,15 @@ class LiteLLMRoutes(enum.Enum): "/usage/alert-rules/{rule_id}", "/usage/alerts", "/usage/alerts/{alert_id}", + "/issues", + "/issues/families", + "/issues/families/{family}", + "/issues/{issue_id}", + "/issues/{issue_id}/signals", + "/issues/{issue_id}/activity", + "/issues/{issue_id}/keys", + "/issues/{issue_id}/users", + "/issues/{issue_id}/impact", # user "/user/new", "/management/v1/users/bulk", @@ -874,6 +892,15 @@ class LiteLLMRoutes(enum.Enum): "/usage/alert-rules/{rule_id}", "/usage/alerts", "/usage/alerts/{alert_id}", + "/issues", + "/issues/families", + "/issues/families/{family}", + "/issues/{issue_id}", + "/issues/{issue_id}/signals", + "/issues/{issue_id}/activity", + "/issues/{issue_id}/keys", + "/issues/{issue_id}/users", + "/issues/{issue_id}/impact", ) self_managed_routes = [ diff --git a/litellm/proxy/alerting_endpoints/issues.py b/litellm/proxy/alerting_endpoints/issues.py new file mode 100644 index 00000000000..30f3d44afe7 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/issues.py @@ -0,0 +1,235 @@ +"""Unified issue inbox and investigations backed by existing daily aggregates.""" + +from datetime import date +from typing import Final + +from fastapi import APIRouter, HTTPException + +from litellm.models.issues import IssueActivityStore, IssueStore +from litellm.proxy.alerting_endpoints.usage_alerts import ( + Limit, + Offset, + ScopeId, + User, + require_repository, + require_scope, +) +from litellm.proxy.deps.repositories import IssueActivityRepositoryDep, IssuesRepositoryDep, UsageAlertsRepositoryDep +from litellm.types.proxy.issues import ( + Issue, + IssueFamily, + IssueFamilyList, + IssueFamilySetting, + IssueFamilyUpdate, + IssueImpact, + IssueKind, + IssueList, + IssueScope, + IssueSignalList, + IssueUpdate, +) +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailyActivityKeyPageResponse, + DailyActivityUserPageResponse, + SpendAnalyticsPaginatedResponse, +) +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import UsageAlertStatus + +router: Final = APIRouter(prefix="/issues", tags=["issues"]) + + +def require_issues(repository: IssueStore | None) -> IssueStore: + if repository is None: + raise HTTPException(status_code=503, detail="Issues require a database connection") + return repository + + +def require_activity(repository: IssueActivityStore | None) -> IssueActivityStore: + if repository is None: + raise HTTPException(status_code=503, detail="Issue Activity requires a database connection") + return repository + + +async def authorized_issue( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + *, + write: bool = False, +) -> Issue: + issue: Final = await require_issues(repository).get_issue(issue_id) + if issue is None: + raise HTTPException(status_code=404, detail="Issue not found") + await require_scope( + user, + require_repository(alerts), + "model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type, + issue.cohort.scope_id, + write=write, + ) + return issue + + +def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]: + start: Final = start_date or issue.range_start + end: Final = end_date or issue.range_end + if start > end or (end - start).days > 365: + raise HTTPException(status_code=422, detail="Choose an ordered date range of at most 366 days") + return start, end + + +@router.get("", response_model=IssueList) +async def list_issues( + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + kind: IssueKind | None = None, + family: IssueFamily | None = None, + status: UsageAlertStatus | None = None, + scope_type: IssueScope | None = None, + scope_id: ScopeId = None, + limit: Limit = 50, + offset: Offset = 0, +) -> IssueList: + await require_scope( + user, require_repository(alerts), "model" if scope_type == "provider" else scope_type, scope_id, write=False + ) + return await require_issues(repository).list_issues(kind, family, status, scope_type, scope_id, limit, offset) + + +@router.get("/families", response_model=IssueFamilyList) +async def list_issue_families( + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> IssueFamilyList: + await require_scope(user, require_repository(alerts), "deployment", None, write=False) + return await require_issues(repository).family_settings() + + +@router.patch("/families/{family}", response_model=IssueFamilySetting) +async def update_issue_family( + family: PerformanceIssueFamily, + data: IssueFamilyUpdate, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> IssueFamilySetting: + await require_scope(user, require_repository(alerts), "deployment", None, write=True) + try: + result: Final = await require_issues(repository).update_family(family, data) + except ValueError as exc: + raise HTTPException( + status_code=422, detail="Invalid family policy; check direction and history bounds" + ) from exc + if result is None: + raise HTTPException(status_code=409, detail="Built-in rules changed or are unavailable; reload and retry") + return result + + +@router.get("/{issue_id}", response_model=Issue) +async def get_issue( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> Issue: + return await authorized_issue(issue_id, user, repository, alerts) + + +@router.patch("/{issue_id}", response_model=Issue) +async def update_issue( + issue_id: str, + data: IssueUpdate, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> Issue: + await authorized_issue(issue_id, user, repository, alerts, write=True) + changed: Final = await require_issues(repository).set_status( + issue_id, data.status, data.expected_status, user.user_id + ) + if changed is None: + raise HTTPException(status_code=409, detail="Issue status changed; reload and retry") + return changed + + +@router.get("/{issue_id}/signals", response_model=IssueSignalList) +async def list_issue_signals( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + limit: Limit = 50, + offset: Offset = 0, +) -> IssueSignalList: + await authorized_issue(issue_id, user, repository, alerts) + return await require_issues(repository).list_signals(issue_id, limit, offset) + + +@router.get("/{issue_id}/activity", response_model=SpendAnalyticsPaginatedResponse) +async def get_issue_activity( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, +) -> SpendAnalyticsPaginatedResponse: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + start, end = activity_range(issue, start_date, end_date) + return await require_activity(activity).daily_activity(issue.cohort, start, end) + + +@router.get("/{issue_id}/keys", response_model=DailyActivityKeyPageResponse) +async def get_issue_keys( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, + limit: Limit = 20, + offset: Offset = 0, +) -> DailyActivityKeyPageResponse: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + start, end = activity_range(issue, start_date, end_date) + return await require_activity(activity).keys(issue.cohort, start, end, limit, offset) + + +@router.get("/{issue_id}/users", response_model=DailyActivityUserPageResponse) +async def get_issue_users( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, + limit: Limit = 20, + offset: Offset = 0, +) -> DailyActivityUserPageResponse: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + if issue.cohort.scope_type == "team": + raise HTTPException(status_code=422, detail="Historical team aggregates do not retain user membership") + start, end = activity_range(issue, start_date, end_date) + return await require_activity(activity).users(issue.cohort, start, end, limit, offset) + + +@router.get("/{issue_id}/impact", response_model=IssueImpact) +async def get_issue_impact( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, +) -> IssueImpact: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + start, end = activity_range(issue, start_date, end_date) + return await require_activity(activity).impact(issue.cohort, start, end) diff --git a/litellm/proxy/deps/repositories.py b/litellm/proxy/deps/repositories.py index b3b3aeafe28..ca12c6ff87c 100644 --- a/litellm/proxy/deps/repositories.py +++ b/litellm/proxy/deps/repositories.py @@ -4,12 +4,15 @@ from fastapi import Depends from litellm.models.deployment_warning import DeploymentWarningIgnoreStore from litellm.models.gateway_activity import GatewayActivityStore +from litellm.models.issues import IssueActivityStore, IssueStore from litellm.models.plugin_upload import PluginUploadStore from litellm.models.usage_alerts import UsageAlertStore from litellm.models.user_banner import UserBannerStore from litellm.proxy.deps.database import DatabaseDep from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository from litellm.repositories.gateway_activity_repository import GatewayActivityRepository +from litellm.repositories.issue_activity import IssueActivityRepository +from litellm.repositories.issues import IssuesRepository from litellm.repositories.plugin_upload_repository import PluginUploadRepository from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.repositories.user_banner_repository import UserBannerRepository @@ -52,3 +55,17 @@ async def get_usage_alerts_repository(database: DatabaseDep) -> UsageAlertStore UsageAlertsRepositoryDep: TypeAlias = Annotated[UsageAlertStore | None, Depends(get_usage_alerts_repository)] + + +async def get_issues_repository(database: DatabaseDep) -> IssueStore | None: + return None if database is None else IssuesRepository.from_prisma(database.prisma) + + +IssuesRepositoryDep: TypeAlias = Annotated[IssueStore | None, Depends(get_issues_repository)] + + +async def get_issue_activity_repository(database: DatabaseDep) -> IssueActivityStore | None: + return None if database is None else IssueActivityRepository.from_prisma(database.prisma) + + +IssueActivityRepositoryDep: TypeAlias = Annotated[IssueActivityStore | None, Depends(get_issue_activity_repository)] diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 6e4ea0bd0cb..1fd3b8bae12 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -1128,7 +1128,7 @@ async def get_daily_activity_aggregated( scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit ), _guardrail_spend_or_empty(repository, scope) - if include_guardrail_breakdown and not scope.model + if include_guardrail_breakdown and not scope.model and scope.provider is None and not scope.model_pairs else asyncio.sleep(0, result=()), ) records: Final = aggregated_rows.grouping_rows diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 1a3445aae52..6cf44e10109 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -136,6 +136,8 @@ def _key_activity_row(row: KeySpendRow, metadata: Mapping[str, KeyMetadataRow]) failed_requests=row.failed_requests, cache_read_input_tokens=row.cache_read_input_tokens, cache_creation_input_tokens=row.cache_creation_input_tokens, + total_response_time_ms=row.total_response_time_ms, + timed_requests=row.timed_requests, ), metadata=_key_metadata(row.api_key, metadata), ) @@ -169,6 +171,8 @@ def _user_activity_row(row: UserSpendRow, metadata: Mapping[str, UserMetadataRow api_requests=row.api_requests, successful_requests=row.successful_requests, failed_requests=row.failed_requests, + total_response_time_ms=row.total_response_time_ms, + timed_requests=row.timed_requests, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d876ce011a4..ad8a1bc67c0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -428,6 +428,7 @@ from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( router as model_deprecation_alerts_router, ) from litellm.proxy.alerting_endpoints.usage_alerts import router as usage_alerts_router +from litellm.proxy.alerting_endpoints.issues import router as issues_router from litellm.proxy.analytics_endpoints.analytics_endpoints import ( router as analytics_router, ) @@ -22440,6 +22441,7 @@ app.include_router(fallback_management_router) app.include_router(cache_settings_router) app.include_router(model_deprecation_alerts_router) app.include_router(usage_alerts_router) +app.include_router(issues_router) app.include_router(coordination_redis_settings_router) app.include_router(user_agent_analytics_router) app.include_router(gateway_request_router) diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 321468c4976..262eb9d7ec1 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -324,6 +324,8 @@ class DailyActivityRepository: ) async def daily_rows(self, scope: DailyActivityScope, *, page: int, page_size: int) -> DailyRowsPage: + if scope.provider is not None or scope.model_pairs: + raise ValueError("Provider cohorts require the aggregated Activity API") table: Final = _daily_rows_table(self._prisma_client, scope.table) adjusted_start, adjusted_end = adjust_dates_for_timezone( scope.start_date, diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 4bb52767de6..c480af53272 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -1,3 +1,4 @@ +import json from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime, timedelta, timezone @@ -75,6 +76,8 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu exclusion_index: Final = entity_index + int(has_entity_array) model_index: Final = exclusion_index + int(bool(scope.exclude_entity_ids)) api_keys_index: Final = model_index + int(bool(scope.model)) + provider_index: Final = api_keys_index + int(bool(scope.api_keys)) + pairs_index: Final = provider_index + int(scope.provider is not None) conditions: Final = ( f"date >= ${start_index}", f"date <= ${start_index + 1}", @@ -85,6 +88,21 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu if has_entity_array else () ), + *( + (f"COALESCE(NULLIF(custom_llm_provider, ''), 'unknown') = ${provider_index}",) + if scope.provider is not None + else () + ), + *( + ( + f"EXISTS (SELECT 1 FROM jsonb_to_recordset(${pairs_index}::jsonb) " + f"AS cohort(match_provider text, match_model text) WHERE " + f"cohort.match_provider=COALESCE(NULLIF(custom_llm_provider, ''), 'unknown') " + f"AND cohort.match_model={_MODEL_EXPR})", + ) + if scope.model_pairs + else () + ), *( ( f'("{scope.entity_id_field}" IS NULL ' @@ -113,6 +131,16 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu *((list(scope.exclude_entity_ids),) if scope.exclude_entity_ids else ()), *((scope.model,) if scope.model else ()), *((list(scope.api_keys),) if scope.api_keys else ()), + *((scope.provider,) if scope.provider is not None else ()), + *( + ( + json.dumps( + tuple({"match_provider": provider, "match_model": model} for provider, model in scope.model_pairs) + ), + ) + if scope.model_pairs + else () + ), ) return " AND ".join(conditions), params @@ -301,7 +329,9 @@ def _key_spend_select() -> str: COALESCE(SUM(successful_requests), 0)::bigint AS successful_requests, COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests, COALESCE(SUM(cache_read_input_tokens), 0)::bigint AS cache_read_input_tokens, - COALESCE(SUM(cache_creation_input_tokens), 0)::bigint AS cache_creation_input_tokens""" + COALESCE(SUM(cache_creation_input_tokens), 0)::bigint AS cache_creation_input_tokens, + COALESCE(SUM(total_response_time_ms), 0)::bigint AS total_response_time_ms, + COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests""" def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery: @@ -339,7 +369,9 @@ def _user_spend_select() -> str: (COALESCE(SUM(prompt_tokens), 0) + COALESCE(SUM(completion_tokens), 0))::bigint AS total_tokens, COALESCE(SUM(api_requests), 0)::bigint AS api_requests, COALESCE(SUM(successful_requests), 0)::bigint AS successful_requests, - COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests""" + COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests, + COALESCE(SUM(total_response_time_ms), 0)::bigint AS total_response_time_ms, + COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests""" def build_user_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery: diff --git a/litellm/repositories/issue_activity.py b/litellm/repositories/issue_activity.py new file mode 100644 index 00000000000..fed59a82cbc --- /dev/null +++ b/litellm/repositories/issue_activity.py @@ -0,0 +1,154 @@ +"""Issue investigations reuse the same aggregate filters as entity Activity.""" + +from dataclasses import asdict, replace +from datetime import date +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # generated Prisma structural database interface + +from pydantic import BaseModel, TypeAdapter + +from litellm.constants import PTU_SENTINEL_API_KEY +from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket +from litellm.repositories.daily_activity_repository import DailyActivityRepository +from litellm.repositories.daily_activity_sql import PRISMA_TO_PG_TABLE, build_where_clause +from litellm.repositories.usage_alerts import AlertDatabase +from litellm.types.proxy.issues import IssueCohort, IssueImpact +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailyActivityKeyPageResponse, + DailyActivityUserPageResponse, + KeyMetadata, + KeySpendActivityRow, + KeySpendMetrics, + SpendAnalyticsPaginatedResponse, + UserActivityRow, +) +from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +class _Counts(BaseModel, frozen=True): + keys: int + entities: int + models: int + requests: int + + +def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyActivityScope: + if start > end or (end - start).days > 365: + raise ValueError("Choose an ordered date range of at most 366 days") + return DailyActivityScope( + table=DailyActivityTable.TEAM if cohort.scope_type == "team" else DailyActivityTable.USER, + entity_id_field="team_id" if cohort.scope_type == "team" else "user_id", + entity_ids=(cohort.scope_id,) if cohort.scope_type in ("team", "user") and cohort.scope_id else None, + exclude_entity_ids=(), + api_keys=(api_key_from_bucket(cohort.scope_id),) if cohort.scope_type == "key" and cohort.scope_id else None, + start_date=start.isoformat(), + end_date=end.isoformat(), + model=cohort.scope_id if cohort.scope_type == "model" else None, + timezone_offset_minutes=0, + include_current_utc_day=True, + provider=cohort.scope_id if cohort.scope_type == "provider" else None, + model_pairs=tuple((pair.provider, pair.model) for pair in cohort.model_pairs), + ) + + +class IssueActivityRepository: + def __init__(self, activity: DailyActivityRepository, database: AlertDatabase) -> None: + self.activity = activity + self.database = database + + @classmethod + def from_prisma(cls, prisma: "PrismaClient") -> "IssueActivityRepository": + from litellm.proxy.management_endpoints.common_daily_activity import daily_activity_repository + + database: Final = cast(AlertDatabase, prisma.db) # cast-ok: generated Prisma query_raw + return cls(daily_activity_repository(prisma), database) + + async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse: + from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated + + return await get_daily_activity_aggregated( + self.activity, issue_activity_scope(cohort, start, end), api_key_limit=20 + ) + + async def keys( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityKeyPageResponse: + page: Final = await self.activity.key_page(issue_activity_scope(cohort, start, end), limit=limit, offset=offset) + metadata: Final = await self.activity.key_metadata( + frozenset(row.api_key for row in page.rows if row.api_key), None + ) + return DailyActivityKeyPageResponse( + api_keys=[ + KeySpendActivityRow( + api_key=api_key_bucket(row.api_key), + metrics=KeySpendMetrics.model_validate(asdict(row)), + metadata=KeyMetadata.model_validate(asdict(metadata[row.api_key])) + if row.api_key in metadata + else KeyMetadata(), + ) + for row in page.rows + ], + total_api_keys=page.total_api_keys, + limit=limit, + offset=offset, + ) + + async def users( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityUserPageResponse: + if cohort.scope_type == "team": + raise ValueError("Historical team aggregates do not retain user membership") + page: Final = await self.activity.user_page( + issue_activity_scope(cohort, start, end), limit=limit, offset=offset + ) + metadata: Final = await self.activity.user_metadata(frozenset(row.user_id for row in page.rows if row.user_id)) + return DailyActivityUserPageResponse( + users=tuple( + UserActivityRow.model_validate( + { + **asdict(row), + **( + asdict(metadata[row.user_id]) if row.user_id is not None and row.user_id in metadata else {} + ), + } + ) + for row in page.rows + ), + total_users=page.total_users, + limit=limit, + offset=offset, + ) + + async def _counts(self, scope: DailyActivityScope) -> _Counts: + where, params = build_where_clause(scope) + rows: Final = TypeAdapter(tuple[_Counts, ...]).validate_python( + await self.database.query_raw( + f'''SELECT COUNT(DISTINCT NULLIF(api_key,'')) FILTER(WHERE api_key<>${len(params) + 1})::int AS keys, + COUNT(DISTINCT NULLIF("{scope.entity_id_field}",'')) + FILTER(WHERE "{scope.entity_id_field}" NOT IN ('default_user_id','litellm-dashboard'))::int AS entities, + COUNT(DISTINCT NULLIF(model,''))::int AS models, + COALESCE(SUM(api_requests),0)::bigint AS requests + FROM "{PRISMA_TO_PG_TABLE[scope.table]}" WHERE {where}''', + *params, + PTU_SENTINEL_API_KEY, + ) + ) + return rows[0] + + async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: + scope: Final = issue_activity_scope(cohort, start, end) + counts: Final = await self._counts(scope) + teams: Final = ( + await self._counts(replace(scope, table=DailyActivityTable.TEAM, entity_id_field="team_id")) + if cohort.scope_type not in ("user", "team") + else None + ) + return IssueImpact( + keys=counts.keys, + models=counts.models, + requests=counts.requests, + users=counts.entities if cohort.scope_type != "team" else None, + teams=counts.entities if cohort.scope_type == "team" else teams.entities if teams else None, + ) diff --git a/litellm/repositories/issue_families.py b/litellm/repositories/issue_families.py new file mode 100644 index 00000000000..05b1e938206 --- /dev/null +++ b/litellm/repositories/issue_families.py @@ -0,0 +1,149 @@ +"""Bounded settings for the built-in performance policies.""" + +import json +from datetime import datetime, timezone +from itertools import chain, product +from typing import Final + +from pydantic import BaseModel, TypeAdapter + +from litellm.repositories.usage_alerts import AlertDatabase +from litellm.types.proxy.issues import ( + IssueFamilyList, + IssueFamilyPolicy, + IssueFamilySetting, + IssueFamilyUpdate, +) +from litellm.types.proxy.usage_alert_defaults import ( + DEFAULT_USAGE_POLICIES, + DEFAULT_USAGE_RULE_IDS, + USAGE_ALERT_SCOPES, + PerformanceIssueFamily, + default_usage_rule_id, +) +from litellm.types.proxy.usage_alerts import UsageAlertRule + + +class _RuleRow(BaseModel, frozen=True): + record: UsageAlertRule + + +class _Count(BaseModel, frozen=True): + count: int + + +def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) -> IssueFamilySetting: + policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family) + groups: Final = tuple( + (policy, tuple(rule for rule in rules if rule.metric == policy.metric and rule.direction == policy.direction)) + for policy in policies + ) + values: Final = tuple( + IssueFamilyPolicy( + direction=policy.direction, + enabled=group[0].enabled if group and len({rule.enabled for rule in group}) == 1 else None, + multiplier=group[0].multiplier if group and len({rule.multiplier for rule in group}) == 1 else None, + min_value=group[0].min_value if group and len({rule.min_value for rule in group}) == 1 else None, + min_requests=group[0].min_requests if group and len({rule.min_requests for rule in group}) == 1 else None, + baseline_days=group[0].baseline_days + if group and len({rule.baseline_days for rule in group}) == 1 + else None, + min_history_days=group[0].min_history_days + if group and len({rule.min_history_days for rule in group}) == 1 + else None, + mixed=len( + { + (r.enabled, r.multiplier, r.min_value, r.min_requests, r.baseline_days, r.min_history_days) + for r in group + } + ) + > 1, + total_rules=len(group), + enabled_rules=sum(rule.enabled for rule in group), + ) + for policy, group in groups + ) + family_rules: Final = tuple(chain.from_iterable(group for _, group in groups)) + return IssueFamilySetting( + family=family, + enabled=family_rules[0].enabled if family_rules and len({r.enabled for r in family_rules}) == 1 else None, + total_rules=len(family_rules), + enabled_rules=sum(rule.enabled for rule in family_rules), + policies=values, + ) + + +async def family_settings(database: AlertDatabase) -> IssueFamilyList: + rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python( + await database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted', + list(DEFAULT_USAGE_RULE_IDS), + ) + ) + return IssueFamilyList( + data=tuple( + _setting(family, tuple(row.record for row in rows)) + for family in ("spend", "latency", "reliability", "traffic") + ) + ) + + +async def update_family( + database: AlertDatabase, + family: PerformanceIssueFamily, + data: IssueFamilyUpdate, +) -> IssueFamilySetting | None: + policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family) + if any(update.direction not in {policy.direction for policy in policies} for update in data.policies): + raise ValueError("This family does not support that direction") + ids: Final = tuple( + default_usage_rule_id(scope, policy.metric, policy.direction) + for policy, scope in product(policies, USAGE_ALERT_SCOPES) + ) + rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python( + await database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted', + list(ids), + ) + ) + timestamp: Final = datetime.now(timezone.utc).isoformat() + updates: Final = { + policy.direction: policy.model_dump(exclude_none=True, exclude={"direction"}) for policy in data.policies + } + records: Final = tuple( + UsageAlertRule.model_validate( + { + **row.record.model_dump(), + **({"enabled": data.enabled} if data.enabled is not None else {}), + **updates.get(row.record.direction, {}), + "updated_at": timestamp, + } + ) + for row in rows + ) + if not records: + return None + changed: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await database.query_raw( + """WITH incoming AS MATERIALIZED ( + SELECT x->'record' AS record,x->>'revision' AS revision FROM jsonb_array_elements($1::jsonb) x + ), eligible AS MATERIALIZED ( + SELECT r.id FROM "LiteLLM_UsageAlertRule" r JOIN incoming x ON r.id=x.record->>'id' + WHERE NOT r.deleted AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz + ORDER BY r.id FOR UPDATE OF r + ), changed AS ( + UPDATE "LiteLLM_UsageAlertRule" r SET record=x.record,enabled=(x.record->>'enabled')::boolean, + updated_at=(x.record->>'updated_at')::timestamptz AT TIME ZONE 'UTC' + FROM incoming x WHERE r.id=x.record->>'id' AND NOT r.deleted + AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz + AND (SELECT count(*) FROM eligible)=(SELECT count(*) FROM incoming) RETURNING r.id + ) SELECT count(*)::int AS count FROM changed""", + json.dumps( + tuple( + {"record": record.model_dump(mode="json"), "revision": row.record.updated_at.isoformat()} + for record, row in zip(records, rows, strict=True) + ) + ), + ) + ) + return _setting(family, records) if changed[0].count == len(records) else None diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py new file mode 100644 index 00000000000..0f9ac3e90f3 --- /dev/null +++ b/litellm/repositories/issues.py @@ -0,0 +1,504 @@ +import json +from collections.abc import Mapping +from datetime import date, datetime, timezone +from typing import ( + TYPE_CHECKING, + Final, + Literal, + cast, # noqa: TID251 # generated Prisma client implements query_raw +) + +from pydantic import BaseModel, TypeAdapter + +from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key +from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository +from litellm.types.proxy.issues import ( + Issue, + IssueDeliveryBinding, + IssueDeliveryInput, + IssueFamily, + IssueFamilyList, + IssueFamilySetting, + IssueFamilyUpdate, + IssueInput, + IssueKind, + IssueList, + IssuePendingDelivery, + IssueProjectionSignal, + IssueScope, + IssueSignal, + IssueSignalInput, + IssueSignalList, +) +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +class _IssueRow(BaseModel, frozen=True): + record: Issue + + +class _SignalRow(BaseModel, frozen=True): + record: IssueSignal + + +class _Count(BaseModel, frozen=True): + count: int + + +class _Identifier(BaseModel, frozen=True): + id: str + + +_ACK: Final = "CASE WHEN jsonb_typeof(c.param_value)='string' THEN (c.param_value#>>'{}')::jsonb ELSE c.param_value END" +_STATUS: Final = f"""CASE WHEN i.kind <> 'model_deprecation' THEN i.status + WHEN NOT i.active THEN CASE WHEN ({_ACK})->>'status'='dismissed' THEN 'dismissed' ELSE 'resolved' END + WHEN COALESCE(({_ACK})->>'status','open')='open' + OR ({_ACK})->>'last_seen_date' IS NULL + OR i.record->'deprecation'->>'last_used_date' > ({_ACK})->>'last_seen_date' + OR (i.record->'deprecation'->>'last_used_date' = ({_ACK})->>'last_seen_date' + AND (i.record->'deprecation'->>'last_day_requests')::bigint + > COALESCE((({_ACK})->>'last_seen_requests')::bigint,0)) THEN 'open' + ELSE ({_ACK})->>'status' END""" +_JOIN: Final = 'FROM "LiteLLM_Issue" i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key' +_RECORD: Final = f"""i.record || jsonb_build_object('status',({_STATUS}),'active',i.active, + 'first_seen',i.first_seen AT TIME ZONE 'UTC','last_seen',i.last_seen AT TIME ZONE 'UTC', + 'updated_at',i.updated_at AT TIME ZONE 'UTC') || CASE WHEN i.kind='model_deprecation' + THEN jsonb_build_object('deprecation',i.record->'deprecation' || + jsonb_build_object('state',COALESCE(({_ACK}),i.record->'deprecation'->'state'))) + ELSE '{{}}'::jsonb END""" + + +def _utc(value: datetime) -> str: + return value.astimezone(timezone.utc).replace(tzinfo=None).isoformat() + + +def _issue_payload(data: IssueInput, status: UsageAlertStatus | None) -> Mapping[str, object]: + initial: Final = Issue.model_validate( + { + **data.model_dump(), + "status": status or "open", + "first_seen": data.observed_at, + "last_seen": data.observed_at, + "updated_at": data.observed_at, + } + ) + return { + "record": initial.model_dump(mode="json"), + "ack_key": deprecation_alert_state_key(data.deprecation.usage_model, data.deprecation.model.deprecation_date) + if data.deprecation + else None, + } + + +class IssuesRepository: + def __init__(self, database: AlertDatabase) -> None: + self.database = database + + @classmethod + def from_prisma(cls, prisma: "PrismaClient") -> "IssuesRepository": + return cls(cast(AlertDatabase, prisma.writer_db)) # cast-ok: generated Prisma delegate implements query_raw + + async def upsert_issue( + self, data: IssueInput, reopen: bool = False, status: UsageAlertStatus | None = None + ) -> Issue: + rows: Final = await self.upsert_issues((data,), reopen=reopen, status=status) + return rows[0] + + async def upsert_issues( + self, + inputs: tuple[IssueInput, ...], + reopen: bool = False, + status: UsageAlertStatus | None = None, + ) -> tuple[Issue, ...]: + if not inputs: + return () + if len(inputs) > 500 or len(frozenset(item.id for item in inputs)) != len(inputs): + raise ValueError("Write at most 500 distinct issues per batch") + payload: Final = tuple(_issue_payload(item, status) for item in inputs) + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw( + f"""WITH incoming AS ( + SELECT x->'record' AS record,x->>'ack_key' AS ack_key FROM jsonb_array_elements($1::jsonb) x + ), changed AS ( + INSERT INTO "LiteLLM_Issue" AS old + (id,cohort_key,kind,family,scope_type,scope_id,status,ack_key,record,first_seen,last_seen,updated_at) + SELECT record->>'id',record->>'cohort_key',record->>'kind',record->>'family', + record->'cohort'->>'scope_type',record->'cohort'->>'scope_id',record->>'status',ack_key,record, + (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC', + (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC', + (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC' FROM incoming + ON CONFLICT(id) DO UPDATE SET + record=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen + THEN EXCLUDED.record ELSE old.record END, + last_seen=GREATEST(old.last_seen,EXCLUDED.last_seen), + active=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen + THEN true ELSE old.active END, + status=CASE WHEN $2 AND old.active AND EXCLUDED.last_seen > old.updated_at + THEN 'open' ELSE old.status END, + updated_at=GREATEST(old.updated_at,EXCLUDED.last_seen) + WHERE old.kind=EXCLUDED.kind AND old.cohort_key=EXCLUDED.cohort_key + AND old.scope_type=EXCLUDED.scope_type AND old.scope_id IS NOT DISTINCT FROM EXCLUDED.scope_id + RETURNING * + ) SELECT {_RECORD} AS record FROM changed i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key""", + json.dumps(payload), + reopen, + ) + ) + if len(rows) != len(inputs): + raise ValueError("An issue ID already belongs to another cohort") + return tuple(row.record for row in rows) + + async def get_issue(self, issue_id: str) -> Issue | None: + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw(f"SELECT {_RECORD} AS record {_JOIN} WHERE i.id=$1", issue_id) + ) + return (await self._labels((rows[0].record,)))[0] if rows else None + + async def list_issues( + self, + kind: IssueKind | None = None, + family: IssueFamily | None = None, + status: UsageAlertStatus | None = None, + scope_type: IssueScope | None = None, + scope_id: str | None = None, + limit: int = 50, + offset: int = 0, + ) -> IssueList: + where: Final = f"""($1::text IS NULL OR i.kind=$1) AND ($2::text IS NULL OR i.family=$2) + AND ($3::text IS NULL OR ({_STATUS})=$3) AND ($4::text IS NULL OR i.scope_type=$4) + AND ($5::text IS NULL OR i.scope_id=$5)""" + args: Final = (kind, family, status, scope_type, scope_id) + count: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw(f"SELECT count(*)::int AS count {_JOIN} WHERE {where}", *args) + ) + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw( + f"SELECT {_RECORD} AS record {_JOIN} WHERE {where} ORDER BY i.last_seen DESC,i.id LIMIT $6 OFFSET $7", + *args, + limit, + offset, + ) + ) + return IssueList( + data=await self._labels(tuple(row.record for row in rows)), total=count[0].count, limit=limit, offset=offset + ) + + async def _labels(self, issues: tuple[Issue, ...]) -> tuple[Issue, ...]: + if not issues: + return () + labels: Final = await UsageAlertsRepository(self.database).resource_labels( + tuple(issue.cohort for issue in issues) + ) + return tuple( + issue.model_copy(update={"scope_label": labels.get((issue.cohort.scope_type, issue.cohort.scope_id))}) + for issue in issues + ) + + async def find_active_issue(self, cohort_key: str) -> Issue | None: + rows: Final = await self.find_active_issues((cohort_key,)) + return rows[0] if rows else None + + async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: + if len(cohort_keys) > 500: + raise ValueError("Read at most 500 issue cohorts per batch") + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw( + f"SELECT DISTINCT ON(i.cohort_key) {_RECORD} AS record {_JOIN} " + "WHERE i.cohort_key=ANY($1::text[]) AND i.active " + "ORDER BY i.cohort_key,i.first_seen DESC,i.id", + list(cohort_keys), + ) + ) + return tuple(row.record for row in rows) + + async def set_status( + self, + issue_id: str, + status: UsageAlertStatus, + expected_status: UsageAlertStatus, + actor: str | None = None, + ) -> Issue | None: + changed: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( + await self.database.query_raw( + f"""WITH candidate AS MATERIALIZED ( + SELECT i.*,({_STATUS}) AS effective_status {_JOIN} WHERE i.id=$1 + ), ack AS ( + INSERT INTO "LiteLLM_Config" AS prior (param_name,param_value) + SELECT ack_key,jsonb_build_object('model',record->'deprecation'->>'usage_model', + 'deprecation_date',record->'deprecation'->'model'->>'deprecation_date', + 'status',$2::text,'last_seen_date',record->'deprecation'->>'last_used_date', + 'last_seen_requests',record->'deprecation'->'last_day_requests', + 'updated_at',CURRENT_TIMESTAMP,'updated_by',$4::text) + FROM candidate WHERE kind='model_deprecation' AND effective_status=$3 + ON CONFLICT(param_name) DO UPDATE SET param_value=EXCLUDED.param_value + WHERE (CASE WHEN COALESCE((CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'status','open')='open' + OR (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' IS NULL + OR EXCLUDED.param_value->>'last_seen_date' > (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' + OR (EXCLUDED.param_value->>'last_seen_date'=(CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' + AND (EXCLUDED.param_value->>'last_seen_requests')::bigint + > COALESCE(((CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_requests')::bigint,0)) + THEN 'open' ELSE (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'status' END)=$3 + RETURNING param_name + ) UPDATE "LiteLLM_Issue" i SET status=$2,updated_at=CURRENT_TIMESTAMP + FROM candidate c WHERE i.id=c.id AND c.effective_status=$3 AND + ((c.kind<>'model_deprecation' AND i.status=$3) OR i.ack_key IN(SELECT param_name FROM ack)) + RETURNING i.id""", + issue_id, + status, + expected_status, + actor, + ) + ) + return await self.get_issue(issue_id) if changed else None + + async def add_signals(self, issue_id: str, signals: tuple[IssueSignalInput, ...]) -> None: + await self.add_signal_batch( + tuple(IssueSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in signals) + ) + + async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: + if not signals: + return + if len(signals) > 500: + raise ValueError("Write at most 500 issue signals per batch") + await self.database.query_raw( + """INSERT INTO "LiteLLM_IssueSignal" (issue_id,source,source_id,record) + SELECT DISTINCT ON(x->>'issue_id',x->>'source',x->>'source_id') + x->>'issue_id',x->>'source',x->>'source_id',x FROM jsonb_array_elements($1::jsonb) x + WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" i WHERE i.id=x->>'issue_id' + AND (i.scope_type NOT IN ('key','team','user') OR + (x->'usage_alert'->>'scope_type'=i.scope_type + AND x->'usage_alert'->>'scope_id'=i.scope_id))) + ON CONFLICT(issue_id,source,source_id) DO NOTHING RETURNING source_id""", + json.dumps(tuple(signal.model_dump(mode="json") for signal in signals)), + ) + + async def list_signals(self, issue_id: str, limit: int = 50, offset: int = 0) -> IssueSignalList: + counts: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + 'SELECT count(*)::int AS count FROM "LiteLLM_IssueSignal" WHERE issue_id=$1', + issue_id, + ) + ) + rows: Final = TypeAdapter(tuple[_SignalRow, ...]).validate_python( + await self.database.query_raw( + 'SELECT record FROM "LiteLLM_IssueSignal" WHERE issue_id=$1 ORDER BY source,source_id LIMIT $2 OFFSET $3', + issue_id, + limit, + offset, + ) + ) + return IssueSignalList( + data=tuple(row.record for row in rows), total=counts[0].count, limit=limit, offset=offset + ) + + async def active_signals( + self, + day: date, + limit: int = 500, + cursor: str | None = None, + scope_type: IssueScope | None = None, + metric: UsageAlertMetric | None = None, + ) -> tuple[IssueProjectionSignal, ...]: + return TypeAdapter(tuple[IssueProjectionSignal, ...]).validate_python( + await self.database.query_raw( + """SELECT a.record AS alert,r.record AS rule,a.evaluation_day + FROM "LiteLLM_UsageAlertIncident" a JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id + WHERE a.evaluation_day=$1 AND a.breached AND r.enabled AND NOT r.deleted + AND NULLIF(a.evaluation_revision,'')::timestamptz=(r.record->>'updated_at')::timestamptz + AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric' + AND a.record->>'direction'=r.record->>'direction' + AND (r.scope_id IS NULL OR a.scope_id=r.scope_id) + AND ($2::text IS NULL OR a.id>$2) AND ($4::text IS NULL OR a.scope_type=$4) + AND ($5::text IS NULL OR a.metric=$5) ORDER BY a.id LIMIT $3""", + day.isoformat(), + cursor, + limit, + scope_type, + metric, + ) + ) + + async def recover_issues(self, day: date) -> int: + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + """WITH changed AS ( + UPDATE "LiteLLM_Issue" i SET active=false, + status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP + WHERE kind='performance' AND active + AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id AND s.source='usage_alert') + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s + LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id + WHERE s.issue_id=i.id AND s.source='usage_alert' + AND (a.id IS NULL OR a.evaluation_day>$1 OR a.breached)) RETURNING i.id + ) SELECT count(*)::int AS count FROM changed""", + day.isoformat(), + ) + ) + return rows[0].count + + async def projection_completed(self, day: date, version: str) -> bool: + rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( + await self.database.query_raw( + 'SELECT id FROM "LiteLLM_UsageAlertCheckpoint" WHERE id=$1 AND completed', + f"issues:{version}:{day.isoformat()}", + ) + ) + return bool(rows) + + async def complete_projection(self, day: date, version: str) -> None: + await self.database.query_raw( + """INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed,expires_at) VALUES($1,true,CURRENT_TIMESTAMP) + ON CONFLICT(id) DO UPDATE SET completed=true,expires_at=CURRENT_TIMESTAMP RETURNING id""", + f"issues:{version}:{day.isoformat()}", + ) + + async def finish_deprecation_projection(self, observed_at: datetime) -> int: + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + """WITH changed AS ( + UPDATE "LiteLLM_Issue" SET active=false, + status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP + WHERE kind='model_deprecation' AND active AND last_seen<$1::timestamp RETURNING id + ) SELECT count(*)::int AS count FROM changed""", + _utc(observed_at), + ) + ) + return rows[0].count + + async def family_settings(self) -> "IssueFamilyList": + from litellm.repositories.issue_families import family_settings + + return await family_settings(self.database) + + async def update_family( + self, family: "PerformanceIssueFamily", data: "IssueFamilyUpdate" + ) -> "IssueFamilySetting | None": + from litellm.repositories.issue_families import update_family + + return await update_family(self.database, family, data) + + async def queue_delivery( + self, + data: IssueDeliveryInput, + rule_id: str, + rule_revision: str, + source_id: str, + ) -> None: + await self.queue_deliveries( + (IssueDeliveryBinding(delivery=data, rule_id=rule_id, rule_revision=rule_revision, source_id=source_id),) + ) + + async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: + if not bindings: + return + if len(bindings) > 500: + raise ValueError("Queue at most 500 delivery bindings per batch") + await self.database.query_raw( + """WITH incoming AS MATERIALIZED ( + SELECT x->'delivery' AS delivery,x->>'rule_id' AS rule_id, + x->>'rule_revision' AS revision,x->>'source_id' AS source_id + FROM jsonb_array_elements($1::jsonb) x + WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" WHERE id=x->'delivery'->>'issue_id') + ), saved AS ( + INSERT INTO "LiteLLM_IssueDelivery" (id,issue_id,record) + SELECT DISTINCT ON(delivery->>'id') delivery->>'id',delivery->>'issue_id',delivery + FROM incoming ON CONFLICT(id) DO NOTHING RETURNING id + ) INSERT INTO "LiteLLM_IssueDeliveryRule" (delivery_id,rule_id,source_id,rule_revision) + SELECT DISTINCT ON(delivery->>'id',rule_id,source_id) delivery->>'id',rule_id,source_id,revision + FROM incoming WHERE delivery->>'id' IN(SELECT id FROM saved) + OR EXISTS(SELECT 1 FROM "LiteLLM_IssueDelivery" d WHERE d.id=delivery->>'id' + AND d.issue_id=delivery->>'issue_id') + ON CONFLICT(delivery_id,rule_id,source_id) DO UPDATE SET rule_revision=EXCLUDED.rule_revision + RETURNING delivery_id""", + json.dumps(tuple(binding.model_dump(mode="json") for binding in bindings)), + ) + + async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: + if not 1 <= limit <= 500: + raise ValueError("Read between one and 500 deliveries") + return TypeAdapter(tuple[IssuePendingDelivery, ...]).validate_python( + await self.database.query_raw( + f"""SELECT d.record || jsonb_build_object('next_attempt',d.next_attempt AT TIME ZONE 'UTC') AS delivery, + {_RECORD} AS issue,bound.rule,bound.alert + FROM "LiteLLM_IssueDelivery" d JOIN "LiteLLM_Issue" i ON i.id=d.issue_id + LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key + JOIN LATERAL ( + SELECT r.record AS rule,a.record AS alert FROM "LiteLLM_IssueDeliveryRule" b + JOIN "LiteLLM_UsageAlertRule" r ON r.id=b.rule_id + JOIN "LiteLLM_UsageAlertIncident" a ON a.id=b.source_id AND a.rule_id=r.id + WHERE b.delivery_id=d.id AND r.enabled AND NOT r.deleted AND a.breached AND a.status='open' + AND (r.record->>'updated_at')::timestamptz=b.rule_revision::timestamptz + AND NULLIF(a.evaluation_revision,'')::timestamptz=b.rule_revision::timestamptz + AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric' + AND a.record->>'direction'=r.record->>'direction' + AND (r.scope_id IS NULL OR a.scope_id=r.scope_id) + AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id + AND s.source='usage_alert' AND s.source_id=a.id) + ORDER BY b.rule_id,b.source_id LIMIT 1 + ) bound ON true + WHERE NOT d.sent AND d.next_attempt<=CURRENT_TIMESTAMP AND i.active AND ({_STATUS})='open' + ORDER BY d.next_attempt,d.id LIMIT $1""", + limit, + ) + ) + + async def mark_delivery_sent(self, delivery_id: str) -> None: + await self.database.query_raw( + """UPDATE "LiteLLM_IssueDelivery" SET sent=true,updated_at=CURRENT_TIMESTAMP, + record=record || jsonb_build_object('last_error',NULL,'attempts', + COALESCE((record->>'attempts')::int,0)+1) + WHERE id=$1 AND NOT sent RETURNING id""", + delivery_id, + ) + + async def defer_delivery( + self, + delivery_id: str, + error: Literal["delivery_failed", "transport_not_configured"], + ) -> None: + await self.database.query_raw( + """UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP+interval '1 hour', + updated_at=CURRENT_TIMESTAMP,record=record || jsonb_build_object('last_error',$2::text, + 'attempts',COALESCE((record->>'attempts')::int,0)+1) + WHERE id=$1 AND NOT sent RETURNING id""", + delivery_id, + error, + ) + + async def prune(self, retention_days: int = 90, limit: int = 500) -> None: + if not 1 <= limit <= 500 or retention_days < 1: + raise ValueError("Retention and cleanup batch must be bounded positive values") + await self.database.query_raw( + """WITH expired AS MATERIALIZED ( + SELECT id FROM "LiteLLM_Issue" WHERE status<>'open' + AND updated_at$1) + ORDER BY updated_at,id LIMIT $2 + ), signals AS ( + DELETE FROM "LiteLLM_IssueSignal" WHERE (issue_id,source,source_id) IN ( + SELECT issue_id,source,source_id FROM "LiteLLM_IssueSignal" + WHERE issue_id IN(SELECT id FROM expired) ORDER BY issue_id,source,source_id LIMIT $2) + RETURNING issue_id + ), bindings AS ( + DELETE FROM "LiteLLM_IssueDeliveryRule" WHERE (delivery_id,rule_id,source_id) IN ( + SELECT b.delivery_id,b.rule_id,b.source_id FROM "LiteLLM_IssueDeliveryRule" b + JOIN "LiteLLM_IssueDelivery" d ON d.id=b.delivery_id + WHERE d.issue_id IN(SELECT id FROM expired) ORDER BY b.delivery_id,b.rule_id,b.source_id LIMIT $2) + RETURNING delivery_id + ), deliveries AS ( + DELETE FROM "LiteLLM_IssueDelivery" d WHERE d.id IN ( + SELECT id FROM "LiteLLM_IssueDelivery" d WHERE issue_id IN(SELECT id FROM expired) + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueDeliveryRule" b WHERE b.delivery_id=d.id) + ORDER BY id LIMIT $2) RETURNING issue_id + ), removed AS ( + DELETE FROM "LiteLLM_Issue" i WHERE id IN(SELECT id FROM expired) + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id) + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueDelivery" d WHERE d.issue_id=i.id) RETURNING id + ) SELECT count(*)::int AS count FROM removed""", + retention_days, + limit, + ) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index c7efc5c77f8..7eb80dae626 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -3,7 +3,6 @@ import json from collections.abc import Mapping from datetime import date, datetime, timedelta, timezone -from itertools import product from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -17,6 +16,7 @@ from uuid import NAMESPACE_URL, uuid4, uuid5 from pydantic import BaseModel, TypeAdapter from litellm.repositories.daily_activity_buckets import api_key_from_bucket +from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS from litellm.types.proxy.usage_alerts import ( UsageAlert, UsageAlertChannel, @@ -49,6 +49,14 @@ class AlertDatabase(Protocol): async def query_raw(self, query: str, *args: object) -> object: ... +class ResourceLabelScope(Protocol): + @property + def scope_type(self) -> str: ... + + @property + def scope_id(self) -> str | None: ... + + class _Record(BaseModel, frozen=True): record: Mapping[str, object] @@ -75,7 +83,7 @@ class _GroupPage(BaseModel, frozen=True): class _ResourceLabel(BaseModel, frozen=True): - scope_type: UsageAlertScope + scope_type: str scope_id: str | None label: str | None @@ -98,14 +106,7 @@ class _Ownership(BaseModel, frozen=True): team_id: str | None -_DEFAULT_RULE_IDS: Final = frozenset( - str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{scope}:{metric}:{direction}")) - for scope, metric, direction in product( - ("deployment", "key", "user", "team", "model"), - ("spend", "requests", "latency", "errors"), - ("increase", "decrease"), - ) -) +_DEFAULT_RULE_IDS: Final = DEFAULT_USAGE_RULE_IDS def _rule(record: Mapping[str, object]) -> UsageAlertRule: @@ -405,8 +406,8 @@ class UsageAlertsRepository: ) ) page: Final = rows[0] - labels: Final[Mapping[tuple[UsageAlertScope, str | None], str | None]] = ( - await self._group_labels(page.data) if page.data else {} + labels: Final[Mapping[tuple[str, str | None], str | None]] = ( + await self.resource_labels(page.data) if page.data else {} ) return UsageAlertGroupList( data=tuple( @@ -434,9 +435,9 @@ class UsageAlertsRepository: offset=offset, ) - async def _group_labels( - self, groups: tuple[_GroupRow, ...] - ) -> Mapping[tuple[UsageAlertScope, str | None], str | None]: + async def resource_labels( + self, groups: tuple[ResourceLabelScope, ...] + ) -> Mapping[tuple[str, str | None], str | None]: rows: Final = TypeAdapter(tuple[_ResourceLabel, ...]).validate_python( await self.database.query_raw( """WITH scopes AS ( @@ -446,7 +447,8 @@ class UsageAlertsRepository: WHEN 'key' THEN COALESCE(NULLIF(k.key_alias,''),NULLIF(k.key_name,'')) WHEN 'team' THEN NULLIF(t.team_alias,'') WHEN 'user' THEN COALESCE(NULLIF(u.user_alias,''),NULLIF(u.user_email,'')) - WHEN 'model' THEN s.scope_id WHEN 'deployment' THEN 'Gateway' END AS label + WHEN 'model' THEN s.scope_id WHEN 'provider' THEN s.scope_id + WHEN 'deployment' THEN 'Gateway' END AS label FROM scopes s LEFT JOIN "LiteLLM_VerificationToken" k ON s.scope_type='key' AND k.token=s.key_id LEFT JOIN "LiteLLM_TeamTable" t ON s.scope_type='team' AND t.team_id=s.scope_id LEFT JOIN "LiteLLM_UserTable" u ON s.scope_type='user' AND u.user_id=s.scope_id""", diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py new file mode 100644 index 00000000000..e0197e6edf2 --- /dev/null +++ b/litellm/types/proxy/issues.py @@ -0,0 +1,260 @@ +from datetime import date, datetime +from typing import Literal, Self, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import ( + UsageAlert, + UsageAlertChannel, + UsageAlertDirection, + UsageAlertMetric, + UsageAlertRule, + UsageAlertStatus, +) + +IssueKind: TypeAlias = Literal["performance", "model_deprecation"] +IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] +IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] +IssueGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"] + + +class IssueModelPair(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + provider: str = Field(min_length=1, max_length=256) + model: str = Field(min_length=1, max_length=1024) + + +class IssueCohort(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + scope_type: IssueScope + scope_id: str | None = Field(default=None, min_length=1, max_length=1024) + model_pairs: tuple[IssueModelPair, ...] = Field(default=(), max_length=100) + + @model_validator(mode="after") + def validate_scope(self) -> Self: + if (self.scope_type == "deployment") != (self.scope_id is None): + raise ValueError("Only a deployment cohort omits scope_id") + if self.scope_type == "key" and self.scope_id and self.scope_id.removeprefix("raw:").startswith("sk-"): + raise ValueError("Use a reporting key ID, never a secret key") + if len(frozenset((pair.provider, pair.model) for pair in self.model_pairs)) != len(self.model_pairs): + raise ValueError("model_pairs must be unique") + return self + + +class IssueComparison(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + metric: UsageAlertMetric + direction: UsageAlertDirection + observed_value: float = Field(ge=0) + baseline_value: float = Field(ge=0) + threshold: float + baseline_start: date + baseline_end: date + baseline_sample_days: int | None = Field(default=None, ge=1, le=30) + + @model_validator(mode="after") + def validate_comparison(self) -> Self: + if self.baseline_start > self.baseline_end: + raise ValueError("baseline_start must not follow baseline_end") + if self.metric == "error_rate" and (self.observed_value > 100 or self.baseline_value > 100): + raise ValueError("error_rate uses percentages from 0 to 100") + return self + + +class IssueImpact(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + keys: int | None = Field(default=None, ge=0) + users: int | None = Field(default=None, ge=0) + teams: int | None = Field(default=None, ge=0) + models: int | None = Field(default=None, ge=0) + requests: int | None = Field(default=None, ge=0) + measured_keys: int | None = Field(default=None, ge=0) + measured_models: int | None = Field(default=None, ge=0) + + +class IssueDeprecation(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + model: ModelDeprecationInfo + usage_model: str = Field(min_length=1, max_length=1024) + requests: int = Field(ge=0) + last_used_date: date + last_day_requests: int = Field(ge=0) + state: DeprecationAlertState + + +class IssueInput(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + id: str = Field(min_length=1, max_length=200) + cohort_key: str = Field(min_length=1, max_length=200) + kind: IssueKind + family: IssueFamily + title: str = Field(min_length=1, max_length=2048) + cohort: IssueCohort + observed_on: date + observed_at: datetime + range_start: date + range_end: date + grouping_reason: IssueGroupingReason + explanation: str = Field(default="", max_length=2000) + comparisons: tuple[IssueComparison, ...] = Field(default=(), max_length=10) + impact: IssueImpact = Field(default_factory=IssueImpact) + deprecation: IssueDeprecation | None = None + + @model_validator(mode="after") + def validate_issue(self) -> Self: + if self.range_start > self.range_end: + raise ValueError("range_start must not follow range_end") + if (self.kind == "model_deprecation") != (self.deprecation is not None): + raise ValueError("Model deprecation issues require deprecation details") + if (self.kind == "model_deprecation") != (self.family == "retirement"): + raise ValueError("Only model deprecation issues use the retirement family") + if self.observed_at.tzinfo is None: + raise ValueError("observed_at must include a timezone") + return self + + +class Issue(IssueInput, frozen=True): + status: UsageAlertStatus + first_seen: datetime + last_seen: datetime + updated_at: datetime + scope_label: str | None = None + active: bool = True + + +class IssueList(BaseModel, frozen=True): + data: tuple[Issue, ...] + total: int + limit: int + offset: int + + +class IssueUpdate(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: UsageAlertStatus + expected_status: UsageAlertStatus + + +class IssueSignalInput(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + source: Literal["usage_alert", "model_deprecation"] + source_id: str = Field(min_length=1, max_length=200) + usage_alert: UsageAlert | None = None + deprecation: IssueDeprecation | None = None + + @model_validator(mode="after") + def validate_signal(self) -> Self: + if self.source == "usage_alert": + if self.usage_alert is None or self.deprecation is not None or self.usage_alert.id != self.source_id: + raise ValueError("Usage signals require their matching source alert") + elif self.deprecation is None or self.usage_alert is not None: + raise ValueError("Deprecation signals require deprecation details") + return self + + +class IssueSignal(IssueSignalInput, frozen=True): + issue_id: str + + +class IssueSignalList(BaseModel, frozen=True): + data: tuple[IssueSignal, ...] + total: int + limit: int + offset: int + + +class IssueProjectionSignal(BaseModel, frozen=True): + alert: UsageAlert + rule: UsageAlertRule + evaluation_day: date + + +class IssueDeliveryInput(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + id: str = Field(min_length=1, max_length=200) + issue_id: str = Field(min_length=1, max_length=200) + channel: UsageAlertChannel + email_recipients: tuple[str, ...] = Field(default=(), max_length=20) + + +class IssueDeliveryBinding(BaseModel, frozen=True): + delivery: IssueDeliveryInput + rule_id: str + rule_revision: str + source_id: str + + +class IssueDelivery(IssueDeliveryInput, frozen=True): + attempts: int = 0 + next_attempt: datetime | None = None + last_error: Literal["delivery_failed", "transport_not_configured"] | None = None + + +class IssuePendingDelivery(BaseModel, frozen=True): + delivery: IssueDelivery + issue: Issue + rule: UsageAlertRule + alert: UsageAlert + + +class IssueFamilyPolicy(BaseModel, frozen=True): + direction: UsageAlertDirection + enabled: bool | None + multiplier: float | None + min_value: float | None + min_requests: int | None + baseline_days: int | None + min_history_days: int | None + mixed: bool + total_rules: int + enabled_rules: int + + +class IssueFamilySetting(BaseModel, frozen=True): + family: PerformanceIssueFamily + enabled: bool | None + total_rules: int + enabled_rules: int + policies: tuple[IssueFamilyPolicy, ...] + + +class IssueFamilyList(BaseModel, frozen=True): + data: tuple[IssueFamilySetting, ...] + + +class IssueFamilyPolicyUpdate(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + direction: UsageAlertDirection + enabled: bool | None = None + multiplier: float | None = Field(default=None, gt=1, le=1000) + min_value: float | None = Field(default=None, ge=0) + min_requests: int | None = Field(default=None, ge=0) + baseline_days: int | None = Field(default=None, ge=1, le=30) + min_history_days: int | None = Field(default=None, ge=1, le=30) + + +class IssueFamilyUpdate(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + enabled: bool | None = None + policies: tuple[IssueFamilyPolicyUpdate, ...] = Field(default=(), max_length=2) + + @model_validator(mode="after") + def validate_update(self) -> Self: + if len(frozenset(policy.direction for policy in self.policies)) != len(self.policies): + raise ValueError("Each direction can occur once") + if self.enabled is None and not self.policies: + raise ValueError("Specify enabled or a policy update") + return self diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 37f59711050..c95df8d6840 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -182,6 +182,8 @@ class KeySpendMetrics(LiteLLMBaseModel): failed_requests: int = 0 cache_read_input_tokens: int = 0 cache_creation_input_tokens: int = 0 + total_response_time_ms: int = 0 + timed_requests: int = 0 class KeySpendActivityRow(LiteLLMBaseModel): @@ -214,6 +216,8 @@ class UserActivityRow(LiteLLMBaseModel): api_requests: int = 0 successful_requests: int = 0 failed_requests: int = 0 + total_response_time_ms: int = 0 + timed_requests: int = 0 class DailyActivityUserPageResponse(LiteLLMBaseModel): diff --git a/litellm/types/proxy/usage_alert_defaults.py b/litellm/types/proxy/usage_alert_defaults.py new file mode 100644 index 00000000000..7b54ac3eb45 --- /dev/null +++ b/litellm/types/proxy/usage_alert_defaults.py @@ -0,0 +1,75 @@ +from dataclasses import dataclass +from itertools import product +from typing import Final, Literal, TypeAlias +from uuid import NAMESPACE_URL, uuid5 + +from litellm.types.proxy.usage_alerts import ( + UsageAlertChannel, + UsageAlertDirection, + UsageAlertMetric, + UsageAlertRule, + UsageAlertRuleInput, + UsageAlertScope, +) + +PerformanceIssueFamily: TypeAlias = Literal["spend", "latency", "reliability", "traffic"] +USAGE_ALERT_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("deployment", "key", "user", "team", "model") + + +@dataclass(frozen=True, slots=True) +class DefaultUsagePolicy: + family: PerformanceIssueFamily + metric: UsageAlertMetric + direction: UsageAlertDirection + multiplier: float + min_value: float + min_requests: int = 0 + min_history_days: int = 3 + + +DEFAULT_USAGE_POLICIES: Final = ( + DefaultUsagePolicy("spend", "spend", "increase", 3, 10), + DefaultUsagePolicy("latency", "latency", "increase", 3, 500, 100), + DefaultUsagePolicy("reliability", "error_rate", "increase", 3, 2, 500), + DefaultUsagePolicy("traffic", "requests", "increase", 10, 1000), + DefaultUsagePolicy("traffic", "requests", "decrease", 5, 0, 500, 7), +) + + +def default_usage_rule_id(scope: UsageAlertScope, metric: UsageAlertMetric, direction: UsageAlertDirection) -> str: + return str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{scope}:{metric}:{direction}")) + + +DEFAULT_USAGE_RULE_IDS: Final = frozenset( + default_usage_rule_id(scope, policy.metric, policy.direction) + for scope, policy in product(USAGE_ALERT_SCOPES, DEFAULT_USAGE_POLICIES) +) + + +def is_default_usage_rule(rule: UsageAlertRule) -> bool: + return rule.id in DEFAULT_USAGE_RULE_IDS and rule.id == default_usage_rule_id( + rule.scope_type, rule.metric, rule.direction + ) + + +def default_usage_rules( + channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False +) -> tuple[UsageAlertRuleInput, ...]: + return tuple( + UsageAlertRuleInput( + name=f"Daily {scope} {policy.metric.replace('_', ' ')} {policy.direction}", + scope_type=scope, + metric=policy.metric, + direction=policy.direction, + multiplier=policy.multiplier, + min_value=policy.min_value, + min_requests=policy.min_requests, + min_history_days=policy.min_history_days, + notification_channels=( + () + if legacy_user_spend_alerts and (scope, policy.metric, policy.direction) == ("user", "spend", "increase") + else channels + ), + ) + for scope, policy in product(USAGE_ALERT_SCOPES, DEFAULT_USAGE_POLICIES) + ) diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index bb719ea48a2..72ccc1f3ef7 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -6,7 +6,7 @@ from typing import Literal, Self, TypeAlias from pydantic import BaseModel, ConfigDict, Field, model_validator UsageAlertScope: TypeAlias = Literal["deployment", "key", "user", "team", "model"] -UsageAlertMetric: TypeAlias = Literal["spend", "requests", "latency", "errors"] +UsageAlertMetric: TypeAlias = Literal["spend", "requests", "latency", "errors", "error_rate"] UsageAlertDirection: TypeAlias = Literal["increase", "decrease"] UsageAlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"] UsageAlertChannel: TypeAlias = Literal["email", "slack"] diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index b013ce43113..938ef0467ed 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -39,6 +39,8 @@ class DailyActivityScope: model: str | None timezone_offset_minutes: int | None include_current_utc_day: bool = False + provider: str | None = None + model_pairs: tuple[tuple[str, str], ...] = () def __post_init__(self) -> None: if self.entity_id_field not in _ENTITY_FIELDS[self.table]: @@ -57,6 +59,8 @@ class KeySpendRow: failed_requests: int cache_read_input_tokens: int cache_creation_input_tokens: int + total_response_time_ms: int = 0 + timed_requests: int = 0 @dataclass(frozen=True, slots=True) @@ -75,6 +79,8 @@ class UserSpendRow: api_requests: int successful_requests: int failed_requests: int + total_response_time_ms: int = 0 + timed_requests: int = 0 @dataclass(frozen=True, slots=True) diff --git a/schema.prisma b/schema.prisma index 46ab28530a9..8a67d1b123a 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2132,11 +2132,13 @@ model LiteLLM_UsageAlertIncident { status String day String evaluation_day String + evaluation_revision String @default("") breached Boolean record Json notified_channels String[] @default([]) updated_at DateTime @default(now()) @@index([rule_id, scope_key, evaluation_day]) + @@index([evaluation_day, breached, id]) @@index([scope_type, scope_id, day]) @@index([status, day]) @@index([updated_at]) @@ -2149,3 +2151,55 @@ model LiteLLM_UsageAlertCheckpoint { expires_at DateTime @default(now()) @@index([expires_at]) } + +model LiteLLM_Issue { + id String @id + cohort_key String + kind String + family String + scope_type String + scope_id String? + status String + active Boolean @default(true) + ack_key String? + record Json + first_seen DateTime + last_seen DateTime + updated_at DateTime @default(now()) + + @@index([status, last_seen, id]) + @@index([kind, family, last_seen, id]) + @@index([scope_type, scope_id, last_seen, id]) + @@index([cohort_key, active]) +} + +model LiteLLM_IssueSignal { + issue_id String + source String + source_id String + record Json + + @@id([issue_id, source, source_id]) + @@index([source, source_id]) +} + +model LiteLLM_IssueDelivery { + id String @id + issue_id String + record Json + sent Boolean @default(false) + next_attempt DateTime @default(now()) + updated_at DateTime @default(now()) + + @@index([sent, next_attempt, id]) + @@index([issue_id]) +} + +model LiteLLM_IssueDeliveryRule { + delivery_id String + rule_id String + source_id String + rule_revision String + + @@id([delivery_id, rule_id, source_id]) +} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 42ae1107a2e..bbd4acaedf8 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8343,6 +8343,160 @@ export interface paths { patch?: never; trace?: never; }; + "/issues": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Issues */ + get: operations["list_issues_issues_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/families": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Issue Families */ + get: operations["list_issue_families_issues_families_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/families/{family}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Issue Family */ + patch: operations["update_issue_family_issues_families__family__patch"]; + trace?: never; + }; + "/issues/{issue_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue */ + get: operations["get_issue_issues__issue_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Issue */ + patch: operations["update_issue_issues__issue_id__patch"]; + trace?: never; + }; + "/issues/{issue_id}/activity": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Activity */ + get: operations["get_issue_activity_issues__issue_id__activity_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/impact": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Impact */ + get: operations["get_issue_impact_issues__issue_id__impact_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/keys": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Keys */ + get: operations["get_issue_keys_issues__issue_id__keys_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/signals": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Issue Signals */ + get: operations["list_issue_signals_issues__issue_id__signals_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/users": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Users */ + get: operations["get_issue_users_issues__issue_id__users_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/jwt/key/mapping/delete": { parameters: { query?: never; @@ -34710,6 +34864,302 @@ export interface components { /** Is Accepted */ is_accepted: boolean; }; + /** Issue */ + Issue: { + /** + * Active + * @default true + */ + active: boolean; + cohort: components["schemas"]["IssueCohort"]; + /** Cohort Key */ + cohort_key: string; + /** + * Comparisons + * @default [] + */ + comparisons: components["schemas"]["IssueComparison"][]; + deprecation?: components["schemas"]["IssueDeprecation"] | null; + /** + * Explanation + * @default + */ + explanation: string; + /** + * Family + * @enum {string} + */ + family: "retirement" | "latency" | "reliability" | "spend" | "traffic"; + /** + * First Seen + * Format: date-time + */ + first_seen: string; + /** + * Grouping Reason + * @enum {string} + */ + grouping_reason: "same_resource" | "dominant_key" | "provider_cohort" | "model_deprecation"; + /** Id */ + id: string; + impact?: components["schemas"]["IssueImpact"]; + /** + * Kind + * @enum {string} + */ + kind: "performance" | "model_deprecation"; + /** + * Last Seen + * Format: date-time + */ + last_seen: string; + /** + * Observed At + * Format: date-time + */ + observed_at: string; + /** + * Observed On + * Format: date + */ + observed_on: string; + /** + * Range End + * Format: date + */ + range_end: string; + /** + * Range Start + * Format: date + */ + range_start: string; + /** Scope Label */ + scope_label?: string | null; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + /** Title */ + title: string; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; + /** IssueCohort */ + IssueCohort: { + /** + * Model Pairs + * @default [] + */ + model_pairs: components["schemas"]["IssueModelPair"][]; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model" | "provider"; + }; + /** IssueComparison */ + IssueComparison: { + /** + * Baseline End + * Format: date + */ + baseline_end: string; + /** Baseline Sample Days */ + baseline_sample_days?: number | null; + /** + * Baseline Start + * Format: date + */ + baseline_start: string; + /** Baseline Value */ + baseline_value: number; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + /** Observed Value */ + observed_value: number; + /** Threshold */ + threshold: number; + }; + /** IssueDeprecation */ + IssueDeprecation: { + /** Last Day Requests */ + last_day_requests: number; + /** + * Last Used Date + * Format: date + */ + last_used_date: string; + model: components["schemas"]["ModelDeprecationInfo"]; + /** Requests */ + requests: number; + state: components["schemas"]["DeprecationAlertState"]; + /** Usage Model */ + usage_model: string; + }; + /** IssueFamilyList */ + IssueFamilyList: { + /** Data */ + data: components["schemas"]["IssueFamilySetting"][]; + }; + /** IssueFamilyPolicy */ + IssueFamilyPolicy: { + /** Baseline Days */ + baseline_days: number | null; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** Enabled */ + enabled: boolean | null; + /** Enabled Rules */ + enabled_rules: number; + /** Min History Days */ + min_history_days: number | null; + /** Min Requests */ + min_requests: number | null; + /** Min Value */ + min_value: number | null; + /** Mixed */ + mixed: boolean; + /** Multiplier */ + multiplier: number | null; + /** Total Rules */ + total_rules: number; + }; + /** IssueFamilyPolicyUpdate */ + IssueFamilyPolicyUpdate: { + /** Baseline Days */ + baseline_days?: number | null; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** Enabled */ + enabled?: boolean | null; + /** Min History Days */ + min_history_days?: number | null; + /** Min Requests */ + min_requests?: number | null; + /** Min Value */ + min_value?: number | null; + /** Multiplier */ + multiplier?: number | null; + }; + /** IssueFamilySetting */ + IssueFamilySetting: { + /** Enabled */ + enabled: boolean | null; + /** Enabled Rules */ + enabled_rules: number; + /** + * Family + * @enum {string} + */ + family: "spend" | "latency" | "reliability" | "traffic"; + /** Policies */ + policies: components["schemas"]["IssueFamilyPolicy"][]; + /** Total Rules */ + total_rules: number; + }; + /** IssueFamilyUpdate */ + IssueFamilyUpdate: { + /** Enabled */ + enabled?: boolean | null; + /** + * Policies + * @default [] + */ + policies: components["schemas"]["IssueFamilyPolicyUpdate"][]; + }; + /** IssueImpact */ + IssueImpact: { + /** Keys */ + keys?: number | null; + /** Measured Keys */ + measured_keys?: number | null; + /** Measured Models */ + measured_models?: number | null; + /** Models */ + models?: number | null; + /** Requests */ + requests?: number | null; + /** Teams */ + teams?: number | null; + /** Users */ + users?: number | null; + }; + /** IssueList */ + IssueList: { + /** Data */ + data: components["schemas"]["Issue"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** IssueModelPair */ + IssueModelPair: { + /** Model */ + model: string; + /** Provider */ + provider: string; + }; + /** IssueSignal */ + IssueSignal: { + deprecation?: components["schemas"]["IssueDeprecation"] | null; + /** Issue Id */ + issue_id: string; + /** + * Source + * @enum {string} + */ + source: "usage_alert" | "model_deprecation"; + /** Source Id */ + source_id: string; + usage_alert?: components["schemas"]["UsageAlert"] | null; + }; + /** IssueSignalList */ + IssueSignalList: { + /** Data */ + data: components["schemas"]["IssueSignal"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** IssueUpdate */ + IssueUpdate: { + /** + * Expected Status + * @enum {string} + */ + expected_status: "open" | "resolved" | "dismissed"; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + }; /** * ItemReference * @description An internal identifier for an item to reference. @@ -34868,6 +35318,16 @@ export interface components { * @default 0 */ successful_requests: number; + /** + * Timed Requests + * @default 0 + */ + timed_requests: number; + /** + * Total Response Time Ms + * @default 0 + */ + total_response_time_ms: number; /** * Total Tokens * @default 0 @@ -50485,7 +50945,7 @@ export interface components { * Metric * @enum {string} */ - metric: "spend" | "requests" | "latency" | "errors"; + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; /** Observed Value */ observed_value: number; /** Rule Id */ @@ -50588,7 +51048,7 @@ export interface components { * Metric * @enum {string} */ - metric: "spend" | "requests" | "latency" | "errors"; + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; representative: components["schemas"]["UsageAlert"]; }; /** UsageAlertGroupUpdate */ @@ -50706,7 +51166,7 @@ export interface components { * Metric * @enum {string} */ - metric: "spend" | "requests" | "latency" | "errors"; + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; /** * Min History Days * @default 3 @@ -50774,7 +51234,7 @@ export interface components { * Metric * @enum {string} */ - metric: "spend" | "requests" | "latency" | "errors"; + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; /** * Min History Days * @default 3 @@ -51349,6 +51809,16 @@ export interface components { * @default 0 */ successful_requests: number; + /** + * Timed Requests + * @default 0 + */ + timed_requests: number; + /** + * Total Response Time Ms + * @default 0 + */ + total_response_time_ms: number; /** * Total Tokens * @default 0 @@ -64994,6 +65464,338 @@ export interface operations { }; }; }; + list_issues_issues_get: { + parameters: { + query?: { + kind?: ("performance" | "model_deprecation") | null; + family?: ("retirement" | "latency" | "reliability" | "spend" | "traffic") | null; + status?: ("open" | "resolved" | "dismissed") | null; + scope_type?: ("deployment" | "key" | "user" | "team" | "model" | "provider") | null; + scope_id?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_issue_families_issues_families_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueFamilyList"]; + }; + }; + }; + }; + update_issue_family_issues_families__family__patch: { + parameters: { + query?: never; + header?: never; + path: { + family: "spend" | "latency" | "reliability" | "traffic"; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["IssueFamilyUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueFamilySetting"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_issues__issue_id__get: { + parameters: { + query?: never; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Issue"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_issue_issues__issue_id__patch: { + parameters: { + query?: never; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["IssueUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Issue"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_activity_issues__issue_id__activity_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_impact_issues__issue_id__impact_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueImpact"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_keys_issues__issue_id__keys_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DailyActivityKeyPageResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_issue_signals_issues__issue_id__signals_get: { + parameters: { + query?: { + limit?: number; + offset?: number; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueSignalList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_users_issues__issue_id__users_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DailyActivityUserPageResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; delete_jwt_key_mapping_jwt_key_mapping_delete_post: { parameters: { query?: never; @@ -79013,7 +79815,7 @@ export interface operations { query?: { scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; scope_id?: string | null; - metric?: ("spend" | "requests" | "latency" | "errors") | null; + metric?: ("spend" | "requests" | "latency" | "errors" | "error_rate") | null; status?: ("open" | "resolved" | "dismissed") | null; start_date?: string | null; end_date?: string | null;