From df62ec53cd7b8d73884507866c3f1ace3dca3c93 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:13:01 +0800 Subject: [PATCH] fix: scope issue evidence and preserve retirement lifecycle --- docs/features/usage-alerts.md | 96 +++++- litellm/models/issues.py | 6 +- litellm/proxy/alerting_endpoints/issues.py | 18 +- litellm/repositories/issues.py | 57 +++- litellm/repositories/usage_alerts.py | 3 +- litellm/types/proxy/issues.py | 7 +- litellm/types/proxy/usage_alerts.py | 2 +- tests/integration/database/test_issues.py | 290 ++++++++++++++++++ .../integration/database/test_usage_alerts.py | 19 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 13 +- 10 files changed, 475 insertions(+), 36 deletions(-) create mode 100644 tests/integration/database/test_issues.py diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index 804a8b44aec..63c353b0fb6 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -1,9 +1,92 @@ -# Daily Performance alerts +# Issues and daily Performance alerts -Performance alerts compare a completed UTC day with earlier recorded days. They cover spend, request count, average latency, and failed-request count for the gateway, a key, a user, a team, or a model. The Performance tab groups alerts for the same resource and original UTC day. Open a resource to inspect its saved comparison and Activity. The API paths remain under `/usage`. +The Issues inbox combines model retirements with daily Performance alerts. Open an issue to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause. + +Performance comparisons cover completed UTC days. Retirement issues show recent model usage and the provider's retirement schedule, including today's usage when present. This feature requires a database. Apply the normal LiteLLM database migrations when deploying a version that includes usage alerts. The API and dashboard use the same persisted rules and incidents; no enterprise license is required. +## Start with the defaults + +Daily Performance defaults cover spend, latency, reliability, and traffic across the gateway, keys, users, teams, and models. They use the previous seven days and the following policies. Retirement uses the existing model-deprecation warning window. + +| Family | Default comparison | Minimum history and volume | +| --- | --- | --- | +| Spend | At least 3× daily spend and a $10 increase | Three recorded reference days with a positive spend baseline | +| Latency | At least 3× average latency and a 500 ms increase | Three recorded reference days and at least 100 timed requests in both periods | +| Reliability | At least 3× error rate and a two percentage-point increase | Three recorded reference days and at least 500 daily requests in both periods | +| Traffic increase | At least 10× requests and an increase of 1,000 requests | Three recorded reference days with a positive request baseline | +| Traffic decrease | At most one fifth of baseline requests | Seven recorded reference days and a baseline of at least 500 daily requests | + +New resources need history before they can produce a default spike. A zero spend, request, or latency baseline does not establish a spike. A zero error-rate baseline is usable only after sufficient historical traffic has been observed; the percentage-point floor still applies. + +Use the family settings to enable or disable a family or adjust its thresholds across the five scopes. Existing custom rules remain independent. An enabled custom rule replaces the matching default for its scope, metric, and direction. A custom wildcard rule replaces that default for every entity. Deleted defaults stay deleted. + +The scheduler wakes hourly and evaluates the latest completed UTC day after a one-hour grace period. A successful aggregate flush after the grace period is required before either direction is finalized. Completed rule versions are skipped on later ticks. The evaluator reads existing daily aggregates in pages of 500 entities; it adds no request-time counters or per-request database writes. + +Decrease rules need a successful background aggregate-flush sample for every hour of the target day and another sample after the grace period. Until that coverage exists, their status is `insufficient_coverage`. A key with no requests can then produce a request or spend drop; missing latency samples never become zero latency. Coverage shows that an aggregate writer was healthy each hour, not that every replica delivered every request. + +The baseline averages recorded reference days; it does not turn missing reference days into zero. Average latency is weighted by timed request count, using the same successful-request duration totals as Activity. `baseline_sample_days` records how many reference days contributed. Cost, traffic, and latency comparisons require a positive baseline. Reliability can compare against a recorded 0% error rate when historical and current request volume meet the rule's minimum. Missing history and low-volume comparisons remain `insufficient_history`. + +## Read and investigate issues + +Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the credential in your environment. + +```bash +curl "$LITELLM_API_BASE/issues?status=open&limit=50" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +``` + +The response contains `data`, `total`, `limit`, and `offset`. Pagination and filtering happen on the combined inbox. Filters are `kind`, `family`, `status`, `scope_type`, and `scope_id`. Families are `retirement`, `latency`, `reliability`, `spend`, and `traffic`; kinds are `model_deprecation` and `performance`. An offset beyond the last page still returns the correct total. There is no default history cutoff on open issues. + +Each issue has an ID, status, readable title, current resource label, typed `cohort`, date range, grouping reason, and saved comparison values. The cohort is one resource, one provider, or exact provider/model pairs. A provider/model pair is an intersection, not a list of independently interchangeable filters. Resource names can change; IDs remain the identity. + +Use the ID from the list to inspect an issue: + +```bash +ISSUE_ID="issue-example" +curl "$LITELLM_API_BASE/issues/$ISSUE_ID" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +curl "$LITELLM_API_BASE/issues/$ISSUE_ID/activity" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +curl "$LITELLM_API_BASE/issues/$ISSUE_ID/signals?limit=50" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +``` + +Replace `issue-example` with an ID returned by `/issues`. Activity uses the issue's saved date range by default. Optional `start_date` and `end_date` are inclusive UTC dates and can span at most 366 days. `/keys` and `/users` provide paginated Activity rows with `limit` and `offset`. Their latency totals are `total_response_time_ms` and `timed_requests`; divide the former by the latter only when timed requests are present. + +`/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown. + +Supporting signals have their own saved comparisons and relationship explanations. Their baseline windows can differ, so do not derive a combined error rate or unit cost by adding their displayed values. Administrators and admin viewers can inspect broader supporting signals. A scoped reader receives only signals for the exact key, user, or team they can access. That restriction applies before the signal count and pagination; broader totals are not disclosed. + +A continuing Performance issue retains its initial comparison and cohort while `last_seen` advances. New supporting observations do not rewrite the original evidence. After a healthy observation ends every linked source episode, a later breach starts a new issue. Retirement details refresh as usage and the provider schedule change. A retirement issue with no recent use becomes inactive; actual new traffic can reopen it. + +Resolve or dismiss an issue using the current status from the read response: + +```bash +curl -X PATCH "$LITELLM_API_BASE/issues/$ISSUE_ID" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \ + -H "Content-Type: application/json" \ + -d '{"expected_status":"open","status":"resolved"}' +``` + +A stale status returns `409`; reload before retrying. Performance acknowledgements do not change other issues that share a supporting observation. Retirement acknowledgements use the same state as the existing model-deprecation API. An inactive retirement cannot be manually reopened. + +## Configure default families + +`GET /issues/families` returns the four Performance families, their enabled counts, and common policy values. A null common value means the underlying scopes have different settings. This is separate from custom rules and the model-retirement warning window. + +This example sets the spend family to a 5× threshold and a $25 minimum absolute increase: + +```bash +curl -X PATCH "$LITELLM_API_BASE/issues/families/spend" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \ + -H "Content-Type: application/json" \ + -d '{"enabled":true,"policies":[{"direction":"increase","multiplier":5,"min_value":25}]}' +``` + +The update applies atomically to registered, non-deleted built-in rules in that family. Omitted settings remain unchanged. It does not change custom rules or recreate deleted defaults. A concurrent rule edit returns `409`; reload and retry. These controls require a gateway administrator. + ## Read resource groups Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the key in your environment. @@ -15,7 +98,7 @@ curl "$LITELLM_API_BASE/usage/alert-groups?status=open&limit=50" \ The response has `data`, `total`, `limit`, and `offset`. `total` counts resource/day groups, including when the requested offset is beyond the last page. Filters are `scope_type`, `scope_id`, `status`, `start_date`, and `end_date`. Status filters apply to individual incidents before grouping, so a group with open and resolved incidents can appear under either filter with the corresponding members. No default date cutoff hides older open incidents. -Each group includes its stable `id`, `scope_type`, exact `scope_id`, current `scope_label` when available, original UTC `day`, `incident_count`, and `activity_path`. Its `metrics` list contains up to four entries with a `metric`, `incident_count`, and the most recently detected matching `representative` incident. These representatives are summaries, not the entire group. Missing or deleted resources keep their original IDs. Names are current display labels, not historical evidence. +Each group includes its stable `id`, `scope_type`, exact `scope_id`, current `scope_label` when available, original UTC `day`, `incident_count`, and `activity_path`. Its `metrics` list contains up to five entries with a `metric`, `incident_count`, and the most recently detected matching `representative` incident. These representatives are summaries, not the entire group. Missing or deleted resources keep their original IDs. Names are current display labels, not historical evidence. To inspect every matching rule, use `GET /usage/alerts` with the group's scope and `start_date` and `end_date` both set to its `day`. Add `metric` and `status` as needed and paginate that response. Rules can have different baseline windows and thresholds, so their comparisons must not be combined into an error rate or a cost-per-request estimate. @@ -69,8 +152,9 @@ The baseline is the mean across recorded days in the preceding window. Missing r | `requests` | Request count | | `latency` | Average response time in milliseconds | | `errors` | Failed-request count | +| `error_rate` | Failed requests divided by requests, as a percentage from 0 to 100 | -Counts come from the existing Activity/spend aggregates, rather than every gateway HTTP outcome. Latency averages successful timed, non-internal requests; it is not time to first token and does not include failed requests. Gateway incidents link to Key Activity, which uses the same aggregates. +Counts come from the existing Activity/spend aggregates, rather than every gateway HTTP outcome. Latency averages successful timed, non-internal requests; it is not time to first token and does not include failed requests. Issue Activity and gateway source incidents use the same aggregates. An error-rate `min_value` of 2 means two percentage points, such as 1% to 3%, not a 2% relative increase. The evaluator runs against daily aggregates. An hourly scheduler does not make these rolling hourly alerts. A complete UTC day and sufficient ingestion coverage are required before detecting drops; a missing flush must not become a false traffic-drop incident. Rule responses expose `last_evaluated_day` and `evaluation_status` so you can distinguish a completed evaluation from insufficient history or coverage. @@ -87,10 +171,12 @@ Use `scope_type: "deployment"` with no `scope_id` for the gateway. For `model`, Consecutive breached days update one incident. Re-evaluating the same day does not reopen a dismissed or resolved incident. After recovery, a later breach starts a new incident. Editing a rule does not rewrite historical comparisons. -Rules can specify `notification_channels: ["slack", "email"]` and `email_recipients: ["alerts@example.com"]`. Notification transports must already be configured on the gateway. An empty channel list keeps incidents available through the API and dashboard. Successful delivery is recorded per channel so a failed channel can retry independently. +Rules can specify `notification_channels: ["slack", "email"]` and `email_recipients: ["alerts@example.com"]`. Notification transports must already be configured on the gateway. An empty channel list keeps incidents available through the API and dashboard. Issue delivery is recorded per channel and destination. A repeated projection does not resend a successful delivery. Failed deliveries retry in the background, and the worker rechecks the issue, source signal, and current rule immediately before transport. Disabling a rule or acknowledging an issue cannot recall a send already in flight. Delivery is at least once: an ambiguous transport timeout, or partial success across several configured Slack webhooks, can cause a retry after a destination accepted a message. Configure `PROXY_BASE_URL` for clickable links in email and Slack. Default rules notify through Slack only when `usage_alerts` is explicitly enabled in the gateway's alert types when those rules are first registered. Otherwise they start with in-app/API incidents. To enable notifications after the defaults already exist, edit their channels through the API or dashboard; changing the gateway alert types does not rewrite saved rules. Explicit rule channels use the configured transport without an additional alert-type switch. If legacy `user_spend_anomalies` notifications are enabled, the default daily user-spend increase rule stays in-app to avoid overlapping notifications. Email requires recipients on the rule. ## Access Gateway administrators can manage every scope. Admin viewers can read every scope. Internal users can manage their own user and key scopes; team administrators can manage their team's scope and keys. Team members can read their team's incidents. Read-only users cannot change rules or incident state, and inference-only keys cannot access these endpoints. Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` indicates that the gateway has no database connection. + +The dashboard's Issues inbox is available to administrators and admin viewers. Other authorized roles can use the scoped API. diff --git a/litellm/models/issues.py b/litellm/models/issues.py index 7d0692d26db..4e5aa4931c0 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -74,7 +74,9 @@ class IssueStore(Protocol): async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: ... - async def list_signals(self, issue_id: str, limit: int = 50, offset: int = 0) -> IssueSignalList: ... + async def list_signals( + self, issue_id: str, limit: int = 50, offset: int = 0, visible_scope: IssueCohort | None = None + ) -> IssueSignalList: ... async def active_signals( self, @@ -111,6 +113,8 @@ class IssueStore(Protocol): async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: ... + async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: ... + async def mark_delivery_sent(self, delivery_id: str) -> None: ... async def defer_delivery( diff --git a/litellm/proxy/alerting_endpoints/issues.py b/litellm/proxy/alerting_endpoints/issues.py index 30f3d44afe7..cca5c730bb5 100644 --- a/litellm/proxy/alerting_endpoints/issues.py +++ b/litellm/proxy/alerting_endpoints/issues.py @@ -15,6 +15,7 @@ from litellm.proxy.alerting_endpoints.usage_alerts import ( require_scope, ) from litellm.proxy.deps.repositories import IssueActivityRepositoryDep, IssuesRepositoryDep, UsageAlertsRepositoryDep +from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.issues import ( Issue, IssueFamily, @@ -147,7 +148,9 @@ async def update_issue( repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, ) -> Issue: - await authorized_issue(issue_id, user, repository, alerts, write=True) + issue: Final = await authorized_issue(issue_id, user, repository, alerts, write=True) + if issue.kind == "model_deprecation" and not issue.active and data.status == "open": + raise HTTPException(status_code=409, detail="This retirement issue reopens when the model receives new traffic") changed: Final = await require_issues(repository).set_status( issue_id, data.status, data.expected_status, user.user_id ) @@ -165,8 +168,17 @@ async def list_issue_signals( limit: Limit = 50, offset: Offset = 0, ) -> IssueSignalList: - await authorized_issue(issue_id, user, repository, alerts) - return await require_issues(repository).list_signals(issue_id, limit, offset) + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + visible_scope: Final = ( + None + if user.user_role + in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ) + else issue.cohort + ) + return await require_issues(repository).list_signals(issue_id, limit, offset, visible_scope) @router.get("/{issue_id}/activity", response_model=SpendAnalyticsPaginatedResponse) diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 0f9ac3e90f3..bec14f68417 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -14,6 +14,7 @@ from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( Issue, + IssueCohort, IssueDeliveryBinding, IssueDeliveryInput, IssueFamily, @@ -54,8 +55,13 @@ class _Identifier(BaseModel, frozen=True): _ACK: Final = "CASE WHEN jsonb_typeof(c.param_value)='string' THEN (c.param_value#>>'{}')::jsonb ELSE c.param_value END" +_ACK_CURRENT: Final = f"""(({_ACK})->>'last_seen_date' > i.record->'deprecation'->>'last_used_date' + OR (({_ACK})->>'last_seen_date' = i.record->'deprecation'->>'last_used_date' + AND COALESCE((({_ACK})->>'last_seen_requests')::bigint,0) + >= (i.record->'deprecation'->>'last_day_requests')::bigint))""" _STATUS: Final = f"""CASE WHEN i.kind <> 'model_deprecation' THEN i.status - WHEN NOT i.active THEN CASE WHEN ({_ACK})->>'status'='dismissed' THEN 'dismissed' ELSE 'resolved' END + WHEN NOT i.active THEN CASE WHEN ({_ACK})->>'status'='dismissed' AND ({_ACK_CURRENT}) + THEN 'dismissed' ELSE 'resolved' END WHEN COALESCE(({_ACK})->>'status','open')='open' OR ({_ACK})->>'last_seen_date' IS NULL OR i.record->'deprecation'->>'last_used_date' > ({_ACK})->>'last_seen_date' @@ -63,6 +69,7 @@ _STATUS: Final = f"""CASE WHEN i.kind <> 'model_deprecation' THEN i.status AND (i.record->'deprecation'->>'last_day_requests')::bigint > COALESCE((({_ACK})->>'last_seen_requests')::bigint,0)) THEN 'open' ELSE ({_ACK})->>'status' END""" +_PRIOR_STATUS: Final = _STATUS.replace(_ACK, _ACK.replace("c.param_value", "prior.param_value")) _JOIN: Final = 'FROM "LiteLLM_Issue" i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key' _RECORD: Final = f"""i.record || jsonb_build_object('status',({_STATUS}),'active',i.active, 'first_seen',i.first_seen AT TIME ZONE 'UTC','last_seen',i.last_seen AT TIME ZONE 'UTC', @@ -226,6 +233,7 @@ class IssuesRepository: await self.database.query_raw( f"""WITH candidate AS MATERIALIZED ( SELECT i.*,({_STATUS}) AS effective_status {_JOIN} WHERE i.id=$1 + AND NOT(i.kind='model_deprecation' AND NOT i.active AND $2='open') ), ack AS ( INSERT INTO "LiteLLM_Config" AS prior (param_name,param_value) SELECT ack_key,jsonb_build_object('model',record->'deprecation'->>'usage_model', @@ -235,13 +243,8 @@ class IssuesRepository: 'updated_at',CURRENT_TIMESTAMP,'updated_by',$4::text) FROM candidate WHERE kind='model_deprecation' AND effective_status=$3 ON CONFLICT(param_name) DO UPDATE SET param_value=EXCLUDED.param_value - WHERE (CASE WHEN COALESCE((CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'status','open')='open' - OR (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' IS NULL - OR EXCLUDED.param_value->>'last_seen_date' > (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' - OR (EXCLUDED.param_value->>'last_seen_date'=(CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_date' - AND (EXCLUDED.param_value->>'last_seen_requests')::bigint - > COALESCE(((CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'last_seen_requests')::bigint,0)) - THEN 'open' ELSE (CASE WHEN jsonb_typeof(prior.param_value)='string' THEN (prior.param_value#>>'{{}}')::jsonb ELSE prior.param_value END)->>'status' END)=$3 + WHERE EXISTS(SELECT 1 FROM candidate i WHERE i.ack_key=prior.param_name + AND ({_PRIOR_STATUS})=$3) RETURNING param_name ) UPDATE "LiteLLM_Issue" i SET status=$2,updated_at=CURRENT_TIMESTAMP FROM candidate c WHERE i.id=c.id AND c.effective_status=$3 AND @@ -269,25 +272,36 @@ class IssuesRepository: """INSERT INTO "LiteLLM_IssueSignal" (issue_id,source,source_id,record) SELECT DISTINCT ON(x->>'issue_id',x->>'source',x->>'source_id') x->>'issue_id',x->>'source',x->>'source_id',x FROM jsonb_array_elements($1::jsonb) x - WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" i WHERE i.id=x->>'issue_id' - AND (i.scope_type NOT IN ('key','team','user') OR - (x->'usage_alert'->>'scope_type'=i.scope_type - AND x->'usage_alert'->>'scope_id'=i.scope_id))) + WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" i WHERE i.id=x->>'issue_id') ON CONFLICT(issue_id,source,source_id) DO NOTHING RETURNING source_id""", json.dumps(tuple(signal.model_dump(mode="json") for signal in signals)), ) - async def list_signals(self, issue_id: str, limit: int = 50, offset: int = 0) -> IssueSignalList: + async def list_signals( + self, + issue_id: str, + limit: int = 50, + offset: int = 0, + visible_scope: IssueCohort | None = None, + ) -> IssueSignalList: + where: Final = """issue_id=$1 AND ($2::text IS NULL OR (source='usage_alert' + AND record->'usage_alert'->>'scope_type'=$2 + AND record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM $3::text))""" + args: Final = ( + issue_id, + visible_scope.scope_type if visible_scope else None, + visible_scope.scope_id if visible_scope else None, + ) counts: Final = TypeAdapter(tuple[_Count, ...]).validate_python( await self.database.query_raw( - 'SELECT count(*)::int AS count FROM "LiteLLM_IssueSignal" WHERE issue_id=$1', - issue_id, + f'SELECT count(*)::int AS count FROM "LiteLLM_IssueSignal" WHERE {where}', + *args, ) ) rows: Final = TypeAdapter(tuple[_SignalRow, ...]).validate_python( await self.database.query_raw( - 'SELECT record FROM "LiteLLM_IssueSignal" WHERE issue_id=$1 ORDER BY source,source_id LIMIT $2 OFFSET $3', - issue_id, + f'SELECT record FROM "LiteLLM_IssueSignal" WHERE {where} ORDER BY source,source_id LIMIT $4 OFFSET $5', + *args, limit, offset, ) @@ -419,6 +433,13 @@ class IssuesRepository: ) async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: + return await self._pending_deliveries(limit, None) + + async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: + rows: Final = await self._pending_deliveries(1, delivery_id) + return rows[0] if rows else None + + async def _pending_deliveries(self, limit: int, delivery_id: str | None) -> tuple[IssuePendingDelivery, ...]: if not 1 <= limit <= 500: raise ValueError("Read between one and 500 deliveries") return TypeAdapter(tuple[IssuePendingDelivery, ...]).validate_python( @@ -442,8 +463,10 @@ class IssuesRepository: ORDER BY b.rule_id,b.source_id LIMIT 1 ) bound ON true WHERE NOT d.sent AND d.next_attempt<=CURRENT_TIMESTAMP AND i.active AND ({_STATUS})='open' + AND ($2::text IS NULL OR d.id=$2) ORDER BY d.next_attempt,d.id LIMIT $1""", limit, + delivery_id, ) ) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 7eb80dae626..25d7a89d077 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -389,7 +389,8 @@ class UsageAlertsRepository: SELECT p.scope_type,p.scope_id,p.day,p.incident_count, jsonb_agg(jsonb_build_object('metric',r.metric,'incident_count',r.incident_count, 'representative',r.record) ORDER BY CASE r.metric - WHEN 'requests' THEN 0 WHEN 'spend' THEN 1 WHEN 'errors' THEN 2 ELSE 3 END) AS metrics + WHEN 'requests' THEN 0 WHEN 'spend' THEN 1 WHEN 'error_rate' THEN 2 + WHEN 'errors' THEN 3 ELSE 4 END) AS metrics FROM page p JOIN ranked r ON p.scope_type=r.scope_type AND p.scope_id IS NOT DISTINCT FROM r.scope_id AND p.day=r.day AND r.rank=1 GROUP BY p.scope_type,p.scope_id,p.day,p.incident_count diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py index e0197e6edf2..1a4104726fb 100644 --- a/litellm/types/proxy/issues.py +++ b/litellm/types/proxy/issues.py @@ -18,6 +18,9 @@ IssueKind: TypeAlias = Literal["performance", "model_deprecation"] IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] IssueGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"] +IssueSignalGroupingReason: TypeAlias = Literal[ + "same_resource", "same_key_traffic", "dominant_key", "provider_cohort", "volume_explains_spend", "model_deprecation" +] class IssueModelPair(BaseModel, frozen=True): @@ -52,7 +55,7 @@ class IssueComparison(BaseModel, frozen=True): direction: UsageAlertDirection observed_value: float = Field(ge=0) baseline_value: float = Field(ge=0) - threshold: float + threshold: float | None = None baseline_start: date baseline_end: date baseline_sample_days: int | None = Field(default=None, ge=1, le=30) @@ -151,6 +154,8 @@ class IssueSignalInput(BaseModel, frozen=True): source_id: str = Field(min_length=1, max_length=200) usage_alert: UsageAlert | None = None deprecation: IssueDeprecation | None = None + grouping_reason: IssueSignalGroupingReason = "same_resource" + explanation: str = Field(default="", max_length=2000) @model_validator(mode="after") def validate_signal(self) -> Self: diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index 72ccc1f3ef7..64336441fbe 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -133,7 +133,7 @@ class UsageAlertGroup(BaseModel, frozen=True): scope_label: str | None day: date incident_count: int - metrics: tuple[UsageAlertGroupMetric, ...] = Field(max_length=4) + metrics: tuple[UsageAlertGroupMetric, ...] = Field(max_length=5) activity_path: str diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py new file mode 100644 index 00000000000..c173cef9e75 --- /dev/null +++ b/tests/integration/database/test_issues.py @@ -0,0 +1,290 @@ +import json +from datetime import date, datetime, timedelta, timezone +from hashlib import sha256 +from typing import Final +from uuid import uuid4 + +import pytest +from integration._support.actors import ManagementDatabase +from integration._support.client import Gateway + +from litellm.repositories.issue_activity import IssueActivityRepository +from litellm.repositories.issues import IssuesRepository +from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.types.proxy.issues import ( + IssueCohort, + IssueComparison, + IssueInput, + IssueList, + IssueModelPair, + IssueSignalInput, + IssueSignalList, +) +from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRuleInput + + +def _issue(cohort: IssueCohort) -> IssueInput: + identity: Final = str(uuid4()) + day: Final = date.today() - timedelta(days=3) + return IssueInput( + id=identity, + cohort_key=identity, + kind="performance", + family="traffic", + title="Requests increased", + cohort=cohort, + observed_on=day, + observed_at=datetime.now(timezone.utc), + range_start=day - timedelta(days=7), + range_end=day, + grouping_reason="same_resource", + explanation="Requests increased for this resource.", + comparisons=( + IssueComparison( + metric="requests", + direction="increase", + observed_value=420, + baseline_value=120, + threshold=360, + baseline_start=day - timedelta(days=7), + baseline_end=day - timedelta(days=1), + baseline_sample_days=7, + ), + ), + ) + + +async def _remove_issues(db: ManagementDatabase, ids: tuple[str, ...]) -> None: + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_IssueDeliveryRule" WHERE delivery_id IN ' + '(SELECT id FROM "LiteLLM_IssueDelivery" WHERE issue_id=ANY($1::text[])) RETURNING delivery_id', + list(ids), + ) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_IssueDelivery" WHERE issue_id=ANY($1::text[]) RETURNING id', list(ids) + ) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_IssueSignal" WHERE issue_id=ANY($1::text[]) RETURNING source_id', list(ids) + ) + await db.prisma.writer_db.query_raw('DELETE FROM "LiteLLM_Issue" WHERE id=ANY($1::text[]) RETURNING id', list(ids)) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gateway, db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + outsider: Final = scenario.user(user_role="internal_user") + viewer: Final = scenario.user(user_role="proxy_admin_viewer") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + outsider_key: Final = scenario.key(user_id=outsider, key_type="management") + viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") + inference_key: Final = scenario.key(user_id=owner, key_type="llm_api", key_alias="Example checkout") + reporting_id: Final = sha256(inference_key.encode()).hexdigest() + first: Final = _issue(IssueCohort(scope_type="key", scope_id=reporting_id)) + second: Final = _issue(first.cohort) + global_issue: Final = _issue(IssueCohort(scope_type="provider", scope_id="example-provider")) + rule: Final = await alerts.save_rule( + UsageAlertRuleInput( + name="Visibility fixture", + scope_type="key", + scope_id=reporting_id, + metric="requests", + direction="increase", + ), + owner, + ) + assert rule is not None + try: + await store.upsert_issues((first, second, global_issue)) + source: Final = ( + await alerts.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=reporting_id, + day=first.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + broader: Final = source.model_copy( + update={ + "id": str(uuid4()), + "scope_type": "deployment", + "scope_id": None, + "observed_value": 99999, + "baseline_value": 9999, + } + ) + await store.add_signals( + first.id, + ( + IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source), + IssueSignalInput( + source="usage_alert", + source_id=broader.id, + usage_alert=broader, + grouping_reason="dominant_key", + explanation="Parent contribution evidence.", + ), + ), + ) + path: Final = f"/issues/{first.id}" + own: Final = gateway.request("GET", path, key=owner_key) + assert own.status_code == 200 and own.json()["scope_label"] == "Example checkout" + assert "99999" not in own.text and "9999" not in own.text + for offset in (0, 1, 2): + page: Final = gateway.request( + "GET", + "/issues", + key=owner_key, + params={"scope_type": "key", "scope_id": reporting_id, "limit": "1", "offset": str(offset)}, + ) + assert page.status_code == 200 + parsed: Final = IssueList.model_validate(page.json()) + assert parsed.total == 2 and len(parsed.data) == (1 if offset < 2 else 0) + assert gateway.request("GET", "/issues", key=owner_key).status_code == 403 + for suffix in ("", "/signals", "/activity", "/keys", "/users", "/impact"): + assert gateway.request("GET", f"/issues/{global_issue.id}{suffix}", key=owner_key).status_code == 403 + assert gateway.request("GET", path + suffix, key=outsider_key).status_code == 403 + assert gateway.request("GET", path + suffix, key=inference_key).status_code == 403 + public_signals: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1"}) + assert public_signals.status_code == 200 + visible: Final = IssueSignalList.model_validate(public_signals.json()) + assert visible.total == 1 and tuple(row.source_id for row in visible.data) == (source.id,) + assert "99999" not in public_signals.text + end: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1", "offset": "1"}) + assert end.json()["total"] == 1 and end.json()["data"] == [] + admin: Final = gateway.request("GET", path + "/signals", key=viewer_key) + assert admin.status_code == 200 and admin.json()["total"] == 2 and "99999" in admin.text + assert ( + gateway.request( + "PATCH", path, {"status": "dismissed", "expected_status": "open"}, key=viewer_key + ).status_code + == 403 + ) + assert ( + gateway.request( + "PATCH", path, {"status": "dismissed", "expected_status": "open"}, key=owner_key + ).status_code + == 200 + ) + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=owner_key + ).status_code + == 409 + ) + assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["status"] == "open" + finally: + await _remove_issues(db, (first.id, second.id, global_issue.id)) + await alerts.delete_rule(rule.id) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id + ) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( + gateway: Gateway, db: ManagementDatabase +) -> None: + repository: Final = IssueActivityRepository.from_prisma(db.prisma) + run_id: Final = str(uuid4()) + day: Final = date.today() - timedelta(days=3) + provider: Final = f"example-provider-{run_id}" + other: Final = f"other-provider-{run_id}" + key: Final = sha256(run_id.encode()).hexdigest() + entries: Final = tuple( + { + "id": f"{run_id}-{index}", + "user_id": f"user-{run_id}", + "date": day.isoformat(), + "api_key": key, + "model": model, + "custom_llm_provider": name, + "api_requests": requests, + "spend": spend, + "successful_requests": requests, + "total_response_time_ms": requests * latency, + "timed_requests": requests, + } + for index, (name, model, requests, spend, latency) in enumerate( + ( + (provider, "example-model-a", 100, 10, 200), + (other, "example-model-a", 1000, 100, 900), + (provider, "example-model-b", 50, 5, 400), + ) + ) + ) + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,api_requests,spend,successful_requests,total_response_time_ms,timed_requests,updated_at) + SELECT id,user_id,date,api_key,model,custom_llm_provider,api_requests,spend,successful_requests,total_response_time_ms,timed_requests,CURRENT_TIMESTAMP + FROM jsonb_to_recordset($1::jsonb) AS x(id text,user_id text,date text,api_key text,model text, + custom_llm_provider text,api_requests int,spend float,successful_requests int,total_response_time_ms bigint,timed_requests int) + RETURNING id""", + json.dumps(entries), + ) + cohort: Final = IssueCohort( + scope_type="provider", + scope_id=provider, + model_pairs=(IssueModelPair(provider=provider, model="example-model-a"),), + ) + store: Final = IssuesRepository.from_prisma(db.prisma) + issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day}) + try: + await store.upsert_issue(issue) + activity_http: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + assert activity_http.status_code == 200 and activity_http.json()["metadata"]["total_api_requests"] == 100 + keys_http: Final = gateway.request("GET", f"/issues/{issue.id}/keys", params={"limit": "1"}) + assert keys_http.status_code == 200 and keys_http.json()["api_keys"][0]["metrics"]["api_requests"] == 100 + users_http: Final = gateway.request("GET", f"/issues/{issue.id}/users") + assert users_http.status_code == 200 and users_http.json()["total_users"] == 1 + impact_http: Final = gateway.request("GET", f"/issues/{issue.id}/impact") + assert impact_http.status_code == 200 and impact_http.json()["requests"] == 100 + daily: Final = await repository.daily_activity(cohort, day, day) + assert daily.metadata.total_api_requests == 100 and daily.metadata.total_spend == 10 + assert daily.metadata.total_response_time_ms == 20000 and daily.metadata.total_timed_requests == 100 + keys: Final = await repository.keys(cohort, day, day, 1, 0) + assert keys.total_api_keys == 1 and keys.api_keys[0].metrics.api_requests == 100 + assert ( + keys.api_keys[0].metrics.total_response_time_ms == 20000 and keys.api_keys[0].metrics.timed_requests == 100 + ) + users: Final = await repository.users(cohort, day, day, 1, 0) + assert users.total_users == 1 and users.users[0].api_requests == 100 and users.users[0].timed_requests == 100 + impact: Final = await repository.impact(cohort, day, day) + assert (impact.keys, impact.users, impact.models, impact.requests) == (1, 1, 1, 100) + pairs: Final = IssueCohort( + scope_type="deployment", + model_pairs=( + IssueModelPair(provider=provider, model="example-model-a"), + IssueModelPair(provider=other, model="example-model-b"), + ), + ) + assert (await repository.daily_activity(pairs, day, day)).metadata.total_api_requests == 100 + injected: Final = IssueCohort(scope_type="provider", scope_id=provider + "' OR true --") + assert (await repository.impact(injected, day, day)).requests == 0 + from litellm.repositories.issue_activity import issue_activity_scope + from litellm.types.repositories.daily_activity import ExportType + + exported: Final = tuple( + [ + row + async for row in repository.activity.export_rows( + issue_activity_scope(cohort, day, day), export_type=ExportType.DAILY + ) + ] + ) + assert len(exported) == 1 and exported[0].api_requests == 100 + finally: + await _remove_issues(db, (issue.id,)) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id=ANY($1::text[]) RETURNING id', + [entry["id"] for entry in entries], + ) diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index a43b5858fe0..51978152ed2 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -276,7 +276,13 @@ async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_action outsider_key: Final = scenario.key(user_id=outsider, key_type="management") viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") inference_key: Final = scenario.key(user_id=owner, key_type="llm_api") - metrics: Final[tuple[UsageAlertMetric, ...]] = (*(("requests",) * 27), "spend", "errors", "latency") + metrics: Final[tuple[UsageAlertMetric, ...]] = ( + *(("requests",) * 27), + "spend", + "errors", + "latency", + "error_rate", + ) rules: Final = tuple( [ await repository.save_rule( @@ -303,9 +309,9 @@ async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_action UsageAlertObservation( entity_id=owner, day=day, - observed=420, - baseline=0 if index == 0 else 120, - threshold=100 if index == 0 else 360, + observed=14 if rule.metric == "error_rate" else 420, + baseline=4 if rule.metric == "error_rate" else 0 if index == 0 else 120, + threshold=12 if rule.metric == "error_rate" else 100 if index == 0 else 360, breached=True, ), ), @@ -367,12 +373,13 @@ async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_action assert page.total == 2 and len(page.data) == 1 group: Final = page.data[0] assert group.day == day and group.scope_label == "Checkout operator" - assert group.incident_count == 30 and len(group.metrics) == 4 + assert group.incident_count == 31 and len(group.metrics) == 5 assert {metric.metric: metric.incident_count for metric in group.metrics} == { "requests": 27, "spend": 1, "errors": 1, "latency": 1, + "error_rate": 1, } assert all(metric.representative.id in {incident.id for incident in incidents} for metric in group.metrics) assert group.activity_path in {incident.activity_path for incident in incidents} @@ -405,7 +412,7 @@ async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_action assert gateway.request("PATCH", "/usage/alert-groups", body, key=forbidden).status_code == 403 assert gateway.request("GET", "/usage/alert-groups", key=viewer_key, params=params).status_code == 200 updated: Final = gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key) - assert updated.status_code == 200 and updated.json()["updated_count"] == 29 + assert updated.status_code == 200 and updated.json()["updated_count"] == 30 assert gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key).json()["updated_count"] == 0 assert await repository.get_alert(old.id) == old assert await repository.get_alert(other.id) == other diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bbd4acaedf8..3517ddb0aef 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -34992,7 +34992,7 @@ export interface components { /** Observed Value */ observed_value: number; /** Threshold */ - threshold: number; + threshold?: number | null; }; /** IssueDeprecation */ IssueDeprecation: { @@ -35125,6 +35125,17 @@ export interface components { /** IssueSignal */ IssueSignal: { deprecation?: components["schemas"]["IssueDeprecation"] | null; + /** + * Explanation + * @default + */ + explanation: string; + /** + * Grouping Reason + * @default same_resource + * @enum {string} + */ + grouping_reason: "same_resource" | "same_key_traffic" | "dominant_key" | "provider_cohort" | "volume_explains_spend" | "model_deprecation"; /** Issue Id */ issue_id: string; /**