Merge be953d8abc into 83dc42f6f9

.github/workflows/test-unit.yml

@@ -550,6 +550,7 @@ jobs:
               tests/unit/proxy/management_endpoints
               tests/unit/proxy/management_helpers
               tests/unit/proxy/list_api
+              tests/unit/proxy/alerting_endpoints
               tests/unit/proxy/analytics_endpoints
               tests/unit/proxy/decisions_endpoints
               tests/unit/proxy/memory

backend/routes/allowlist.py

@@ -110,6 +110,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/v1/rust_control_plane/",
     # Alerting / email / IP allowlist
     "/alerting/",
+    "/issues/",
     "/email/",
     "/add/allowed_ip",
     "/delete/allowed_ip",
@@ -153,6 +154,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
     {
         "/",
         "/routes",
+        "/issues",
         "/lens",
         "/openapi.json",
         "/docs",

docs/features/usage-alerts.md

@@ -0,0 +1,186 @@
+# Alerts and daily Performance alerts
+
+The Alerts page combines model retirements with daily Performance alerts. Open an issue to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause.
+
+Performance comparisons cover completed UTC days. Retirement issues show recent model usage and the provider's retirement schedule, including today's usage when present.
+
+This feature requires a database. Apply the normal LiteLLM database migrations when deploying a version that includes usage alerts. The API and dashboard use the same persisted rules and incidents; no enterprise license is required.
+
+## Start with the defaults
+
+Daily Performance defaults cover spend, latency, reliability, and traffic across the gateway, keys, users, teams, and models. They use the previous seven days and the following policies. Retirement uses the existing model-deprecation warning window.
+
+| Family | Default comparison | Minimum history and volume |
+| --- | --- | --- |
+| Spend | At least 3× daily spend and a $10 increase | Three recorded reference days with a positive spend baseline |
+| Latency | At least 3× average latency and a 500 ms increase | Three recorded reference days and at least 100 timed requests in both periods |
+| Reliability | At least 3× error rate and a two percentage-point increase | Three recorded reference days and at least 500 daily requests in both periods |
+| Traffic increase | At least 10× requests and an increase of 1,000 requests | Three recorded reference days with a positive request baseline |
+| Traffic decrease | At most one fifth of baseline requests | Seven recorded reference days and a baseline of at least 500 daily requests |
+
+New resources need history before they can produce a default spike. A zero spend, request, or latency baseline does not establish a spike. A zero error-rate baseline is usable only after sufficient historical traffic has been observed; the percentage-point floor still applies.
+
+Use the family settings to enable or disable a family or adjust its thresholds across the five scopes. Existing custom rules remain independent. An enabled custom rule replaces the matching default for its scope, metric, and direction. A custom wildcard rule replaces that default for every entity. Deleted defaults stay deleted.
+
+The scheduler wakes hourly and evaluates the latest completed UTC day after a one-hour grace period. A successful aggregate flush after the grace period is required before either direction is finalized. Completed rule versions are skipped on later ticks. The evaluator reads existing daily aggregates in pages of 500 entities; it adds no request-time counters or per-request database writes.
+
+Decrease rules need a successful background aggregate-flush sample for every hour of the target day and another sample after the grace period. Until that coverage exists, their status is `insufficient_coverage`. A key with no requests can then produce a request or spend drop; missing latency samples never become zero latency. Coverage shows that an aggregate writer was healthy each hour, not that every replica delivered every request.
+
+The baseline averages recorded reference days; it does not turn missing reference days into zero. Average latency is weighted by timed request count, using the same successful-request duration totals as Activity. `baseline_sample_days` records how many reference days contributed. Cost, traffic, and latency comparisons require a positive baseline. Reliability can compare against a recorded 0% error rate when historical and current request volume meet the rule's minimum. Missing history and low-volume comparisons remain `insufficient_history`.
+
+## Read and investigate issues
+
+Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the credential in your environment.
+
+```bash
+curl "$LITELLM_API_BASE/issues?status=open&limit=50" \
+  -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY"
+```
+
+The response contains `data`, `total`, `limit`, and `offset`. Pagination and filtering happen on the combined inbox. Filters are `kind`, `family`, `status`, `scope_type`, and `scope_id`. Families are `retirement`, `latency`, `reliability`, `spend`, and `traffic`; kinds are `model_deprecation` and `performance`. An offset beyond the last page still returns the correct total. There is no default history cutoff on open issues.
+
+Each issue has an ID, status, readable title, current resource label, typed `cohort`, date range, grouping reason, and saved comparison values. The cohort is one resource, one provider, or exact provider/model pairs. A provider/model pair is an intersection, not a list of independently interchangeable filters. Resource names can change; IDs remain the identity.
+
+Use the ID from the list to inspect an issue:
+
+```bash
+ISSUE_ID="issue-example"
+curl "$LITELLM_API_BASE/issues/$ISSUE_ID" \
+  -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY"
+curl "$LITELLM_API_BASE/issues/$ISSUE_ID/activity" \
+  -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY"
+curl "$LITELLM_API_BASE/issues/$ISSUE_ID/signals?limit=50" \
+  -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY"
+```
+
+Replace `issue-example` with an ID returned by `/issues`. Activity uses the issue's saved date range by default. Optional `start_date` and `end_date` are inclusive UTC dates and can span at most 366 days. `/keys` and `/users` provide paginated Activity rows with `limit` and `offset`. Their latency totals are `total_response_time_ms` and `timed_requests`; divide the former by the latter only when timed requests are present.
+
+Alert Activity returns exact daily totals in `results` and exact range totals in `metadata`. It does not return the general Usage API's per-day dimension dictionaries. Key issues also return `model_breakdown`, with up to 20 models by range spend by default. Set `model_limit` between 1 and 100 to change that limit. `total_models` counts every matching model; `other` contains the combined metrics and `model_count` of all remaining models, or null when none remain. It is an aggregate, not a model identifier. Adding the displayed models and `other` recovers the full totals. Model identifiers longer than 1,024 characters are included in `other` without truncation. Non-key issues return null for `model_breakdown`. Activity queries use a bounded execution budget. A busy or over-budget query returns `503` with `Retry-After: 5`; retry later or choose a shorter date range.
+
+`/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Resource pages fetch labels only for the selected page, using the latest deleted-key label when needed. They do not load arbitrary key or user metadata. Display aliases are limited to 256 characters; email labels longer than 320 characters are omitted. Resource pages omit IDs longer than 1,024 characters or 4,096 bytes instead of truncating them. Key IDs that need a `raw:` escape prefix reserve four characters within that limit. Oversized linked user or team IDs are omitted from resource metadata. Activity totals still include traffic from omitted resources. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown.
+
+Supporting signals have their own saved comparisons and relationship explanations. Their baseline windows can differ, so do not derive a combined error rate or unit cost by adding their displayed values. Administrators and admin viewers can inspect broader supporting signals. A scoped reader receives only signals for the exact key, user, or team they can access. That restriction applies before the signal count and pagination; broader totals are not disclosed.
+
+A continuing Performance issue retains its initial comparison and cohort while `last_seen` advances. New supporting observations do not rewrite the original evidence. Resource issues recover when their primary metric recovers for that resource; broader supporting observations do not keep them open. Provider issues require a fresh comparison of the saved provider/model cohort. Missing history or coverage does not establish recovery. A later breach starts a new issue. Retirement details refresh as usage and the provider schedule change. A retirement issue with no recent use becomes inactive; actual new traffic can reopen it.
+
+Resolve or dismiss an issue using the current status from the read response:
+
+```bash
+curl -X PATCH "$LITELLM_API_BASE/issues/$ISSUE_ID" \
+  -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"expected_status":"open","status":"resolved"}'
+```
+
+A stale status returns `409`; reload before retrying. Performance acknowledgements do not change other issues that share a supporting observation. Retirement acknowledgements use the same state as the existing model-deprecation API. An inactive retirement cannot be manually reopened.
+
+The issue-detail response includes `can_update_status` and `can_view_activity` for the current reader. Clients should show status actions only when `can_update_status` is true and request Activity or resource breakdowns only when `can_view_activity` is true. These fields are null in responses where permission has not been evaluated, such as inbox lists. Every endpoint still checks authorization on each request. Team Activity requires team-admin access or the explicit `/team/daily/activity` permission; ordinary membership can read saved alert evidence but does not grant other members’ activity or key details.
+
+## Configure default families
+
+`GET /issues/families` returns the four Performance families, their enabled counts, and common policy values. A null common value means the underlying scopes have different settings. This is separate from custom rules and the model-retirement warning window.
+
+This example sets the spend family to a 5× threshold and a $25 minimum absolute increase:
+
+```bash
+curl -X PATCH "$LITELLM_API_BASE/issues/families/spend" \
+  -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"enabled":true,"policies":[{"direction":"increase","multiplier":5,"min_value":25}]}'
+```
+
+The update applies atomically to registered, non-deleted built-in rules in that family. Omitted settings remain unchanged. It does not change custom rules or recreate deleted defaults. A concurrent rule edit returns `409`; reload and retry. These controls require a gateway administrator.
+
+## Read resource groups
+
+Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the key in your environment.
+
+```bash
+curl "$LITELLM_API_BASE/usage/alert-groups?status=open&limit=50" \
+  -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY"
+```
+
+The response has `data`, `total`, `limit`, and `offset`. `total` counts resource/day groups, including when the requested offset is beyond the last page. Filters are `scope_type`, `scope_id`, `status`, `start_date`, and `end_date`. Status filters apply to individual incidents before grouping, so a group with open and resolved incidents can appear under either filter with the corresponding members. No default date cutoff hides older open incidents.
+
+Each group includes its stable `id`, `scope_type`, exact `scope_id`, current `scope_label` when available, original UTC `day`, `incident_count`, and `activity_path`. Its `metrics` list contains up to five entries with a `metric`, `incident_count`, and the most recently detected matching `representative` incident. These representatives are summaries, not the entire group. Missing or deleted resources keep their original IDs. Names are current display labels, not historical evidence.
+
+To inspect every matching rule, use `GET /usage/alerts` with the group's scope and `start_date` and `end_date` both set to its `day`. Add `metric` and `status` as needed and paginate that response. Rules can have different baseline windows and thresholds, so their comparisons must not be combined into an error rate or a cost-per-request estimate.
+
+The group Activity link uses a representative's saved date range and alert context. The path is relative to the dashboard mount: for example, `/api-keys?...` is opened below the dashboard's `/ui` mount. A continuing episode retains its original day and evidence; the grouped row does not describe today's traffic.
+
+Resolve or dismiss all matching incidents in one resource/day group with an expected current status:
+
+```bash
+curl -X PATCH "$LITELLM_API_BASE/usage/alert-groups" \
+  -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"scope_type":"team","scope_id":"team-example","day":"2026-11-04","expected_status":"open","status":"resolved"}'
+```
+
+The response contains `updated_count`. Only incidents in that exact group that still have `expected_status` are updated atomically. Other statuses remain unchanged; repeating the same request returns zero after all matching incidents have changed. Use `scope_type: "deployment"` with `scope_id: null` for the gateway.
+
+## Read individual incidents
+
+`GET /usage/alerts` retains the individual-incident view. Each incident includes its scope, metric, direction, status, UTC day, observed value, baseline value, threshold, baseline date range, and `activity_path`.
+
+`GET /usage/alerts/{alert_id}` returns one incident. List filters are `scope_type`, `scope_id`, `metric`, `status`, `start_date`, and `end_date`. Dates use `YYYY-MM-DD`; date filters include both endpoints. The maximum page size is 100.
+
+## Create a rule
+
+This example alerts when a team's daily spend is at least three times its baseline and the absolute change is at least $10. Replace `team-example` with the team's ID.
+
+```bash
+curl "$LITELLM_API_BASE/usage/alert-rules" \
+  -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "name": "Team daily spend",
+    "scope_type": "team",
+    "scope_id": "team-example",
+    "metric": "spend",
+    "direction": "increase",
+    "multiplier": 3,
+    "baseline_days": 7,
+    "min_history_days": 3,
+    "min_requests": 100,
+    "min_value": 10,
+    "enabled": true
+  }'
+```
+
+The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value at or below half the baseline. `min_value` is the minimum absolute difference in the metric's own units.
+
+| Metric | Unit |
+| --- | --- |
+| `spend` | USD |
+| `requests` | Request count |
+| `latency` | Average response time in milliseconds |
+| `errors` | Failed-request count |
+| `error_rate` | Failed requests divided by requests, as a percentage from 0 to 100 |
+
+Counts come from the existing Activity/spend aggregates, rather than every gateway HTTP outcome. Latency averages successful timed, non-internal requests; it is not time to first token and does not include failed requests. Issue Activity and gateway source incidents use the same aggregates. An error-rate `min_value` of 2 means two percentage points, such as 1% to 3%, not a 2% relative increase.
+
+The evaluator runs against daily aggregates. An hourly scheduler does not make these rolling hourly alerts. A complete UTC day and sufficient ingestion coverage are required before detecting drops; a missing flush must not become a false traffic-drop incident. Rule responses expose `last_evaluated_day` and `evaluation_status` so you can distinguish a completed evaluation from insufficient history or coverage.
+
+Use `scope_type: "deployment"` with no `scope_id` for the gateway. For `model`, use the underlying model identity recorded in Activity. For `key`, use its reporting ID, never the secret API key. Omitting `scope_id` for key, user, team, or model evaluates all entities of that type and requires a gateway administrator.
+
+`GET /usage/alert-models?limit=20&search=example` returns paginated model identities recorded in the most recent 30 UTC days. Its `data` entries contain `scope_id` and `label`; `start_date` and `end_date` state the searched window. Pass `next_cursor` as `cursor` to read the next page. A null cursor ends the result. Search matches literal text, not SQL wildcards. This endpoint requires gateway admin or admin-viewer access. Configured aliases and deployment IDs are not reporting-model identities. An existing rule's saved model remains valid even when it falls outside the options window.
+
+## Manage rules and incidents
+
+- `GET /usage/alert-rules` lists rules; `GET /usage/alert-rules/{rule_id}` reads one. Use `kind=custom`, `kind=default`, or `kind=all` to choose which rules to list. Each rule exposes `is_default`. The list's `default_summary` counts total and enabled defaults within the requested scope, independent of `kind`, and reports evaluation-status counts among enabled defaults.
+- `PUT /usage/alert-rules/{rule_id}` replaces the rule configuration. Send the configuration fields from the create example, including any notification settings you want to retain. Set `enabled` to `false` to pause it. Built-in rules keep their scope, metric, and direction; create a custom rule to change those fields. Their names, thresholds, notification settings, and enabled state remain editable.
+- `DELETE /usage/alert-rules/{rule_id}` removes a rule from evaluation and preserves existing incidents. Deleted defaults stay deleted.
+- `PATCH /usage/alerts/{alert_id}` with `{"status":"dismissed"}` dismisses an incident. `open` and `resolved` are also accepted.
+
+Consecutive breached days update one incident. Re-evaluating the same day does not reopen a dismissed or resolved incident. After recovery, a later breach starts a new incident. Editing a rule does not rewrite historical comparisons.
+
+Rules can specify `notification_channels: ["slack", "email"]` and `email_recipients: ["alerts@example.com"]`. Notification transports must already be configured on the gateway. An empty channel list keeps incidents available through the API and dashboard. Issue delivery is recorded per channel and destination. A repeated projection does not resend a successful delivery. Failed deliveries retry in the background, and the worker rechecks the issue, source signal, and current rule immediately before transport. Disabling a rule or acknowledging an issue cannot recall a send already in flight. Delivery is at least once: an ambiguous transport timeout, or partial success across several configured Slack webhooks, can cause a retry after a destination accepted a message. Configure `PROXY_BASE_URL` for clickable links in email and Slack. Default rules notify through Slack only when `usage_alerts` is explicitly enabled in the gateway's alert types when those rules are first registered. Otherwise they start with in-app/API incidents. To enable notifications after the defaults already exist, edit their channels through the API or dashboard; changing the gateway alert types does not rewrite saved rules. Explicit rule channels use the configured transport without an additional alert-type switch. If legacy `user_spend_anomalies` notifications are enabled, the default daily user-spend increase rule stays in-app to avoid overlapping notifications. Email requires recipients on the rule.
+
+## Access
+
+Gateway administrators can manage every scope. Admin viewers can read every scope. Internal users can manage their own user and key scopes; team administrators can manage their team's scope and keys. Team members can read their team's incidents. Read-only users cannot change rules or incident state, and inference-only keys cannot access these endpoints.
+
+Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` can indicate an unavailable database or exhausted analytics capacity; a `Retry-After` header indicates that the client should retry later.
+
+The dashboard's Alerts page is available to administrators and admin viewers. Other authorized roles can use the scoped API.

litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql

@@ -0,0 +1,26 @@
+SET LOCAL lock_timeout = '10s';
+
+CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertRule" (
+ "id" TEXT PRIMARY KEY, "scope_type" TEXT NOT NULL, "scope_id" TEXT,
+ "enabled" BOOLEAN NOT NULL DEFAULT true, "record" JSONB NOT NULL,
+ "deleted" BOOLEAN NOT NULL DEFAULT false
+);
+CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertRule_enabled_deleted_id_idx" ON "LiteLLM_UsageAlertRule" ("enabled", "deleted", "id");
+CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertRule_scope_type_scope_id_deleted_idx" ON "LiteLLM_UsageAlertRule" ("scope_type", "scope_id", "deleted");
+CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertIncident" (
+ "id" TEXT PRIMARY KEY, "rule_id" TEXT NOT NULL, "scope_key" TEXT NOT NULL,
+ "scope_type" TEXT NOT NULL, "scope_id" TEXT, "metric" TEXT NOT NULL,
+ "status" TEXT NOT NULL, "day" TEXT NOT NULL, "evaluation_day" TEXT NOT NULL,
+ "breached" BOOLEAN NOT NULL, "record" JSONB NOT NULL,
+ "notified_channels" TEXT[] DEFAULT ARRAY[]::TEXT[],
+ "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_rule_id_scope_key_evaluation_day_idx" ON "LiteLLM_UsageAlertIncident" ("rule_id", "scope_key", "evaluation_day");
+CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_scope_type_scope_id_day_idx" ON "LiteLLM_UsageAlertIncident" ("scope_type", "scope_id", "day");
+CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_status_day_idx" ON "LiteLLM_UsageAlertIncident" ("status", "day");
+CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_updated_at_idx" ON "LiteLLM_UsageAlertIncident" ("updated_at");
+CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertCheckpoint" (
+ "id" TEXT PRIMARY KEY, "completed" BOOLEAN NOT NULL DEFAULT false,
+ "owner" TEXT, "expires_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertCheckpoint_expires_at_idx" ON "LiteLLM_UsageAlertCheckpoint" ("expires_at");

litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql

@@ -0,0 +1,51 @@
+SET LOCAL lock_timeout = '10s';
+
+CREATE TABLE IF NOT EXISTS "LiteLLM_Issue" (
+  "id" TEXT NOT NULL,
+  "cohort_key" TEXT NOT NULL,
+  "kind" TEXT NOT NULL,
+  "family" TEXT NOT NULL,
+  "scope_type" TEXT NOT NULL,
+  "scope_id" TEXT,
+  "status" TEXT NOT NULL,
+  "active" BOOLEAN NOT NULL DEFAULT true,
+  "ack_key" TEXT,
+  "record" JSONB NOT NULL,
+  "first_seen" TIMESTAMP(3) NOT NULL,
+  "last_seen" TIMESTAMP(3) NOT NULL,
+  "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  CONSTRAINT "LiteLLM_Issue_pkey" PRIMARY KEY ("id")
+);
+CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_status_last_seen_id_idx" ON "LiteLLM_Issue"("status", "last_seen", "id");
+CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_kind_family_last_seen_id_idx" ON "LiteLLM_Issue"("kind", "family", "last_seen", "id");
+CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_scope_type_scope_id_last_seen_id_idx" ON "LiteLLM_Issue"("scope_type", "scope_id", "last_seen", "id");
+CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_cohort_key_active_idx" ON "LiteLLM_Issue"("cohort_key", "active");
+CREATE TABLE IF NOT EXISTS "LiteLLM_IssueSignal" (
+  "issue_id" TEXT NOT NULL,
+  "source" TEXT NOT NULL,
+  "source_id" TEXT NOT NULL,
+  "record" JSONB NOT NULL,
+  CONSTRAINT "LiteLLM_IssueSignal_pkey" PRIMARY KEY ("issue_id", "source", "source_id")
+);
+CREATE INDEX IF NOT EXISTS "LiteLLM_IssueSignal_source_source_id_idx" ON "LiteLLM_IssueSignal"("source", "source_id");
+CREATE TABLE IF NOT EXISTS "LiteLLM_IssueDelivery" (
+  "id" TEXT NOT NULL,
+  "issue_id" TEXT NOT NULL,
+  "record" JSONB NOT NULL,
+  "sent" BOOLEAN NOT NULL DEFAULT false,
+  "next_attempt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  CONSTRAINT "LiteLLM_IssueDelivery_pkey" PRIMARY KEY ("id")
+);
+CREATE INDEX IF NOT EXISTS "LiteLLM_IssueDelivery_sent_next_attempt_id_idx" ON "LiteLLM_IssueDelivery"("sent", "next_attempt", "id");
+CREATE INDEX IF NOT EXISTS "LiteLLM_IssueDelivery_issue_id_idx" ON "LiteLLM_IssueDelivery"("issue_id");
+CREATE TABLE IF NOT EXISTS "LiteLLM_IssueDeliveryRule" (
+  "delivery_id" TEXT NOT NULL,
+  "rule_id" TEXT NOT NULL,
+  "source_id" TEXT NOT NULL,
+  "rule_revision" TEXT NOT NULL,
+  CONSTRAINT "LiteLLM_IssueDeliveryRule_pkey" PRIMARY KEY ("delivery_id", "rule_id", "source_id")
+);
+ALTER TABLE "LiteLLM_UsageAlertIncident" ADD COLUMN IF NOT EXISTS "evaluation_revision" TEXT NOT NULL DEFAULT '';
+CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_evaluation_day_breached_id_idx"
+  ON "LiteLLM_UsageAlertIncident"("evaluation_day", "breached", "id");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -2168,3 +2168,96 @@ model LiteLLM_SubscriptionAccountTable {
 
   @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
 }
+
+model LiteLLM_UsageAlertRule {
+  id String @id
+  scope_type String
+  scope_id String?
+  enabled Boolean @default(true)
+  record Json
+  deleted Boolean @default(false)
+  @@index([enabled, deleted, id])
+  @@index([scope_type, scope_id, deleted])
+}
+
+model LiteLLM_UsageAlertIncident {
+  id String @id
+  rule_id String
+  scope_key String
+  scope_type String
+  scope_id String?
+  metric String
+  status String
+  day String
+  evaluation_day String
+  evaluation_revision String @default("")
+  breached Boolean
+  record Json
+  notified_channels String[] @default([])
+  updated_at DateTime @default(now())
+  @@index([rule_id, scope_key, evaluation_day])
+  @@index([evaluation_day, breached, id])
+  @@index([scope_type, scope_id, day])
+  @@index([status, day])
+  @@index([updated_at])
+}
+
+model LiteLLM_UsageAlertCheckpoint {
+  id String @id
+  completed Boolean @default(false)
+  owner String?
+  expires_at DateTime @default(now())
+  @@index([expires_at])
+}
+
+model LiteLLM_Issue {
+  id String @id
+  cohort_key String
+  kind String
+  family String
+  scope_type String
+  scope_id String?
+  status String
+  active Boolean @default(true)
+  ack_key String?
+  record Json
+  first_seen DateTime
+  last_seen DateTime
+  updated_at DateTime @default(now())
+
+  @@index([status, last_seen, id])
+  @@index([kind, family, last_seen, id])
+  @@index([scope_type, scope_id, last_seen, id])
+  @@index([cohort_key, active])
+}
+
+model LiteLLM_IssueSignal {
+  issue_id String
+  source String
+  source_id String
+  record Json
+
+  @@id([issue_id, source, source_id])
+  @@index([source, source_id])
+}
+
+model LiteLLM_IssueDelivery {
+  id String @id
+  issue_id String
+  record Json
+  sent Boolean @default(false)
+  next_attempt DateTime @default(now())
+  updated_at DateTime @default(now())
+
+  @@index([sent, next_attempt, id])
+  @@index([issue_id])
+}
+
+model LiteLLM_IssueDeliveryRule {
+  delivery_id String
+  rule_id String
+  source_id String
+  rule_revision String
+
+  @@id([delivery_id, rule_id, source_id])
+}

litellm/integrations/otel/model/spans.py

@@ -367,6 +367,13 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_SubscriptionAccountTable",
         "LiteLLM_LensSignalConfig",
         "LiteLLM_LensTraceSignal",
+        "LiteLLM_UsageAlertRule",
+        "LiteLLM_UsageAlertIncident",
+        "LiteLLM_UsageAlertCheckpoint",
+        "LiteLLM_Issue",
+        "LiteLLM_IssueSignal",
+        "LiteLLM_IssueDelivery",
+        "LiteLLM_IssueDeliveryRule",
     )
 )
 PRISMA_RELATIONS: Final[frozenset[str]] = _PRISMA_MODELS | _PRISMA_VIEWS

litellm/models/issues.py

@@ -0,0 +1,151 @@
+from datetime import date, datetime
+from typing import Literal, Protocol
+
+from litellm.models.team import LiteLLM_TeamTable
+from litellm.types.proxy.issues import (
+    Issue,
+    IssueActivityResponse,
+    IssueCohort,
+    IssueDeliveryBinding,
+    IssueDeliveryInput,
+    IssueFamily,
+    IssueFamilyList,
+    IssueFamilySetting,
+    IssueFamilyUpdate,
+    IssueImpact,
+    IssueInput,
+    IssueKind,
+    IssueList,
+    IssuePendingDelivery,
+    IssueProjectionSignal,
+    IssueScope,
+    IssueSignal,
+    IssueSignalInput,
+    IssueSignalList,
+)
+from litellm.types.proxy.management_endpoints.common_daily_activity import (
+    DailyActivityKeyPageResponse,
+    DailyActivityUserPageResponse,
+)
+from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily
+from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus
+
+
+class IssueStore(Protocol):
+    async def get_issue(self, issue_id: str) -> Issue | None: ...
+
+    async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: ...
+
+    async def list_issues(
+        self,
+        kind: IssueKind | None = None,
+        family: IssueFamily | None = None,
+        status: UsageAlertStatus | None = None,
+        scope_type: IssueScope | None = None,
+        scope_id: str | None = None,
+        limit: int = 50,
+        offset: int = 0,
+    ) -> IssueList: ...
+
+    async def set_status(
+        self,
+        issue_id: str,
+        status: UsageAlertStatus,
+        expected_status: UsageAlertStatus,
+        actor: str | None = None,
+        authorized_scopes: tuple[IssueCohort, ...] | None = None,
+    ) -> Issue | None: ...
+
+    async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]: ...
+
+    async def upsert_issue(
+        self,
+        data: IssueInput,
+        reopen: bool = False,
+        status: UsageAlertStatus | None = None,
+    ) -> Issue: ...
+
+    async def upsert_issues(
+        self,
+        inputs: tuple[IssueInput, ...],
+        reopen: bool = False,
+        status: UsageAlertStatus | None = None,
+    ) -> tuple[Issue, ...]: ...
+
+    async def find_active_issue(self, cohort_key: str) -> Issue | None: ...
+
+    async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: ...
+
+    async def add_signals(self, issue_id: str, signals: tuple[IssueSignalInput, ...]) -> None: ...
+
+    async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: ...
+
+    async def list_signals(
+        self, issue_id: str, limit: int = 50, offset: int = 0, visible_scope: IssueCohort | None = None
+    ) -> IssueSignalList: ...
+
+    async def active_signals(
+        self,
+        day: date,
+        limit: int = 500,
+        cursor: str | None = None,
+        scope_type: IssueScope | None = None,
+        metric: UsageAlertMetric | None = None,
+        scope_ids: tuple[str, ...] | None = None,
+    ) -> tuple[IssueProjectionSignal, ...]: ...
+
+    async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: ...
+
+    async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: ...
+
+    async def recover_issue_ids(self, ids: tuple[str, ...], observed_at: datetime) -> int: ...
+
+    async def projection_completed(self, day: date, version: str) -> bool: ...
+
+    async def complete_projection(self, day: date, version: str) -> None: ...
+
+    async def finish_deprecation_projection(self, observed_at: datetime) -> int: ...
+
+    async def family_settings(self) -> IssueFamilyList: ...
+
+    async def update_family(
+        self, family: PerformanceIssueFamily, data: IssueFamilyUpdate
+    ) -> IssueFamilySetting | None: ...
+
+    async def queue_delivery(
+        self,
+        data: IssueDeliveryInput,
+        rule_id: str,
+        rule_revision: str,
+        source_id: str,
+    ) -> None: ...
+
+    async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: ...
+
+    async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: ...
+
+    async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: ...
+
+    async def mark_delivery_sent(self, delivery_id: str) -> None: ...
+
+    async def defer_delivery(
+        self, delivery_id: str, error: Literal["delivery_failed", "transport_not_configured"]
+    ) -> None: ...
+
+    async def prune(self, retention_days: int = 90, limit: int = 500) -> None: ...
+
+
+class IssueActivityStore(Protocol):
+    async def daily_activity(
+        self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20
+    ) -> IssueActivityResponse: ...
+
+    async def keys(
+        self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
+    ) -> DailyActivityKeyPageResponse: ...
+
+    async def users(
+        self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
+    ) -> DailyActivityUserPageResponse: ...
+
+    async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: ...

litellm/models/usage_alerts.py

@@ -0,0 +1,77 @@
+from datetime import date
+from typing import Protocol
+
+from litellm.types.proxy.usage_alerts import (
+    UsageAlert,
+    UsageAlertGroupList,
+    UsageAlertGroupUpdate,
+    UsageAlertGroupUpdateResult,
+    UsageAlertList,
+    UsageAlertMetric,
+    UsageAlertModelList,
+    UsageAlertRule,
+    UsageAlertRuleInput,
+    UsageAlertRuleKind,
+    UsageAlertRuleList,
+    UsageAlertScope,
+    UsageAlertStatus,
+)
+
+
+class UsageAlertStore(Protocol):
+    async def get_rule(self, rule_id: str) -> UsageAlertRule | None: ...
+
+    async def save_rule(
+        self,
+        data: UsageAlertRuleInput,
+        created_by: str | None,
+        existing: UsageAlertRule | None = None,
+        *,
+        overrides_defaults: bool = False,
+    ) -> UsageAlertRule | None: ...
+
+    async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: ...
+
+    async def list_rules(
+        self,
+        scope_type: UsageAlertScope | None = None,
+        scope_id: str | None = None,
+        limit: int = 50,
+        offset: int = 0,
+        kind: UsageAlertRuleKind = "all",
+    ) -> UsageAlertRuleList: ...
+
+    async def get_alert(self, alert_id: str) -> UsageAlert | None: ...
+
+    async def list_alerts(
+        self,
+        scope_type: UsageAlertScope | None = None,
+        scope_id: str | None = None,
+        metric: UsageAlertMetric | None = None,
+        status: UsageAlertStatus | None = None,
+        start_date: date | None = None,
+        end_date: date | None = None,
+        limit: int = 50,
+        offset: int = 0,
+    ) -> UsageAlertList: ...
+
+    async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None: ...
+
+    async def list_groups(
+        self,
+        scope_type: UsageAlertScope | None = None,
+        scope_id: str | None = None,
+        status: UsageAlertStatus | None = None,
+        start_date: date | None = None,
+        end_date: date | None = None,
+        limit: int = 50,
+        offset: int = 0,
+    ) -> UsageAlertGroupList: ...
+
+    async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult: ...
+
+    async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList: ...
+
+    async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None: ...
+
+    async def team_role(self, team_id: str, user_id: str) -> str | None: ...

litellm/proxy/_lazy_openapi_snapshot.json

@@ -4488,6 +4488,16 @@
               "title": "Successful Requests",
               "type": "integer"
             },
+            "timed_requests": {
+              "default": 0,
+              "title": "Timed Requests",
+              "type": "integer"
+            },
+            "total_response_time_ms": {
+              "default": 0,
+              "title": "Total Response Time Ms",
+              "type": "integer"
+            },
             "total_tokens": {
               "default": 0,
               "title": "Total Tokens",

litellm/proxy/_types.py

@@ -595,6 +595,21 @@ class LiteLLMRoutes(enum.Enum):
     )
     llm_api_routes_without_mcp = sorted(set(llm_api_routes) - set(mcp_inference_routes))
     info_routes = [
+        "/usage/alert-groups",
+        "/usage/alert-models",
+        "/usage/alert-rules",
+        "/usage/alert-rules/{rule_id}",
+        "/usage/alerts",
+        "/usage/alerts/{alert_id}",
+        "/issues",
+        "/issues/families",
+        "/issues/families/{family}",
+        "/issues/{issue_id}",
+        "/issues/{issue_id}/signals",
+        "/issues/{issue_id}/activity",
+        "/issues/{issue_id}/keys",
+        "/issues/{issue_id}/users",
+        "/issues/{issue_id}/impact",
         "/key/info",
         "/key/health",
         "/team/info",
@@ -662,6 +677,21 @@ class LiteLLMRoutes(enum.Enum):
 
     management_routes = (
         [
+            "/usage/alert-groups",
+            "/usage/alert-models",
+            "/usage/alert-rules",
+            "/usage/alert-rules/{rule_id}",
+            "/usage/alerts",
+            "/usage/alerts/{alert_id}",
+            "/issues",
+            "/issues/families",
+            "/issues/families/{family}",
+            "/issues/{issue_id}",
+            "/issues/{issue_id}/signals",
+            "/issues/{issue_id}/activity",
+            "/issues/{issue_id}/keys",
+            "/issues/{issue_id}/users",
+            "/issues/{issue_id}/impact",
             # user
             "/user/new",
             "/management/v1/users/bulk",
@@ -872,7 +902,26 @@ class LiteLLMRoutes(enum.Enum):
         ]
     )
 
+    usage_alert_routes = (
+        "/usage/alert-groups",
+        "/usage/alert-models",
+        "/usage/alert-rules",
+        "/usage/alert-rules/{rule_id}",
+        "/usage/alerts",
+        "/usage/alerts/{alert_id}",
+        "/issues",
+        "/issues/families",
+        "/issues/families/{family}",
+        "/issues/{issue_id}",
+        "/issues/{issue_id}/signals",
+        "/issues/{issue_id}/activity",
+        "/issues/{issue_id}/keys",
+        "/issues/{issue_id}/users",
+        "/issues/{issue_id}/impact",
+    )
+
     self_managed_routes = [
+        *usage_alert_routes,
         "/lens",
         "/lens/{path:path}",
         "/telemetry/ui_events",

litellm/proxy/alerting_endpoints/issues.py

@@ -0,0 +1,325 @@
+"""Unified issue inbox and investigations backed by existing daily aggregates."""
+
+from collections.abc import Generator
+from contextlib import contextmanager
+from datetime import date
+from typing import Final
+
+from fastapi import APIRouter, HTTPException
+
+from litellm.models.issues import IssueActivityStore, IssueStore
+from litellm.proxy.alerting_endpoints.usage_alerts import (
+    Limit,
+    Offset,
+    ScopeId,
+    User,
+    require_repository,
+    require_scope,
+    require_write_access,
+)
+from litellm.proxy.deps.repositories import IssueActivityRepositoryDep, IssuesRepositoryDep, UsageAlertsRepositoryDep
+from litellm.proxy.management.teams.authz import is_team_admin
+from litellm.proxy.management_endpoints.common_utils import team_member_has_permission
+from litellm.repositories.analytics_budget import AnalyticsCapacityError
+from litellm.types.proxy.auth.user_roles import LitellmUserRoles
+from litellm.types.proxy.issues import (
+    Issue,
+    IssueActivityResponse,
+    IssueCohort,
+    IssueFamily,
+    IssueFamilyList,
+    IssueFamilySetting,
+    IssueFamilyUpdate,
+    IssueImpact,
+    IssueKind,
+    IssueList,
+    IssueScope,
+    IssueSignalList,
+    IssueUpdate,
+)
+from litellm.types.proxy.management_endpoints.common_daily_activity import (
+    DailyActivityKeyPageResponse,
+    DailyActivityUserPageResponse,
+)
+from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily
+from litellm.types.proxy.usage_alerts import UsageAlertStatus
+
+router: Final = APIRouter(prefix="/issues", tags=["issues"])
+
+
+def require_issues(repository: IssueStore | None) -> IssueStore:
+    if repository is None:
+        raise HTTPException(status_code=503, detail="Issues require a database connection")
+    return repository
+
+
+def require_activity(repository: IssueActivityStore | None) -> IssueActivityStore:
+    if repository is None:
+        raise HTTPException(status_code=503, detail="Issue Activity requires a database connection")
+    return repository
+
+
+@contextmanager
+def activity_budget() -> Generator[None]:
+    try:
+        yield
+    except AnalyticsCapacityError as exc:
+        raise HTTPException(status_code=503, detail=str(exc), headers={"Retry-After": "5"}) from exc
+
+
+async def authorized_issue(
+    issue_id: str,
+    user: User,
+    repository: IssuesRepositoryDep,
+    alerts: UsageAlertsRepositoryDep,
+    *,
+    write: bool = False,
+) -> Issue:
+    issue: Final = await require_issues(repository).get_issue(issue_id)
+    if issue is None:
+        raise HTTPException(status_code=404, detail="Issue not found")
+    await require_scope(
+        user,
+        require_repository(alerts),
+        "model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type,
+        issue.cohort.scope_id,
+        write=write,
+    )
+    return issue
+
+
+async def can_view_activity(issue: Issue, user: User, repository: IssueStore) -> bool:
+    if issue.cohort.scope_type != "team" or user.user_role in (
+        LitellmUserRoles.PROXY_ADMIN,
+        LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
+    ):
+        return True
+    if user.user_id is None or issue.cohort.scope_id is None:
+        return False
+    team: Final = await repository.team_activity_access(issue.cohort.scope_id, user.user_id)
+    return team is not None and (
+        is_team_admin(user, team) or team_member_has_permission(user, team, "/team/daily/activity")
+    )
+
+
+async def require_activity_access(issue: Issue, user: User, repository: IssuesRepositoryDep) -> None:
+    if not await can_view_activity(issue, user, require_issues(repository)):
+        raise HTTPException(status_code=403, detail="Full team Activity permission is required for this investigation")
+
+
+async def require_status_scopes(
+    issue: Issue, user: User, repository: IssueStore, alerts: UsageAlertsRepositoryDep
+) -> tuple[IssueCohort, ...] | None:
+    store: Final = require_repository(alerts)
+    await require_scope(
+        user,
+        store,
+        "model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type,
+        issue.cohort.scope_id,
+        write=True,
+    )
+    if user.user_role == LitellmUserRoles.PROXY_ADMIN:
+        return None
+    scopes: Final = await repository.status_scopes(issue.id)
+    if len(scopes) > 10:
+        raise HTTPException(status_code=403, detail="An administrator must update this alert's status")
+    for scope in scopes:
+        await require_scope(
+            user, store, "model" if scope.scope_type == "provider" else scope.scope_type, scope.scope_id, write=True
+        )
+    return scopes
+
+
+def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]:
+    start: Final = start_date or issue.range_start
+    end: Final = end_date or issue.range_end
+    if start > end or (end - start).days > 365:
+        raise HTTPException(status_code=422, detail="Choose an ordered date range of at most 366 days")
+    return start, end
+
+
+@router.get("", response_model=IssueList)
+async def list_issues(
+    user: User,
+    repository: IssuesRepositoryDep,
+    alerts: UsageAlertsRepositoryDep,
+    kind: IssueKind | None = None,
+    family: IssueFamily | None = None,
+    status: UsageAlertStatus | None = None,
+    scope_type: IssueScope | None = None,
+    scope_id: ScopeId = None,
+    limit: Limit = 50,
+    offset: Offset = 0,
+) -> IssueList:
+    await require_scope(
+        user, require_repository(alerts), "model" if scope_type == "provider" else scope_type, scope_id, write=False
+    )
+    return await require_issues(repository).list_issues(kind, family, status, scope_type, scope_id, limit, offset)
+
+
+@router.get("/families", response_model=IssueFamilyList)
+async def list_issue_families(
+    user: User,
+    repository: IssuesRepositoryDep,
+    alerts: UsageAlertsRepositoryDep,
+) -> IssueFamilyList:
+    await require_scope(user, require_repository(alerts), "deployment", None, write=False)
+    return await require_issues(repository).family_settings()
+
+
+@router.patch("/families/{family}", response_model=IssueFamilySetting)
+async def update_issue_family(
+    family: PerformanceIssueFamily,
+    data: IssueFamilyUpdate,
+    user: User,
+    repository: IssuesRepositoryDep,
+    alerts: UsageAlertsRepositoryDep,
+) -> IssueFamilySetting:
+    require_write_access(user, f"/issues/families/{family}")
+    await require_scope(user, require_repository(alerts), "deployment", None, write=True)
+    try:
+        result: Final = await require_issues(repository).update_family(family, data)
+    except ValueError as exc:
+        raise HTTPException(
+            status_code=422, detail="Invalid family policy; check direction and history bounds"
+        ) from exc
+    if result is None:
+        raise HTTPException(status_code=409, detail="Built-in rules changed or are unavailable; reload and retry")
+    return result
+
+
+@router.get("/{issue_id}", response_model=Issue)
+async def get_issue(
+    issue_id: str,
+    user: User,
+    repository: IssuesRepositoryDep,
+    alerts: UsageAlertsRepositoryDep,
+) -> Issue:
+    issue: Final = await authorized_issue(issue_id, user, repository, alerts)
+    activity_allowed: Final = await can_view_activity(issue, user, require_issues(repository))
+    try:
+        require_write_access(user, f"/issues/{issue_id}")
+        await require_status_scopes(issue, user, require_issues(repository), alerts)
+    except HTTPException as exc:
+        if exc.status_code != 403:
+            raise
+        return issue.model_copy(update={"can_update_status": False, "can_view_activity": activity_allowed})
+    return issue.model_copy(update={"can_update_status": True, "can_view_activity": activity_allowed})
+
+
+@router.patch("/{issue_id}", response_model=Issue)
+async def update_issue(
+    issue_id: str,
+    data: IssueUpdate,
+    user: User,
+    repository: IssuesRepositoryDep,
+    alerts: UsageAlertsRepositoryDep,
+) -> Issue:
+    require_write_access(user, f"/issues/{issue_id}")
+    issue: Final = await authorized_issue(issue_id, user, repository, alerts)
+    scopes: Final = await require_status_scopes(issue, user, require_issues(repository), alerts)
+    if issue.kind == "model_deprecation" and not issue.active and data.status == "open":
+        raise HTTPException(status_code=409, detail="This retirement issue reopens when the model receives new traffic")
+    changed: Final = await require_issues(repository).set_status(
+        issue_id, data.status, data.expected_status, user.user_id, authorized_scopes=scopes
+    )
+    if changed is None:
+        raise HTTPException(status_code=409, detail="Issue status changed; reload and retry")
+    return changed
+
+
+@router.get("/{issue_id}/signals", response_model=IssueSignalList)
+async def list_issue_signals(
+    issue_id: str,
+    user: User,
+    repository: IssuesRepositoryDep,
+    alerts: UsageAlertsRepositoryDep,
+    limit: Limit = 50,
+    offset: Offset = 0,
+) -> IssueSignalList:
+    issue: Final = await authorized_issue(issue_id, user, repository, alerts)
+    visible_scope: Final = (
+        None
+        if user.user_role
+        in (
+            LitellmUserRoles.PROXY_ADMIN,
+            LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
+        )
+        else issue.cohort
+    )
+    return await require_issues(repository).list_signals(issue_id, limit, offset, visible_scope)
+
+
+@router.get("/{issue_id}/activity", response_model=IssueActivityResponse)
+async def get_issue_activity(
+    issue_id: str,
+    user: User,
+    repository: IssuesRepositoryDep,
+    alerts: UsageAlertsRepositoryDep,
+    activity: IssueActivityRepositoryDep,
+    start_date: date | None = None,
+    end_date: date | None = None,
+    model_limit: Limit = 20,
+) -> IssueActivityResponse:
+    issue: Final = await authorized_issue(issue_id, user, repository, alerts)
+    await require_activity_access(issue, user, repository)
+    start, end = activity_range(issue, start_date, end_date)
+    with activity_budget():
+        return await require_activity(activity).daily_activity(issue.cohort, start, end, model_limit)
+
+
+@router.get("/{issue_id}/keys", response_model=DailyActivityKeyPageResponse)
+async def get_issue_keys(
+    issue_id: str,
+    user: User,
+    repository: IssuesRepositoryDep,
+    alerts: UsageAlertsRepositoryDep,
+    activity: IssueActivityRepositoryDep,
+    start_date: date | None = None,
+    end_date: date | None = None,
+    limit: Limit = 20,
+    offset: Offset = 0,
+) -> DailyActivityKeyPageResponse:
+    issue: Final = await authorized_issue(issue_id, user, repository, alerts)
+    await require_activity_access(issue, user, repository)
+    start, end = activity_range(issue, start_date, end_date)
+    with activity_budget():
+        return await require_activity(activity).keys(issue.cohort, start, end, limit, offset)
+
+
+@router.get("/{issue_id}/users", response_model=DailyActivityUserPageResponse)
+async def get_issue_users(
+    issue_id: str,
+    user: User,
+    repository: IssuesRepositoryDep,
+    alerts: UsageAlertsRepositoryDep,
+    activity: IssueActivityRepositoryDep,
+    start_date: date | None = None,
+    end_date: date | None = None,
+    limit: Limit = 20,
+    offset: Offset = 0,
+) -> DailyActivityUserPageResponse:
+    issue: Final = await authorized_issue(issue_id, user, repository, alerts)
+    await require_activity_access(issue, user, repository)
+    if issue.cohort.scope_type == "team":
+        raise HTTPException(status_code=422, detail="Historical team aggregates do not retain user membership")
+    start, end = activity_range(issue, start_date, end_date)
+    with activity_budget():
+        return await require_activity(activity).users(issue.cohort, start, end, limit, offset)
+
+
+@router.get("/{issue_id}/impact", response_model=IssueImpact)
+async def get_issue_impact(
+    issue_id: str,
+    user: User,
+    repository: IssuesRepositoryDep,
+    alerts: UsageAlertsRepositoryDep,
+    activity: IssueActivityRepositoryDep,
+    start_date: date | None = None,
+    end_date: date | None = None,
+) -> IssueImpact:
+    issue: Final = await authorized_issue(issue_id, user, repository, alerts)
+    await require_activity_access(issue, user, repository)
+    start, end = activity_range(issue, start_date, end_date)
+    with activity_budget():
+        return await require_activity(activity).impact(issue.cohort, start, end)

litellm/proxy/alerting_endpoints/model_deprecation_alerts.py

@@ -1,7 +1,5 @@
 """Shared acknowledgement state for model migration alerts."""
 
-import hashlib
-import json
 from datetime import date, datetime, timezone
 from typing import Annotated, Final
 
@@ -10,6 +8,7 @@ from pydantic import TypeAdapter
 
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key
 from litellm.proxy.deps.database import DatabaseDep
 from litellm.repositories.config_repository import ConfigRepository
 from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket
@@ -32,12 +31,6 @@ def alert_tag_repository(database: DatabaseDep) -> DailyTagSpendRepository:
     return DailyTagSpendRepository(database.prisma)
 
 
-def _state_key(model: str, deprecation_date: date) -> str:
-    identity: Final = json.dumps([model, deprecation_date.isoformat()])
-    digest: Final = hashlib.sha256(identity.encode()).hexdigest()
-    return f"model_deprecation_alert:{digest}"
-
-
 def _require_admin(user: UserAPIKeyAuth, *, write: bool) -> None:
     if user.user_role == LitellmUserRoles.PROXY_ADMIN:
         return
@@ -54,7 +47,7 @@ async def get_deprecation_alert_state(
     repository: Annotated[ConfigRepository, Depends(alert_state_repository)],
 ) -> DeprecationAlertState:
     _require_admin(user, write=False)
-    record: Final = await repository.get_param(_state_key(model, deprecation_date))
+    record: Final = await repository.get_param(deprecation_alert_state_key(model, deprecation_date))
     if record is None:
         return DeprecationAlertState(model=model, deprecation_date=deprecation_date)
     return DeprecationAlertState.model_validate(record.param_value)
@@ -70,7 +63,9 @@ async def update_deprecation_alert_state(
     state: Final = DeprecationAlertState(
         **data.model_dump(), updated_at=datetime.now(timezone.utc), updated_by=user.user_id
     )
-    await repository.set_param(_state_key(data.model, data.deprecation_date), state.model_dump(mode="json"))
+    await repository.set_param(
+        deprecation_alert_state_key(data.model, data.deprecation_date), state.model_dump(mode="json")
+    )
     return state
 
 

litellm/proxy/alerting_endpoints/usage_alerts.py

@@ -0,0 +1,293 @@
+"""Public management API for daily usage alert rules and incidents."""
+
+from datetime import date
+from typing import Annotated, Final, TypeAlias
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Response
+from pydantic import BaseModel, TypeAdapter
+
+from litellm.models.usage_alerts import UsageAlertStore
+from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth
+from litellm.proxy.auth.route_checks import RouteChecks
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.deps.repositories import UsageAlertsRepositoryDep
+from litellm.repositories.daily_activity_buckets import api_key_from_bucket
+from litellm.types.proxy.auth.user_roles import LitellmUserRoles
+from litellm.types.proxy.usage_alerts import (
+    UsageAlert,
+    UsageAlertGroupList,
+    UsageAlertGroupUpdate,
+    UsageAlertGroupUpdateResult,
+    UsageAlertList,
+    UsageAlertMetric,
+    UsageAlertModelList,
+    UsageAlertRule,
+    UsageAlertRuleInput,
+    UsageAlertRuleKind,
+    UsageAlertRuleList,
+    UsageAlertScope,
+    UsageAlertStatus,
+    UsageAlertUpdate,
+)
+
+router: Final = APIRouter(prefix="/usage", tags=["usage alerts"])
+
+
+def require_repository(repository: UsageAlertStore | None) -> UsageAlertStore:
+    if repository is None:
+        raise HTTPException(status_code=503, detail="Usage alerts require a database connection")
+    return repository
+
+
+User: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)]
+Repository: TypeAlias = UsageAlertsRepositoryDep
+Limit: TypeAlias = Annotated[int, Query(ge=1, le=100)]
+Offset: TypeAlias = Annotated[int, Query(ge=0, le=100000)]
+ScopeId: TypeAlias = Annotated[str | None, Query(min_length=1, max_length=1024)]
+
+
+def _allows_write(grant: str, route: str) -> bool:
+    if grant == "info_routes":
+        return False
+    paths: Final = (
+        TypeAdapter(tuple[str, ...]).validate_python(LiteLLMRoutes[grant].value)
+        if grant in LiteLLMRoutes.__members__
+        else (grant,)
+    )
+    return RouteChecks.check_route_access(route, paths)
+
+
+class _WriteGrants(BaseModel, frozen=True):
+    allowed_routes: tuple[str, ...] | None = None
+
+
+def require_write_access(user: UserAPIKeyAuth, route: str) -> None:
+    grants: Final = _WriteGrants.model_validate(user, from_attributes=True).allowed_routes
+    if not grants:
+        return
+    if not any(_allows_write(grant, route) for grant in grants):
+        raise HTTPException(status_code=403, detail="This credential does not grant alert write access")
+
+
+def require_notification_access(
+    user: UserAPIKeyAuth, data: UsageAlertRuleInput, existing: UsageAlertRule | None = None
+) -> None:
+    if user.user_role == LitellmUserRoles.PROXY_ADMIN:
+        return
+    if data.notification_channels or data.email_recipients:
+        raise HTTPException(status_code=403, detail="Only proxy admins can configure notification delivery")
+    if existing is not None and (existing.notification_channels or existing.overrides_defaults):
+        raise HTTPException(status_code=403, detail="Only proxy admins can change operator-managed alert rules")
+
+
+async def require_scope(
+    user: UserAPIKeyAuth,
+    repository: UsageAlertStore,
+    scope_type: UsageAlertScope | None,
+    scope_id: str | None,
+    *,
+    write: bool,
+) -> None:
+    if user.key_type == "llm_api" or (write and user.key_type == "read_only"):
+        raise HTTPException(status_code=403, detail="A management-capable credential is required")
+    if user.user_role == LitellmUserRoles.PROXY_ADMIN:
+        return
+    if user.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY:
+        if not write:
+            return
+        raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts")
+    if user.user_role not in (
+        LitellmUserRoles.INTERNAL_USER,
+        LitellmUserRoles.INTERNAL_USER_VIEW_ONLY,
+        LitellmUserRoles.ORG_ADMIN,
+        LitellmUserRoles.TEAM,
+    ):
+        raise HTTPException(status_code=403, detail="Usage alerts require a management role")
+    if write and user.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY:
+        raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts")
+    if scope_id is None:
+        raise HTTPException(status_code=403, detail="Specify a scope you can access")
+    if scope_type == "user" and user.user_id is not None and scope_id == user.user_id:
+        return
+    if scope_type == "team":
+        role: Final = await repository.team_role(scope_id, user.user_id) if user.user_id else None
+        if role == "admin" or (
+            not write and (role == "user" or (user.user_role == LitellmUserRoles.TEAM and user.team_id == scope_id))
+        ):
+            return
+    if scope_type == "key":
+        ownership: Final = await repository.key_ownership(api_key_from_bucket(scope_id))
+        if ownership is not None:
+            owner_id, team_id = ownership
+            if user.user_id is not None and owner_id == user.user_id:
+                return
+            if user.user_id and team_id and await repository.team_role(team_id, user.user_id) == "admin":
+                return
+    raise HTTPException(status_code=403, detail="You cannot access this alert scope")
+
+
+@router.get("/alert-rules", response_model=UsageAlertRuleList)
+async def list_usage_alert_rules(
+    user: User,
+    repository: Repository,
+    scope_type: UsageAlertScope | None = None,
+    scope_id: ScopeId = None,
+    limit: Limit = 50,
+    offset: Offset = 0,
+    kind: UsageAlertRuleKind = "all",
+) -> UsageAlertRuleList:
+    store: Final = require_repository(repository)
+    await require_scope(user, store, scope_type, scope_id, write=False)
+    return await store.list_rules(scope_type, scope_id, limit, offset, kind)
+
+
+@router.get("/alert-groups", response_model=UsageAlertGroupList)
+async def list_usage_alert_groups(
+    user: User,
+    repository: Repository,
+    scope_type: UsageAlertScope | None = None,
+    scope_id: ScopeId = None,
+    status: UsageAlertStatus | None = None,
+    start_date: date | None = None,
+    end_date: date | None = None,
+    limit: Limit = 50,
+    offset: Offset = 0,
+) -> UsageAlertGroupList:
+    store: Final = require_repository(repository)
+    await require_scope(user, store, scope_type, scope_id, write=False)
+    if start_date is not None and end_date is not None and start_date > end_date:
+        raise HTTPException(status_code=422, detail="start_date must not be after end_date")
+    return await store.list_groups(scope_type, scope_id, status, start_date, end_date, limit, offset)
+
+
+@router.patch("/alert-groups", response_model=UsageAlertGroupUpdateResult)
+async def update_usage_alert_group(
+    data: UsageAlertGroupUpdate, user: User, repository: Repository
+) -> UsageAlertGroupUpdateResult:
+    store: Final = require_repository(repository)
+    require_write_access(user, "/usage/alert-groups")
+    await require_scope(user, store, data.scope_type, data.scope_id, write=True)
+    return await store.set_group_status(data)
+
+
+@router.get("/alert-models", response_model=UsageAlertModelList)
+async def list_usage_alert_models(
+    user: User,
+    repository: Repository,
+    search: ScopeId = None,
+    cursor: ScopeId = None,
+    limit: Limit = 20,
+) -> UsageAlertModelList:
+    store: Final = require_repository(repository)
+    await require_scope(user, store, "model", None, write=False)
+    return await store.list_reported_models(search, cursor, limit)
+
+
+@router.post("/alert-rules", response_model=UsageAlertRule, status_code=201)
+async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, repository: Repository) -> UsageAlertRule:
+    """Create a daily usage alert rule for a scope the caller can manage."""
+    store: Final = require_repository(repository)
+    require_write_access(user, "/usage/alert-rules")
+    await require_scope(user, store, data.scope_type, data.scope_id, write=True)
+    require_notification_access(user, data)
+    saved: Final = await store.save_rule(
+        data, user.user_id, overrides_defaults=user.user_role == LitellmUserRoles.PROXY_ADMIN
+    )
+    if saved is None:
+        raise HTTPException(status_code=409, detail="Alert rule could not be saved")
+    return saved
+
+
+@router.get("/alert-rules/{rule_id}", response_model=UsageAlertRule)
+async def get_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> UsageAlertRule:
+    store: Final = require_repository(repository)
+    rule: Final = await store.get_rule(rule_id)
+    if rule is None:
+        raise HTTPException(status_code=404, detail="Alert rule not found")
+    await require_scope(user, store, rule.scope_type, rule.scope_id, write=False)
+    return rule
+
+
+@router.put("/alert-rules/{rule_id}", response_model=UsageAlertRule)
+async def update_usage_alert_rule(
+    rule_id: str, data: UsageAlertRuleInput, user: User, repository: Repository
+) -> UsageAlertRule:
+    store: Final = require_repository(repository)
+    require_write_access(user, f"/usage/alert-rules/{rule_id}")
+    rule: Final = await store.get_rule(rule_id)
+    if rule is None:
+        raise HTTPException(status_code=404, detail="Alert rule not found")
+    await require_scope(user, store, rule.scope_type, rule.scope_id, write=True)
+    if rule.is_default and (rule.scope_type, rule.scope_id, rule.metric, rule.direction) != (
+        data.scope_type,
+        data.scope_id,
+        data.metric,
+        data.direction,
+    ):
+        raise HTTPException(
+            status_code=422, detail="Create a custom rule to change a built-in rule's scope, metric, or direction"
+        )
+    await require_scope(user, store, data.scope_type, data.scope_id, write=True)
+    require_notification_access(user, data, rule)
+    saved: Final = await store.save_rule(data, user.user_id, existing=rule, overrides_defaults=rule.overrides_defaults)
+    if saved is None:
+        raise HTTPException(status_code=409, detail="Alert rule changed while being updated. Refresh and try again")
+    return saved
+
+
+@router.delete("/alert-rules/{rule_id}", status_code=204)
+async def delete_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> Response:
+    store: Final = require_repository(repository)
+    require_write_access(user, f"/usage/alert-rules/{rule_id}")
+    rule: Final = await store.get_rule(rule_id)
+    if rule is None:
+        raise HTTPException(status_code=404, detail="Alert rule not found")
+    await require_scope(user, store, rule.scope_type, rule.scope_id, write=True)
+    require_notification_access(user, rule, rule)
+    if not await store.delete_rule(rule_id, existing=rule):
+        raise HTTPException(status_code=409, detail="Alert rule changed while being deleted. Refresh and try again")
+    return Response(status_code=204)
+
+
+@router.get("/alerts", response_model=UsageAlertList)
+async def list_usage_alerts(
+    user: User,
+    repository: Repository,
+    scope_type: UsageAlertScope | None = None,
+    scope_id: ScopeId = None,
+    metric: UsageAlertMetric | None = None,
+    status: UsageAlertStatus | None = None,
+    start_date: date | None = None,
+    end_date: date | None = None,
+    limit: Limit = 50,
+    offset: Offset = 0,
+) -> UsageAlertList:
+    store: Final = require_repository(repository)
+    await require_scope(user, store, scope_type, scope_id, write=False)
+    if start_date is not None and end_date is not None and start_date > end_date:
+        raise HTTPException(status_code=422, detail="start_date must not be after end_date")
+    return await store.list_alerts(scope_type, scope_id, metric, status, start_date, end_date, limit, offset)
+
+
+@router.get("/alerts/{alert_id}", response_model=UsageAlert)
+async def get_usage_alert(alert_id: str, user: User, repository: Repository) -> UsageAlert:
+    store: Final = require_repository(repository)
+    alert: Final = await store.get_alert(alert_id)
+    if alert is None:
+        raise HTTPException(status_code=404, detail="Usage alert not found")
+    await require_scope(user, store, alert.scope_type, alert.scope_id, write=False)
+    return alert
+
+
+@router.patch("/alerts/{alert_id}", response_model=UsageAlert)
+async def update_usage_alert(alert_id: str, data: UsageAlertUpdate, user: User, repository: Repository) -> UsageAlert:
+    store: Final = require_repository(repository)
+    require_write_access(user, f"/usage/alerts/{alert_id}")
+    alert: Final = await store.get_alert(alert_id)
+    if alert is None:
+        raise HTTPException(status_code=404, detail="Usage alert not found")
+    await require_scope(user, store, alert.scope_type, alert.scope_id, write=True)
+    updated: Final = await store.set_status(alert_id, data.status)
+    if updated is None:
+        raise HTTPException(status_code=404, detail="Usage alert not found")
+    return updated

litellm/proxy/common_utils/model_deprecation.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import hashlib
+import json
 import logging
 from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass
@@ -28,6 +30,11 @@ _SHARED_SCHEDULE_PROVIDER: Final[Mapping[str, str]] = MappingProxyType({"azure_a
 LoadedCatalog: TypeAlias = Callable[[], Mapping[str, Mapping[str, object]]]
 
 
+def deprecation_alert_state_key(model: str, deprecation_date: date) -> str:
+    identity: Final = json.dumps([model, deprecation_date.isoformat()])
+    return f"model_deprecation_alert:{hashlib.sha256(identity.encode()).hexdigest()}"
+
+
 @dataclass(frozen=True, slots=True)
 class _ResolvedDeprecation:
     deprecation_date: date

litellm/proxy/deps/repositories.py

@@ -4,14 +4,19 @@ from fastapi import Depends
 
 from litellm.models.deployment_warning import DeploymentWarningIgnoreStore
 from litellm.models.gateway_activity import GatewayActivityStore
+from litellm.models.issues import IssueActivityStore, IssueStore
 from litellm.models.plugin_upload import PluginUploadStore
 from litellm.models.skill_submission import SkillSubmissionStore
+from litellm.models.usage_alerts import UsageAlertStore
 from litellm.models.user_banner import UserBannerStore
 from litellm.proxy.deps.database import DatabaseDep
 from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository
 from litellm.repositories.gateway_activity_repository import GatewayActivityRepository
+from litellm.repositories.issue_activity import IssueActivityRepository
+from litellm.repositories.issues import IssuesRepository
 from litellm.repositories.plugin_upload_repository import PluginUploadRepository
 from litellm.repositories.skill_submission_repository import SkillSubmissionRepository
+from litellm.repositories.usage_alerts import UsageAlertsRepository
 from litellm.repositories.user_banner_repository import UserBannerRepository
 
 
@@ -54,3 +59,24 @@ async def get_skill_submission_repository(database: DatabaseDep) -> SkillSubmiss
 SkillSubmissionRepositoryDep: TypeAlias = Annotated[
     SkillSubmissionStore | None, Depends(get_skill_submission_repository)
 ]
+
+
+async def get_usage_alerts_repository(database: DatabaseDep) -> UsageAlertStore | None:
+    return None if database is None else UsageAlertsRepository.from_prisma(database.prisma)
+
+
+UsageAlertsRepositoryDep: TypeAlias = Annotated[UsageAlertStore | None, Depends(get_usage_alerts_repository)]
+
+
+async def get_issues_repository(database: DatabaseDep) -> IssueStore | None:
+    return None if database is None else IssuesRepository.from_prisma(database.prisma)
+
+
+IssuesRepositoryDep: TypeAlias = Annotated[IssueStore | None, Depends(get_issues_repository)]
+
+
+async def get_issue_activity_repository(database: DatabaseDep) -> IssueActivityStore | None:
+    return None if database is None else IssueActivityRepository.from_prisma(database.prisma)
+
+
+IssueActivityRepositoryDep: TypeAlias = Annotated[IssueActivityStore | None, Depends(get_issue_activity_repository)]

litellm/proxy/management_endpoints/common_daily_activity.py

@@ -1162,7 +1162,7 @@ async def get_daily_activity_aggregated(
                 scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit
             ),
             _guardrail_spend_or_empty(repository, scope)
-            if include_guardrail_breakdown and not scope.model
+            if include_guardrail_breakdown and not scope.model and scope.provider is None and not scope.model_pairs
             else asyncio.sleep(0, result=()),
         )
         records: Final = aggregated_rows.grouping_rows

litellm/proxy/management_endpoints/daily_activity_routes.py

@@ -141,6 +141,8 @@ def _key_activity_row(row: KeySpendRow, metadata: Mapping[str, KeyMetadataRow])
             failed_requests=row.failed_requests,
             cache_read_input_tokens=row.cache_read_input_tokens,
             cache_creation_input_tokens=row.cache_creation_input_tokens,
+            total_response_time_ms=row.total_response_time_ms,
+            timed_requests=row.timed_requests,
         ),
         metadata=_key_metadata(row.api_key, metadata),
     )
@@ -175,6 +177,8 @@ def _user_activity_row(row: UserSpendRow, metadata: Mapping[str, UserMetadataRow
         api_requests=row.api_requests,
         successful_requests=row.successful_requests,
         failed_requests=row.failed_requests,
+        total_response_time_ms=row.total_response_time_ms,
+        timed_requests=row.timed_requests,
     )
 
 

litellm/proxy/proxy_server.py

@@ -427,9 +427,11 @@ from litellm.messages.dispatch import messages_target
 from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot
 from litellm.proxy._types import *
 from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling
+from litellm.proxy.alerting_endpoints.issues import router as issues_router
 from litellm.proxy.alerting_endpoints.model_deprecation_alerts import (
     router as model_deprecation_alerts_router,
 )
+from litellm.proxy.alerting_endpoints.usage_alerts import router as usage_alerts_router
 from litellm.proxy.analytics_endpoints.analytics_endpoints import (
     router as analytics_router,
 )
@@ -22507,6 +22509,8 @@ app.include_router(router_settings_router)
 app.include_router(fallback_management_router)
 app.include_router(cache_settings_router)
 app.include_router(model_deprecation_alerts_router)
+app.include_router(usage_alerts_router)
+app.include_router(issues_router)
 app.include_router(coordination_redis_settings_router)
 app.include_router(user_agent_analytics_router)
 app.include_router(gateway_request_router)

litellm/proxy/schema.prisma

@@ -2168,3 +2168,96 @@ model LiteLLM_SubscriptionAccountTable {
 
   @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
 }
+
+model LiteLLM_UsageAlertRule {
+  id String @id
+  scope_type String
+  scope_id String?
+  enabled Boolean @default(true)
+  record Json
+  deleted Boolean @default(false)
+  @@index([enabled, deleted, id])
+  @@index([scope_type, scope_id, deleted])
+}
+
+model LiteLLM_UsageAlertIncident {
+  id String @id
+  rule_id String
+  scope_key String
+  scope_type String
+  scope_id String?
+  metric String
+  status String
+  day String
+  evaluation_day String
+  evaluation_revision String @default("")
+  breached Boolean
+  record Json
+  notified_channels String[] @default([])
+  updated_at DateTime @default(now())
+  @@index([rule_id, scope_key, evaluation_day])
+  @@index([evaluation_day, breached, id])
+  @@index([scope_type, scope_id, day])
+  @@index([status, day])
+  @@index([updated_at])
+}
+
+model LiteLLM_UsageAlertCheckpoint {
+  id String @id
+  completed Boolean @default(false)
+  owner String?
+  expires_at DateTime @default(now())
+  @@index([expires_at])
+}
+
+model LiteLLM_Issue {
+  id String @id
+  cohort_key String
+  kind String
+  family String
+  scope_type String
+  scope_id String?
+  status String
+  active Boolean @default(true)
+  ack_key String?
+  record Json
+  first_seen DateTime
+  last_seen DateTime
+  updated_at DateTime @default(now())
+
+  @@index([status, last_seen, id])
+  @@index([kind, family, last_seen, id])
+  @@index([scope_type, scope_id, last_seen, id])
+  @@index([cohort_key, active])
+}
+
+model LiteLLM_IssueSignal {
+  issue_id String
+  source String
+  source_id String
+  record Json
+
+  @@id([issue_id, source, source_id])
+  @@index([source, source_id])
+}
+
+model LiteLLM_IssueDelivery {
+  id String @id
+  issue_id String
+  record Json
+  sent Boolean @default(false)
+  next_attempt DateTime @default(now())
+  updated_at DateTime @default(now())
+
+  @@index([sent, next_attempt, id])
+  @@index([issue_id])
+}
+
+model LiteLLM_IssueDeliveryRule {
+  delivery_id String
+  rule_id String
+  source_id String
+  rule_revision String
+
+  @@id([delivery_id, rule_id, source_id])
+}

litellm/repositories/config_repository.py

@@ -30,7 +30,7 @@ class _ConfigRow(Protocol):
 class _ConfigTable(Protocol):
     async def find_unique(self, *, where: Mapping[str, str]) -> _ConfigRow | None: ...
 
-    async def find_many(self) -> Sequence[_ConfigRow]: ...
+    async def find_many(self, *, where: Mapping[str, object] | None = None) -> Sequence[_ConfigRow]: ...
 
     async def upsert(self, *, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]]) -> _ConfigRow: ...
 
@@ -83,6 +83,18 @@ class ConfigRepository:
             param_value = _decoded_json(param_value)
         return ConfigParam(param_name=param_name, param_value=param_value)
 
+    async def get_params(self, param_names: Sequence[str]) -> tuple[ConfigParam, ...]:
+        if not param_names:
+            return ()
+        records: Final = await self._config_table.find_many(where={"param_name": {"in": tuple(param_names)}})
+        return tuple(
+            ConfigParam(
+                record.param_name,
+                _decoded_json(record.param_value) if isinstance(record.param_value, str) else record.param_value,
+            )
+            for record in records
+        )
+
     async def set_param(self, param_name: str, param_value: object) -> ConfigParam:
         """Set a config parameter in the database."""
         value_json: Final = json.dumps(param_value) if not isinstance(param_value, str) else param_value

litellm/repositories/daily_activity_repository.py

@@ -216,8 +216,10 @@ class DailyActivityRepository:
         rows: Final = _KEY_SPEND_ADAPTER.validate_python(await self._query(query))
         return tuple(row.api_key for row in rows)
 
-    async def key_page(self, scope: DailyActivityScope, *, offset: int, limit: int) -> KeyPage:
-        query: Final = build_key_page_sql(scope, offset=offset, limit=limit)
+    async def key_page(
+        self, scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False
+    ) -> KeyPage:
+        query: Final = build_key_page_sql(scope, offset=offset, limit=limit, bounded_identifiers=bounded_identifiers)
         result: Final = await self._query(query)
         total_api_keys_value: Final = result[0].get("total_api_keys") if result else 0
         total_api_keys: Final = (
@@ -226,8 +228,10 @@ class DailyActivityRepository:
         rows: Final = _KEY_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("api_key") is not None))
         return KeyPage(rows=rows, total_api_keys=total_api_keys)
 
-    async def user_page(self, scope: DailyActivityScope, *, offset: int, limit: int) -> UserPage:
-        query: Final = build_user_page_sql(scope, offset=offset, limit=limit)
+    async def user_page(
+        self, scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False
+    ) -> UserPage:
+        query: Final = build_user_page_sql(scope, offset=offset, limit=limit, bounded_identifiers=bounded_identifiers)
         result: Final = await self._query(query)
         total_users_value: Final = result[0].get("total_users") if result else 0
         total_users: Final = (
@@ -335,6 +339,8 @@ class DailyActivityRepository:
         )
 
     async def daily_rows(self, scope: DailyActivityScope, *, page: int, page_size: int) -> DailyRowsPage:
+        if scope.provider is not None or scope.model_pairs:
+            raise ValueError("Provider cohorts require the aggregated Activity API")
         table: Final = _daily_rows_table(self._prisma_client, scope.table)
         adjusted_start, adjusted_end = adjust_dates_for_timezone(
             scope.start_date,

litellm/repositories/daily_activity_sql.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Mapping
 from dataclasses import dataclass, replace
 from datetime import date, datetime, timedelta, timezone
@@ -75,6 +76,8 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu
     exclusion_index: Final = entity_index + int(has_entity_array)
     model_index: Final = exclusion_index + int(bool(scope.exclude_entity_ids))
     api_keys_index: Final = model_index + int(bool(scope.model))
+    provider_index: Final = api_keys_index + int(bool(scope.api_keys))
+    pairs_index: Final = provider_index + int(scope.provider is not None)
     conditions: Final = (
         f"date >= ${start_index}",
         f"date <= ${start_index + 1}",
@@ -85,6 +88,21 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu
             if has_entity_array
             else ()
         ),
+        *(
+            (f"COALESCE(NULLIF(custom_llm_provider, ''), 'unknown') = ${provider_index}",)
+            if scope.provider is not None
+            else ()
+        ),
+        *(
+            (
+                f"EXISTS (SELECT 1 FROM jsonb_to_recordset(${pairs_index}::jsonb) "
+                f"AS cohort(match_provider text, match_model text) WHERE "
+                f"cohort.match_provider=COALESCE(NULLIF(custom_llm_provider, ''), 'unknown') "
+                f"AND cohort.match_model={_MODEL_EXPR})",
+            )
+            if scope.model_pairs
+            else ()
+        ),
         *(
             (
                 f'("{scope.entity_id_field}" IS NULL '
@@ -113,6 +131,16 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu
         *((list(scope.exclude_entity_ids),) if scope.exclude_entity_ids else ()),
         *((scope.model,) if scope.model else ()),
         *((list(scope.api_keys),) if scope.api_keys else ()),
+        *((scope.provider,) if scope.provider is not None else ()),
+        *(
+            (
+                json.dumps(
+                    tuple({"match_provider": provider, "match_model": model} for provider, model in scope.model_pairs)
+                ),
+            )
+            if scope.model_pairs
+            else ()
+        ),
     )
     return " AND ".join(conditions), params
 
@@ -378,10 +406,14 @@ def _key_spend_select() -> str:
             COALESCE(SUM(successful_requests), 0)::bigint AS successful_requests,
             COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests,
             COALESCE(SUM(cache_read_input_tokens), 0)::bigint AS cache_read_input_tokens,
-            COALESCE(SUM(cache_creation_input_tokens), 0)::bigint AS cache_creation_input_tokens"""
+            COALESCE(SUM(cache_creation_input_tokens), 0)::bigint AS cache_creation_input_tokens,
+            COALESCE(SUM(total_response_time_ms), 0)::bigint AS total_response_time_ms,
+            COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests"""
 
 
-def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery:
+def build_key_page_sql(
+    scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False
+) -> SqlQuery:
     if not 1 <= limit <= constants.USAGE_KEY_PAGE_MAX:
         raise ValueError(f"limit must be between 1 and {constants.USAGE_KEY_PAGE_MAX}")
     if offset < 0:
@@ -390,11 +422,17 @@ def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) ->
     sentinel_param: Final = len(where_params) + 1
     limit_param: Final = sentinel_param + 1
     offset_param: Final = limit_param + 1
+    identity_filter: Final = (
+        "AND CASE WHEN octet_length(api_key)<=4096 THEN "
+        "length(api_key)<=CASE WHEN api_key LIKE 'raw:%' THEN 1020 ELSE 1024 END ELSE false END"
+        if bounded_identifiers
+        else ""
+    )
     sql: Final = f"""
         WITH ranked AS (
             SELECT api_key,{_key_spend_select()}, SUM(spend::numeric) AS rank_spend
             FROM "{PRISMA_TO_PG_TABLE[scope.table]}"
-            WHERE {where_clause} AND api_key <> ${sentinel_param}
+            WHERE {where_clause} AND api_key <> ${sentinel_param} {identity_filter}
             GROUP BY api_key
         )
         SELECT (SELECT COUNT(*) FROM ranked)::bigint AS total_api_keys, page.*
@@ -416,10 +454,14 @@ def _user_spend_select() -> str:
             (COALESCE(SUM(prompt_tokens), 0) + COALESCE(SUM(completion_tokens), 0))::bigint AS total_tokens,
             COALESCE(SUM(api_requests), 0)::bigint AS api_requests,
             COALESCE(SUM(successful_requests), 0)::bigint AS successful_requests,
-            COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests"""
+            COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests,
+            COALESCE(SUM(total_response_time_ms), 0)::bigint AS total_response_time_ms,
+            COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests"""
 
 
-def build_user_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery:
+def build_user_page_sql(
+    scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False
+) -> SqlQuery:
     if not 1 <= limit <= constants.USAGE_USER_PAGE_MAX:
         raise ValueError(f"limit must be between 1 and {constants.USAGE_USER_PAGE_MAX}")
     if offset < 0:
@@ -428,11 +470,17 @@ def build_user_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -
     limit_param: Final = len(where_params) + 1
     offset_param: Final = limit_param + 1
     group_expr: Final = f"NULLIF(\"{scope.entity_id_field}\", '')"
+    identity_filter: Final = (
+        f"AND ({group_expr} IS NULL OR CASE WHEN octet_length({group_expr})<=4096 "
+        f"THEN length({group_expr})<=1024 ELSE false END)"
+        if bounded_identifiers
+        else ""
+    )
     sql: Final = f"""
         WITH ranked AS (
             SELECT {group_expr} AS user_id,{_user_spend_select()}, SUM(spend::numeric) AS rank_spend
             FROM "{PRISMA_TO_PG_TABLE[scope.table]}"
-            WHERE {where_clause}
+            WHERE {where_clause} {identity_filter}
             GROUP BY {group_expr}
         )
         SELECT (SELECT COUNT(*) FROM ranked)::bigint AS total_users, page.*

litellm/repositories/issue_activity.py

@@ -0,0 +1,223 @@
+"""Issue investigations reuse the same aggregate filters as entity Activity."""
+
+from collections.abc import Mapping
+from dataclasses import asdict, replace
+from datetime import date
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, cast  # noqa: TID251  # generated Prisma structural database interface
+
+from pydantic import BaseModel, TypeAdapter
+
+from litellm.constants import PTU_SENTINEL_API_KEY
+from litellm.repositories.analytics_budget import AnalyticsDatabase, BoundedAnalyticsDatabase
+from litellm.repositories.bounded_daily_activity import BoundedDailyActivityRepository
+from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket
+from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase, DailyActivityRepository
+from litellm.repositories.daily_activity_sql import PRISMA_TO_PG_TABLE, build_where_clause
+from litellm.types.proxy.issues import (
+    IssueActivityResponse,
+    IssueCohort,
+    IssueImpact,
+)
+from litellm.types.proxy.management_endpoints.common_daily_activity import (
+    DailyActivityKeyPageResponse,
+    DailyActivityUserPageResponse,
+    KeyMetadata,
+    KeySpendActivityRow,
+    KeySpendMetrics,
+    UserActivityRow,
+)
+from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable
+
+if TYPE_CHECKING:
+    from litellm.proxy.utils import PrismaClient
+
+
+class _Counts(BaseModel, frozen=True):
+    keys: int
+    entities: int
+    models: int
+    requests: int
+
+
+class _KeyLabel(BaseModel, frozen=True):
+    api_key: str
+    metadata: KeyMetadata
+
+
+class _UserLabel(BaseModel, frozen=True):
+    user_id: str
+    user_alias: str | None
+    user_email: str | None
+
+
+def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyActivityScope:
+    if start > end or (end - start).days > 365:
+        raise ValueError("Choose an ordered date range of at most 366 days")
+    return DailyActivityScope(
+        table=DailyActivityTable.TEAM if cohort.scope_type == "team" else DailyActivityTable.USER,
+        entity_id_field="team_id" if cohort.scope_type == "team" else "user_id",
+        entity_ids=(cohort.scope_id,) if cohort.scope_type in ("team", "user") and cohort.scope_id else None,
+        exclude_entity_ids=(),
+        api_keys=(api_key_from_bucket(cohort.scope_id),) if cohort.scope_type == "key" and cohort.scope_id else None,
+        start_date=start.isoformat(),
+        end_date=end.isoformat(),
+        model=cohort.scope_id if cohort.scope_type == "model" else None,
+        timezone_offset_minutes=0,
+        include_current_utc_day=True,
+        provider=cohort.scope_id if cohort.scope_type == "provider" else None,
+        model_pairs=tuple((pair.provider, pair.model) for pair in cohort.model_pairs),
+    )
+
+
+class IssueActivityRepository:
+    def __init__(self, activity: DailyActivityRepository, database: DailyActivityQueryDatabase) -> None:
+        self.activity = activity
+        self.database = database
+
+    @classmethod
+    def from_prisma(cls, prisma: "PrismaClient") -> "IssueActivityRepository":
+        from litellm.proxy.management_endpoints.common_daily_activity import daily_activity_repository
+
+        database: Final = BoundedAnalyticsDatabase(cast(AnalyticsDatabase, prisma.db))  # cast-ok: generated Prisma tx
+        return cls(daily_activity_repository(prisma, query_database=database), database)
+
+    async def daily_activity(
+        self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20
+    ) -> IssueActivityResponse:
+        return await BoundedDailyActivityRepository(self.database).aggregate(
+            issue_activity_scope(cohort, start, end), model_limit=model_limit if cohort.scope_type == "key" else None
+        )
+
+    async def keys(
+        self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
+    ) -> DailyActivityKeyPageResponse:
+        if not 1 <= limit <= 100 or offset < 0:
+            raise ValueError("Read between one and 100 keys with a nonnegative offset")
+        page: Final = await self.activity.key_page(
+            issue_activity_scope(cohort, start, end), limit=limit, offset=offset, bounded_identifiers=True
+        )
+        metadata: Final = await self._key_labels(tuple(row.api_key for row in page.rows if row.api_key))
+        return DailyActivityKeyPageResponse(
+            api_keys=[
+                KeySpendActivityRow(
+                    api_key=api_key_bucket(row.api_key),
+                    metrics=KeySpendMetrics.model_validate(asdict(row)),
+                    metadata=metadata.get(row.api_key, KeyMetadata()),
+                )
+                for row in page.rows
+            ],
+            total_api_keys=page.total_api_keys,
+            limit=limit,
+            offset=offset,
+        )
+
+    async def users(
+        self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int
+    ) -> DailyActivityUserPageResponse:
+        if not 1 <= limit <= 100 or offset < 0:
+            raise ValueError("Read between one and 100 users with a nonnegative offset")
+        if cohort.scope_type == "team":
+            raise ValueError("Historical team aggregates do not retain user membership")
+        page: Final = await self.activity.user_page(
+            issue_activity_scope(cohort, start, end), limit=limit, offset=offset, bounded_identifiers=True
+        )
+        metadata: Final = await self._user_labels(tuple(row.user_id for row in page.rows if row.user_id))
+        return DailyActivityUserPageResponse(
+            users=tuple(
+                UserActivityRow.model_validate(
+                    {
+                        **asdict(row),
+                        **(
+                            metadata[row.user_id].model_dump()
+                            if row.user_id is not None and row.user_id in metadata
+                            else {}
+                        ),
+                    }
+                )
+                for row in page.rows
+            ),
+            total_users=page.total_users,
+            limit=limit,
+            offset=offset,
+        )
+
+    async def _key_labels(self, ids: tuple[str, ...]) -> Mapping[str, KeyMetadata]:
+        if len(ids) > 100:
+            raise ValueError("Read at most 100 key labels")
+        if not ids:
+            return MappingProxyType({})
+        rows: Final = TypeAdapter(tuple[_KeyLabel, ...]).validate_python(
+            await self.database.query_raw(
+                """SELECT w.api_key,jsonb_build_object(
+                    'key_alias',LEFT(COALESCE(k.key_alias,d.key_alias),256),
+                    'team_id',CASE WHEN octet_length(COALESCE(k.team_id,d.team_id))<=4096
+                        THEN CASE WHEN length(COALESCE(k.team_id,d.team_id))<=1024
+                            THEN COALESCE(k.team_id,d.team_id) END END,
+                    'user_id',CASE WHEN octet_length(COALESCE(k.user_id,d.user_id))<=4096
+                        THEN CASE WHEN length(COALESCE(k.user_id,d.user_id))<=1024
+                            THEN COALESCE(k.user_id,d.user_id) END END,
+                    'user_email',CASE WHEN octet_length(u.user_email)<=1280
+                        THEN CASE WHEN length(u.user_email)<=320 THEN u.user_email END END,
+                    'key_exists',CASE WHEN k.token IS NOT NULL THEN true WHEN d.id IS NOT NULL THEN false END
+                ) AS metadata
+                FROM unnest($1::text[]) w(api_key)
+                LEFT JOIN "LiteLLM_VerificationToken" k ON k.token=w.api_key
+                LEFT JOIN LATERAL (
+                    SELECT id,key_alias,team_id,user_id FROM "LiteLLM_DeletedVerificationToken"
+                    WHERE token=w.api_key AND k.token IS NULL ORDER BY deleted_at DESC,id DESC LIMIT 1
+                ) d ON true
+                LEFT JOIN "LiteLLM_UserTable" u ON u.user_id=COALESCE(k.user_id,d.user_id)
+                LIMIT 100""",
+                list(ids),
+            )
+        )
+        return MappingProxyType({row.api_key: row.metadata for row in rows})
+
+    async def _user_labels(self, ids: tuple[str, ...]) -> Mapping[str, _UserLabel]:
+        if len(ids) > 100:
+            raise ValueError("Read at most 100 user labels")
+        if not ids:
+            return MappingProxyType({})
+        rows: Final = TypeAdapter(tuple[_UserLabel, ...]).validate_python(
+            await self.database.query_raw(
+                """SELECT user_id,LEFT(user_alias,256) AS user_alias,
+                    CASE WHEN octet_length(user_email)<=1280
+                        THEN CASE WHEN length(user_email)<=320 THEN user_email END END AS user_email
+                FROM "LiteLLM_UserTable" WHERE user_id=ANY($1::text[]) LIMIT 100""",
+                list(ids),
+            )
+        )
+        return MappingProxyType({row.user_id: row for row in rows})
+
+    async def _counts(self, scope: DailyActivityScope) -> _Counts:
+        where, params = build_where_clause(scope)
+        rows: Final = TypeAdapter(tuple[_Counts, ...]).validate_python(
+            await self.database.query_raw(
+                f'''SELECT COUNT(DISTINCT NULLIF(api_key,'')) FILTER(WHERE api_key<>${len(params) + 1})::int AS keys,
+                COUNT(DISTINCT NULLIF("{scope.entity_id_field}",''))
+                    FILTER(WHERE "{scope.entity_id_field}" NOT IN ('default_user_id','litellm-dashboard'))::int AS entities,
+                COUNT(DISTINCT NULLIF(model,''))::int AS models,
+                COALESCE(SUM(api_requests),0)::bigint AS requests
+                FROM "{PRISMA_TO_PG_TABLE[scope.table]}" WHERE {where}''',
+                *params,
+                PTU_SENTINEL_API_KEY,
+            )
+        )
+        return rows[0]
+
+    async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact:
+        scope: Final = issue_activity_scope(cohort, start, end)
+        counts: Final = await self._counts(scope)
+        teams: Final = (
+            await self._counts(replace(scope, table=DailyActivityTable.TEAM, entity_id_field="team_id"))
+            if cohort.scope_type not in ("user", "team")
+            else None
+        )
+        return IssueImpact(
+            keys=counts.keys,
+            models=counts.models,
+            requests=counts.requests,
+            users=counts.entities if cohort.scope_type != "team" else None,
+            teams=counts.entities if cohort.scope_type == "team" else teams.entities if teams else None,
+        )

litellm/repositories/issue_families.py

@@ -0,0 +1,165 @@
+"""Bounded settings for the built-in performance policies."""
+
+import json
+from collections import Counter
+from datetime import datetime, timezone
+from itertools import chain, product
+from typing import Final
+
+from pydantic import BaseModel, TypeAdapter
+
+from litellm.repositories.usage_alerts import AlertDatabase
+from litellm.types.proxy.issues import (
+    IssueFamilyList,
+    IssueFamilyPolicy,
+    IssueFamilySetting,
+    IssueFamilyUpdate,
+)
+from litellm.types.proxy.usage_alert_defaults import (
+    DEFAULT_USAGE_POLICIES,
+    DEFAULT_USAGE_RULE_IDS,
+    USAGE_ALERT_SCOPES,
+    PerformanceIssueFamily,
+    default_usage_rule_id,
+)
+from litellm.types.proxy.usage_alerts import UsageAlertEvaluationCounts, UsageAlertRule
+
+
+class _RuleRow(BaseModel, frozen=True):
+    record: UsageAlertRule
+
+
+class _Count(BaseModel, frozen=True):
+    count: int
+
+
+class _ProjectionState(BaseModel, frozen=True):
+    pending: bool
+
+
+def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) -> IssueFamilySetting:
+    policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family)
+    groups: Final = tuple(
+        (policy, tuple(rule for rule in rules if rule.metric == policy.metric and rule.direction == policy.direction))
+        for policy in policies
+    )
+    values: Final = tuple(
+        IssueFamilyPolicy(
+            direction=policy.direction,
+            enabled=group[0].enabled if group and len({rule.enabled for rule in group}) == 1 else None,
+            multiplier=group[0].multiplier if group and len({rule.multiplier for rule in group}) == 1 else None,
+            min_value=group[0].min_value if group and len({rule.min_value for rule in group}) == 1 else None,
+            min_requests=group[0].min_requests if group and len({rule.min_requests for rule in group}) == 1 else None,
+            baseline_days=group[0].baseline_days
+            if group and len({rule.baseline_days for rule in group}) == 1
+            else None,
+            min_history_days=group[0].min_history_days
+            if group and len({rule.min_history_days for rule in group}) == 1
+            else None,
+            mixed=len(
+                {
+                    (r.enabled, r.multiplier, r.min_value, r.min_requests, r.baseline_days, r.min_history_days)
+                    for r in group
+                }
+            )
+            > 1,
+            total_rules=len(group),
+            enabled_rules=sum(rule.enabled for rule in group),
+        )
+        for policy, group in groups
+    )
+    family_rules: Final = tuple(chain.from_iterable(group for _, group in groups))
+    return IssueFamilySetting(
+        family=family,
+        enabled=family_rules[0].enabled if family_rules and len({r.enabled for r in family_rules}) == 1 else None,
+        total_rules=len(family_rules),
+        enabled_rules=sum(rule.enabled for rule in family_rules),
+        policies=values,
+        evaluation_status_counts=UsageAlertEvaluationCounts.model_validate(
+            Counter(rule.evaluation_status for rule in family_rules if rule.enabled)
+        ),
+    )
+
+
+async def family_settings(database: AlertDatabase) -> IssueFamilyList:
+    rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python(
+        await database.query_raw(
+            'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted',
+            list(DEFAULT_USAGE_RULE_IDS),
+        )
+    )
+    projection: Final = TypeAdapter(tuple[_ProjectionState, ...]).validate_python(
+        await database.query_raw(
+            """SELECT EXISTS(SELECT 1 FROM "LiteLLM_UsageAlertCheckpoint"
+                WHERE id LIKE 'performance-dirty:%' AND NOT completed LIMIT 1) AS pending"""
+        )
+    )
+    return IssueFamilyList(
+        data=tuple(
+            _setting(family, tuple(row.record for row in rows))
+            for family in ("spend", "latency", "reliability", "traffic")
+        ),
+        projection_pending=projection[0].pending,
+    )
+
+
+async def update_family(
+    database: AlertDatabase,
+    family: PerformanceIssueFamily,
+    data: IssueFamilyUpdate,
+) -> IssueFamilySetting | None:
+    policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family)
+    if any(update.direction not in {policy.direction for policy in policies} for update in data.policies):
+        raise ValueError("This family does not support that direction")
+    ids: Final = tuple(
+        default_usage_rule_id(scope, policy.metric, policy.direction)
+        for policy, scope in product(policies, USAGE_ALERT_SCOPES)
+    )
+    rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python(
+        await database.query_raw(
+            'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted',
+            list(ids),
+        )
+    )
+    timestamp: Final = datetime.now(timezone.utc).isoformat()
+    updates: Final = {
+        policy.direction: policy.model_dump(exclude_none=True, exclude={"direction"}) for policy in data.policies
+    }
+    records: Final = tuple(
+        UsageAlertRule.model_validate(
+            {
+                **row.record.model_dump(),
+                **({"enabled": data.enabled} if data.enabled is not None else {}),
+                **updates.get(row.record.direction, {}),
+                "updated_at": timestamp,
+                "evaluation_status": "pending",
+                "last_evaluated_day": None,
+            }
+        )
+        for row in rows
+    )
+    if not records:
+        return None
+    changed: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
+        await database.query_raw(
+            """WITH incoming AS MATERIALIZED (
+            SELECT x->'record' AS record,x->>'revision' AS revision FROM jsonb_array_elements($1::jsonb) x
+        ), eligible AS MATERIALIZED (
+            SELECT r.id FROM "LiteLLM_UsageAlertRule" r JOIN incoming x ON r.id=x.record->>'id'
+            WHERE NOT r.deleted AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz
+            ORDER BY r.id FOR UPDATE OF r
+        ), changed AS (
+            UPDATE "LiteLLM_UsageAlertRule" r SET record=x.record,enabled=(x.record->>'enabled')::boolean
+            FROM incoming x WHERE r.id=x.record->>'id' AND NOT r.deleted
+                AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz
+                AND (SELECT count(*) FROM eligible)=(SELECT count(*) FROM incoming) RETURNING r.id
+        ) SELECT count(*)::int AS count FROM changed""",
+            json.dumps(
+                tuple(
+                    {"record": record.model_dump(mode="json"), "revision": row.record.updated_at.isoformat()}
+                    for record, row in zip(records, rows, strict=True)
+                )
+            ),
+        )
+    )
+    return _setting(family, records) if changed[0].count == len(records) else None

litellm/repositories/issues.py

@@ -0,0 +1,680 @@
+import json
+from collections.abc import Mapping
+from datetime import date, datetime, timezone
+from typing import (
+    TYPE_CHECKING,
+    Final,
+    Literal,
+    cast,  # noqa: TID251  # generated Prisma client implements query_raw
+)
+
+from pydantic import BaseModel, TypeAdapter
+
+from litellm.models.team import LiteLLM_TeamTable
+from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key
+from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository
+from litellm.types.proxy.issues import (
+    Issue,
+    IssueCohort,
+    IssueDeliveryBinding,
+    IssueDeliveryInput,
+    IssueFamily,
+    IssueFamilyList,
+    IssueFamilySetting,
+    IssueFamilyUpdate,
+    IssueInput,
+    IssueKind,
+    IssueList,
+    IssuePendingDelivery,
+    IssueProjectionSignal,
+    IssueScope,
+    IssueSignal,
+    IssueSignalInput,
+    IssueSignalList,
+)
+from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS, PerformanceIssueFamily
+from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus
+
+if TYPE_CHECKING:
+    from litellm.proxy.utils import PrismaClient
+
+
+class _IssueRow(BaseModel, frozen=True):
+    record: Issue
+
+
+class _SignalRow(BaseModel, frozen=True):
+    record: IssueSignal
+
+
+class _Count(BaseModel, frozen=True):
+    count: int
+
+
+class _Identifier(BaseModel, frozen=True):
+    id: str
+
+
+_ACK: Final = "CASE WHEN jsonb_typeof(c.param_value)='string' THEN (c.param_value#>>'{}')::jsonb ELSE c.param_value END"
+_ACK_CURRENT: Final = f"""(({_ACK})->>'last_seen_date' > i.record->'deprecation'->>'last_used_date'
+    OR (({_ACK})->>'last_seen_date' = i.record->'deprecation'->>'last_used_date'
+        AND COALESCE((({_ACK})->>'last_seen_requests')::bigint,0)
+            >= (i.record->'deprecation'->>'last_day_requests')::bigint))"""
+_STATUS: Final = f"""CASE WHEN i.kind <> 'model_deprecation' THEN i.status
+    WHEN NOT i.active THEN CASE WHEN ({_ACK})->>'status'='dismissed' AND ({_ACK_CURRENT})
+        THEN 'dismissed' ELSE 'resolved' END
+    WHEN COALESCE(({_ACK})->>'status','open')='open'
+        OR ({_ACK})->>'last_seen_date' IS NULL
+        OR i.record->'deprecation'->>'last_used_date' > ({_ACK})->>'last_seen_date'
+        OR (i.record->'deprecation'->>'last_used_date' = ({_ACK})->>'last_seen_date'
+            AND (i.record->'deprecation'->>'last_day_requests')::bigint
+                > COALESCE((({_ACK})->>'last_seen_requests')::bigint,0)) THEN 'open'
+    ELSE ({_ACK})->>'status' END"""
+_PRIOR_STATUS: Final = _STATUS.replace(_ACK, _ACK.replace("c.param_value", "prior.param_value"))
+_JOIN: Final = 'FROM "LiteLLM_Issue" i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key'
+_RECORD: Final = f"""i.record || jsonb_build_object('status',({_STATUS}),'active',i.active,
+    'first_seen',i.first_seen AT TIME ZONE 'UTC','last_seen',i.last_seen AT TIME ZONE 'UTC',
+    'updated_at',i.updated_at AT TIME ZONE 'UTC') || CASE WHEN i.kind='model_deprecation'
+    THEN jsonb_build_object('deprecation',i.record->'deprecation' ||
+        jsonb_build_object('state',COALESCE(({_ACK}),i.record->'deprecation'->'state')))
+    ELSE '{{}}'::jsonb END"""
+
+
+def _utc(value: datetime) -> str:
+    return value.astimezone(timezone.utc).replace(tzinfo=None).isoformat()
+
+
+def _issue_payload(data: IssueInput, status: UsageAlertStatus | None) -> Mapping[str, object]:
+    initial: Final = Issue.model_validate(
+        {
+            **data.model_dump(),
+            "status": status or "open",
+            "first_seen": data.observed_at,
+            "last_seen": data.observed_at,
+            "updated_at": data.observed_at,
+        }
+    )
+    return {
+        "record": initial.model_dump(mode="json"),
+        "ack_key": deprecation_alert_state_key(data.deprecation.usage_model, data.deprecation.model.deprecation_date)
+        if data.deprecation
+        else None,
+    }
+
+
+class IssuesRepository:
+    def __init__(self, database: AlertDatabase) -> None:
+        self.database = database
+
+    @classmethod
+    def from_prisma(cls, prisma: "PrismaClient") -> "IssuesRepository":
+        return cls(cast(AlertDatabase, prisma.writer_db))  # cast-ok: generated Prisma delegate implements query_raw
+
+    async def upsert_issue(
+        self, data: IssueInput, reopen: bool = False, status: UsageAlertStatus | None = None
+    ) -> Issue:
+        rows: Final = await self.upsert_issues((data,), reopen=reopen, status=status)
+        return rows[0]
+
+    async def upsert_issues(
+        self,
+        inputs: tuple[IssueInput, ...],
+        reopen: bool = False,
+        status: UsageAlertStatus | None = None,
+    ) -> tuple[Issue, ...]:
+        if not inputs:
+            return ()
+        if len(inputs) > 500 or len(frozenset(item.id for item in inputs)) != len(inputs):
+            raise ValueError("Write at most 500 distinct issues per batch")
+        payload: Final = tuple(_issue_payload(item, status) for item in inputs)
+        rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
+            await self.database.query_raw(
+                f"""WITH incoming AS (
+                SELECT x->'record' AS record,x->>'ack_key' AS ack_key FROM jsonb_array_elements($1::jsonb) x
+            ), changed AS (
+                INSERT INTO "LiteLLM_Issue" AS old
+                    (id,cohort_key,kind,family,scope_type,scope_id,status,ack_key,record,first_seen,last_seen,updated_at)
+                SELECT record->>'id',record->>'cohort_key',record->>'kind',record->>'family',
+                    record->'cohort'->>'scope_type',record->'cohort'->>'scope_id',record->>'status',ack_key,record,
+                    (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC',
+                    (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC',
+                    (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC' FROM incoming
+                ON CONFLICT(id) DO UPDATE SET
+                    record=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen
+                        THEN EXCLUDED.record ELSE old.record END,
+                    last_seen=GREATEST(old.last_seen,EXCLUDED.last_seen),
+                    active=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen
+                        THEN true ELSE old.active END,
+                    status=CASE WHEN $2 AND old.active AND EXCLUDED.last_seen > old.updated_at
+                        THEN 'open' ELSE old.status END,
+                    updated_at=GREATEST(old.updated_at,EXCLUDED.last_seen)
+                WHERE old.kind=EXCLUDED.kind AND old.cohort_key=EXCLUDED.cohort_key
+                    AND old.scope_type=EXCLUDED.scope_type AND old.scope_id IS NOT DISTINCT FROM EXCLUDED.scope_id
+                RETURNING *
+            ) SELECT {_RECORD} AS record FROM changed i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key""",
+                json.dumps(payload),
+                reopen,
+            )
+        )
+        if len(rows) != len(inputs):
+            raise ValueError("An issue ID already belongs to another cohort")
+        return tuple(row.record for row in rows)
+
+    async def get_issue(self, issue_id: str) -> Issue | None:
+        rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
+            await self.database.query_raw(f"SELECT {_RECORD} AS record {_JOIN} WHERE i.id=$1", issue_id)
+        )
+        return (await self._labels((rows[0].record,)))[0] if rows else None
+
+    async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None:
+        rows: Final = TypeAdapter(tuple[LiteLLM_TeamTable, ...]).validate_python(
+            await self.database.query_raw(
+                """SELECT t.team_id,
+                    CASE WHEN m.role IS NULL THEN '[]'::jsonb
+                        ELSE jsonb_build_array(jsonb_build_object('user_id',$2::text,'role',m.role)) END
+                        AS members_with_roles,
+                    CASE WHEN '/team/daily/activity'=ANY(t.team_member_permissions)
+                        THEN ARRAY['/team/daily/activity'] ELSE ARRAY[]::text[] END AS team_member_permissions
+                FROM "LiteLLM_TeamTable" t LEFT JOIN LATERAL (
+                    SELECT member->>'role' AS role FROM jsonb_array_elements(t.members_with_roles) member
+                    WHERE member->>'user_id'=$2 ORDER BY (member->>'role'='admin') DESC LIMIT 1
+                ) m ON true WHERE t.team_id=$1""",
+                team_id,
+                user_id,
+            )
+        )
+        return rows[0] if rows else None
+
+    async def list_issues(
+        self,
+        kind: IssueKind | None = None,
+        family: IssueFamily | None = None,
+        status: UsageAlertStatus | None = None,
+        scope_type: IssueScope | None = None,
+        scope_id: str | None = None,
+        limit: int = 50,
+        offset: int = 0,
+    ) -> IssueList:
+        where: Final = f"""($1::text IS NULL OR i.kind=$1) AND ($2::text IS NULL OR i.family=$2)
+            AND ($3::text IS NULL OR ({_STATUS})=$3) AND ($4::text IS NULL OR i.scope_type=$4)
+            AND ($5::text IS NULL OR i.scope_id=$5)"""
+        args: Final = (kind, family, status, scope_type, scope_id)
+        count: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
+            await self.database.query_raw(f"SELECT count(*)::int AS count {_JOIN} WHERE {where}", *args)
+        )
+        rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
+            await self.database.query_raw(
+                f"SELECT {_RECORD} AS record {_JOIN} WHERE {where} ORDER BY i.last_seen DESC,i.id LIMIT $6 OFFSET $7",
+                *args,
+                limit,
+                offset,
+            )
+        )
+        return IssueList(
+            data=await self._labels(tuple(row.record for row in rows)), total=count[0].count, limit=limit, offset=offset
+        )
+
+    async def _labels(self, issues: tuple[Issue, ...]) -> tuple[Issue, ...]:
+        if not issues:
+            return ()
+        labels: Final = await UsageAlertsRepository(self.database).resource_labels(
+            tuple(issue.cohort for issue in issues)
+        )
+        return tuple(
+            issue.model_copy(update={"scope_label": labels.get((issue.cohort.scope_type, issue.cohort.scope_id))})
+            for issue in issues
+        )
+
+    async def find_active_issue(self, cohort_key: str) -> Issue | None:
+        rows: Final = await self.find_active_issues((cohort_key,))
+        return rows[0] if rows else None
+
+    async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]:
+        if len(cohort_keys) > 500:
+            raise ValueError("Read at most 500 issue cohorts per batch")
+        rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
+            await self.database.query_raw(
+                f"SELECT DISTINCT ON(i.cohort_key) {_RECORD} AS record {_JOIN} "
+                "WHERE i.cohort_key=ANY($1::text[]) AND i.active "
+                "ORDER BY i.cohort_key,i.first_seen DESC,i.id",
+                list(cohort_keys),
+            )
+        )
+        return tuple(row.record for row in rows)
+
+    async def set_status(
+        self,
+        issue_id: str,
+        status: UsageAlertStatus,
+        expected_status: UsageAlertStatus,
+        actor: str | None = None,
+        authorized_scopes: tuple[IssueCohort, ...] | None = None,
+    ) -> Issue | None:
+        if authorized_scopes is not None and len(authorized_scopes) > 10:
+            raise ValueError("Authorize at most 10 alert source scopes")
+        changed: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python(
+            await self.database.query_raw(
+                f"""WITH candidate AS MATERIALIZED (
+                    SELECT i.*,({_STATUS}) AS effective_status {_JOIN} WHERE i.id=$1
+                        AND NOT(i.kind='model_deprecation' AND NOT i.active AND $2='open')
+                        AND ($5::jsonb IS NULL OR NOT EXISTS (
+                            SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id AND NOT EXISTS (
+                                SELECT 1 FROM jsonb_to_recordset($5::jsonb) AS x(scope_type text,scope_id text)
+                                WHERE s.source='usage_alert' AND x.scope_type=s.record->'usage_alert'->>'scope_type'
+                                    AND x.scope_id IS NOT DISTINCT FROM s.record->'usage_alert'->>'scope_id')))
+                ), ack AS (
+                    INSERT INTO "LiteLLM_Config" AS prior (param_name,param_value)
+                    SELECT ack_key,jsonb_build_object('model',record->'deprecation'->>'usage_model',
+                        'deprecation_date',record->'deprecation'->'model'->>'deprecation_date',
+                        'status',$2::text,'last_seen_date',record->'deprecation'->>'last_used_date',
+                        'last_seen_requests',record->'deprecation'->'last_day_requests',
+                        'updated_at',CURRENT_TIMESTAMP,'updated_by',$4::text)
+                    FROM candidate WHERE kind='model_deprecation' AND effective_status=$3
+                    ON CONFLICT(param_name) DO UPDATE SET param_value=EXCLUDED.param_value
+                    WHERE EXISTS(SELECT 1 FROM candidate i WHERE i.ack_key=prior.param_name
+                        AND ({_PRIOR_STATUS})=$3)
+                    RETURNING param_name
+                ) UPDATE "LiteLLM_Issue" i SET status=$2,
+                    updated_at=GREATEST(CURRENT_TIMESTAMP,i.updated_at+interval '1 millisecond')
+                FROM candidate c WHERE i.id=c.id AND (c.kind='model_deprecation' OR i.updated_at=c.updated_at)
+                    AND c.effective_status=$3 AND
+                    ((c.kind<>'model_deprecation' AND i.status=$3) OR i.ack_key IN(SELECT param_name FROM ack))
+                RETURNING i.id""",
+                issue_id,
+                status,
+                expected_status,
+                actor,
+                json.dumps(tuple(scope.model_dump(mode="json") for scope in authorized_scopes))
+                if authorized_scopes is not None
+                else None,
+            )
+        )
+        return await self.get_issue(issue_id) if changed else None
+
+    async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]:
+        return TypeAdapter(tuple[IssueCohort, ...]).validate_python(
+            await self.database.query_raw(
+                """SELECT DISTINCT CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_type'
+                    ELSE 'deployment' END AS scope_type,
+                    CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_id' END AS scope_id
+                FROM "LiteLLM_IssueSignal" WHERE issue_id=$1 ORDER BY scope_type,scope_id LIMIT 11""",
+                issue_id,
+            )
+        )
+
+    async def add_signals(self, issue_id: str, signals: tuple[IssueSignalInput, ...]) -> None:
+        await self.add_signal_batch(
+            tuple(IssueSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in signals)
+        )
+
+    async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None:
+        if not signals:
+            return
+        if len(signals) > 500:
+            raise ValueError("Write at most 500 issue signals per batch")
+        await self.database.query_raw(
+            """WITH incoming AS MATERIALIZED (
+                SELECT DISTINCT ON(x->>'issue_id',x->>'source',x->>'source_id') x AS record
+                FROM jsonb_array_elements($1::jsonb) x
+            ), parents AS MATERIALIZED (
+                SELECT i.id FROM "LiteLLM_Issue" i WHERE i.id IN(SELECT record->>'issue_id' FROM incoming)
+                ORDER BY i.id FOR UPDATE
+            ), added AS (
+                INSERT INTO "LiteLLM_IssueSignal" (issue_id,source,source_id,record)
+                SELECT x.record->>'issue_id',x.record->>'source',x.record->>'source_id',x.record
+                FROM incoming x JOIN parents p ON p.id=x.record->>'issue_id'
+                ON CONFLICT(issue_id,source,source_id) DO NOTHING RETURNING issue_id,source,record
+            ) UPDATE "LiteLLM_Issue" i SET
+                updated_at=GREATEST(CURRENT_TIMESTAMP,i.updated_at+interval '1 millisecond'),
+                status=CASE WHEN i.kind='performance' AND i.active AND i.status<>'open' AND EXISTS (
+                    SELECT 1 FROM added a WHERE a.issue_id=i.id AND a.source='usage_alert'
+                        AND (a.record->'usage_alert'->>'scope_type'<>i.scope_type
+                            OR a.record->'usage_alert'->>'scope_id' IS DISTINCT FROM i.scope_id)
+                        AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" prior
+                            WHERE prior.issue_id=i.id AND prior.source='usage_alert'
+                                AND prior.record->'usage_alert'->>'scope_type'=a.record->'usage_alert'->>'scope_type'
+                                AND prior.record->'usage_alert'->>'scope_id'
+                                    IS NOT DISTINCT FROM a.record->'usage_alert'->>'scope_id')
+                ) THEN 'open' ELSE i.status END
+            WHERE i.id IN(SELECT issue_id FROM added) RETURNING i.id""",
+            json.dumps(tuple(signal.model_dump(mode="json") for signal in signals)),
+        )
+
+    async def list_signals(
+        self,
+        issue_id: str,
+        limit: int = 50,
+        offset: int = 0,
+        visible_scope: IssueCohort | None = None,
+    ) -> IssueSignalList:
+        where: Final = """issue_id=$1 AND ($2::text IS NULL OR (source='usage_alert'
+            AND record->'usage_alert'->>'scope_type'=$2
+            AND record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM $3::text))"""
+        args: Final = (
+            issue_id,
+            visible_scope.scope_type if visible_scope else None,
+            visible_scope.scope_id if visible_scope else None,
+        )
+        counts: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
+            await self.database.query_raw(
+                f'SELECT count(*)::int AS count FROM "LiteLLM_IssueSignal" WHERE {where}',
+                *args,
+            )
+        )
+        rows: Final = TypeAdapter(tuple[_SignalRow, ...]).validate_python(
+            await self.database.query_raw(
+                f'SELECT record FROM "LiteLLM_IssueSignal" WHERE {where} ORDER BY source,source_id LIMIT $4 OFFSET $5',
+                *args,
+                limit,
+                offset,
+            )
+        )
+        return IssueSignalList(
+            data=tuple(row.record for row in rows), total=counts[0].count, limit=limit, offset=offset
+        )
+
+    async def active_signals(
+        self,
+        day: date,
+        limit: int = 500,
+        cursor: str | None = None,
+        scope_type: IssueScope | None = None,
+        metric: UsageAlertMetric | None = None,
+        scope_ids: tuple[str, ...] | None = None,
+    ) -> tuple[IssueProjectionSignal, ...]:
+        if not 1 <= limit <= 500:
+            raise ValueError("Read between one and 500 source signals")
+        if scope_ids == ():
+            return ()
+        if scope_ids is not None and len(scope_ids) > 500:
+            raise ValueError("Filter at most 500 source scopes")
+        return TypeAdapter(tuple[IssueProjectionSignal, ...]).validate_python(
+            await self.database.query_raw(
+                """SELECT a.record AS alert,r.record AS rule,a.evaluation_day
+                FROM "LiteLLM_UsageAlertIncident" a JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id
+                WHERE a.evaluation_day=$1 AND a.breached AND r.enabled AND NOT r.deleted
+                    AND NULLIF(a.evaluation_revision,'')::timestamptz=(r.record->>'updated_at')::timestamptz
+                    AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric'
+                    AND a.record->>'direction'=r.record->>'direction'
+                    AND (r.scope_id IS NULL OR a.scope_id=r.scope_id)
+                    AND ($2::text IS NULL OR a.id>$2) AND ($4::text IS NULL OR a.scope_type=$4)
+                    AND ($5::text IS NULL OR a.metric=$5)
+                    AND ($6::text[] IS NULL OR a.scope_id=ANY($6::text[])) ORDER BY a.id LIMIT $3""",
+                day.isoformat(),
+                cursor,
+                limit,
+                scope_type,
+                metric,
+                list(scope_ids) if scope_ids is not None else None,
+            )
+        )
+
+    async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int:
+        if issue_ids == ():
+            return 0
+        if issue_ids is not None and len(issue_ids) > 10:
+            raise ValueError("Recover at most 10 resource alerts")
+        selection: Final = "i.id=ANY($3::text[])" if issue_ids is not None else "$3::text[] IS NULL"
+        rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
+            await self.database.query_raw(
+                f"""WITH primary_signals AS (
+                SELECT i.id,a.id AS source_id,a.breached,a.evaluation_day,a.evaluation_revision,
+                    max(a.evaluation_day) OVER (PARTITION BY i.id) AS latest_observation_day,
+                    r.id AS rule_id,r.record AS rule_record,
+                    COALESCE(r.enabled AND NOT r.deleted
+                        AND r.scope_type=a.scope_type AND r.record->>'metric'=a.metric
+                        AND r.record->>'direction'=a.record->>'direction'
+                        AND (r.scope_id IS NULL OR r.scope_id=a.scope_id),false) AS eligible,
+                    COALESCE(r.id=ANY($2::text[]) AND EXISTS (
+                            SELECT 1 FROM "LiteLLM_UsageAlertRule" custom WHERE custom.enabled AND NOT custom.deleted
+                            AND COALESCE((custom.record->>'overrides_defaults')::boolean,false)
+                            AND NOT(custom.id=ANY($2::text[])) AND custom.scope_type=r.scope_type
+                            AND custom.record->>'metric'=r.record->>'metric'
+                            AND custom.record->>'direction'=r.record->>'direction'
+                            AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id)),false) AS suppressed
+                FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id
+                LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id
+                LEFT JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id
+                WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='usage_alert'
+                    AND ({selection})
+                    AND s.record->'usage_alert'->>'scope_type'=i.scope_type
+                    AND s.record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM i.scope_id
+                    AND ((i.family='spend' AND s.record->'usage_alert'->>'metric'='spend')
+                        OR (i.family='latency' AND s.record->'usage_alert'->>'metric'='latency')
+                        OR (i.family='traffic' AND s.record->'usage_alert'->>'metric'='requests')
+                        OR (i.family='reliability' AND s.record->'usage_alert'->>'metric' IN ('errors','error_rate')))
+            ), observations AS (
+                SELECT *,COALESCE(source_id IS NOT NULL AND NOT breached AND evaluation_day<=$1
+                    AND evaluation_day>=latest_observation_day
+                    AND NULLIF(evaluation_revision,'')::timestamptz=(rule_record->>'updated_at')::timestamptz,false) AS healthy
+                FROM primary_signals
+            ), healthy AS (
+                SELECT id FROM observations GROUP BY id
+                HAVING bool_or(eligible AND healthy) AND bool_and(NOT eligible OR suppressed OR healthy)
+            ), changed AS (
+                UPDATE "LiteLLM_Issue" i SET active=false,
+                    status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP
+                WHERE id IN(SELECT id FROM healthy) AND active RETURNING id
+            ) SELECT count(*)::int AS count FROM changed""",
+                day.isoformat(),
+                tuple(DEFAULT_USAGE_RULE_IDS),
+                issue_ids,
+            )
+        )
+        return rows[0].count
+
+    async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]:
+        if not 1 <= limit <= 500:
+            raise ValueError("Read between one and 500 provider issues")
+        rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python(
+            await self.database.query_raw(
+                f"""SELECT {_RECORD} AS record {_JOIN}
+                WHERE i.kind='performance' AND i.scope_type='provider' AND i.active
+                    AND ($2::text IS NULL OR i.id>$2) ORDER BY i.id LIMIT $1""",
+                limit,
+                cursor,
+            )
+        )
+        return tuple(row.record for row in rows)
+
+    async def recover_issue_ids(self, ids: tuple[str, ...], observed_at: datetime) -> int:
+        if len(ids) > 500 or observed_at.tzinfo is None:
+            raise ValueError("Recover at most 500 issues with a timezone-aware observation")
+        if not ids:
+            return 0
+        rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
+            await self.database.query_raw(
+                """WITH changed AS (
+                UPDATE "LiteLLM_Issue" SET active=false,
+                    status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP
+                WHERE id=ANY($1::text[]) AND kind='performance' AND active AND last_seen<$2::timestamp(3)
+                RETURNING id
+            ) SELECT count(*)::int AS count FROM changed""",
+                list(ids),
+                _utc(observed_at),
+            )
+        )
+        return rows[0].count
+
+    async def projection_completed(self, day: date, version: str) -> bool:
+        rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python(
+            await self.database.query_raw(
+                'SELECT id FROM "LiteLLM_UsageAlertCheckpoint" WHERE id=$1 AND completed',
+                f"issues:{version}:{day.isoformat()}",
+            )
+        )
+        return bool(rows)
+
+    async def complete_projection(self, day: date, version: str) -> None:
+        await self.database.query_raw(
+            """INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed,expires_at) VALUES($1,true,CURRENT_TIMESTAMP)
+            ON CONFLICT(id) DO UPDATE SET completed=true,expires_at=CURRENT_TIMESTAMP RETURNING id""",
+            f"issues:{version}:{day.isoformat()}",
+        )
+
+    async def finish_deprecation_projection(self, observed_at: datetime) -> int:
+        rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
+            await self.database.query_raw(
+                """WITH changed AS (
+                UPDATE "LiteLLM_Issue" SET active=false,
+                    status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP
+                WHERE kind='model_deprecation' AND active AND last_seen<$1::timestamp(3) RETURNING id
+            ) SELECT count(*)::int AS count FROM changed""",
+                _utc(observed_at),
+            )
+        )
+        return rows[0].count
+
+    async def family_settings(self) -> "IssueFamilyList":
+        from litellm.repositories.issue_families import family_settings
+
+        return await family_settings(self.database)
+
+    async def update_family(
+        self, family: "PerformanceIssueFamily", data: "IssueFamilyUpdate"
+    ) -> "IssueFamilySetting | None":
+        from litellm.repositories.issue_families import update_family
+
+        return await update_family(self.database, family, data)
+
+    async def queue_delivery(
+        self,
+        data: IssueDeliveryInput,
+        rule_id: str,
+        rule_revision: str,
+        source_id: str,
+    ) -> None:
+        await self.queue_deliveries(
+            (IssueDeliveryBinding(delivery=data, rule_id=rule_id, rule_revision=rule_revision, source_id=source_id),)
+        )
+
+    async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None:
+        if not bindings:
+            return
+        if len(bindings) > 500:
+            raise ValueError("Queue at most 500 delivery bindings per batch")
+        await self.database.query_raw(
+            """WITH incoming AS MATERIALIZED (
+                SELECT x->'delivery' AS delivery,x->>'rule_id' AS rule_id,
+                    x->>'rule_revision' AS revision,x->>'source_id' AS source_id
+                FROM jsonb_array_elements($1::jsonb) x
+                WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" WHERE id=x->'delivery'->>'issue_id')
+            ), saved AS (
+                INSERT INTO "LiteLLM_IssueDelivery" (id,issue_id,record)
+                SELECT DISTINCT ON(delivery->>'id') delivery->>'id',delivery->>'issue_id',delivery
+                    FROM incoming ON CONFLICT(id) DO NOTHING RETURNING id
+            ) INSERT INTO "LiteLLM_IssueDeliveryRule" (delivery_id,rule_id,source_id,rule_revision)
+            SELECT DISTINCT ON(delivery->>'id',rule_id,source_id) delivery->>'id',rule_id,source_id,revision
+            FROM incoming WHERE delivery->>'id' IN(SELECT id FROM saved)
+                OR EXISTS(SELECT 1 FROM "LiteLLM_IssueDelivery" d WHERE d.id=delivery->>'id'
+                    AND d.issue_id=delivery->>'issue_id')
+            ON CONFLICT(delivery_id,rule_id,source_id) DO UPDATE SET rule_revision=EXCLUDED.rule_revision
+            RETURNING delivery_id""",
+            json.dumps(tuple(binding.model_dump(mode="json") for binding in bindings)),
+        )
+
+    async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]:
+        return await self._pending_deliveries(limit, None)
+
+    async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None:
+        rows: Final = await self._pending_deliveries(1, delivery_id)
+        return rows[0] if rows else None
+
+    async def _pending_deliveries(self, limit: int, delivery_id: str | None) -> tuple[IssuePendingDelivery, ...]:
+        if not 1 <= limit <= 500:
+            raise ValueError("Read between one and 500 deliveries")
+        return TypeAdapter(tuple[IssuePendingDelivery, ...]).validate_python(
+            await self.database.query_raw(
+                f"""SELECT d.record || jsonb_build_object('next_attempt',d.next_attempt AT TIME ZONE 'UTC') AS delivery,
+                {_RECORD} AS issue,bound.rule,bound.alert
+            FROM "LiteLLM_IssueDelivery" d JOIN "LiteLLM_Issue" i ON i.id=d.issue_id
+            LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key
+            JOIN LATERAL (
+                SELECT r.record AS rule,a.record AS alert FROM "LiteLLM_IssueDeliveryRule" b
+                JOIN "LiteLLM_UsageAlertRule" r ON r.id=b.rule_id
+                JOIN "LiteLLM_UsageAlertIncident" a ON a.id=b.source_id AND a.rule_id=r.id
+                WHERE b.delivery_id=d.id AND r.enabled AND NOT r.deleted AND a.breached AND a.status='open'
+                    AND (r.record->'notification_channels') ? (d.record->>'channel')
+                    AND (d.record->>'channel'<>'email' OR
+                        (jsonb_array_length(d.record->'email_recipients')>0
+                         AND (d.record->'email_recipients') <@ (r.record->'email_recipients')))
+                    AND (r.record->>'updated_at')::timestamptz=b.rule_revision::timestamptz
+                    AND NULLIF(a.evaluation_revision,'')::timestamptz=b.rule_revision::timestamptz
+                    AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric'
+                    AND a.record->>'direction'=r.record->>'direction'
+                    AND (r.scope_id IS NULL OR a.scope_id=r.scope_id)
+                    AND (NOT(r.id=ANY($3::text[])) OR NOT EXISTS (
+                        SELECT 1 FROM "LiteLLM_UsageAlertRule" custom WHERE custom.enabled AND NOT custom.deleted
+                        AND COALESCE((custom.record->>'overrides_defaults')::boolean,false)
+                        AND NOT(custom.id=ANY($3::text[])) AND custom.scope_type=r.scope_type
+                        AND custom.record->>'metric'=r.record->>'metric'
+                        AND custom.record->>'direction'=r.record->>'direction'
+                        AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id)))
+                    AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id
+                        AND s.source='usage_alert' AND s.source_id=a.id)
+                ORDER BY b.rule_id,b.source_id LIMIT 1
+            ) bound ON true
+            WHERE NOT d.sent AND d.next_attempt<=CURRENT_TIMESTAMP AND i.active AND ({_STATUS})='open'
+                AND ($2::text IS NULL OR d.id=$2)
+            ORDER BY d.next_attempt,d.id LIMIT $1""",
+                limit,
+                delivery_id,
+                tuple(DEFAULT_USAGE_RULE_IDS),
+            )
+        )
+
+    async def mark_delivery_sent(self, delivery_id: str) -> None:
+        await self.database.query_raw(
+            """UPDATE "LiteLLM_IssueDelivery" SET sent=true,updated_at=CURRENT_TIMESTAMP,
+                record=record || jsonb_build_object('last_error',NULL,'attempts',
+                    COALESCE((record->>'attempts')::int,0)+1)
+            WHERE id=$1 AND NOT sent RETURNING id""",
+            delivery_id,
+        )
+
+    async def defer_delivery(
+        self,
+        delivery_id: str,
+        error: Literal["delivery_failed", "transport_not_configured"],
+    ) -> None:
+        await self.database.query_raw(
+            """UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP+interval '1 hour',
+                updated_at=CURRENT_TIMESTAMP,record=record || jsonb_build_object('last_error',$2::text,
+                    'attempts',COALESCE((record->>'attempts')::int,0)+1)
+            WHERE id=$1 AND NOT sent RETURNING id""",
+            delivery_id,
+            error,
+        )
+
+    async def prune(self, retention_days: int = 90, limit: int = 500) -> None:
+        if not 1 <= limit <= 500 or retention_days < 1:
+            raise ValueError("Retention and cleanup batch must be bounded positive values")
+        await self.database.query_raw(
+            """WITH expired AS MATERIALIZED (
+                SELECT id FROM "LiteLLM_Issue" WHERE status<>'open'
+                    AND updated_at<CURRENT_TIMESTAMP-make_interval(days=>$1::int)
+                ORDER BY updated_at,id LIMIT $2
+            ), signals AS (
+                DELETE FROM "LiteLLM_IssueSignal" WHERE (issue_id,source,source_id) IN (
+                    SELECT issue_id,source,source_id FROM "LiteLLM_IssueSignal"
+                    WHERE issue_id IN(SELECT id FROM expired) ORDER BY issue_id,source,source_id LIMIT $2)
+                RETURNING issue_id
+            ), bindings AS (
+                DELETE FROM "LiteLLM_IssueDeliveryRule" WHERE (delivery_id,rule_id,source_id) IN (
+                    SELECT b.delivery_id,b.rule_id,b.source_id FROM "LiteLLM_IssueDeliveryRule" b
+                    JOIN "LiteLLM_IssueDelivery" d ON d.id=b.delivery_id
+                    WHERE d.issue_id IN(SELECT id FROM expired) ORDER BY b.delivery_id,b.rule_id,b.source_id LIMIT $2)
+                RETURNING delivery_id
+            ), deliveries AS (
+                DELETE FROM "LiteLLM_IssueDelivery" d WHERE d.id IN (
+                    SELECT id FROM "LiteLLM_IssueDelivery" d WHERE issue_id IN(SELECT id FROM expired)
+                        AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueDeliveryRule" b WHERE b.delivery_id=d.id)
+                    ORDER BY id LIMIT $2) RETURNING issue_id
+            ), removed AS (
+                DELETE FROM "LiteLLM_Issue" i WHERE id IN(SELECT id FROM expired)
+                    AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id)
+                    AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueDelivery" d WHERE d.issue_id=i.id) RETURNING id
+            ) SELECT count(*)::int AS count FROM removed""",
+            retention_days,
+            limit,
+        )

litellm/repositories/usage_alerts.py

@@ -0,0 +1,752 @@
+"""Bounded, writer-consistent storage for daily alert rules and episodes."""
+
+import json
+from collections.abc import Mapping
+from datetime import date, datetime, timedelta, timezone
+from types import MappingProxyType
+from typing import (
+    TYPE_CHECKING,
+    Final,
+    Protocol,
+    cast,  # noqa: TID251  # PrismaWrapper forwards generated client methods dynamically
+)
+from urllib.parse import urlencode
+from uuid import NAMESPACE_URL, uuid4, uuid5
+
+from pydantic import BaseModel, TypeAdapter
+
+from litellm.repositories.daily_activity_buckets import api_key_from_bucket
+from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS
+from litellm.types.proxy.usage_alerts import (
+    UsageAlert,
+    UsageAlertChannel,
+    UsageAlertDefaultSummary,
+    UsageAlertEvaluationStatus,
+    UsageAlertGroup,
+    UsageAlertGroupList,
+    UsageAlertGroupMetric,
+    UsageAlertGroupUpdate,
+    UsageAlertGroupUpdateResult,
+    UsageAlertList,
+    UsageAlertMetric,
+    UsageAlertModelList,
+    UsageAlertModelOption,
+    UsageAlertNotification,
+    UsageAlertObservation,
+    UsageAlertRule,
+    UsageAlertRuleInput,
+    UsageAlertRuleKind,
+    UsageAlertRuleList,
+    UsageAlertScope,
+    UsageAlertStatus,
+)
+
+if TYPE_CHECKING:
+    from litellm.proxy.utils import PrismaClient
+
+
+class AlertDatabase(Protocol):
+    async def query_raw(self, query: str, *args: object) -> object: ...
+
+
+class ResourceLabelScope(Protocol):
+    @property
+    def scope_type(self) -> str: ...
+
+    @property
+    def scope_id(self) -> str | None: ...
+
+
+class _Record(BaseModel, frozen=True):
+    record: Mapping[str, object]
+
+
+class _Count(BaseModel, frozen=True):
+    count: int
+
+
+class _RuleCounts(_Count, frozen=True):
+    default_summary: UsageAlertDefaultSummary
+
+
+class _GroupRow(BaseModel, frozen=True):
+    scope_type: UsageAlertScope
+    scope_id: str | None
+    day: date
+    incident_count: int
+    metrics: tuple[UsageAlertGroupMetric, ...]
+
+
+class _GroupPage(BaseModel, frozen=True):
+    total: int
+    data: tuple[_GroupRow, ...]
+
+
+class _ResourceLabel(BaseModel, frozen=True):
+    scope_type: str
+    scope_id: str | None
+    label: str | None
+
+
+class _ModelIdentity(BaseModel, frozen=True):
+    model: str
+
+
+class _Pending(_Record, frozen=True):
+    rule: Mapping[str, object]
+    notified_channels: tuple[UsageAlertChannel, ...]
+
+
+class _Ownership(BaseModel, frozen=True):
+    user_id: str | None
+    team_id: str | None
+
+
+_DEFAULT_RULE_IDS: Final = DEFAULT_USAGE_RULE_IDS
+
+
+def _rule(record: Mapping[str, object]) -> UsageAlertRule:
+    return UsageAlertRule.model_validate({**record, "is_default": str(record.get("id")) in _DEFAULT_RULE_IDS})
+
+
+def activity_path(
+    scope: UsageAlertScope, entity_id: str | None, day: date, alert_id: str, baseline_days: int = 7
+) -> str:
+    routes: Final = {
+        "key": ("/api-keys", "key"),
+        "team": ("/teams", "team"),
+        "user": ("/users", "user"),
+        "model": ("/models-and-endpoints", "activity_model"),
+        "deployment": ("/usage", ""),
+    }
+    route, parameter = routes[scope]
+    if scope == "deployment":
+        return (
+            route
+            + "?"
+            + urlencode(
+                {
+                    "tab": "keys",
+                    "from": (day - timedelta(days=baseline_days)).isoformat(),
+                    "to": day.isoformat(),
+                    "alert": alert_id,
+                }
+            )
+        )
+    query: Final = {
+        "activity_from": (day - timedelta(days=baseline_days)).isoformat(),
+        "activity_to": day.isoformat(),
+        "alert": alert_id,
+    }
+    if parameter and entity_id and not entity_id.startswith("sk-"):
+        query[parameter] = entity_id
+        query["detail_tab"] = "activity"
+    return route + "?" + urlencode(query)
+
+
+def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now: datetime) -> Mapping[str, object]:
+    alert_id: Final = str(
+        uuid5(
+            NAMESPACE_URL,
+            json.dumps(
+                [
+                    "litellm:usage-alert",
+                    rule.id,
+                    rule.scope_type,
+                    item.entity_id,
+                    rule.metric,
+                    rule.direction,
+                    item.day.isoformat(),
+                    rule.updated_at.isoformat(),
+                ]
+            ),
+        )
+    )
+    incident: Final = UsageAlert(
+        id=alert_id,
+        rule_id=rule.id,
+        rule_name=rule.name,
+        scope_type=rule.scope_type,
+        scope_id=item.entity_id,
+        metric=rule.metric,
+        direction=rule.direction,
+        status="open",
+        day=item.day,
+        baseline_start=item.day - timedelta(days=rule.baseline_days),
+        baseline_end=item.day - timedelta(days=1),
+        baseline_sample_days=item.baseline_sample_days,
+        observed_value=item.observed,
+        baseline_value=item.baseline,
+        threshold=item.threshold,
+        first_seen=now,
+        last_seen=now,
+        activity_path=activity_path(rule.scope_type, item.entity_id, item.day, alert_id, rule.baseline_days),
+    )
+    return {
+        "scope_key": item.entity_id or "",
+        "breached": item.breached,
+        "day": item.day.isoformat(),
+        "record": incident.model_dump(mode="json"),
+    }
+
+
+class UsageAlertsRepository:
+    def __init__(self, database: AlertDatabase) -> None:
+        self.database: Final = database
+
+    @classmethod
+    def from_prisma(cls, prisma: "PrismaClient") -> "UsageAlertsRepository":
+        return cls(
+            cast(AlertDatabase, prisma.writer_db)  # cast-ok: PrismaWrapper forwards query_raw to its generated client
+        )
+
+    async def get_rule(self, rule_id: str) -> UsageAlertRule | None:
+        rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
+            await self.database.query_raw(
+                'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND NOT deleted', rule_id
+            )
+        )
+        return _rule(rows[0].record) if rows else None
+
+    async def save_rule(
+        self,
+        data: UsageAlertRuleInput,
+        created_by: str | None,
+        existing: UsageAlertRule | None = None,
+        *,
+        overrides_defaults: bool = False,
+    ) -> UsageAlertRule | None:
+        now: Final = datetime.now(timezone.utc)
+        rule: Final = UsageAlertRule.model_validate(
+            {
+                **data.model_dump(),
+                "id": existing.id if existing else str(uuid4()),
+                "created_by": existing.created_by if existing else created_by,
+                "created_at": existing.created_at if existing else now,
+                "updated_at": now,
+                "overrides_defaults": overrides_defaults,
+            }
+        )
+        rows: Final = await self.database.query_raw(
+            """INSERT INTO "LiteLLM_UsageAlertRule" (id,scope_type,scope_id,enabled,record) VALUES ($1,$2,$3,$4,$5::jsonb)
+            ON CONFLICT (id) DO UPDATE SET scope_type=EXCLUDED.scope_type,scope_id=EXCLUDED.scope_id,enabled=EXCLUDED.enabled,record=EXCLUDED.record
+            WHERE NOT "LiteLLM_UsageAlertRule".deleted
+                AND ("LiteLLM_UsageAlertRule".record->>'updated_at')::timestamptz=$6::text::timestamptz
+                AND "LiteLLM_UsageAlertRule".scope_type=$7
+                AND "LiteLLM_UsageAlertRule".scope_id IS NOT DISTINCT FROM $8::text
+            RETURNING record""",
+            rule.id,
+            rule.scope_type,
+            rule.scope_id,
+            rule.enabled,
+            json.dumps({**rule.model_dump(mode="json"), "overrides_defaults": overrides_defaults}),
+            existing.updated_at.isoformat() if existing else None,
+            existing.scope_type if existing else None,
+            existing.scope_id if existing else None,
+        )
+        saved: Final = TypeAdapter(tuple[_Record, ...]).validate_python(rows)
+        return _rule(saved[0].record) if saved else None
+
+    async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool:
+        return bool(
+            await self.database.query_raw(
+                """UPDATE "LiteLLM_UsageAlertRule" SET deleted=true,enabled=false
+            WHERE id=$1 AND NOT deleted AND ($2::text::timestamptz IS NULL OR (
+                (record->>'updated_at')::timestamptz=$2::text::timestamptz
+                AND scope_type=$3 AND scope_id IS NOT DISTINCT FROM $4::text)) RETURNING id""",
+                rule_id,
+                existing.updated_at.isoformat() if existing else None,
+                existing.scope_type if existing else None,
+                existing.scope_id if existing else None,
+            )
+        )
+
+    async def list_rules(
+        self,
+        scope_type: UsageAlertScope | None = None,
+        scope_id: str | None = None,
+        limit: int = 50,
+        offset: int = 0,
+        kind: UsageAlertRuleKind = "all",
+    ) -> UsageAlertRuleList:
+        args: Final = (scope_type, scope_id, json.dumps(tuple(sorted(_DEFAULT_RULE_IDS))), kind)
+        where: Final = "NOT deleted AND ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2)"
+        builtin: Final = "id IN (SELECT jsonb_array_elements_text($3::jsonb))"
+        count: Final = TypeAdapter(tuple[_RuleCounts, ...]).validate_python(
+            await self.database.query_raw(
+                f"""WITH scoped AS (
+                    SELECT enabled,record,({builtin}) AS builtin FROM "LiteLLM_UsageAlertRule" WHERE {where}
+                ) SELECT count(*) FILTER (WHERE $4='all' OR builtin=($4='default'))::int AS count,
+                    jsonb_build_object('total',count(*) FILTER (WHERE builtin),
+                        'enabled',count(*) FILTER (WHERE builtin AND enabled),
+                        'evaluation_status_counts',jsonb_build_object(
+                            'pending',count(*) FILTER (WHERE builtin AND enabled AND COALESCE(record->>'evaluation_status','pending')='pending'),
+                            'complete',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='complete'),
+                            'insufficient_coverage',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='insufficient_coverage'),
+                            'insufficient_history',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='insufficient_history')
+                        )) AS default_summary FROM scoped""",
+                *args,
+            )
+        )
+        rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
+            await self.database.query_raw(
+                f"SELECT record FROM \"LiteLLM_UsageAlertRule\" WHERE {where} AND ($4='all' OR ({builtin})=($4='default')) ORDER BY id LIMIT $5 OFFSET $6",
+                *args,
+                limit,
+                offset,
+            )
+        )
+        return UsageAlertRuleList(
+            data=tuple(_rule(row.record) for row in rows),
+            total=count[0].count,
+            limit=limit,
+            offset=offset,
+            default_summary=count[0].default_summary,
+        )
+
+    async def list_enabled_rules(self, limit: int = 100, offset: int = 0) -> tuple[UsageAlertRule, ...]:
+        rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
+            await self.database.query_raw(
+                'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE enabled AND NOT deleted ORDER BY id LIMIT $1 OFFSET $2',
+                limit,
+                offset,
+            )
+        )
+        return tuple(UsageAlertRule.model_validate(row.record) for row in rows)
+
+    async def get_alert(self, alert_id: str) -> UsageAlert | None:
+        rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
+            await self.database.query_raw('SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE id=$1', alert_id)
+        )
+        return UsageAlert.model_validate(rows[0].record) if rows else None
+
+    async def list_alerts(
+        self,
+        scope_type: UsageAlertScope | None = None,
+        scope_id: str | None = None,
+        metric: UsageAlertMetric | None = None,
+        status: UsageAlertStatus | None = None,
+        start_date: date | None = None,
+        end_date: date | None = None,
+        limit: int = 50,
+        offset: int = 0,
+    ) -> UsageAlertList:
+        args: Final = (
+            scope_type,
+            scope_id,
+            metric,
+            status,
+            start_date.isoformat() if start_date else None,
+            end_date.isoformat() if end_date else None,
+        )
+        where: Final = "($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2) AND ($3::text IS NULL OR metric=$3) AND ($4::text IS NULL OR status=$4) AND ($5::text IS NULL OR day >= $5) AND ($6::text IS NULL OR day <= $6)"
+        count: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
+            await self.database.query_raw(
+                f'SELECT count(*)::int AS count FROM "LiteLLM_UsageAlertIncident" WHERE {where}', *args
+            )
+        )
+        rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
+            await self.database.query_raw(
+                f'SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE {where} ORDER BY day DESC,id LIMIT $7 OFFSET $8',
+                *args,
+                limit,
+                offset,
+            )
+        )
+        return UsageAlertList(
+            data=tuple(UsageAlert.model_validate(row.record) for row in rows),
+            total=count[0].count,
+            limit=limit,
+            offset=offset,
+        )
+
+    async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None:
+        rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
+            await self.database.query_raw(
+                """UPDATE "LiteLLM_UsageAlertIncident" SET status=$2, record=jsonb_set(record,'{status}',to_jsonb($2::text)),updated_at=CURRENT_TIMESTAMP WHERE id=$1 RETURNING record""",
+                alert_id,
+                status,
+            )
+        )
+        return UsageAlert.model_validate(rows[0].record) if rows else None
+
+    async def list_groups(
+        self,
+        scope_type: UsageAlertScope | None = None,
+        scope_id: str | None = None,
+        status: UsageAlertStatus | None = None,
+        start_date: date | None = None,
+        end_date: date | None = None,
+        limit: int = 50,
+        offset: int = 0,
+    ) -> UsageAlertGroupList:
+        rows: Final = TypeAdapter(tuple[_GroupPage, ...]).validate_python(
+            await self.database.query_raw(
+                """WITH groups AS MATERIALIZED (
+                    SELECT scope_type,scope_id,day,count(*)::int AS incident_count FROM "LiteLLM_UsageAlertIncident"
+                    WHERE ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2)
+                        AND ($3::text IS NULL OR status=$3) AND ($4::text IS NULL OR day >= $4)
+                        AND ($5::text IS NULL OR day <= $5)
+                    GROUP BY scope_type,scope_id,day
+                ), page AS MATERIALIZED (
+                    SELECT * FROM groups ORDER BY day DESC,scope_type,scope_id NULLS FIRST LIMIT $6 OFFSET $7
+                ), selected AS (
+                    SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i
+                        ON i.scope_type=p.scope_type AND i.scope_id=p.scope_id AND i.day=p.day
+                    WHERE ($3::text IS NULL OR i.status=$3)
+                    UNION ALL
+                    SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i
+                        ON i.scope_type=p.scope_type AND i.scope_id IS NULL AND i.day=p.day
+                    WHERE p.scope_id IS NULL AND ($3::text IS NULL OR i.status=$3)
+                ), ranked AS (
+                    SELECT scope_type,scope_id,day,metric,record,
+                        count(*) OVER (PARTITION BY scope_type,scope_id,day,metric)::int AS incident_count,
+                        row_number() OVER (PARTITION BY scope_type,scope_id,day,metric
+                            ORDER BY record->>'first_seen' DESC,id) AS rank
+                    FROM selected
+                ), packed AS (
+                    SELECT p.scope_type,p.scope_id,p.day,p.incident_count,
+                        jsonb_agg(jsonb_build_object('metric',r.metric,'incident_count',r.incident_count,
+                            'representative',r.record) ORDER BY CASE r.metric
+                                WHEN 'requests' THEN 0 WHEN 'spend' THEN 1 WHEN 'error_rate' THEN 2
+                                WHEN 'errors' THEN 3 ELSE 4 END) AS metrics
+                    FROM page p JOIN ranked r ON p.scope_type=r.scope_type
+                        AND p.scope_id IS NOT DISTINCT FROM r.scope_id AND p.day=r.day AND r.rank=1
+                    GROUP BY p.scope_type,p.scope_id,p.day,p.incident_count
+                ) SELECT (SELECT count(*)::int FROM groups) AS total,
+                    COALESCE((SELECT jsonb_agg(to_jsonb(packed) ORDER BY day DESC,scope_type,scope_id NULLS FIRST)
+                        FROM packed),'[]'::jsonb) AS data""",
+                scope_type,
+                scope_id,
+                status,
+                start_date.isoformat() if start_date else None,
+                end_date.isoformat() if end_date else None,
+                limit,
+                offset,
+            )
+        )
+        page: Final = rows[0]
+        labels: Final[Mapping[tuple[str, str | None], str | None]] = (
+            await self.resource_labels(page.data) if page.data else {}
+        )
+        return UsageAlertGroupList(
+            data=tuple(
+                UsageAlertGroup(
+                    id=str(
+                        uuid5(
+                            NAMESPACE_URL,
+                            json.dumps(
+                                ["litellm:performance-group", row.scope_type, row.scope_id, row.day.isoformat()]
+                            ),
+                        )
+                    ),
+                    scope_type=row.scope_type,
+                    scope_id=row.scope_id,
+                    scope_label=labels.get((row.scope_type, row.scope_id)),
+                    day=row.day,
+                    incident_count=row.incident_count,
+                    metrics=row.metrics,
+                    activity_path=row.metrics[0].representative.activity_path,
+                )
+                for row in page.data
+            ),
+            total=page.total,
+            limit=limit,
+            offset=offset,
+        )
+
+    async def resource_labels(
+        self, groups: tuple[ResourceLabelScope, ...]
+    ) -> Mapping[tuple[str, str | None], str | None]:
+        rows: Final = TypeAdapter(tuple[_ResourceLabel, ...]).validate_python(
+            await self.database.query_raw(
+                """WITH scopes AS (
+                    SELECT DISTINCT scope_type,scope_id,key_id FROM jsonb_to_recordset($1::jsonb)
+                    AS x(scope_type text,scope_id text,key_id text)
+                ) SELECT s.scope_type,s.scope_id,LEFT(CASE s.scope_type
+                    WHEN 'key' THEN COALESCE(NULLIF(k.key_alias,''),NULLIF(k.key_name,''))
+                    WHEN 'team' THEN NULLIF(t.team_alias,'')
+                    WHEN 'user' THEN COALESCE(NULLIF(u.user_alias,''),NULLIF(u.user_email,''))
+                    WHEN 'model' THEN s.scope_id WHEN 'provider' THEN s.scope_id
+                    WHEN 'deployment' THEN 'Gateway' END,256) AS label
+                FROM scopes s LEFT JOIN "LiteLLM_VerificationToken" k ON s.scope_type='key' AND k.token=s.key_id
+                    LEFT JOIN "LiteLLM_TeamTable" t ON s.scope_type='team' AND t.team_id=s.scope_id
+                    LEFT JOIN "LiteLLM_UserTable" u ON s.scope_type='user' AND u.user_id=s.scope_id""",
+                json.dumps(
+                    tuple(
+                        {
+                            "scope_type": row.scope_type,
+                            "scope_id": row.scope_id,
+                            "key_id": api_key_from_bucket(row.scope_id)
+                            if row.scope_type == "key" and row.scope_id
+                            else None,
+                        }
+                        for row in groups
+                    )
+                ),
+            )
+        )
+        return MappingProxyType({(row.scope_type, row.scope_id): row.label for row in rows})
+
+    async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult:
+        rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
+            await self.database.query_raw(
+                """WITH changed AS (
+                    UPDATE "LiteLLM_UsageAlertIncident" SET status=$5,
+                        record=jsonb_set(record,'{status}',to_jsonb($5::text)),updated_at=CURRENT_TIMESTAMP
+                    WHERE scope_type=$1 AND scope_id IS NOT DISTINCT FROM $2::text AND day=$3 AND status=$4
+                    RETURNING id
+                ) SELECT count(*)::int AS count FROM changed""",
+                data.scope_type,
+                data.scope_id,
+                data.day.isoformat(),
+                data.expected_status,
+                data.status,
+            )
+        )
+        return UsageAlertGroupUpdateResult(updated_count=rows[0].count)
+
+    async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList:
+        today: Final = datetime.now(timezone.utc).date()
+        rows: Final = TypeAdapter(tuple[_ModelIdentity, ...]).validate_python(
+            await self.database.query_raw(
+                """SELECT DISTINCT model FROM "LiteLLM_DailyUserSpend"
+                WHERE date >= $1 AND date <= $2 AND model IS NOT NULL AND model <> ''
+                    AND ($3::text IS NULL OR strpos(lower(model),lower($3))>0)
+                    AND ($4::text IS NULL OR model>$4)
+                ORDER BY model LIMIT $5""",
+                (today - timedelta(days=29)).isoformat(),
+                today.isoformat(),
+                search,
+                cursor,
+                limit + 1,
+            )
+        )
+        selected: Final = rows[:limit]
+        return UsageAlertModelList(
+            data=tuple(UsageAlertModelOption(scope_id=row.model, label=row.model) for row in selected),
+            limit=limit,
+            next_cursor=selected[-1].model if len(rows) > limit else None,
+            start_date=today - timedelta(days=29),
+            end_date=today,
+        )
+
+    async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None:
+        rows: Final = TypeAdapter(tuple[_Ownership, ...]).validate_python(
+            await self.database.query_raw(
+                """SELECT user_id,team_id FROM "LiteLLM_VerificationToken" WHERE token=$1
+                UNION ALL
+                (SELECT user_id,team_id FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1
+                    AND NOT EXISTS(SELECT 1 FROM "LiteLLM_VerificationToken" WHERE token=$1)
+                    ORDER BY deleted_at DESC,id DESC LIMIT 1)""",
+                key_id,
+            )
+        )
+        return (rows[0].user_id, rows[0].team_id) if rows else None
+
+    async def claim_job(self, owner: str, lease_seconds: int = 900) -> bool:
+        rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
+            await self.database.query_raw(
+                """INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,owner,expires_at) VALUES ('worker',$1,CURRENT_TIMESTAMP+($2::int * INTERVAL '1 second'))
+            ON CONFLICT(id) DO UPDATE SET owner=EXCLUDED.owner,expires_at=EXCLUDED.expires_at WHERE "LiteLLM_UsageAlertCheckpoint".expires_at<CURRENT_TIMESTAMP OR "LiteLLM_UsageAlertCheckpoint".owner=$1 RETURNING 1 AS count""",
+                owner,
+                lease_seconds,
+            )
+        )
+        return bool(rows)
+
+    async def release_job(self, owner: str) -> None:
+        await self.database.query_raw(
+            "DELETE FROM \"LiteLLM_UsageAlertCheckpoint\" WHERE id='worker' AND owner=$1 RETURNING id", owner
+        )
+
+    @staticmethod
+    def _checkpoint(rule: UsageAlertRule, day: date) -> str:
+        return f"{rule.id}:{rule.updated_at.isoformat()}:{day.isoformat()}"
+
+    async def is_day_complete(self, rule: UsageAlertRule, day: date) -> bool:
+        rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
+            await self.database.query_raw(
+                'SELECT 1 AS count FROM "LiteLLM_UsageAlertCheckpoint" WHERE id=$1 AND completed',
+                self._checkpoint(rule, day),
+            )
+        )
+        return bool(rows)
+
+    async def finish_day(self, rule: UsageAlertRule, day: date) -> None:
+        await self.database.query_raw(
+            'INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed) VALUES ($1,true) ON CONFLICT(id) DO NOTHING RETURNING id',
+            self._checkpoint(rule, day),
+        )
+
+    async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None:
+        for rule in rules:
+            await self._ensure_default_rule(rule)
+
+    async def _ensure_default_rule(self, rule: UsageAlertRuleInput) -> None:
+        rule_id: Final = str(
+            uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}")
+        )
+        now: Final = datetime.now(timezone.utc)
+        record: Final = UsageAlertRule.model_validate(
+            {**rule.model_dump(), "id": rule_id, "created_at": now, "updated_at": now, "created_by": None}
+        )
+        await self.database.query_raw(
+            'INSERT INTO "LiteLLM_UsageAlertRule" (id,scope_type,scope_id,enabled,record) VALUES ($1,$2,$3,$4,$5::jsonb) ON CONFLICT(id) DO NOTHING RETURNING id',
+            rule_id,
+            rule.scope_type,
+            rule.scope_id,
+            rule.enabled,
+            record.model_dump_json(),
+        )
+
+    async def list_breached_entity_ids(self, rule_id: str, limit: int = 500, offset: int = 0) -> tuple[str | None, ...]:
+        rows: Final = TypeAdapter(tuple[Mapping[str, str | None], ...]).validate_python(
+            await self.database.query_raw(
+                """SELECT DISTINCT i.scope_id FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id WHERE i.rule_id=$1 AND i.breached AND i.scope_type=r.scope_type AND i.metric=r.record->>'metric' AND i.record->>'direction'=r.record->>'direction' AND (r.scope_id IS NULL OR i.scope_id=r.scope_id) ORDER BY i.scope_id NULLS FIRST LIMIT $2 OFFSET $3""",
+                rule_id,
+                limit,
+                offset,
+            )
+        )
+        return tuple(row["scope_id"] for row in rows)
+
+    async def record_evaluation_status(
+        self, rule: UsageAlertRule, day: date, status: UsageAlertEvaluationStatus
+    ) -> None:
+        await self.database.query_raw(
+            """UPDATE "LiteLLM_UsageAlertRule" SET record=record || jsonb_build_object('last_evaluated_day',$2::text,'evaluation_status',$3::text) WHERE id=$1 AND (record->>'updated_at')::timestamptz=$4::timestamptz AND NOT deleted RETURNING id""",
+            rule.id,
+            day.isoformat(),
+            status,
+            rule.updated_at.isoformat(),
+        )
+
+    async def record_observation(
+        self,
+        rule: UsageAlertRule,
+        entity_id: str | None,
+        day: date,
+        observed: float,
+        baseline: float,
+        threshold: float,
+        breached: bool,
+    ) -> UsageAlert | None:
+        observation: Final = UsageAlertObservation(
+            entity_id=entity_id, day=day, observed=observed, baseline=baseline, threshold=threshold, breached=breached
+        )
+        changed: Final = await self.record_observations(rule, (observation,))
+        if changed:
+            return changed[0]
+        rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
+            await self.database.query_raw(
+                """SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=$2 AND scope_type=$3 AND metric=$4 AND record->>'direction'=$5 ORDER BY evaluation_day DESC,id DESC LIMIT 1""",
+                rule.id,
+                entity_id or "",
+                rule.scope_type,
+                rule.metric,
+                rule.direction,
+            )
+        )
+        return UsageAlert.model_validate(rows[0].record) if rows else None
+
+    async def record_observations(
+        self, rule: UsageAlertRule, observations: tuple[UsageAlertObservation, ...]
+    ) -> tuple[UsageAlert, ...]:
+        if not observations:
+            return ()
+        if len(observations) > 500 or len({item.entity_id for item in observations}) != len(observations):
+            raise ValueError("An observation batch must contain at most 500 distinct entities")
+        now: Final = datetime.now(timezone.utc)
+        records: Final = tuple(_observation_payload(rule, item, now) for item in observations)
+        rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python(
+            await self.database.query_raw(
+                """
+            WITH input AS (SELECT * FROM jsonb_to_recordset($2::jsonb) AS x(scope_key text, breached boolean, day text, record jsonb)),
+            prior AS (
+                SELECT x.*,p.id AS prior_id,p.evaluation_day,p.evaluation_revision AS prior_revision,
+                    p.breached AS prior_breached,p.record AS prior_record
+                FROM input x LEFT JOIN LATERAL (
+                    SELECT * FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=x.scope_key
+                      AND scope_type=x.record->>'scope_type' AND scope_id IS NOT DISTINCT FROM x.record->>'scope_id'
+                      AND metric=x.record->>'metric' AND record->>'direction'=x.record->>'direction'
+                    ORDER BY evaluation_day DESC,id DESC LIMIT 1
+                ) p ON true
+                WHERE EXISTS (SELECT 1 FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND enabled AND NOT deleted AND (record->>'updated_at')::timestamptz=$3::timestamptz)
+            ), updated AS (
+                UPDATE "LiteLLM_UsageAlertIncident" i SET
+                    evaluation_day=p.day, evaluation_revision=$3::text, breached=p.breached,
+                    record=i.record || jsonb_build_object(
+                        'last_seen',CASE WHEN p.breached THEN p.record->'last_seen' ELSE i.record->'last_seen' END,
+                        'status',CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END),
+                    status=CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END,
+                    updated_at=CURRENT_TIMESTAMP
+                FROM prior p WHERE i.id=p.prior_id
+                    AND (p.evaluation_day<p.day OR (p.evaluation_day=p.day AND p.prior_revision IS DISTINCT FROM $3))
+                    AND (NOT p.breached OR p.prior_breached)
+                RETURNING i.record
+            ), inserted AS (
+                INSERT INTO "LiteLLM_UsageAlertIncident" (id,rule_id,scope_key,scope_type,scope_id,metric,status,day,evaluation_day,evaluation_revision,breached,record)
+                SELECT p.record->>'id',$1,p.scope_key,p.record->>'scope_type',p.record->>'scope_id',p.record->>'metric','open',p.day,p.day,$3,true,p.record
+                FROM prior p WHERE p.breached AND (p.prior_id IS NULL OR (NOT p.prior_breached AND
+                    (p.evaluation_day<p.day OR (p.evaluation_day=p.day AND p.prior_revision IS DISTINCT FROM $3))))
+                ON CONFLICT(id) DO NOTHING RETURNING record
+            ) SELECT record FROM updated UNION ALL SELECT record FROM inserted
+            """,
+                rule.id,
+                json.dumps(records),
+                rule.updated_at.isoformat(),
+            )
+        )
+        return tuple(UsageAlert.model_validate(row.record) for row in rows)
+
+    async def pending_notifications(self, limit: int = 100) -> tuple[UsageAlertNotification, ...]:
+        rows: Final = TypeAdapter(tuple[_Pending, ...]).validate_python(
+            await self.database.query_raw(
+                """SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id
+            WHERE i.status='open' AND r.enabled AND NOT r.deleted AND EXISTS (
+                SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels)))
+            ORDER BY i.updated_at,i.id LIMIT $1""",
+                limit,
+            )
+        )
+        return tuple(
+            UsageAlertNotification(
+                incident=UsageAlert.model_validate(row.record),
+                rule=UsageAlertRule.model_validate(row.rule),
+                pending_channels=tuple(
+                    channel
+                    for channel in UsageAlertRule.model_validate(row.rule).notification_channels
+                    if channel not in row.notified_channels
+                ),
+            )
+            for row in rows
+        )
+
+    async def mark_notified(self, incident_id: str, channel: UsageAlertChannel) -> None:
+        await self.database.query_raw(
+            'UPDATE "LiteLLM_UsageAlertIncident" SET notified_channels=array_append(notified_channels,$2::text) WHERE id=$1 AND NOT ($2::text=ANY(notified_channels)) RETURNING id',
+            incident_id,
+            channel,
+        )
+
+    async def prune(self, retention_days: int = 90, limit: int = 500) -> None:
+        await self.database.query_raw(
+            """DELETE FROM "LiteLLM_UsageAlertIncident" WHERE id IN (SELECT id FROM "LiteLLM_UsageAlertIncident" WHERE status<>'open' AND updated_at<CURRENT_TIMESTAMP-($1::int * INTERVAL '1 day') ORDER BY updated_at LIMIT $2) RETURNING id""",
+            retention_days,
+            limit,
+        )
+        await self.database.query_raw(
+            """DELETE FROM "LiteLLM_UsageAlertCheckpoint" WHERE id IN (SELECT id FROM "LiteLLM_UsageAlertCheckpoint" WHERE completed AND expires_at<CURRENT_TIMESTAMP-INTERVAL '45 days' ORDER BY expires_at LIMIT $1) RETURNING id""",
+            limit,
+        )
+
+    async def team_role(self, team_id: str, user_id: str) -> str | None:
+        rows: Final = TypeAdapter(tuple[Mapping[str, str], ...]).validate_python(
+            await self.database.query_raw(
+                """SELECT member->>'role' AS role FROM "LiteLLM_TeamTable", jsonb_array_elements(members_with_roles) member WHERE team_id=$1 AND member->>'user_id'=$2 LIMIT 1""",
+                team_id,
+                user_id,
+            )
+        )
+        return rows[0]["role"] if rows else None

litellm/types/proxy/issues.py

@@ -0,0 +1,280 @@
+from datetime import date, datetime
+from typing import Literal, TypeAlias
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from typing_extensions import Self
+
+from litellm.types.proxy.management_endpoints.common_daily_activity import (
+    BoundedDailyActivityResponse,
+)
+from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo
+from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily
+from litellm.types.proxy.usage_alerts import (
+    UsageAlert,
+    UsageAlertChannel,
+    UsageAlertDirection,
+    UsageAlertEvaluationCounts,
+    UsageAlertMetric,
+    UsageAlertRule,
+    UsageAlertStatus,
+)
+
+IssueActivityResponse: TypeAlias = BoundedDailyActivityResponse
+
+IssueKind: TypeAlias = Literal["performance", "model_deprecation"]
+IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"]
+IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"]
+IssueGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"]
+IssueSignalGroupingReason: TypeAlias = Literal[
+    "same_resource", "same_key_traffic", "dominant_key", "provider_cohort", "volume_explains_spend", "model_deprecation"
+]
+
+
+class IssueModelPair(BaseModel, frozen=True):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    provider: str = Field(min_length=1, max_length=256)
+    model: str = Field(min_length=1, max_length=1024)
+
+
+class IssueCohort(BaseModel, frozen=True):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    scope_type: IssueScope
+    scope_id: str | None = Field(default=None, min_length=1, max_length=1024)
+    model_pairs: tuple[IssueModelPair, ...] = Field(default=(), max_length=100)
+
+    @model_validator(mode="after")
+    def validate_scope(self) -> Self:
+        if (self.scope_type == "deployment") != (self.scope_id is None):
+            raise ValueError("Only a deployment cohort omits scope_id")
+        if self.scope_type == "key" and self.scope_id and self.scope_id.removeprefix("raw:").startswith("sk-"):
+            raise ValueError("Use a reporting key ID, never a secret key")
+        if len(frozenset((pair.provider, pair.model) for pair in self.model_pairs)) != len(self.model_pairs):
+            raise ValueError("model_pairs must be unique")
+        return self
+
+
+class IssueComparison(BaseModel, frozen=True):
+    model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False)
+
+    metric: UsageAlertMetric
+    direction: UsageAlertDirection
+    observed_value: float = Field(ge=0)
+    baseline_value: float = Field(ge=0)
+    threshold: float | None = None
+    baseline_start: date
+    baseline_end: date
+    baseline_sample_days: int | None = Field(default=None, ge=1, le=30)
+
+    @model_validator(mode="after")
+    def validate_comparison(self) -> Self:
+        if self.baseline_start > self.baseline_end:
+            raise ValueError("baseline_start must not follow baseline_end")
+        if self.metric == "error_rate" and (self.observed_value > 100 or self.baseline_value > 100):
+            raise ValueError("error_rate uses percentages from 0 to 100")
+        return self
+
+
+class IssueImpact(BaseModel, frozen=True):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    keys: int | None = Field(default=None, ge=0)
+    users: int | None = Field(default=None, ge=0)
+    teams: int | None = Field(default=None, ge=0)
+    models: int | None = Field(default=None, ge=0)
+    requests: int | None = Field(default=None, ge=0)
+    measured_keys: int | None = Field(default=None, ge=0)
+    measured_models: int | None = Field(default=None, ge=0)
+
+
+class IssueDeprecation(BaseModel, frozen=True):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    model: ModelDeprecationInfo
+    usage_model: str = Field(min_length=1, max_length=1024)
+    requests: int = Field(ge=0)
+    last_used_date: date
+    last_day_requests: int = Field(ge=0)
+    state: DeprecationAlertState
+
+
+class IssueInput(BaseModel, frozen=True):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    id: str = Field(min_length=1, max_length=200)
+    cohort_key: str = Field(min_length=1, max_length=200)
+    kind: IssueKind
+    family: IssueFamily
+    title: str = Field(min_length=1, max_length=2048)
+    cohort: IssueCohort
+    observed_on: date
+    observed_at: datetime
+    range_start: date
+    range_end: date
+    grouping_reason: IssueGroupingReason
+    explanation: str = Field(default="", max_length=2000)
+    comparisons: tuple[IssueComparison, ...] = Field(default=(), max_length=10)
+    impact: IssueImpact = Field(default_factory=IssueImpact)
+    deprecation: IssueDeprecation | None = None
+
+    @model_validator(mode="after")
+    def validate_issue(self) -> Self:
+        if self.range_start > self.range_end:
+            raise ValueError("range_start must not follow range_end")
+        if (self.kind == "model_deprecation") != (self.deprecation is not None):
+            raise ValueError("Model deprecation issues require deprecation details")
+        if (self.kind == "model_deprecation") != (self.family == "retirement"):
+            raise ValueError("Only model deprecation issues use the retirement family")
+        if self.observed_at.tzinfo is None:
+            raise ValueError("observed_at must include a timezone")
+        return self
+
+
+class Issue(IssueInput, frozen=True):
+    status: UsageAlertStatus
+    first_seen: datetime
+    last_seen: datetime
+    updated_at: datetime
+    scope_label: str | None = None
+    active: bool = True
+    can_update_status: bool | None = Field(
+        default=None, description="Current reader's status permission, evaluated by the issue-detail endpoint"
+    )
+    can_view_activity: bool | None = Field(
+        default=None, description="Current reader's permission to inspect Activity, keys, users and impact"
+    )
+
+
+class IssueList(BaseModel, frozen=True):
+    data: tuple[Issue, ...]
+    total: int
+    limit: int
+    offset: int
+
+
+class IssueUpdate(BaseModel, frozen=True):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    status: UsageAlertStatus
+    expected_status: UsageAlertStatus
+
+
+class IssueSignalInput(BaseModel, frozen=True):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    source: Literal["usage_alert", "model_deprecation"]
+    source_id: str = Field(min_length=1, max_length=200)
+    usage_alert: UsageAlert | None = None
+    deprecation: IssueDeprecation | None = None
+    grouping_reason: IssueSignalGroupingReason = "same_resource"
+    explanation: str = Field(default="", max_length=2000)
+
+    @model_validator(mode="after")
+    def validate_signal(self) -> Self:
+        if self.source == "usage_alert":
+            if self.usage_alert is None or self.deprecation is not None or self.usage_alert.id != self.source_id:
+                raise ValueError("Usage signals require their matching source alert")
+        elif self.deprecation is None or self.usage_alert is not None:
+            raise ValueError("Deprecation signals require deprecation details")
+        return self
+
+
+class IssueSignal(IssueSignalInput, frozen=True):
+    issue_id: str
+
+
+class IssueSignalList(BaseModel, frozen=True):
+    data: tuple[IssueSignal, ...]
+    total: int
+    limit: int
+    offset: int
+
+
+class IssueProjectionSignal(BaseModel, frozen=True):
+    alert: UsageAlert
+    rule: UsageAlertRule
+    evaluation_day: date
+
+
+class IssueDeliveryInput(BaseModel, frozen=True):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    id: str = Field(min_length=1, max_length=200)
+    issue_id: str = Field(min_length=1, max_length=200)
+    channel: UsageAlertChannel
+    email_recipients: tuple[str, ...] = Field(default=(), max_length=20)
+
+
+class IssueDeliveryBinding(BaseModel, frozen=True):
+    delivery: IssueDeliveryInput
+    rule_id: str
+    rule_revision: str
+    source_id: str
+
+
+class IssueDelivery(IssueDeliveryInput, frozen=True):
+    attempts: int = 0
+    next_attempt: datetime | None = None
+    last_error: Literal["delivery_failed", "transport_not_configured"] | None = None
+
+
+class IssuePendingDelivery(BaseModel, frozen=True):
+    delivery: IssueDelivery
+    issue: Issue
+    rule: UsageAlertRule
+    alert: UsageAlert
+
+
+class IssueFamilyPolicy(BaseModel, frozen=True):
+    direction: UsageAlertDirection
+    enabled: bool | None
+    multiplier: float | None
+    min_value: float | None
+    min_requests: int | None
+    baseline_days: int | None
+    min_history_days: int | None
+    mixed: bool
+    total_rules: int
+    enabled_rules: int
+
+
+class IssueFamilySetting(BaseModel, frozen=True):
+    family: PerformanceIssueFamily
+    enabled: bool | None
+    total_rules: int
+    enabled_rules: int
+    policies: tuple[IssueFamilyPolicy, ...]
+    evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts)
+
+
+class IssueFamilyList(BaseModel, frozen=True):
+    data: tuple[IssueFamilySetting, ...]
+    projection_pending: bool = False
+
+
+class IssueFamilyPolicyUpdate(BaseModel, frozen=True):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    direction: UsageAlertDirection
+    enabled: bool | None = None
+    multiplier: float | None = Field(default=None, gt=1, le=1000)
+    min_value: float | None = Field(default=None, ge=0)
+    min_requests: int | None = Field(default=None, ge=0)
+    baseline_days: int | None = Field(default=None, ge=1, le=30)
+    min_history_days: int | None = Field(default=None, ge=1, le=30)
+
+
+class IssueFamilyUpdate(BaseModel, frozen=True):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    enabled: bool | None = None
+    policies: tuple[IssueFamilyPolicyUpdate, ...] = Field(default=(), max_length=2)
+
+    @model_validator(mode="after")
+    def validate_update(self) -> Self:
+        if len(frozenset(policy.direction for policy in self.policies)) != len(self.policies):
+            raise ValueError("Each direction can occur once")
+        if self.enabled is None and not self.policies:
+            raise ValueError("Specify enabled or a policy update")
+        return self

litellm/types/proxy/management_endpoints/common_daily_activity.py

@@ -182,6 +182,8 @@ class KeySpendMetrics(LiteLLMBaseModel):
     failed_requests: int = 0
     cache_read_input_tokens: int = 0
     cache_creation_input_tokens: int = 0
+    total_response_time_ms: int = 0
+    timed_requests: int = 0
 
 
 class KeySpendActivityRow(LiteLLMBaseModel):
@@ -214,6 +216,8 @@ class UserActivityRow(LiteLLMBaseModel):
     api_requests: int = 0
     successful_requests: int = 0
     failed_requests: int = 0
+    total_response_time_ms: int = 0
+    timed_requests: int = 0
 
 
 class DailyActivityUserPageResponse(LiteLLMBaseModel):

litellm/types/proxy/model_deprecation.py

@@ -112,3 +112,15 @@ class DeprecationTagUsage(LiteLLMBaseModel):
 
     api_key: str
     tag: str
+
+
+class ModelDeprecationUsage(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    model: str
+    requests: int = Field(ge=0)
+    last_used_date: date
+    last_day_requests: int = Field(ge=0)
+    keys: int = Field(ge=0)
+    users: int = Field(ge=0)
+    teams: int = Field(ge=0)

litellm/types/proxy/usage_alert_defaults.py

@@ -0,0 +1,76 @@
+from dataclasses import dataclass
+from itertools import product
+from typing import Final, Literal, TypeAlias
+from uuid import NAMESPACE_URL, uuid5
+
+from litellm.types.proxy.usage_alerts import (
+    UsageAlertChannel,
+    UsageAlertDirection,
+    UsageAlertMetric,
+    UsageAlertRule,
+    UsageAlertRuleInput,
+    UsageAlertScope,
+)
+
+PerformanceIssueFamily: TypeAlias = Literal["spend", "latency", "reliability", "traffic"]
+USAGE_ALERT_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("deployment", "key", "user", "team", "model")
+
+
+@dataclass(frozen=True, slots=True)
+class DefaultUsagePolicy:
+    family: PerformanceIssueFamily
+    metric: UsageAlertMetric
+    direction: UsageAlertDirection
+    multiplier: float
+    min_value: float
+    min_requests: int = 0
+    min_history_days: int = 3
+
+
+DEFAULT_USAGE_POLICIES: Final = (
+    DefaultUsagePolicy("spend", "spend", "increase", 3, 10),
+    DefaultUsagePolicy("latency", "latency", "increase", 3, 500, 100),
+    DefaultUsagePolicy("reliability", "error_rate", "increase", 3, 2, 500),
+    DefaultUsagePolicy("traffic", "requests", "increase", 10, 1000),
+    DefaultUsagePolicy("traffic", "requests", "decrease", 5, 0, 500, 7),
+)
+
+
+def default_usage_rule_id(scope: UsageAlertScope, metric: UsageAlertMetric, direction: UsageAlertDirection) -> str:
+    return str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{scope}:{metric}:{direction}"))
+
+
+DEFAULT_USAGE_RULE_IDS: Final = frozenset(
+    default_usage_rule_id(scope, policy.metric, policy.direction)
+    for scope, policy in product(USAGE_ALERT_SCOPES, DEFAULT_USAGE_POLICIES)
+)
+
+
+def is_default_usage_rule(rule: UsageAlertRule) -> bool:
+    return rule.id in DEFAULT_USAGE_RULE_IDS and rule.id == default_usage_rule_id(
+        rule.scope_type, rule.metric, rule.direction
+    )
+
+
+def default_usage_rules(
+    channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False
+) -> tuple[UsageAlertRuleInput, ...]:
+    return tuple(
+        UsageAlertRuleInput(
+            name=f"Daily {scope} {policy.metric.replace('_', ' ')} {policy.direction}",
+            scope_type=scope,
+            metric=policy.metric,
+            direction=policy.direction,
+            multiplier=policy.multiplier,
+            min_value=policy.min_value,
+            min_requests=policy.min_requests,
+            min_history_days=policy.min_history_days,
+            notification_channels=(
+                ()
+                if legacy_user_spend_alerts
+                and (scope, policy.metric, policy.direction) == ("user", "spend", "increase")
+                else channels
+            ),
+        )
+        for scope, policy in product(USAGE_ALERT_SCOPES, DEFAULT_USAGE_POLICIES)
+    )

litellm/types/proxy/usage_alerts.py

@@ -0,0 +1,198 @@
+"""Daily usage alert rules and incidents shared by the API and evaluator."""
+
+from datetime import date, datetime
+from typing import Literal, TypeAlias
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from typing_extensions import Self
+
+UsageAlertScope: TypeAlias = Literal["deployment", "key", "user", "team", "model"]
+UsageAlertMetric: TypeAlias = Literal["spend", "requests", "latency", "errors", "error_rate"]
+UsageAlertDirection: TypeAlias = Literal["increase", "decrease"]
+UsageAlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"]
+UsageAlertChannel: TypeAlias = Literal["email", "slack"]
+UsageAlertEvaluationStatus: TypeAlias = Literal["pending", "complete", "insufficient_coverage", "insufficient_history"]
+UsageAlertRuleKind: TypeAlias = Literal["all", "default", "custom"]
+
+
+class UsageAlertRuleInput(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False)
+
+    name: str = Field(min_length=1, max_length=200, pattern=r"^[^\x00-\x1f\x7f]+$")
+    scope_type: UsageAlertScope
+    scope_id: str | None = Field(default=None, min_length=1, max_length=1024)
+    metric: UsageAlertMetric
+    direction: UsageAlertDirection
+    multiplier: float = Field(default=3, gt=1, le=1000)
+    baseline_days: int = Field(default=7, ge=1, le=30)
+    min_history_days: int = Field(default=3, ge=1, le=30)
+    min_requests: int = Field(default=100, ge=0)
+    min_value: float = Field(default=0, ge=0, description="Minimum absolute change in the metric's units")
+    enabled: bool = True
+    notification_channels: tuple[UsageAlertChannel, ...] = Field(default=(), max_length=2)
+    email_recipients: tuple[str, ...] = Field(default=(), max_length=20)
+
+    @model_validator(mode="after")
+    def validate_rule(self) -> Self:
+        if self.min_history_days > self.baseline_days:
+            raise ValueError("min_history_days must not exceed baseline_days")
+        if self.scope_type == "deployment" and self.scope_id is not None:
+            raise ValueError("Deployment alerts cover the whole gateway; omit scope_id")
+        if (
+            self.scope_type == "key"
+            and self.scope_id is not None
+            and self.scope_id.removeprefix("raw:").startswith("sk-")
+        ):
+            raise ValueError("Use the key's reporting ID, never the secret key")
+        if len(set(self.notification_channels)) != len(self.notification_channels):
+            raise ValueError("notification_channels must be unique")
+        if "email" in self.notification_channels and not self.email_recipients:
+            raise ValueError("Email notifications require at least one recipient")
+        if any(
+            "@" not in email or "\n" in email or "\r" in email or len(email) > 254 for email in self.email_recipients
+        ):
+            raise ValueError("Invalid email recipient")
+        return self
+
+
+class UsageAlertRule(UsageAlertRuleInput):
+    overrides_defaults: bool = Field(default=False, exclude=True)
+    id: str
+    created_at: datetime
+    updated_at: datetime
+    created_by: str | None
+    last_evaluated_day: date | None = None
+    evaluation_status: UsageAlertEvaluationStatus = "pending"
+    is_default: bool = False
+
+
+class UsageAlert(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False)
+
+    id: str
+    rule_id: str
+    rule_name: str
+    scope_type: UsageAlertScope
+    scope_id: str | None
+    metric: UsageAlertMetric
+    direction: UsageAlertDirection
+    status: UsageAlertStatus
+    day: date
+    baseline_start: date
+    baseline_end: date
+    baseline_sample_days: int | None = Field(default=None, ge=1, le=30)
+    observed_value: float
+    baseline_value: float
+    threshold: float
+    first_seen: datetime
+    last_seen: datetime
+    activity_path: str
+
+
+class UsageAlertUpdate(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+    status: UsageAlertStatus
+
+
+class UsageAlertEvaluationCounts(BaseModel, frozen=True):
+    pending: int = 0
+    complete: int = 0
+    insufficient_coverage: int = 0
+    insufficient_history: int = 0
+
+
+class UsageAlertDefaultSummary(BaseModel, frozen=True):
+    total: int = 0
+    enabled: int = 0
+    evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts)
+
+
+class UsageAlertRuleList(BaseModel, frozen=True):
+    data: tuple[UsageAlertRule, ...]
+    total: int
+    limit: int
+    offset: int
+    default_summary: UsageAlertDefaultSummary = Field(default_factory=UsageAlertDefaultSummary)
+
+
+class UsageAlertList(BaseModel, frozen=True):
+    data: tuple[UsageAlert, ...]
+    total: int
+    limit: int
+    offset: int
+
+
+class UsageAlertGroupMetric(BaseModel, frozen=True):
+    metric: UsageAlertMetric
+    incident_count: int
+    representative: UsageAlert
+
+
+class UsageAlertGroup(BaseModel, frozen=True):
+    id: str
+    scope_type: UsageAlertScope
+    scope_id: str | None
+    scope_label: str | None
+    day: date
+    incident_count: int
+    metrics: tuple[UsageAlertGroupMetric, ...] = Field(max_length=5)
+    activity_path: str
+
+
+class UsageAlertGroupList(BaseModel, frozen=True):
+    data: tuple[UsageAlertGroup, ...]
+    total: int
+    limit: int
+    offset: int
+
+
+class UsageAlertGroupUpdate(BaseModel, frozen=True):
+    model_config = ConfigDict(extra="forbid")
+    scope_type: UsageAlertScope
+    scope_id: str | None = Field(default=None, min_length=1, max_length=1024)
+    day: date
+    expected_status: UsageAlertStatus
+    status: UsageAlertStatus
+
+    @model_validator(mode="after")
+    def validate_group(self) -> Self:
+        if (self.scope_type == "deployment") != (self.scope_id is None):
+            raise ValueError("Specify a resource ID except for deployment alerts")
+        if self.expected_status == self.status:
+            raise ValueError("The new status must differ from the expected status")
+        return self
+
+
+class UsageAlertGroupUpdateResult(BaseModel, frozen=True):
+    updated_count: int
+
+
+class UsageAlertModelOption(BaseModel, frozen=True):
+    scope_id: str
+    label: str
+
+
+class UsageAlertModelList(BaseModel, frozen=True):
+    data: tuple[UsageAlertModelOption, ...]
+    limit: int
+    next_cursor: str | None
+    start_date: date
+    end_date: date
+
+
+class UsageAlertNotification(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    incident: UsageAlert
+    rule: UsageAlertRule
+    pending_channels: tuple[UsageAlertChannel, ...]
+
+
+class UsageAlertObservation(BaseModel):
+    model_config = ConfigDict(frozen=True, allow_inf_nan=False)
+    entity_id: str | None
+    day: date
+    observed: float
+    baseline: float
+    baseline_sample_days: int | None = Field(default=None, ge=1, le=30)
+    threshold: float
+    breached: bool

litellm/types/repositories/daily_activity.py

@@ -39,6 +39,8 @@ class DailyActivityScope:
     model: str | None
     timezone_offset_minutes: int | None
     include_current_utc_day: bool = False
+    provider: str | None = None
+    model_pairs: tuple[tuple[str, str], ...] = ()
 
     def __post_init__(self) -> None:
         if self.entity_id_field not in _ENTITY_FIELDS[self.table]:
@@ -57,6 +59,8 @@ class KeySpendRow:
     failed_requests: int
     cache_read_input_tokens: int
     cache_creation_input_tokens: int
+    total_response_time_ms: int = 0
+    timed_requests: int = 0
 
 
 @dataclass(frozen=True, slots=True)
@@ -75,6 +79,8 @@ class UserSpendRow:
     api_requests: int
     successful_requests: int
     failed_requests: int
+    total_response_time_ms: int = 0
+    timed_requests: int = 0
 
 
 @dataclass(frozen=True, slots=True)

schema.prisma

@@ -2168,3 +2168,96 @@ model LiteLLM_SubscriptionAccountTable {
 
   @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
 }
+
+model LiteLLM_UsageAlertRule {
+  id String @id
+  scope_type String
+  scope_id String?
+  enabled Boolean @default(true)
+  record Json
+  deleted Boolean @default(false)
+  @@index([enabled, deleted, id])
+  @@index([scope_type, scope_id, deleted])
+}
+
+model LiteLLM_UsageAlertIncident {
+  id String @id
+  rule_id String
+  scope_key String
+  scope_type String
+  scope_id String?
+  metric String
+  status String
+  day String
+  evaluation_day String
+  evaluation_revision String @default("")
+  breached Boolean
+  record Json
+  notified_channels String[] @default([])
+  updated_at DateTime @default(now())
+  @@index([rule_id, scope_key, evaluation_day])
+  @@index([evaluation_day, breached, id])
+  @@index([scope_type, scope_id, day])
+  @@index([status, day])
+  @@index([updated_at])
+}
+
+model LiteLLM_UsageAlertCheckpoint {
+  id String @id
+  completed Boolean @default(false)
+  owner String?
+  expires_at DateTime @default(now())
+  @@index([expires_at])
+}
+
+model LiteLLM_Issue {
+  id String @id
+  cohort_key String
+  kind String
+  family String
+  scope_type String
+  scope_id String?
+  status String
+  active Boolean @default(true)
+  ack_key String?
+  record Json
+  first_seen DateTime
+  last_seen DateTime
+  updated_at DateTime @default(now())
+
+  @@index([status, last_seen, id])
+  @@index([kind, family, last_seen, id])
+  @@index([scope_type, scope_id, last_seen, id])
+  @@index([cohort_key, active])
+}
+
+model LiteLLM_IssueSignal {
+  issue_id String
+  source String
+  source_id String
+  record Json
+
+  @@id([issue_id, source, source_id])
+  @@index([source, source_id])
+}
+
+model LiteLLM_IssueDelivery {
+  id String @id
+  issue_id String
+  record Json
+  sent Boolean @default(false)
+  next_attempt DateTime @default(now())
+  updated_at DateTime @default(now())
+
+  @@index([sent, next_attempt, id])
+  @@index([issue_id])
+}
+
+model LiteLLM_IssueDeliveryRule {
+  delivery_id String
+  rule_id String
+  source_id String
+  rule_revision String
+
+  @@id([delivery_id, rule_id, source_id])
+}

scripts/check_openapi_docs.py

@@ -7,7 +7,7 @@ count them per rule the same delta-vs-base way as the other lint gates. The rule
 OA001  no request body schema (the handler reads ``request.body()`` and never told FastAPI
        what is in it, see ``inline_request_body`` in custom_openapi_spec.py for the fix)
 OA002  no description (the handler has no docstring and the route no ``description=``)
-OA003  the 200 response has no schema (no ``response_model`` and no ``responses={200: ...}``)
+OA003  the 200 or 201 response has no schema (no ``response_model`` or explicit response schema)
 """
 
 from __future__ import annotations
@@ -45,7 +45,8 @@ def _operation_codes(operation: Mapping[str, object]) -> Iterator[str]:
     description: Final = operation.get("description")
     if not isinstance(description, str) or not description.strip():
         yield "OA002"
-    if not _content_schema(_as_mapping(operation.get("responses")).get("200")):
+    responses: Final = _as_mapping(operation.get("responses"))
+    if not _content_schema(responses.get("200", responses.get("201"))):
         yield "OA003"
 
 

scripts/openapi_docs_gate.py

@@ -5,7 +5,7 @@ Sibling of scripts/test_quality_gate.py, pointed at the proxy's OpenAPI document
 instead of a source tree. Each rule is counted over every POST operation the proxy
 serves at HEAD and at the merge-base with the branch this change merges into, and the
 gate fails only when a rule grew past the merge-base count, so a new endpoint has to
-ship with a request schema, a description and a 200 schema while the undocumented
+ship with a request schema, a description and a 200/201 schema while the undocumented
 routes that already exist are only ever allowed to shrink.
 
 The merge-base counts come from scripts/lint_base_counts.py: the disk cache, then the
@@ -167,7 +167,7 @@ def cmd_check(base: str) -> None:
     print(
         "Give every new POST route a request body schema (a typed body parameter, or "
         "`openapi_extra={'requestBody': inline_request_body(Model, example)}` when the handler reads "
-        "`request.body()`), a docstring, and a 200 schema (`response_model=` or `responses={200: {'model': ...}}`), "
+        "`request.body()`), a docstring, and a 200/201 schema (`response_model=` or an explicit response schema), "
         "or document an equal number of existing routes; the ceiling is the merge-base count. Run "
         "`python scripts/check_openapi_docs.py` to see every finding."
     )

tests/integration/database/test_usage_alerts.py

@@ -0,0 +1,693 @@
+from datetime import date, timedelta
+from hashlib import sha256
+from typing import Final
+from uuid import uuid4
+
+import pytest
+from integration._support.actors import ManagementDatabase
+from integration._support.client import Gateway, string_value
+
+from litellm.repositories.usage_alerts import UsageAlertsRepository
+from litellm.types.proxy.usage_alerts import (
+    UsageAlertGroupList,
+    UsageAlertMetric,
+    UsageAlertModelList,
+    UsageAlertObservation,
+    UsageAlertRule,
+    UsageAlertRuleInput,
+)
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_alert_resource_labels_bound_legacy_display_values(db: ManagementDatabase) -> None:
+    repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
+    identity: Final = "label-bound-" + uuid4().hex
+    scope: Final = UsageAlertRuleInput(
+        name="Display label", scope_type="user", scope_id=identity, metric="spend", direction="increase"
+    )
+    await repository.database.query_raw(
+        'INSERT INTO "LiteLLM_UserTable" (user_id,user_alias) VALUES ($1,repeat(chr(128640),65536))', identity
+    )
+    try:
+        labels: Final = await repository.resource_labels((scope,))
+        assert labels == {("user", identity): chr(128640) * 256}
+        stored: Final = await repository.database.query_raw(
+            'SELECT length(user_alias) AS characters FROM "LiteLLM_UserTable" WHERE user_id=$1', identity
+        )
+        assert stored[0]["characters"] == 65536
+    finally:
+        await repository.database.query_raw('DELETE FROM "LiteLLM_UserTable" WHERE user_id=$1', identity)
+
+
+def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gateway: Gateway) -> None:
+    with gateway.scenario() as scenario:
+        owner: Final = scenario.user(user_role="internal_user")
+        outsider: Final = scenario.user(user_role="internal_user")
+        viewer: Final = scenario.user(user_role="proxy_admin_viewer")
+        owner_key: Final = scenario.key(user_id=owner, key_type="management")
+        outsider_key: Final = scenario.key(user_id=outsider, key_type="management")
+        viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only")
+        inference_key: Final = scenario.key(user_id=owner, key_type="llm_api")
+        config: Final = {
+            "name": "Daily key spend",
+            "scope_type": "key",
+            "scope_id": sha256(inference_key.encode()).hexdigest(),
+            "metric": "spend",
+            "direction": "increase",
+            "multiplier": 3,
+        }
+        created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key)
+        assert created.status_code == 201, created.text
+        rule_id: Final = string_value(created.json()["id"])
+        path: Final = f"/usage/alert-rules/{rule_id}"
+        try:
+            assert gateway.request("GET", path, key=owner_key).status_code == 200
+            assert gateway.request("GET", path, key=outsider_key).status_code == 403
+            assert gateway.request("GET", path, key=inference_key).status_code == 403
+            assert gateway.request("GET", path, key=viewer_key).status_code == 200
+            assert gateway.request("PUT", path, config, key=viewer_key).status_code == 403
+            assert gateway.request("PUT", path, config, key=outsider_key).status_code == 403
+            assert (
+                gateway.request("POST", "/usage/alert-rules", {**config, "scope_id": None}, key=owner_key).status_code
+                == 403
+            )
+            disabled: Final = gateway.request("PUT", path, {**config, "enabled": False}, key=owner_key)
+            assert disabled.status_code == 200 and disabled.json()["enabled"] is False
+            page: Final = gateway.request(
+                "GET",
+                "/usage/alert-rules",
+                key=owner_key,
+                params={"scope_type": "key", "scope_id": str(config["scope_id"]), "limit": "1"},
+            )
+            assert page.status_code == 200 and page.json()["total"] == 1
+            assert page.json()["data"][0]["id"] == rule_id
+        finally:
+            assert gateway.request("DELETE", path, key=owner_key).status_code == 204
+        assert gateway.request("GET", path, key=owner_key).status_code == 404
+
+
+def test_admin_edit_preserves_owner_rule_access(gateway: Gateway) -> None:
+    with gateway.scenario() as scenario:
+        owner: Final = scenario.user(user_role="internal_user")
+        owner_key: Final = scenario.key(user_id=owner, key_type="management")
+        config: Final = {
+            "name": "Owner policy",
+            "scope_type": "user",
+            "scope_id": owner,
+            "metric": "spend",
+            "direction": "increase",
+        }
+        created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key)
+        assert created.status_code == 201, created.text
+        path: Final = f"/usage/alert-rules/{created.json()['id']}"
+        try:
+            edited: Final = gateway.request("PUT", path, {**config, "name": "Renamed by admin"})
+            assert edited.status_code == 200, edited.text
+            owner_edit: Final = gateway.request("PUT", path, {**config, "multiplier": 5}, key=owner_key)
+            assert owner_edit.status_code == 200, "Admin edit must preserve the owner's rule access"
+        finally:
+            assert gateway.request("DELETE", path).status_code == 204
+
+
+def test_team_admin_can_manage_team_alerts_but_member_can_only_read(gateway: Gateway) -> None:
+    with gateway.scenario() as scenario:
+        admin_id: Final = scenario.user(user_role="internal_user")
+        member_id: Final = scenario.user(user_role="internal_user")
+        team_id: Final = scenario.team(
+            members_with_roles=[{"user_id": admin_id, "role": "admin"}, {"user_id": member_id, "role": "user"}]
+        )
+        admin_key: Final = scenario.key(user_id=admin_id, key_type="management")
+        member_key: Final = scenario.key(user_id=member_id, key_type="management")
+        config: Final = {
+            "name": "Daily team requests",
+            "scope_type": "team",
+            "scope_id": team_id,
+            "metric": "requests",
+            "direction": "decrease",
+            "multiplier": 2,
+        }
+        created: Final = gateway.request("POST", "/usage/alert-rules", config, key=admin_key)
+        assert created.status_code == 201, created.text
+        rule_id: Final = string_value(created.json()["id"])
+        path: Final = f"/usage/alert-rules/{rule_id}"
+        try:
+            assert gateway.request("GET", path, key=member_key).status_code == 200
+            assert gateway.request("PUT", path, config, key=member_key).status_code == 403
+            assert gateway.request("DELETE", path, key=member_key).status_code == 403
+        finally:
+            assert gateway.request("DELETE", path, key=admin_key).status_code == 204
+
+
+def test_info_only_credentials_cannot_mutate_alerts(gateway: Gateway) -> None:
+    with gateway.scenario() as scenario:
+        owner: Final = scenario.user(user_role="internal_user")
+        admin: Final = scenario.user(user_role="proxy_admin")
+        owner_key: Final = scenario.key(user_id=owner, key_type="management")
+        reader: Final = scenario.key(user_id=owner, key_type="default", allowed_routes=["info_routes"])
+        admin_reader: Final = scenario.key(user_id=admin, key_type="default", allowed_routes=["info_routes"])
+        config: Final = {
+            "name": "Read-only route grant",
+            "scope_type": "user",
+            "scope_id": owner,
+            "metric": "spend",
+            "direction": "increase",
+        }
+        created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key)
+        assert created.status_code == 201
+        path: Final = f"/usage/alert-rules/{created.json()['id']}"
+        try:
+            assert gateway.request("GET", path, key=reader).status_code == 200
+            assert gateway.request("POST", "/usage/alert-rules", config, key=reader).status_code == 403
+            assert gateway.request("PUT", path, config, key=reader).status_code == 403
+            assert gateway.request("DELETE", path, key=reader).status_code == 403
+            assert (
+                gateway.request("PATCH", "/issues/families/spend", {"enabled": False}, key=admin_reader).status_code
+                == 403
+            )
+            assert (
+                gateway.request(
+                    "PATCH", "/issues/unavailable", {"status": "resolved", "expected_status": "open"}, key=admin_reader
+                ).status_code
+                == 403
+            )
+        finally:
+            assert gateway.request("DELETE", path, key=owner_key).status_code == 204
+
+
+def test_only_admins_can_configure_operator_notifications(gateway: Gateway) -> None:
+    with gateway.scenario() as scenario:
+        owner: Final = scenario.user(user_role="internal_user")
+        owner_key: Final = scenario.key(user_id=owner, key_type="management")
+        config: Final = {
+            "name": "Operator delivery",
+            "scope_type": "user",
+            "scope_id": owner,
+            "metric": "spend",
+            "direction": "increase",
+        }
+        for notification in (
+            {"notification_channels": ["slack"]},
+            {"notification_channels": ["email"], "email_recipients": ["ops@example.com"]},
+        ):
+            assert (
+                gateway.request("POST", "/usage/alert-rules", {**config, **notification}, key=owner_key).status_code
+                == 403
+            )
+        assert (
+            gateway.request(
+                "POST", "/usage/alert-rules", {**config, "overrides_defaults": True}, key=owner_key
+            ).status_code
+            == 422
+        )
+        assert (
+            gateway.request("POST", "/usage/alert-rules", {**config, "name": "Spend\r\nInjected: header"}).status_code
+            == 422
+        )
+        created: Final = gateway.request("POST", "/usage/alert-rules", {**config, "notification_channels": ["slack"]})
+        assert created.status_code == 201
+        assert "overrides_defaults" not in created.json()
+        path: Final = f"/usage/alert-rules/{created.json()['id']}"
+        try:
+            assert gateway.request("PUT", path, config, key=owner_key).status_code == 403
+            assert gateway.request("DELETE", path, key=owner_key).status_code == 403
+        finally:
+            assert gateway.request("DELETE", path).status_code == 204
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gateway, db: ManagementDatabase) -> None:
+    """Populate an explicit storage fixture, then exercise the real incident HTTP API."""
+    repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
+    with gateway.scenario() as scenario:
+        owner: Final = scenario.user(user_role="internal_user")
+        outsider: Final = scenario.user(user_role="internal_user")
+        viewer: Final = scenario.user(user_role="proxy_admin_viewer")
+        owner_key: Final = scenario.key(user_id=owner, key_type="management")
+        outsider_key: Final = scenario.key(user_id=outsider, key_type="management")
+        viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only")
+        created: Final = gateway.request(
+            "POST",
+            "/usage/alert-rules",
+            {
+                "name": "Incident API fixture",
+                "scope_type": "user",
+                "scope_id": owner,
+                "metric": "requests",
+                "direction": "increase",
+                "multiplier": 3,
+            },
+            key=owner_key,
+        )
+        assert created.status_code == 201
+        rule: Final = UsageAlertRule.model_validate(created.json())
+        try:
+            incident: Final = (
+                await repository.record_observations(
+                    rule,
+                    (
+                        UsageAlertObservation(
+                            entity_id=owner,
+                            day=date(2026, 11, 4),
+                            observed=420,
+                            baseline=120,
+                            baseline_sample_days=7,
+                            threshold=360,
+                            breached=True,
+                        ),
+                    ),
+                )
+            )[0]
+            path: Final = f"/usage/alerts/{incident.id}"
+            params: Final = {"scope_type": "user", "scope_id": owner, "limit": "1"}
+            page: Final = gateway.request("GET", "/usage/alerts", key=owner_key, params=params)
+            assert page.status_code == 200 and page.json()["data"][0]["id"] == incident.id
+            detail: Final = gateway.request("GET", path, key=owner_key)
+            assert detail.status_code == 200
+            assert detail.json()["observed_value"] == 420
+            assert detail.json()["baseline_value"] == 120
+            assert detail.json()["threshold"] == 360
+            assert detail.json()["baseline_sample_days"] == 7
+            assert "detail_tab=activity" in detail.json()["activity_path"]
+            assert gateway.request("GET", "/usage/alerts", key=outsider_key, params=params).status_code == 403
+            assert gateway.request("GET", path, key=outsider_key).status_code == 403
+            assert gateway.request("PATCH", path, {"status": "dismissed"}, key=outsider_key).status_code == 403
+            assert gateway.request("GET", path, key=viewer_key).status_code == 200
+            assert gateway.request("PATCH", path, {"status": "dismissed"}, key=viewer_key).status_code == 403
+            dismissed: Final = gateway.request("PATCH", path, {"status": "dismissed"}, key=owner_key)
+            assert dismissed.status_code == 200 and dismissed.json()["status"] == "dismissed"
+            assert gateway.request("GET", path, key=owner_key).json()["status"] == "dismissed"
+        finally:
+            assert gateway.request("DELETE", f"/usage/alert-rules/{rule.id}", key=owner_key).status_code == 204
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_ongoing_episode_preserves_its_original_evidence_and_activity_link(db: ManagementDatabase) -> None:
+    repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
+    definition: Final = UsageAlertRuleInput(
+        name="Original episode", scope_type="user", scope_id="episode-example", metric="requests", direction="increase"
+    )
+    rule: Final = await repository.save_rule(definition, "qa-admin")
+    assert rule is not None
+    initial: Final = UsageAlertObservation(
+        entity_id=rule.scope_id,
+        day=date(2026, 11, 2),
+        observed=420,
+        baseline=120,
+        baseline_sample_days=7,
+        threshold=360,
+        breached=True,
+    )
+    try:
+        original: Final = (await repository.record_observations(rule, (initial,)))[0]
+        edited: Final = await repository.save_rule(
+            definition.model_copy(update={"name": "Edited rule", "multiplier": 4}), "qa-admin", rule
+        )
+        assert edited is not None
+        later: Final = initial.model_copy(
+            update={
+                "day": date(2026, 11, 3),
+                "observed": 600,
+                "baseline": 100,
+                "threshold": 400,
+                "baseline_sample_days": 4,
+            }
+        )
+        ongoing: Final = (await repository.record_observations(edited, (later,)))[0]
+        assert ongoing.last_seen >= original.last_seen
+        assert ongoing.model_copy(update={"last_seen": original.last_seen}) == original
+        stored: Final = await repository.get_alert(original.id)
+        assert stored == ongoing
+        healthy: Final = later.model_copy(update={"day": date(2026, 11, 4), "breached": False})
+        resolved: Final = (await repository.record_observations(edited, (healthy,)))[0]
+        assert resolved.model_copy(update={"status": "open", "last_seen": original.last_seen}) == original
+        repeated: Final = later.model_copy(update={"day": date(2026, 11, 5)})
+        new_episode: Final = (await repository.record_observations(edited, (repeated,)))[0]
+        assert new_episode.id != original.id
+        assert new_episode.day == repeated.day and new_episode.observed_value == 600
+        assert new_episode.threshold == 400 and new_episode.rule_name == "Edited rule"
+        assert new_episode.activity_path != original.activity_path
+    finally:
+        await repository.delete_rule(rule.id)
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_rule_scope_changes_reject_stale_updates_and_deletes(db: ManagementDatabase) -> None:
+    repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
+    definition: Final = UsageAlertRuleInput(
+        name="Concurrent scope edit", scope_type="user", scope_id="original-owner", metric="spend", direction="increase"
+    )
+    original: Final = await repository.save_rule(definition, "qa-admin")
+    assert original is not None
+    try:
+        moved: Final = await repository.save_rule(
+            definition.model_copy(update={"scope_id": "different-owner"}), "qa-admin", existing=original
+        )
+        assert moved is not None
+        assert await repository.save_rule(definition, "original-owner", existing=original) is None
+        assert not await repository.delete_rule(original.id, existing=original)
+        assert await repository.get_rule(original.id) == moved
+        current: Final = await repository.save_rule(
+            definition.model_copy(update={"scope_id": moved.scope_id, "name": "Current authorized edit"}),
+            "qa-admin",
+            existing=moved,
+        )
+        assert current is not None
+        assert not await repository.delete_rule(original.id, existing=moved)
+        assert await repository.delete_rule(original.id, existing=current)
+        assert await repository.save_rule(definition, "qa-admin", existing=current) is None
+        assert await repository.get_rule(original.id) is None
+    finally:
+        await repository.delete_rule(original.id)
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.parametrize(
+    "changed_field,new_value",
+    [("metric", "latency"), ("direction", "decrease"), ("scope_type", "user"), ("scope_id", "semantic-other")],
+)
+async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency(
+    db: ManagementDatabase, changed_field: str, new_value: str
+) -> None:
+    repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
+    definition: Final = UsageAlertRuleInput(
+        name="Semantic identity",
+        scope_type="team",
+        scope_id="semantic-example",
+        metric="spend",
+        direction="increase",
+        notification_channels=("slack",),
+    )
+    rule: Final = await repository.save_rule(definition, "qa-admin")
+    assert rule is not None
+    initial: Final = UsageAlertObservation(
+        entity_id=rule.scope_id, day=date(2026, 11, 4), observed=40, baseline=5, threshold=15, breached=True
+    )
+    try:
+        original: Final = (await repository.record_observations(rule, (initial,)))[0]
+        edited: Final = await repository.save_rule(
+            UsageAlertRuleInput.model_validate({**definition.model_dump(), changed_field: new_value}), "qa-admin", rule
+        )
+        assert edited is not None
+        observation: Final = initial.model_copy(update={"entity_id": edited.scope_id})
+        replacement: Final = (await repository.record_observations(edited, (observation,)))[0]
+        assert replacement.id != original.id
+        assert await repository.get_alert(original.id) == original
+        before_page: Final = await repository.list_alerts(
+            scope_type=original.scope_type, scope_id=original.scope_id, metric=original.metric
+        )
+        after_page: Final = await repository.list_alerts(
+            scope_type=replacement.scope_type, scope_id=replacement.scope_id, metric=replacement.metric
+        )
+        assert original.id in {item.id for item in before_page.data}
+        assert replacement.id in {item.id for item in after_page.data}
+        assert all(
+            item.scope_type == original.scope_type and item.metric == original.metric for item in before_page.data
+        )
+        healthy: Final = observation.model_copy(update={"day": date(2026, 11, 5), "breached": False})
+        await repository.record_observations(edited, (healthy,))
+        assert (await repository.get_alert(original.id)) == original
+        resolved: Final = await repository.get_alert(replacement.id)
+        assert resolved is not None and resolved.status == "resolved"
+    finally:
+        await repository.delete_rule(rule.id)
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_actions(
+    gateway: Gateway, db: ManagementDatabase
+) -> None:
+    repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
+    day: Final = date(2026, 11, 4)
+    with gateway.scenario() as scenario:
+        owner: Final = scenario.user(user_role="internal_user", user_alias="Checkout operator")
+        outsider: Final = scenario.user(user_role="internal_user")
+        viewer: Final = scenario.user(user_role="proxy_admin_viewer")
+        owner_key: Final = scenario.key(user_id=owner, key_type="management")
+        outsider_key: Final = scenario.key(user_id=outsider, key_type="management")
+        viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only")
+        inference_key: Final = scenario.key(user_id=owner, key_type="llm_api")
+        metrics: Final[tuple[UsageAlertMetric, ...]] = (
+            *(("requests",) * 27),
+            "spend",
+            "errors",
+            "latency",
+            "error_rate",
+        )
+        rules: Final = tuple(
+            [
+                await repository.save_rule(
+                    UsageAlertRuleInput(
+                        name=f"Group rule {index}",
+                        scope_type="user",
+                        scope_id=owner,
+                        metric=metric,
+                        direction="increase",
+                        baseline_days=7 if index % 2 else 14,
+                    ),
+                    owner,
+                )
+                for index, metric in enumerate(metrics)
+            ]
+        )
+        assert all(rule is not None for rule in rules)
+        incidents: Final = tuple(
+            [
+                (
+                    await repository.record_observations(
+                        rule,
+                        (
+                            UsageAlertObservation(
+                                entity_id=owner,
+                                day=day,
+                                observed=14 if rule.metric == "error_rate" else 420,
+                                baseline=4 if rule.metric == "error_rate" else 0 if index == 0 else 120,
+                                threshold=12 if rule.metric == "error_rate" else 100 if index == 0 else 360,
+                                breached=True,
+                            ),
+                        ),
+                    )
+                )[0]
+                for index, rule in enumerate(rules)
+                if rule is not None
+            ]
+        )
+        old_rule: Final = await repository.save_rule(
+            UsageAlertRuleInput(
+                name="Earlier day", scope_type="user", scope_id=owner, metric="requests", direction="increase"
+            ),
+            owner,
+        )
+        assert old_rule is not None
+        old: Final = (
+            await repository.record_observations(
+                old_rule,
+                (
+                    UsageAlertObservation(
+                        entity_id=owner,
+                        day=day - timedelta(days=1),
+                        observed=420,
+                        baseline=120,
+                        threshold=360,
+                        breached=True,
+                    ),
+                ),
+            )
+        )[0]
+        other_rule: Final = await repository.save_rule(
+            UsageAlertRuleInput(
+                name="Other resource", scope_type="user", scope_id=outsider, metric="requests", direction="increase"
+            ),
+            outsider,
+        )
+        assert other_rule is not None
+        other: Final = (
+            await repository.record_observations(
+                other_rule,
+                (
+                    UsageAlertObservation(
+                        entity_id=outsider,
+                        day=day,
+                        observed=420,
+                        baseline=120,
+                        threshold=360,
+                        breached=True,
+                    ),
+                ),
+            )
+        )[0]
+        params: Final = {"scope_type": "user", "scope_id": owner, "status": "open", "limit": "1"}
+        try:
+            response: Final = gateway.request("GET", "/usage/alert-groups", key=owner_key, params=params)
+            assert response.status_code == 200, response.text
+            page: Final = UsageAlertGroupList.model_validate(response.json())
+            assert page.total == 2 and len(page.data) == 1
+            group: Final = page.data[0]
+            assert group.day == day and group.scope_label == "Checkout operator"
+            assert group.incident_count == 31 and len(group.metrics) == 5
+            assert {metric.metric: metric.incident_count for metric in group.metrics} == {
+                "requests": 27,
+                "spend": 1,
+                "errors": 1,
+                "latency": 1,
+                "error_rate": 1,
+            }
+            assert all(metric.representative.id in {incident.id for incident in incidents} for metric in group.metrics)
+            assert group.activity_path in {incident.activity_path for incident in incidents}
+            second: Final = UsageAlertGroupList.model_validate(
+                gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "1"}).json()
+            )
+            assert second.total == 2 and second.data[0].day == old.day
+            empty: Final = UsageAlertGroupList.model_validate(
+                gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "2"}).json()
+            )
+            assert empty.total == 2 and empty.data == ()
+            history: Final = gateway.request(
+                "GET",
+                "/usage/alerts",
+                key=owner_key,
+                params={**params, "start_date": day.isoformat(), "end_date": day.isoformat(), "metric": "requests"},
+            )
+            assert history.status_code == 200 and history.json()["total"] == 27
+            await repository.set_status(incidents[0].id, "dismissed")
+            body: Final = {
+                "scope_type": "user",
+                "scope_id": owner,
+                "day": day.isoformat(),
+                "expected_status": "open",
+                "status": "resolved",
+            }
+            for forbidden in (outsider_key, inference_key):
+                assert gateway.request("GET", "/usage/alert-groups", key=forbidden, params=params).status_code == 403
+            for forbidden in (outsider_key, inference_key, viewer_key):
+                assert gateway.request("PATCH", "/usage/alert-groups", body, key=forbidden).status_code == 403
+            assert gateway.request("GET", "/usage/alert-groups", key=viewer_key, params=params).status_code == 200
+            updated: Final = gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key)
+            assert updated.status_code == 200 and updated.json()["updated_count"] == 30
+            assert gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key).json()["updated_count"] == 0
+            assert await repository.get_alert(old.id) == old
+            assert await repository.get_alert(other.id) == other
+            dismissed: Final = await repository.get_alert(incidents[0].id)
+            assert dismissed is not None and dismissed.status == "dismissed" and dismissed.baseline_value == 0
+            resolved: Final = await repository.get_alert(incidents[-1].id)
+            assert resolved == incidents[-1].model_copy(update={"status": "resolved"})
+            assert (
+                gateway.request(
+                    "GET", "/usage/alert-groups", params={"scope_type": "user", "scope_id": "x' OR 1=1 --"}
+                ).json()["total"]
+                == 0
+            )
+            assert (
+                gateway.request(
+                    "GET", "/usage/alert-groups", params={"start_date": "2026-11-05", "end_date": "2026-11-04"}
+                ).status_code
+                == 422
+            )
+        finally:
+            for rule in (*rules, old_rule, other_rule):
+                if rule is not None:
+                    await repository.delete_rule(rule.id)
+    deleted: Final = await repository.list_groups(scope_type="user", scope_id=owner)
+    assert deleted.total == 2 and all(group.scope_label is None and group.scope_id == owner for group in deleted.data)
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_default_rule_summary_and_kind_filters_do_not_require_loading_every_rule(
+    gateway: Gateway, db: ManagementDatabase
+) -> None:
+    repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
+    default: Final = UsageAlertRuleInput(
+        name="Built-in daily model requests", scope_type="model", metric="requests", direction="increase"
+    )
+    await repository.ensure_default_rules((default,))
+    custom: Final = await repository.save_rule(
+        default.model_copy(update={"scope_id": f"model-{uuid4().hex}"}), "qa-admin"
+    )
+    assert custom is not None
+    try:
+        all_rules: Final = await repository.list_rules(scope_type="model", limit=1)
+        builtins: Final = await repository.list_rules(scope_type="model", limit=1, kind="default")
+        customs: Final = await repository.list_rules(scope_type="model", limit=1, kind="custom")
+        assert builtins.total >= 1 and all(rule.is_default for rule in builtins.data)
+        assert customs.total >= 1 and all(not rule.is_default for rule in customs.data)
+        assert all_rules.total == builtins.total + customs.total
+        assert all_rules.default_summary == builtins.default_summary == customs.default_summary
+        assert builtins.default_summary.total == builtins.total
+        assert (
+            sum(builtins.default_summary.evaluation_status_counts.model_dump().values())
+            == builtins.default_summary.enabled
+        )
+        assert (await repository.list_rules(scope_type="model", kind="default", offset=builtins.total)).data == ()
+        stored: Final = await repository.get_rule(builtins.data[0].id)
+        assert stored is not None and stored.is_default
+        filtered: Final = gateway.request(
+            "GET", "/usage/alert-rules", params={"scope_type": "model", "kind": "custom", "limit": "1"}
+        )
+        assert filtered.status_code == 200 and filtered.json()["total"] == customs.total
+        assert all(not rule["is_default"] for rule in filtered.json()["data"])
+        configuration: Final = stored.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields))
+        path: Final = f"/usage/alert-rules/{stored.id}"
+        for change in (
+            {"scope_type": "key"},
+            {"scope_id": "model-example"},
+            {"metric": "requests" if stored.metric != "requests" else "spend"},
+            {"direction": "increase" if stored.direction == "decrease" else "decrease"},
+        ):
+            assert gateway.request("PUT", path, {**configuration, **change}).status_code == 422
+        assert await repository.get_rule(stored.id) == stored
+        try:
+            updated: Final = gateway.request("PUT", path, {**configuration, "min_value": stored.min_value + 1})
+            assert updated.status_code == 200 and updated.json()["is_default"] is True
+            assert updated.json()["min_value"] == stored.min_value + 1
+        finally:
+            assert gateway.request("PUT", path, configuration).status_code == 200
+        custom_input: Final = custom.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields))
+        assert (
+            gateway.request("PUT", f"/usage/alert-rules/{custom.id}", {**custom_input, "metric": "latency"}).status_code
+            == 200
+        )
+    finally:
+        await repository.delete_rule(custom.id)
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_reported_model_search_preserves_exact_identity_and_cursor(
+    gateway: Gateway, db: ManagementDatabase
+) -> None:
+    repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
+    prefix: Final = f"reported-{uuid4().hex}"
+    initial: Final = await repository.list_reported_models(prefix, None, 1)
+    assert initial.data == ()
+    models: Final = (f"{prefix}/first", f"{prefix}/literal%quote'", f"{prefix}/second")
+    try:
+        for index, model in enumerate((*models, models[0])):
+            await db.prisma.writer_db.execute_raw(
+                'INSERT INTO "LiteLLM_DailyUserSpend" (id,date,api_key,model,updated_at) VALUES ($1,$2,$3,$4,CURRENT_TIMESTAMP)',
+                f"{prefix}-{index}",
+                initial.end_date.isoformat(),
+                prefix,
+                model,
+            )
+        await db.prisma.writer_db.execute_raw(
+            'INSERT INTO "LiteLLM_DailyUserSpend" (id,date,api_key,model,updated_at) VALUES ($1,$2,$3,$4,CURRENT_TIMESTAMP)',
+            f"{prefix}-old",
+            (initial.start_date - timedelta(days=1)).isoformat(),
+            prefix,
+            f"{prefix}/old",
+        )
+        response: Final = gateway.request("GET", "/usage/alert-models", params={"search": prefix, "limit": "1"})
+        assert response.status_code == 200, response.text
+        first: Final = UsageAlertModelList.model_validate(response.json())
+        assert first.data[0].scope_id == models[0] and first.next_cursor == models[0]
+        second: Final = await repository.list_reported_models(prefix, first.next_cursor, 1)
+        third: Final = await repository.list_reported_models(prefix, second.next_cursor, 1)
+        assert second.data[0].scope_id == models[1] and third.data[0].scope_id == models[2]
+        assert third.next_cursor is None
+        literal: Final = await repository.list_reported_models(f"{prefix}/literal%quote'", None, 20)
+        assert tuple(option.scope_id for option in literal.data) == (models[1],)
+        assert literal.data[0].label == models[1]
+        with gateway.scenario() as scenario:
+            user: Final = scenario.user(user_role="internal_user")
+            key: Final = scenario.key(user_id=user, key_type="management")
+            assert gateway.request("GET", "/usage/alert-models", key=key).status_code == 403
+    finally:
+        await db.prisma.writer_db.execute_raw('DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', prefix)

tests/unit/proxy/alerting_endpoints/test_usage_alerts.py

@@ -0,0 +1,87 @@
+from datetime import date
+from typing import Final
+from urllib.parse import parse_qs, urlsplit
+
+import pytest
+from fastapi import HTTPException
+
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.alerting_endpoints.usage_alerts import require_scope, require_write_access
+from litellm.repositories.usage_alerts import UsageAlertsRepository, activity_path
+from litellm.types.proxy.auth.user_roles import LitellmUserRoles
+
+
+class NoDatabase:
+    async def query_raw(self, query: str, *args: object) -> object:
+        raise AssertionError("Unauthorized callers must not reach the database")
+
+
+@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER])
+@pytest.mark.parametrize("key_type", [None, "default", "management"])
+def test_info_grants_do_not_authorize_mutations(role: LitellmUserRoles, key_type: str | None) -> None:
+    user: Final = UserAPIKeyAuth(user_role=role, key_type=key_type, allowed_routes=["info_routes", "openai_routes"])
+    with pytest.raises(HTTPException) as error:
+        require_write_access(user, "/usage/alert-rules/example")
+    assert error.value.status_code == 403
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "role,key_type,write",
+    [
+        (LitellmUserRoles.PROXY_ADMIN, "llm_api", False),
+        (LitellmUserRoles.PROXY_ADMIN, "read_only", True),
+        (LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, None, True),
+        (LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, None, True),
+        (LitellmUserRoles.INTERNAL_USER, None, False),
+        (LitellmUserRoles.CUSTOMER, None, False),
+    ],
+)
+async def test_global_access_rejects_inference_keys_and_unprivileged_roles(
+    role: LitellmUserRoles, key_type: str | None, write: bool
+) -> None:
+    user: Final = UserAPIKeyAuth(user_id="qa-user", user_role=role, key_type=key_type)
+    with pytest.raises(HTTPException) as error:
+        await require_scope(user, UsageAlertsRepository(NoDatabase()), "deployment", None, write=write)
+    assert error.value.status_code == 403
+
+
+@pytest.mark.asyncio
+async def test_internal_user_can_read_and_manage_only_their_user_scope() -> None:
+    user: Final = UserAPIKeyAuth(user_id="qa-user", user_role=LitellmUserRoles.INTERNAL_USER)
+    repository: Final = UsageAlertsRepository(NoDatabase())
+    await require_scope(user, repository, "user", "qa-user", write=True)
+    with pytest.raises(HTTPException) as error:
+        await require_scope(user, repository, "user", "other-user", write=False)
+    assert error.value.status_code == 403
+
+
+def test_activity_link_escapes_entity_identity_and_includes_baseline() -> None:
+    path: Final = activity_path("model", "provider/model?foo=bar&other=1", date(2026, 11, 4), "qa-alert", 7)
+    parsed: Final = urlsplit(path)
+    query: Final = parse_qs(parsed.query)
+    assert parsed.path == "/models-and-endpoints"
+    assert query["activity_model"] == ["provider/model?foo=bar&other=1"]
+    assert query["activity_from"] == ["2026-10-28"]
+    assert query["activity_to"] == ["2026-11-04"]
+    assert query["alert"] == ["qa-alert"]
+    assert query["detail_tab"] == ["activity"]
+
+
+def test_global_activity_link_uses_the_usage_page_date_contract() -> None:
+    query: Final = parse_qs(urlsplit(activity_path("deployment", None, date(2026, 11, 4), "qa-alert")).query)
+    assert query["tab"] == ["keys"]
+    assert "range" not in query
+    assert query["from"] == ["2026-10-28"]
+    assert query["to"] == ["2026-11-04"]
+
+
+@pytest.mark.asyncio
+async def test_no_database_returns_an_actionable_service_error() -> None:
+    from litellm.proxy.alerting_endpoints.usage_alerts import list_usage_alerts
+
+    user: Final = UserAPIKeyAuth(user_id="qa-admin", user_role=LitellmUserRoles.PROXY_ADMIN)
+    with pytest.raises(HTTPException) as error:
+        await list_usage_alerts(user, None)
+    assert error.value.status_code == 503
+    assert error.value.detail == "Usage alerts require a database connection"

tests/unit/proxy/management_endpoints/test_daily_activity_routes.py

@@ -424,6 +424,8 @@ def _user_spend_row(user_id: str, rows: Sequence[_Activity]) -> UserSpendRow:
         api_requests=sum(row.api_requests for row in matching),
         successful_requests=sum(row.successful_requests for row in matching),
         failed_requests=sum(row.failed_requests for row in matching),
+        total_response_time_ms=sum(row.total_response_time_ms for row in matching),
+        timed_requests=sum(row.timed_requests for row in matching),
     )
 
 
@@ -448,6 +450,8 @@ def _key_spend_row(api_key: str, rows: Sequence[_Activity]) -> KeySpendRow:
         api_requests=sum(row.api_requests for row in matching),
         successful_requests=sum(row.successful_requests for row in matching),
         failed_requests=sum(row.failed_requests for row in matching),
+        total_response_time_ms=sum(row.total_response_time_ms for row in matching),
+        timed_requests=sum(row.timed_requests for row in matching),
         cache_read_input_tokens=sum(row.cache_read_input_tokens for row in matching),
         cache_creation_input_tokens=sum(row.cache_creation_input_tokens for row in matching),
     )
@@ -873,6 +877,8 @@ def test_key_page_routes_map_ranked_rows_and_totals(
                     "api_requests": 1,
                     "successful_requests": 1,
                     "failed_requests": 0,
+                    "total_response_time_ms": 100,
+                    "timed_requests": 1,
                     "cache_read_input_tokens": 0,
                     "cache_creation_input_tokens": 1,
                 },
@@ -894,6 +900,8 @@ def test_key_page_routes_map_ranked_rows_and_totals(
                     "api_requests": 2,
                     "successful_requests": 2,
                     "failed_requests": 0,
+                    "total_response_time_ms": 200,
+                    "timed_requests": 2,
                     "cache_read_input_tokens": 0,
                     "cache_creation_input_tokens": 2,
                 },
@@ -1008,6 +1016,8 @@ def test_user_page_route_ranks_users_by_spend_and_attaches_metadata(
                 "api_requests": 1,
                 "successful_requests": 1,
                 "failed_requests": 0,
+                "total_response_time_ms": 100,
+                "timed_requests": 1,
             },
             {
                 "user_id": "user-a",
@@ -1020,6 +1030,8 @@ def test_user_page_route_ranks_users_by_spend_and_attaches_metadata(
                 "api_requests": 6,
                 "successful_requests": 6,
                 "failed_requests": 0,
+                "total_response_time_ms": 600,
+                "timed_requests": 6,
             },
         ],
         "total_users": 2,
@@ -1380,6 +1392,8 @@ def test_model_top_routes_rank_keys_and_include_metadata(
         api_requests=1,
         successful_requests=1,
         failed_requests=0,
+        total_response_time_ms=100,
+        timed_requests=1,
         cache_read_input_tokens=0,
         cache_creation_input_tokens=1,
     )
@@ -1675,6 +1689,8 @@ def test_user_cache_leakage_route_returns_cache_keys_and_metadata(
         api_requests=1,
         successful_requests=1,
         failed_requests=0,
+        total_response_time_ms=100,
+        timed_requests=1,
         cache_read_input_tokens=20,
         cache_creation_input_tokens=1,
     )

tests/unit/repositories/test_config_repository.py

@@ -0,0 +1,27 @@
+from types import SimpleNamespace
+from typing import Final
+from unittest.mock import AsyncMock
+
+import pytest
+
+from litellm.repositories.config_repository import ConfigRepository
+
+
+@pytest.mark.asyncio
+async def test_batch_reads_decode_legacy_json_and_current_objects_from_writer() -> None:
+    find_many: Final = AsyncMock(
+        return_value=(
+            SimpleNamespace(param_name="first", param_value='{"status":"resolved"}'),
+            SimpleNamespace(param_name="second", param_value={"status": "dismissed"}),
+        )
+    )
+    prisma: Final = SimpleNamespace(writer_db=SimpleNamespace(litellm_config=SimpleNamespace(find_many=find_many)))
+    repository: Final = ConfigRepository(prisma, use_writer=True)
+    rows: Final = await repository.get_params(("first", "second", "absent"))
+    assert {row.param_name: row.param_value for row in rows} == {
+        "first": {"status": "resolved"},
+        "second": {"status": "dismissed"},
+    }
+    find_many.assert_awaited_once_with(where={"param_name": {"in": ("first", "second", "absent")}})
+    assert await repository.get_params(()) == ()
+    assert find_many.await_count == 1

tests/unit/repositories/test_issue_families.py

@@ -0,0 +1,63 @@
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Final, get_args
+
+import pytest
+
+from litellm.repositories.issue_families import family_settings
+from litellm.types.proxy.usage_alerts import (
+    UsageAlertEvaluationStatus,
+    UsageAlertMetric,
+    UsageAlertRule,
+    UsageAlertScope,
+)
+
+
+@dataclass
+class RuleRows:
+    rules: tuple[UsageAlertRule, ...]
+    calls: int = 0
+
+    async def query_raw(self, query: str, *args: object) -> object:
+        self.calls += 1
+        if "SELECT EXISTS" in query:
+            return ({"pending": False},)
+        return tuple({"record": rule.model_dump(mode="json")} for rule in self.rules)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("status", get_args(UsageAlertEvaluationStatus))
+async def test_family_counts_only_its_enabled_rules_without_an_extra_query(status: UsageAlertEvaluationStatus) -> None:
+    timestamp: Final = datetime(2026, 11, 5, tzinfo=timezone.utc)
+    inputs: Final[tuple[tuple[UsageAlertScope, UsageAlertMetric, bool], ...]] = (
+        ("key", "spend", True),
+        ("team", "spend", True),
+        ("user", "spend", False),
+        ("model", "latency", True),
+    )
+    rules: Final = tuple(
+        UsageAlertRule(
+            id=f"test-{scope}-{metric}",
+            name="Daily monitoring",
+            scope_type=scope,
+            metric=metric,
+            direction="increase",
+            enabled=enabled,
+            evaluation_status=status,
+            created_at=timestamp,
+            updated_at=timestamp,
+            created_by="test-admin",
+        )
+        for scope, metric, enabled in inputs
+    )
+
+    database: Final = RuleRows(rules)
+    result: Final = next(setting for setting in (await family_settings(database)).data if setting.family == "spend")
+
+    assert database.calls == 2  # existing rule rows plus the separate projection-backlog read
+    assert result.enabled_rules == 2
+    assert result.total_rules == 3
+    assert result.evaluation_status_counts.model_dump() == {
+        **dict.fromkeys(get_args(UsageAlertEvaluationStatus), 0),
+        status: 2,
+    }

tests/unit/test_check_openapi_docs.py

@@ -40,6 +40,23 @@ def test_each_missing_piece_is_its_own_rule() -> None:
     assert codes == {"OA001", "OA002", "OA003"}
 
 
+def test_created_response_schema_satisfies_the_success_contract() -> None:
+    operation: Final = {**_DOCUMENTED, "responses": {"201": _RESPONSES["200"]}}
+    assert checker.violations({"paths": {"/v1/thing": {"post": operation}}}) == ()
+
+
+@pytest.mark.parametrize("responses", [
+    {"201": {"description": "Created"}},
+    {"400": _RESPONSES["200"]},
+    {"200": {}, "201": _RESPONSES["200"]},
+])
+def test_success_response_must_still_have_a_schema(responses: Mapping[str, object]) -> None:
+    operation: Final = {**_DOCUMENTED, "responses": responses}
+    assert checker.violations({"paths": {"/v1/thing": {"post": operation}}}) == (
+        checker.Violation("POST", "/v1/thing", "OA003"),
+    )
+
+
 @pytest.mark.parametrize("body", [{}, {"content": {"multipart/form-data": {}}}])
 def test_only_the_missing_piece_is_reported(body: Mapping[str, object]) -> None:
     no_body: Final = {**_DOCUMENTED, "requestBody": body}

tests/unit/types/proxy/test_usage_alerts.py

@@ -0,0 +1,64 @@
+from typing import Final
+
+import pytest
+from pydantic import ValidationError
+
+from litellm.types.proxy.usage_alerts import UsageAlertGroupUpdate, UsageAlertRuleInput
+
+RULE: Final = {
+    "name": "Daily spend",
+    "scope_type": "team",
+    "scope_id": "qa-team",
+    "metric": "spend",
+    "direction": "increase",
+}
+
+
+@pytest.mark.parametrize(
+    "changes",
+    [
+        {"multiplier": 1},
+        {"multiplier": float("inf")},
+        {"min_value": float("nan")},
+        {"baseline_days": 2, "min_history_days": 3},
+        {"baseline_days": 31},
+        {"scope_type": "deployment"},
+        {"scope_type": "key", "scope_id": "sk-test-placeholder"},
+        {"notification_channels": ["email"]},
+        {"notification_channels": ["slack", "slack"]},
+        {"email_recipients": ["ops@example.com\r\nBcc: other@example.com"]},
+        {"name": "Spend\r\nBcc: other@example.com"},
+        {"name": "Spend\x00notification"},
+        {"overrides_defaults": True},
+    ],
+)
+def test_invalid_rules_cannot_be_stored(changes: dict[str, object]) -> None:
+    with pytest.raises(ValidationError):
+        UsageAlertRuleInput.model_validate({**RULE, **changes})
+
+
+def test_zero_request_floor_and_daily_drop_multiplier_are_supported() -> None:
+    rule: Final = UsageAlertRuleInput.model_validate(
+        {**RULE, "metric": "requests", "direction": "decrease", "multiplier": 2, "min_requests": 0}
+    )
+    assert rule.direction == "decrease"
+    assert rule.multiplier == 2
+    assert rule.min_requests == 0
+
+
+@pytest.mark.parametrize(
+    "changes",
+    [{"scope_id": None}, {"scope_type": "deployment"}, {"status": "open"}, {"status": "unknown"}],
+)
+def test_group_updates_require_exact_resource_and_a_different_valid_status(changes: dict[str, object]) -> None:
+    with pytest.raises(ValidationError):
+        UsageAlertGroupUpdate.model_validate(
+            {
+                "scope_type": "team",
+                "scope_id": "team-example",
+                "day": "2026-11-04",
+                "expected_status": "open",
+                "status": "resolved",
+                **changes,
+            }
+        )