diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index fb29a92e306..905c6d4ff77 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -550,6 +550,7 @@ jobs: tests/unit/proxy/management_endpoints tests/unit/proxy/management_helpers tests/unit/proxy/list_api + tests/unit/proxy/alerting_endpoints tests/unit/proxy/analytics_endpoints tests/unit/proxy/decisions_endpoints tests/unit/proxy/memory diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py index 2d3ccf40175..414c205b2b3 100644 --- a/backend/routes/allowlist.py +++ b/backend/routes/allowlist.py @@ -110,6 +110,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = ( "/v1/rust_control_plane/", # Alerting / email / IP allowlist "/alerting/", + "/issues/", "/email/", "/add/allowed_ip", "/delete/allowed_ip", @@ -153,6 +154,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset( { "/", "/routes", + "/issues", "/lens", "/openapi.json", "/docs", diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md new file mode 100644 index 00000000000..af900e3ffdf --- /dev/null +++ b/docs/features/usage-alerts.md @@ -0,0 +1,186 @@ +# Alerts and daily Performance alerts + +The Alerts page combines model retirements with daily Performance alerts. Open an issue to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause. + +Performance comparisons cover completed UTC days. Retirement issues show recent model usage and the provider's retirement schedule, including today's usage when present. + +This feature requires a database. Apply the normal LiteLLM database migrations when deploying a version that includes usage alerts. The API and dashboard use the same persisted rules and incidents; no enterprise license is required. + +## Start with the defaults + +Daily Performance defaults cover spend, latency, reliability, and traffic across the gateway, keys, users, teams, and models. They use the previous seven days and the following policies. Retirement uses the existing model-deprecation warning window. + +| Family | Default comparison | Minimum history and volume | +| --- | --- | --- | +| Spend | At least 3× daily spend and a $10 increase | Three recorded reference days with a positive spend baseline | +| Latency | At least 3× average latency and a 500 ms increase | Three recorded reference days and at least 100 timed requests in both periods | +| Reliability | At least 3× error rate and a two percentage-point increase | Three recorded reference days and at least 500 daily requests in both periods | +| Traffic increase | At least 10× requests and an increase of 1,000 requests | Three recorded reference days with a positive request baseline | +| Traffic decrease | At most one fifth of baseline requests | Seven recorded reference days and a baseline of at least 500 daily requests | + +New resources need history before they can produce a default spike. A zero spend, request, or latency baseline does not establish a spike. A zero error-rate baseline is usable only after sufficient historical traffic has been observed; the percentage-point floor still applies. + +Use the family settings to enable or disable a family or adjust its thresholds across the five scopes. Existing custom rules remain independent. An enabled custom rule replaces the matching default for its scope, metric, and direction. A custom wildcard rule replaces that default for every entity. Deleted defaults stay deleted. + +The scheduler wakes hourly and evaluates the latest completed UTC day after a one-hour grace period. A successful aggregate flush after the grace period is required before either direction is finalized. Completed rule versions are skipped on later ticks. The evaluator reads existing daily aggregates in pages of 500 entities; it adds no request-time counters or per-request database writes. + +Decrease rules need a successful background aggregate-flush sample for every hour of the target day and another sample after the grace period. Until that coverage exists, their status is `insufficient_coverage`. A key with no requests can then produce a request or spend drop; missing latency samples never become zero latency. Coverage shows that an aggregate writer was healthy each hour, not that every replica delivered every request. + +The baseline averages recorded reference days; it does not turn missing reference days into zero. Average latency is weighted by timed request count, using the same successful-request duration totals as Activity. `baseline_sample_days` records how many reference days contributed. Cost, traffic, and latency comparisons require a positive baseline. Reliability can compare against a recorded 0% error rate when historical and current request volume meet the rule's minimum. Missing history and low-volume comparisons remain `insufficient_history`. + +## Read and investigate issues + +Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the credential in your environment. + +```bash +curl "$LITELLM_API_BASE/issues?status=open&limit=50" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +``` + +The response contains `data`, `total`, `limit`, and `offset`. Pagination and filtering happen on the combined inbox. Filters are `kind`, `family`, `status`, `scope_type`, and `scope_id`. Families are `retirement`, `latency`, `reliability`, `spend`, and `traffic`; kinds are `model_deprecation` and `performance`. An offset beyond the last page still returns the correct total. There is no default history cutoff on open issues. + +Each issue has an ID, status, readable title, current resource label, typed `cohort`, date range, grouping reason, and saved comparison values. The cohort is one resource, one provider, or exact provider/model pairs. A provider/model pair is an intersection, not a list of independently interchangeable filters. Resource names can change; IDs remain the identity. + +Use the ID from the list to inspect an issue: + +```bash +ISSUE_ID="issue-example" +curl "$LITELLM_API_BASE/issues/$ISSUE_ID" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +curl "$LITELLM_API_BASE/issues/$ISSUE_ID/activity" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +curl "$LITELLM_API_BASE/issues/$ISSUE_ID/signals?limit=50" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +``` + +Replace `issue-example` with an ID returned by `/issues`. Activity uses the issue's saved date range by default. Optional `start_date` and `end_date` are inclusive UTC dates and can span at most 366 days. `/keys` and `/users` provide paginated Activity rows with `limit` and `offset`. Their latency totals are `total_response_time_ms` and `timed_requests`; divide the former by the latter only when timed requests are present. + +Alert Activity returns exact daily totals in `results` and exact range totals in `metadata`. It does not return the general Usage API's per-day dimension dictionaries. Key issues also return `model_breakdown`, with up to 20 models by range spend by default. Set `model_limit` between 1 and 100 to change that limit. `total_models` counts every matching model; `other` contains the combined metrics and `model_count` of all remaining models, or null when none remain. It is an aggregate, not a model identifier. Adding the displayed models and `other` recovers the full totals. Model identifiers longer than 1,024 characters are included in `other` without truncation. Non-key issues return null for `model_breakdown`. Activity queries use a bounded execution budget. A busy or over-budget query returns `503` with `Retry-After: 5`; retry later or choose a shorter date range. + +`/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Resource pages fetch labels only for the selected page, using the latest deleted-key label when needed. They do not load arbitrary key or user metadata. Display aliases are limited to 256 characters; email labels longer than 320 characters are omitted. Resource pages omit IDs longer than 1,024 characters or 4,096 bytes instead of truncating them. Key IDs that need a `raw:` escape prefix reserve four characters within that limit. Oversized linked user or team IDs are omitted from resource metadata. Activity totals still include traffic from omitted resources. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown. + +Supporting signals have their own saved comparisons and relationship explanations. Their baseline windows can differ, so do not derive a combined error rate or unit cost by adding their displayed values. Administrators and admin viewers can inspect broader supporting signals. A scoped reader receives only signals for the exact key, user, or team they can access. That restriction applies before the signal count and pagination; broader totals are not disclosed. + +A continuing Performance issue retains its initial comparison and cohort while `last_seen` advances. New supporting observations do not rewrite the original evidence. Resource issues recover when their primary metric recovers for that resource; broader supporting observations do not keep them open. Provider issues require a fresh comparison of the saved provider/model cohort. Missing history or coverage does not establish recovery. A later breach starts a new issue. Retirement details refresh as usage and the provider schedule change. A retirement issue with no recent use becomes inactive; actual new traffic can reopen it. + +Resolve or dismiss an issue using the current status from the read response: + +```bash +curl -X PATCH "$LITELLM_API_BASE/issues/$ISSUE_ID" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \ + -H "Content-Type: application/json" \ + -d '{"expected_status":"open","status":"resolved"}' +``` + +A stale status returns `409`; reload before retrying. Performance acknowledgements do not change other issues that share a supporting observation. Retirement acknowledgements use the same state as the existing model-deprecation API. An inactive retirement cannot be manually reopened. + +The issue-detail response includes `can_update_status` and `can_view_activity` for the current reader. Clients should show status actions only when `can_update_status` is true and request Activity or resource breakdowns only when `can_view_activity` is true. These fields are null in responses where permission has not been evaluated, such as inbox lists. Every endpoint still checks authorization on each request. Team Activity requires team-admin access or the explicit `/team/daily/activity` permission; ordinary membership can read saved alert evidence but does not grant other members’ activity or key details. + +## Configure default families + +`GET /issues/families` returns the four Performance families, their enabled counts, and common policy values. A null common value means the underlying scopes have different settings. This is separate from custom rules and the model-retirement warning window. + +This example sets the spend family to a 5× threshold and a $25 minimum absolute increase: + +```bash +curl -X PATCH "$LITELLM_API_BASE/issues/families/spend" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \ + -H "Content-Type: application/json" \ + -d '{"enabled":true,"policies":[{"direction":"increase","multiplier":5,"min_value":25}]}' +``` + +The update applies atomically to registered, non-deleted built-in rules in that family. Omitted settings remain unchanged. It does not change custom rules or recreate deleted defaults. A concurrent rule edit returns `409`; reload and retry. These controls require a gateway administrator. + +## Read resource groups + +Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the key in your environment. + +```bash +curl "$LITELLM_API_BASE/usage/alert-groups?status=open&limit=50" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" +``` + +The response has `data`, `total`, `limit`, and `offset`. `total` counts resource/day groups, including when the requested offset is beyond the last page. Filters are `scope_type`, `scope_id`, `status`, `start_date`, and `end_date`. Status filters apply to individual incidents before grouping, so a group with open and resolved incidents can appear under either filter with the corresponding members. No default date cutoff hides older open incidents. + +Each group includes its stable `id`, `scope_type`, exact `scope_id`, current `scope_label` when available, original UTC `day`, `incident_count`, and `activity_path`. Its `metrics` list contains up to five entries with a `metric`, `incident_count`, and the most recently detected matching `representative` incident. These representatives are summaries, not the entire group. Missing or deleted resources keep their original IDs. Names are current display labels, not historical evidence. + +To inspect every matching rule, use `GET /usage/alerts` with the group's scope and `start_date` and `end_date` both set to its `day`. Add `metric` and `status` as needed and paginate that response. Rules can have different baseline windows and thresholds, so their comparisons must not be combined into an error rate or a cost-per-request estimate. + +The group Activity link uses a representative's saved date range and alert context. The path is relative to the dashboard mount: for example, `/api-keys?...` is opened below the dashboard's `/ui` mount. A continuing episode retains its original day and evidence; the grouped row does not describe today's traffic. + +Resolve or dismiss all matching incidents in one resource/day group with an expected current status: + +```bash +curl -X PATCH "$LITELLM_API_BASE/usage/alert-groups" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \ + -H "Content-Type: application/json" \ + -d '{"scope_type":"team","scope_id":"team-example","day":"2026-11-04","expected_status":"open","status":"resolved"}' +``` + +The response contains `updated_count`. Only incidents in that exact group that still have `expected_status` are updated atomically. Other statuses remain unchanged; repeating the same request returns zero after all matching incidents have changed. Use `scope_type: "deployment"` with `scope_id: null` for the gateway. + +## Read individual incidents + +`GET /usage/alerts` retains the individual-incident view. Each incident includes its scope, metric, direction, status, UTC day, observed value, baseline value, threshold, baseline date range, and `activity_path`. + +`GET /usage/alerts/{alert_id}` returns one incident. List filters are `scope_type`, `scope_id`, `metric`, `status`, `start_date`, and `end_date`. Dates use `YYYY-MM-DD`; date filters include both endpoints. The maximum page size is 100. + +## Create a rule + +This example alerts when a team's daily spend is at least three times its baseline and the absolute change is at least $10. Replace `team-example` with the team's ID. + +```bash +curl "$LITELLM_API_BASE/usage/alert-rules" \ + -H "Authorization: Bearer $LITELLM_MANAGEMENT_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Team daily spend", + "scope_type": "team", + "scope_id": "team-example", + "metric": "spend", + "direction": "increase", + "multiplier": 3, + "baseline_days": 7, + "min_history_days": 3, + "min_requests": 100, + "min_value": 10, + "enabled": true + }' +``` + +The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value at or below half the baseline. `min_value` is the minimum absolute difference in the metric's own units. + +| Metric | Unit | +| --- | --- | +| `spend` | USD | +| `requests` | Request count | +| `latency` | Average response time in milliseconds | +| `errors` | Failed-request count | +| `error_rate` | Failed requests divided by requests, as a percentage from 0 to 100 | + +Counts come from the existing Activity/spend aggregates, rather than every gateway HTTP outcome. Latency averages successful timed, non-internal requests; it is not time to first token and does not include failed requests. Issue Activity and gateway source incidents use the same aggregates. An error-rate `min_value` of 2 means two percentage points, such as 1% to 3%, not a 2% relative increase. + +The evaluator runs against daily aggregates. An hourly scheduler does not make these rolling hourly alerts. A complete UTC day and sufficient ingestion coverage are required before detecting drops; a missing flush must not become a false traffic-drop incident. Rule responses expose `last_evaluated_day` and `evaluation_status` so you can distinguish a completed evaluation from insufficient history or coverage. + +Use `scope_type: "deployment"` with no `scope_id` for the gateway. For `model`, use the underlying model identity recorded in Activity. For `key`, use its reporting ID, never the secret API key. Omitting `scope_id` for key, user, team, or model evaluates all entities of that type and requires a gateway administrator. + +`GET /usage/alert-models?limit=20&search=example` returns paginated model identities recorded in the most recent 30 UTC days. Its `data` entries contain `scope_id` and `label`; `start_date` and `end_date` state the searched window. Pass `next_cursor` as `cursor` to read the next page. A null cursor ends the result. Search matches literal text, not SQL wildcards. This endpoint requires gateway admin or admin-viewer access. Configured aliases and deployment IDs are not reporting-model identities. An existing rule's saved model remains valid even when it falls outside the options window. + +## Manage rules and incidents + +- `GET /usage/alert-rules` lists rules; `GET /usage/alert-rules/{rule_id}` reads one. Use `kind=custom`, `kind=default`, or `kind=all` to choose which rules to list. Each rule exposes `is_default`. The list's `default_summary` counts total and enabled defaults within the requested scope, independent of `kind`, and reports evaluation-status counts among enabled defaults. +- `PUT /usage/alert-rules/{rule_id}` replaces the rule configuration. Send the configuration fields from the create example, including any notification settings you want to retain. Set `enabled` to `false` to pause it. Built-in rules keep their scope, metric, and direction; create a custom rule to change those fields. Their names, thresholds, notification settings, and enabled state remain editable. +- `DELETE /usage/alert-rules/{rule_id}` removes a rule from evaluation and preserves existing incidents. Deleted defaults stay deleted. +- `PATCH /usage/alerts/{alert_id}` with `{"status":"dismissed"}` dismisses an incident. `open` and `resolved` are also accepted. + +Consecutive breached days update one incident. Re-evaluating the same day does not reopen a dismissed or resolved incident. After recovery, a later breach starts a new incident. Editing a rule does not rewrite historical comparisons. + +Rules can specify `notification_channels: ["slack", "email"]` and `email_recipients: ["alerts@example.com"]`. Notification transports must already be configured on the gateway. An empty channel list keeps incidents available through the API and dashboard. Issue delivery is recorded per channel and destination. A repeated projection does not resend a successful delivery. Failed deliveries retry in the background, and the worker rechecks the issue, source signal, and current rule immediately before transport. Disabling a rule or acknowledging an issue cannot recall a send already in flight. Delivery is at least once: an ambiguous transport timeout, or partial success across several configured Slack webhooks, can cause a retry after a destination accepted a message. Configure `PROXY_BASE_URL` for clickable links in email and Slack. Default rules notify through Slack only when `usage_alerts` is explicitly enabled in the gateway's alert types when those rules are first registered. Otherwise they start with in-app/API incidents. To enable notifications after the defaults already exist, edit their channels through the API or dashboard; changing the gateway alert types does not rewrite saved rules. Explicit rule channels use the configured transport without an additional alert-type switch. If legacy `user_spend_anomalies` notifications are enabled, the default daily user-spend increase rule stays in-app to avoid overlapping notifications. Email requires recipients on the rule. + +## Access + +Gateway administrators can manage every scope. Admin viewers can read every scope. Internal users can manage their own user and key scopes; team administrators can manage their team's scope and keys. Team members can read their team's incidents. Read-only users cannot change rules or incident state, and inference-only keys cannot access these endpoints. + +Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` can indicate an unavailable database or exhausted analytics capacity; a `Retry-After` header indicates that the client should retry later. + +The dashboard's Alerts page is available to administrators and admin viewers. Other authorized roles can use the scoped API. diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql new file mode 100644 index 00000000000..f01406ae385 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_add_daily_usage_alerts/migration.sql @@ -0,0 +1,26 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertRule" ( + "id" TEXT PRIMARY KEY, "scope_type" TEXT NOT NULL, "scope_id" TEXT, + "enabled" BOOLEAN NOT NULL DEFAULT true, "record" JSONB NOT NULL, + "deleted" BOOLEAN NOT NULL DEFAULT false +); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertRule_enabled_deleted_id_idx" ON "LiteLLM_UsageAlertRule" ("enabled", "deleted", "id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertRule_scope_type_scope_id_deleted_idx" ON "LiteLLM_UsageAlertRule" ("scope_type", "scope_id", "deleted"); +CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertIncident" ( + "id" TEXT PRIMARY KEY, "rule_id" TEXT NOT NULL, "scope_key" TEXT NOT NULL, + "scope_type" TEXT NOT NULL, "scope_id" TEXT, "metric" TEXT NOT NULL, + "status" TEXT NOT NULL, "day" TEXT NOT NULL, "evaluation_day" TEXT NOT NULL, + "breached" BOOLEAN NOT NULL, "record" JSONB NOT NULL, + "notified_channels" TEXT[] DEFAULT ARRAY[]::TEXT[], + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_rule_id_scope_key_evaluation_day_idx" ON "LiteLLM_UsageAlertIncident" ("rule_id", "scope_key", "evaluation_day"); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_scope_type_scope_id_day_idx" ON "LiteLLM_UsageAlertIncident" ("scope_type", "scope_id", "day"); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_status_day_idx" ON "LiteLLM_UsageAlertIncident" ("status", "day"); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_updated_at_idx" ON "LiteLLM_UsageAlertIncident" ("updated_at"); +CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertCheckpoint" ( + "id" TEXT PRIMARY KEY, "completed" BOOLEAN NOT NULL DEFAULT false, + "owner" TEXT, "expires_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertCheckpoint_expires_at_idx" ON "LiteLLM_UsageAlertCheckpoint" ("expires_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql new file mode 100644 index 00000000000..6afa8f18a0e --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105200000_issues_inbox/migration.sql @@ -0,0 +1,51 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_Issue" ( + "id" TEXT NOT NULL, + "cohort_key" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "family" TEXT NOT NULL, + "scope_type" TEXT NOT NULL, + "scope_id" TEXT, + "status" TEXT NOT NULL, + "active" BOOLEAN NOT NULL DEFAULT true, + "ack_key" TEXT, + "record" JSONB NOT NULL, + "first_seen" TIMESTAMP(3) NOT NULL, + "last_seen" TIMESTAMP(3) NOT NULL, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "LiteLLM_Issue_pkey" PRIMARY KEY ("id") +); +CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_status_last_seen_id_idx" ON "LiteLLM_Issue"("status", "last_seen", "id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_kind_family_last_seen_id_idx" ON "LiteLLM_Issue"("kind", "family", "last_seen", "id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_scope_type_scope_id_last_seen_id_idx" ON "LiteLLM_Issue"("scope_type", "scope_id", "last_seen", "id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_Issue_cohort_key_active_idx" ON "LiteLLM_Issue"("cohort_key", "active"); +CREATE TABLE IF NOT EXISTS "LiteLLM_IssueSignal" ( + "issue_id" TEXT NOT NULL, + "source" TEXT NOT NULL, + "source_id" TEXT NOT NULL, + "record" JSONB NOT NULL, + CONSTRAINT "LiteLLM_IssueSignal_pkey" PRIMARY KEY ("issue_id", "source", "source_id") +); +CREATE INDEX IF NOT EXISTS "LiteLLM_IssueSignal_source_source_id_idx" ON "LiteLLM_IssueSignal"("source", "source_id"); +CREATE TABLE IF NOT EXISTS "LiteLLM_IssueDelivery" ( + "id" TEXT NOT NULL, + "issue_id" TEXT NOT NULL, + "record" JSONB NOT NULL, + "sent" BOOLEAN NOT NULL DEFAULT false, + "next_attempt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "LiteLLM_IssueDelivery_pkey" PRIMARY KEY ("id") +); +CREATE INDEX IF NOT EXISTS "LiteLLM_IssueDelivery_sent_next_attempt_id_idx" ON "LiteLLM_IssueDelivery"("sent", "next_attempt", "id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_IssueDelivery_issue_id_idx" ON "LiteLLM_IssueDelivery"("issue_id"); +CREATE TABLE IF NOT EXISTS "LiteLLM_IssueDeliveryRule" ( + "delivery_id" TEXT NOT NULL, + "rule_id" TEXT NOT NULL, + "source_id" TEXT NOT NULL, + "rule_revision" TEXT NOT NULL, + CONSTRAINT "LiteLLM_IssueDeliveryRule_pkey" PRIMARY KEY ("delivery_id", "rule_id", "source_id") +); +ALTER TABLE "LiteLLM_UsageAlertIncident" ADD COLUMN IF NOT EXISTS "evaluation_revision" TEXT NOT NULL DEFAULT ''; +CREATE INDEX IF NOT EXISTS "LiteLLM_UsageAlertIncident_evaluation_day_breached_id_idx" + ON "LiteLLM_UsageAlertIncident"("evaluation_day", "breached", "id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 818f2dc4858..21ca3d46844 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2168,3 +2168,96 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +model LiteLLM_UsageAlertRule { + id String @id + scope_type String + scope_id String? + enabled Boolean @default(true) + record Json + deleted Boolean @default(false) + @@index([enabled, deleted, id]) + @@index([scope_type, scope_id, deleted]) +} + +model LiteLLM_UsageAlertIncident { + id String @id + rule_id String + scope_key String + scope_type String + scope_id String? + metric String + status String + day String + evaluation_day String + evaluation_revision String @default("") + breached Boolean + record Json + notified_channels String[] @default([]) + updated_at DateTime @default(now()) + @@index([rule_id, scope_key, evaluation_day]) + @@index([evaluation_day, breached, id]) + @@index([scope_type, scope_id, day]) + @@index([status, day]) + @@index([updated_at]) +} + +model LiteLLM_UsageAlertCheckpoint { + id String @id + completed Boolean @default(false) + owner String? + expires_at DateTime @default(now()) + @@index([expires_at]) +} + +model LiteLLM_Issue { + id String @id + cohort_key String + kind String + family String + scope_type String + scope_id String? + status String + active Boolean @default(true) + ack_key String? + record Json + first_seen DateTime + last_seen DateTime + updated_at DateTime @default(now()) + + @@index([status, last_seen, id]) + @@index([kind, family, last_seen, id]) + @@index([scope_type, scope_id, last_seen, id]) + @@index([cohort_key, active]) +} + +model LiteLLM_IssueSignal { + issue_id String + source String + source_id String + record Json + + @@id([issue_id, source, source_id]) + @@index([source, source_id]) +} + +model LiteLLM_IssueDelivery { + id String @id + issue_id String + record Json + sent Boolean @default(false) + next_attempt DateTime @default(now()) + updated_at DateTime @default(now()) + + @@index([sent, next_attempt, id]) + @@index([issue_id]) +} + +model LiteLLM_IssueDeliveryRule { + delivery_id String + rule_id String + source_id String + rule_revision String + + @@id([delivery_id, rule_id, source_id]) +} diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index 0f98a8fce0e..5ca3d4e5d51 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -367,6 +367,13 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_SubscriptionAccountTable", "LiteLLM_LensSignalConfig", "LiteLLM_LensTraceSignal", + "LiteLLM_UsageAlertRule", + "LiteLLM_UsageAlertIncident", + "LiteLLM_UsageAlertCheckpoint", + "LiteLLM_Issue", + "LiteLLM_IssueSignal", + "LiteLLM_IssueDelivery", + "LiteLLM_IssueDeliveryRule", ) ) PRISMA_RELATIONS: Final[frozenset[str]] = _PRISMA_MODELS | _PRISMA_VIEWS diff --git a/litellm/models/issues.py b/litellm/models/issues.py new file mode 100644 index 00000000000..22043c234d6 --- /dev/null +++ b/litellm/models/issues.py @@ -0,0 +1,151 @@ +from datetime import date, datetime +from typing import Literal, Protocol + +from litellm.models.team import LiteLLM_TeamTable +from litellm.types.proxy.issues import ( + Issue, + IssueActivityResponse, + IssueCohort, + IssueDeliveryBinding, + IssueDeliveryInput, + IssueFamily, + IssueFamilyList, + IssueFamilySetting, + IssueFamilyUpdate, + IssueImpact, + IssueInput, + IssueKind, + IssueList, + IssuePendingDelivery, + IssueProjectionSignal, + IssueScope, + IssueSignal, + IssueSignalInput, + IssueSignalList, +) +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailyActivityKeyPageResponse, + DailyActivityUserPageResponse, +) +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus + + +class IssueStore(Protocol): + async def get_issue(self, issue_id: str) -> Issue | None: ... + + async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: ... + + async def list_issues( + self, + kind: IssueKind | None = None, + family: IssueFamily | None = None, + status: UsageAlertStatus | None = None, + scope_type: IssueScope | None = None, + scope_id: str | None = None, + limit: int = 50, + offset: int = 0, + ) -> IssueList: ... + + async def set_status( + self, + issue_id: str, + status: UsageAlertStatus, + expected_status: UsageAlertStatus, + actor: str | None = None, + authorized_scopes: tuple[IssueCohort, ...] | None = None, + ) -> Issue | None: ... + + async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]: ... + + async def upsert_issue( + self, + data: IssueInput, + reopen: bool = False, + status: UsageAlertStatus | None = None, + ) -> Issue: ... + + async def upsert_issues( + self, + inputs: tuple[IssueInput, ...], + reopen: bool = False, + status: UsageAlertStatus | None = None, + ) -> tuple[Issue, ...]: ... + + async def find_active_issue(self, cohort_key: str) -> Issue | None: ... + + async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: ... + + async def add_signals(self, issue_id: str, signals: tuple[IssueSignalInput, ...]) -> None: ... + + async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: ... + + async def list_signals( + self, issue_id: str, limit: int = 50, offset: int = 0, visible_scope: IssueCohort | None = None + ) -> IssueSignalList: ... + + async def active_signals( + self, + day: date, + limit: int = 500, + cursor: str | None = None, + scope_type: IssueScope | None = None, + metric: UsageAlertMetric | None = None, + scope_ids: tuple[str, ...] | None = None, + ) -> tuple[IssueProjectionSignal, ...]: ... + + async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: ... + + async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: ... + + async def recover_issue_ids(self, ids: tuple[str, ...], observed_at: datetime) -> int: ... + + async def projection_completed(self, day: date, version: str) -> bool: ... + + async def complete_projection(self, day: date, version: str) -> None: ... + + async def finish_deprecation_projection(self, observed_at: datetime) -> int: ... + + async def family_settings(self) -> IssueFamilyList: ... + + async def update_family( + self, family: PerformanceIssueFamily, data: IssueFamilyUpdate + ) -> IssueFamilySetting | None: ... + + async def queue_delivery( + self, + data: IssueDeliveryInput, + rule_id: str, + rule_revision: str, + source_id: str, + ) -> None: ... + + async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: ... + + async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: ... + + async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: ... + + async def mark_delivery_sent(self, delivery_id: str) -> None: ... + + async def defer_delivery( + self, delivery_id: str, error: Literal["delivery_failed", "transport_not_configured"] + ) -> None: ... + + async def prune(self, retention_days: int = 90, limit: int = 500) -> None: ... + + +class IssueActivityStore(Protocol): + async def daily_activity( + self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 + ) -> IssueActivityResponse: ... + + async def keys( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityKeyPageResponse: ... + + async def users( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityUserPageResponse: ... + + async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: ... diff --git a/litellm/models/usage_alerts.py b/litellm/models/usage_alerts.py new file mode 100644 index 00000000000..63e2e05b024 --- /dev/null +++ b/litellm/models/usage_alerts.py @@ -0,0 +1,77 @@ +from datetime import date +from typing import Protocol + +from litellm.types.proxy.usage_alerts import ( + UsageAlert, + UsageAlertGroupList, + UsageAlertGroupUpdate, + UsageAlertGroupUpdateResult, + UsageAlertList, + UsageAlertMetric, + UsageAlertModelList, + UsageAlertRule, + UsageAlertRuleInput, + UsageAlertRuleKind, + UsageAlertRuleList, + UsageAlertScope, + UsageAlertStatus, +) + + +class UsageAlertStore(Protocol): + async def get_rule(self, rule_id: str) -> UsageAlertRule | None: ... + + async def save_rule( + self, + data: UsageAlertRuleInput, + created_by: str | None, + existing: UsageAlertRule | None = None, + *, + overrides_defaults: bool = False, + ) -> UsageAlertRule | None: ... + + async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: ... + + async def list_rules( + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + limit: int = 50, + offset: int = 0, + kind: UsageAlertRuleKind = "all", + ) -> UsageAlertRuleList: ... + + async def get_alert(self, alert_id: str) -> UsageAlert | None: ... + + async def list_alerts( + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + metric: UsageAlertMetric | None = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: int = 50, + offset: int = 0, + ) -> UsageAlertList: ... + + async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None: ... + + async def list_groups( + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: int = 50, + offset: int = 0, + ) -> UsageAlertGroupList: ... + + async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult: ... + + async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList: ... + + async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None: ... + + async def team_role(self, team_id: str, user_id: str) -> str | None: ... diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index d8d3d563783..482a172c924 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -4488,6 +4488,16 @@ "title": "Successful Requests", "type": "integer" }, + "timed_requests": { + "default": 0, + "title": "Timed Requests", + "type": "integer" + }, + "total_response_time_ms": { + "default": 0, + "title": "Total Response Time Ms", + "type": "integer" + }, "total_tokens": { "default": 0, "title": "Total Tokens", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e53b5d4ad52..5d42ce4767d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -595,6 +595,21 @@ class LiteLLMRoutes(enum.Enum): ) llm_api_routes_without_mcp = sorted(set(llm_api_routes) - set(mcp_inference_routes)) info_routes = [ + "/usage/alert-groups", + "/usage/alert-models", + "/usage/alert-rules", + "/usage/alert-rules/{rule_id}", + "/usage/alerts", + "/usage/alerts/{alert_id}", + "/issues", + "/issues/families", + "/issues/families/{family}", + "/issues/{issue_id}", + "/issues/{issue_id}/signals", + "/issues/{issue_id}/activity", + "/issues/{issue_id}/keys", + "/issues/{issue_id}/users", + "/issues/{issue_id}/impact", "/key/info", "/key/health", "/team/info", @@ -662,6 +677,21 @@ class LiteLLMRoutes(enum.Enum): management_routes = ( [ + "/usage/alert-groups", + "/usage/alert-models", + "/usage/alert-rules", + "/usage/alert-rules/{rule_id}", + "/usage/alerts", + "/usage/alerts/{alert_id}", + "/issues", + "/issues/families", + "/issues/families/{family}", + "/issues/{issue_id}", + "/issues/{issue_id}/signals", + "/issues/{issue_id}/activity", + "/issues/{issue_id}/keys", + "/issues/{issue_id}/users", + "/issues/{issue_id}/impact", # user "/user/new", "/management/v1/users/bulk", @@ -872,7 +902,26 @@ class LiteLLMRoutes(enum.Enum): ] ) + usage_alert_routes = ( + "/usage/alert-groups", + "/usage/alert-models", + "/usage/alert-rules", + "/usage/alert-rules/{rule_id}", + "/usage/alerts", + "/usage/alerts/{alert_id}", + "/issues", + "/issues/families", + "/issues/families/{family}", + "/issues/{issue_id}", + "/issues/{issue_id}/signals", + "/issues/{issue_id}/activity", + "/issues/{issue_id}/keys", + "/issues/{issue_id}/users", + "/issues/{issue_id}/impact", + ) + self_managed_routes = [ + *usage_alert_routes, "/lens", "/lens/{path:path}", "/telemetry/ui_events", diff --git a/litellm/proxy/alerting_endpoints/issues.py b/litellm/proxy/alerting_endpoints/issues.py new file mode 100644 index 00000000000..27b663abc18 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/issues.py @@ -0,0 +1,325 @@ +"""Unified issue inbox and investigations backed by existing daily aggregates.""" + +from collections.abc import Generator +from contextlib import contextmanager +from datetime import date +from typing import Final + +from fastapi import APIRouter, HTTPException + +from litellm.models.issues import IssueActivityStore, IssueStore +from litellm.proxy.alerting_endpoints.usage_alerts import ( + Limit, + Offset, + ScopeId, + User, + require_repository, + require_scope, + require_write_access, +) +from litellm.proxy.deps.repositories import IssueActivityRepositoryDep, IssuesRepositoryDep, UsageAlertsRepositoryDep +from litellm.proxy.management.teams.authz import is_team_admin +from litellm.proxy.management_endpoints.common_utils import team_member_has_permission +from litellm.repositories.analytics_budget import AnalyticsCapacityError +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.issues import ( + Issue, + IssueActivityResponse, + IssueCohort, + IssueFamily, + IssueFamilyList, + IssueFamilySetting, + IssueFamilyUpdate, + IssueImpact, + IssueKind, + IssueList, + IssueScope, + IssueSignalList, + IssueUpdate, +) +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailyActivityKeyPageResponse, + DailyActivityUserPageResponse, +) +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import UsageAlertStatus + +router: Final = APIRouter(prefix="/issues", tags=["issues"]) + + +def require_issues(repository: IssueStore | None) -> IssueStore: + if repository is None: + raise HTTPException(status_code=503, detail="Issues require a database connection") + return repository + + +def require_activity(repository: IssueActivityStore | None) -> IssueActivityStore: + if repository is None: + raise HTTPException(status_code=503, detail="Issue Activity requires a database connection") + return repository + + +@contextmanager +def activity_budget() -> Generator[None]: + try: + yield + except AnalyticsCapacityError as exc: + raise HTTPException(status_code=503, detail=str(exc), headers={"Retry-After": "5"}) from exc + + +async def authorized_issue( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + *, + write: bool = False, +) -> Issue: + issue: Final = await require_issues(repository).get_issue(issue_id) + if issue is None: + raise HTTPException(status_code=404, detail="Issue not found") + await require_scope( + user, + require_repository(alerts), + "model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type, + issue.cohort.scope_id, + write=write, + ) + return issue + + +async def can_view_activity(issue: Issue, user: User, repository: IssueStore) -> bool: + if issue.cohort.scope_type != "team" or user.user_role in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + return True + if user.user_id is None or issue.cohort.scope_id is None: + return False + team: Final = await repository.team_activity_access(issue.cohort.scope_id, user.user_id) + return team is not None and ( + is_team_admin(user, team) or team_member_has_permission(user, team, "/team/daily/activity") + ) + + +async def require_activity_access(issue: Issue, user: User, repository: IssuesRepositoryDep) -> None: + if not await can_view_activity(issue, user, require_issues(repository)): + raise HTTPException(status_code=403, detail="Full team Activity permission is required for this investigation") + + +async def require_status_scopes( + issue: Issue, user: User, repository: IssueStore, alerts: UsageAlertsRepositoryDep +) -> tuple[IssueCohort, ...] | None: + store: Final = require_repository(alerts) + await require_scope( + user, + store, + "model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type, + issue.cohort.scope_id, + write=True, + ) + if user.user_role == LitellmUserRoles.PROXY_ADMIN: + return None + scopes: Final = await repository.status_scopes(issue.id) + if len(scopes) > 10: + raise HTTPException(status_code=403, detail="An administrator must update this alert's status") + for scope in scopes: + await require_scope( + user, store, "model" if scope.scope_type == "provider" else scope.scope_type, scope.scope_id, write=True + ) + return scopes + + +def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]: + start: Final = start_date or issue.range_start + end: Final = end_date or issue.range_end + if start > end or (end - start).days > 365: + raise HTTPException(status_code=422, detail="Choose an ordered date range of at most 366 days") + return start, end + + +@router.get("", response_model=IssueList) +async def list_issues( + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + kind: IssueKind | None = None, + family: IssueFamily | None = None, + status: UsageAlertStatus | None = None, + scope_type: IssueScope | None = None, + scope_id: ScopeId = None, + limit: Limit = 50, + offset: Offset = 0, +) -> IssueList: + await require_scope( + user, require_repository(alerts), "model" if scope_type == "provider" else scope_type, scope_id, write=False + ) + return await require_issues(repository).list_issues(kind, family, status, scope_type, scope_id, limit, offset) + + +@router.get("/families", response_model=IssueFamilyList) +async def list_issue_families( + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> IssueFamilyList: + await require_scope(user, require_repository(alerts), "deployment", None, write=False) + return await require_issues(repository).family_settings() + + +@router.patch("/families/{family}", response_model=IssueFamilySetting) +async def update_issue_family( + family: PerformanceIssueFamily, + data: IssueFamilyUpdate, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> IssueFamilySetting: + require_write_access(user, f"/issues/families/{family}") + await require_scope(user, require_repository(alerts), "deployment", None, write=True) + try: + result: Final = await require_issues(repository).update_family(family, data) + except ValueError as exc: + raise HTTPException( + status_code=422, detail="Invalid family policy; check direction and history bounds" + ) from exc + if result is None: + raise HTTPException(status_code=409, detail="Built-in rules changed or are unavailable; reload and retry") + return result + + +@router.get("/{issue_id}", response_model=Issue) +async def get_issue( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> Issue: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + activity_allowed: Final = await can_view_activity(issue, user, require_issues(repository)) + try: + require_write_access(user, f"/issues/{issue_id}") + await require_status_scopes(issue, user, require_issues(repository), alerts) + except HTTPException as exc: + if exc.status_code != 403: + raise + return issue.model_copy(update={"can_update_status": False, "can_view_activity": activity_allowed}) + return issue.model_copy(update={"can_update_status": True, "can_view_activity": activity_allowed}) + + +@router.patch("/{issue_id}", response_model=Issue) +async def update_issue( + issue_id: str, + data: IssueUpdate, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, +) -> Issue: + require_write_access(user, f"/issues/{issue_id}") + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + scopes: Final = await require_status_scopes(issue, user, require_issues(repository), alerts) + if issue.kind == "model_deprecation" and not issue.active and data.status == "open": + raise HTTPException(status_code=409, detail="This retirement issue reopens when the model receives new traffic") + changed: Final = await require_issues(repository).set_status( + issue_id, data.status, data.expected_status, user.user_id, authorized_scopes=scopes + ) + if changed is None: + raise HTTPException(status_code=409, detail="Issue status changed; reload and retry") + return changed + + +@router.get("/{issue_id}/signals", response_model=IssueSignalList) +async def list_issue_signals( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + limit: Limit = 50, + offset: Offset = 0, +) -> IssueSignalList: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + visible_scope: Final = ( + None + if user.user_role + in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ) + else issue.cohort + ) + return await require_issues(repository).list_signals(issue_id, limit, offset, visible_scope) + + +@router.get("/{issue_id}/activity", response_model=IssueActivityResponse) +async def get_issue_activity( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, + model_limit: Limit = 20, +) -> IssueActivityResponse: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) + start, end = activity_range(issue, start_date, end_date) + with activity_budget(): + return await require_activity(activity).daily_activity(issue.cohort, start, end, model_limit) + + +@router.get("/{issue_id}/keys", response_model=DailyActivityKeyPageResponse) +async def get_issue_keys( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, + limit: Limit = 20, + offset: Offset = 0, +) -> DailyActivityKeyPageResponse: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) + start, end = activity_range(issue, start_date, end_date) + with activity_budget(): + return await require_activity(activity).keys(issue.cohort, start, end, limit, offset) + + +@router.get("/{issue_id}/users", response_model=DailyActivityUserPageResponse) +async def get_issue_users( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, + limit: Limit = 20, + offset: Offset = 0, +) -> DailyActivityUserPageResponse: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) + if issue.cohort.scope_type == "team": + raise HTTPException(status_code=422, detail="Historical team aggregates do not retain user membership") + start, end = activity_range(issue, start_date, end_date) + with activity_budget(): + return await require_activity(activity).users(issue.cohort, start, end, limit, offset) + + +@router.get("/{issue_id}/impact", response_model=IssueImpact) +async def get_issue_impact( + issue_id: str, + user: User, + repository: IssuesRepositoryDep, + alerts: UsageAlertsRepositoryDep, + activity: IssueActivityRepositoryDep, + start_date: date | None = None, + end_date: date | None = None, +) -> IssueImpact: + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) + start, end = activity_range(issue, start_date, end_date) + with activity_budget(): + return await require_activity(activity).impact(issue.cohort, start, end) diff --git a/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py index a69dd6ee025..f0a057c4be2 100644 --- a/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py +++ b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py @@ -1,7 +1,5 @@ """Shared acknowledgement state for model migration alerts.""" -import hashlib -import json from datetime import date, datetime, timezone from typing import Annotated, Final @@ -10,6 +8,7 @@ from pydantic import TypeAdapter from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key from litellm.proxy.deps.database import DatabaseDep from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket @@ -32,12 +31,6 @@ def alert_tag_repository(database: DatabaseDep) -> DailyTagSpendRepository: return DailyTagSpendRepository(database.prisma) -def _state_key(model: str, deprecation_date: date) -> str: - identity: Final = json.dumps([model, deprecation_date.isoformat()]) - digest: Final = hashlib.sha256(identity.encode()).hexdigest() - return f"model_deprecation_alert:{digest}" - - def _require_admin(user: UserAPIKeyAuth, *, write: bool) -> None: if user.user_role == LitellmUserRoles.PROXY_ADMIN: return @@ -54,7 +47,7 @@ async def get_deprecation_alert_state( repository: Annotated[ConfigRepository, Depends(alert_state_repository)], ) -> DeprecationAlertState: _require_admin(user, write=False) - record: Final = await repository.get_param(_state_key(model, deprecation_date)) + record: Final = await repository.get_param(deprecation_alert_state_key(model, deprecation_date)) if record is None: return DeprecationAlertState(model=model, deprecation_date=deprecation_date) return DeprecationAlertState.model_validate(record.param_value) @@ -70,7 +63,9 @@ async def update_deprecation_alert_state( state: Final = DeprecationAlertState( **data.model_dump(), updated_at=datetime.now(timezone.utc), updated_by=user.user_id ) - await repository.set_param(_state_key(data.model, data.deprecation_date), state.model_dump(mode="json")) + await repository.set_param( + deprecation_alert_state_key(data.model, data.deprecation_date), state.model_dump(mode="json") + ) return state diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py new file mode 100644 index 00000000000..c954c40cc48 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -0,0 +1,293 @@ +"""Public management API for daily usage alert rules and incidents.""" + +from datetime import date +from typing import Annotated, Final, TypeAlias + +from fastapi import APIRouter, Depends, HTTPException, Query, Response +from pydantic import BaseModel, TypeAdapter + +from litellm.models.usage_alerts import UsageAlertStore +from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth +from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.deps.repositories import UsageAlertsRepositoryDep +from litellm.repositories.daily_activity_buckets import api_key_from_bucket +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.usage_alerts import ( + UsageAlert, + UsageAlertGroupList, + UsageAlertGroupUpdate, + UsageAlertGroupUpdateResult, + UsageAlertList, + UsageAlertMetric, + UsageAlertModelList, + UsageAlertRule, + UsageAlertRuleInput, + UsageAlertRuleKind, + UsageAlertRuleList, + UsageAlertScope, + UsageAlertStatus, + UsageAlertUpdate, +) + +router: Final = APIRouter(prefix="/usage", tags=["usage alerts"]) + + +def require_repository(repository: UsageAlertStore | None) -> UsageAlertStore: + if repository is None: + raise HTTPException(status_code=503, detail="Usage alerts require a database connection") + return repository + + +User: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] +Repository: TypeAlias = UsageAlertsRepositoryDep +Limit: TypeAlias = Annotated[int, Query(ge=1, le=100)] +Offset: TypeAlias = Annotated[int, Query(ge=0, le=100000)] +ScopeId: TypeAlias = Annotated[str | None, Query(min_length=1, max_length=1024)] + + +def _allows_write(grant: str, route: str) -> bool: + if grant == "info_routes": + return False + paths: Final = ( + TypeAdapter(tuple[str, ...]).validate_python(LiteLLMRoutes[grant].value) + if grant in LiteLLMRoutes.__members__ + else (grant,) + ) + return RouteChecks.check_route_access(route, paths) + + +class _WriteGrants(BaseModel, frozen=True): + allowed_routes: tuple[str, ...] | None = None + + +def require_write_access(user: UserAPIKeyAuth, route: str) -> None: + grants: Final = _WriteGrants.model_validate(user, from_attributes=True).allowed_routes + if not grants: + return + if not any(_allows_write(grant, route) for grant in grants): + raise HTTPException(status_code=403, detail="This credential does not grant alert write access") + + +def require_notification_access( + user: UserAPIKeyAuth, data: UsageAlertRuleInput, existing: UsageAlertRule | None = None +) -> None: + if user.user_role == LitellmUserRoles.PROXY_ADMIN: + return + if data.notification_channels or data.email_recipients: + raise HTTPException(status_code=403, detail="Only proxy admins can configure notification delivery") + if existing is not None and (existing.notification_channels or existing.overrides_defaults): + raise HTTPException(status_code=403, detail="Only proxy admins can change operator-managed alert rules") + + +async def require_scope( + user: UserAPIKeyAuth, + repository: UsageAlertStore, + scope_type: UsageAlertScope | None, + scope_id: str | None, + *, + write: bool, +) -> None: + if user.key_type == "llm_api" or (write and user.key_type == "read_only"): + raise HTTPException(status_code=403, detail="A management-capable credential is required") + if user.user_role == LitellmUserRoles.PROXY_ADMIN: + return + if user.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: + if not write: + return + raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts") + if user.user_role not in ( + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + LitellmUserRoles.ORG_ADMIN, + LitellmUserRoles.TEAM, + ): + raise HTTPException(status_code=403, detail="Usage alerts require a management role") + if write and user.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY: + raise HTTPException(status_code=403, detail="Read-only users cannot manage usage alerts") + if scope_id is None: + raise HTTPException(status_code=403, detail="Specify a scope you can access") + if scope_type == "user" and user.user_id is not None and scope_id == user.user_id: + return + if scope_type == "team": + role: Final = await repository.team_role(scope_id, user.user_id) if user.user_id else None + if role == "admin" or ( + not write and (role == "user" or (user.user_role == LitellmUserRoles.TEAM and user.team_id == scope_id)) + ): + return + if scope_type == "key": + ownership: Final = await repository.key_ownership(api_key_from_bucket(scope_id)) + if ownership is not None: + owner_id, team_id = ownership + if user.user_id is not None and owner_id == user.user_id: + return + if user.user_id and team_id and await repository.team_role(team_id, user.user_id) == "admin": + return + raise HTTPException(status_code=403, detail="You cannot access this alert scope") + + +@router.get("/alert-rules", response_model=UsageAlertRuleList) +async def list_usage_alert_rules( + user: User, + repository: Repository, + scope_type: UsageAlertScope | None = None, + scope_id: ScopeId = None, + limit: Limit = 50, + offset: Offset = 0, + kind: UsageAlertRuleKind = "all", +) -> UsageAlertRuleList: + store: Final = require_repository(repository) + await require_scope(user, store, scope_type, scope_id, write=False) + return await store.list_rules(scope_type, scope_id, limit, offset, kind) + + +@router.get("/alert-groups", response_model=UsageAlertGroupList) +async def list_usage_alert_groups( + user: User, + repository: Repository, + scope_type: UsageAlertScope | None = None, + scope_id: ScopeId = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: Limit = 50, + offset: Offset = 0, +) -> UsageAlertGroupList: + store: Final = require_repository(repository) + await require_scope(user, store, scope_type, scope_id, write=False) + if start_date is not None and end_date is not None and start_date > end_date: + raise HTTPException(status_code=422, detail="start_date must not be after end_date") + return await store.list_groups(scope_type, scope_id, status, start_date, end_date, limit, offset) + + +@router.patch("/alert-groups", response_model=UsageAlertGroupUpdateResult) +async def update_usage_alert_group( + data: UsageAlertGroupUpdate, user: User, repository: Repository +) -> UsageAlertGroupUpdateResult: + store: Final = require_repository(repository) + require_write_access(user, "/usage/alert-groups") + await require_scope(user, store, data.scope_type, data.scope_id, write=True) + return await store.set_group_status(data) + + +@router.get("/alert-models", response_model=UsageAlertModelList) +async def list_usage_alert_models( + user: User, + repository: Repository, + search: ScopeId = None, + cursor: ScopeId = None, + limit: Limit = 20, +) -> UsageAlertModelList: + store: Final = require_repository(repository) + await require_scope(user, store, "model", None, write=False) + return await store.list_reported_models(search, cursor, limit) + + +@router.post("/alert-rules", response_model=UsageAlertRule, status_code=201) +async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, repository: Repository) -> UsageAlertRule: + """Create a daily usage alert rule for a scope the caller can manage.""" + store: Final = require_repository(repository) + require_write_access(user, "/usage/alert-rules") + await require_scope(user, store, data.scope_type, data.scope_id, write=True) + require_notification_access(user, data) + saved: Final = await store.save_rule( + data, user.user_id, overrides_defaults=user.user_role == LitellmUserRoles.PROXY_ADMIN + ) + if saved is None: + raise HTTPException(status_code=409, detail="Alert rule could not be saved") + return saved + + +@router.get("/alert-rules/{rule_id}", response_model=UsageAlertRule) +async def get_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> UsageAlertRule: + store: Final = require_repository(repository) + rule: Final = await store.get_rule(rule_id) + if rule is None: + raise HTTPException(status_code=404, detail="Alert rule not found") + await require_scope(user, store, rule.scope_type, rule.scope_id, write=False) + return rule + + +@router.put("/alert-rules/{rule_id}", response_model=UsageAlertRule) +async def update_usage_alert_rule( + rule_id: str, data: UsageAlertRuleInput, user: User, repository: Repository +) -> UsageAlertRule: + store: Final = require_repository(repository) + require_write_access(user, f"/usage/alert-rules/{rule_id}") + rule: Final = await store.get_rule(rule_id) + if rule is None: + raise HTTPException(status_code=404, detail="Alert rule not found") + await require_scope(user, store, rule.scope_type, rule.scope_id, write=True) + if rule.is_default and (rule.scope_type, rule.scope_id, rule.metric, rule.direction) != ( + data.scope_type, + data.scope_id, + data.metric, + data.direction, + ): + raise HTTPException( + status_code=422, detail="Create a custom rule to change a built-in rule's scope, metric, or direction" + ) + await require_scope(user, store, data.scope_type, data.scope_id, write=True) + require_notification_access(user, data, rule) + saved: Final = await store.save_rule(data, user.user_id, existing=rule, overrides_defaults=rule.overrides_defaults) + if saved is None: + raise HTTPException(status_code=409, detail="Alert rule changed while being updated. Refresh and try again") + return saved + + +@router.delete("/alert-rules/{rule_id}", status_code=204) +async def delete_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> Response: + store: Final = require_repository(repository) + require_write_access(user, f"/usage/alert-rules/{rule_id}") + rule: Final = await store.get_rule(rule_id) + if rule is None: + raise HTTPException(status_code=404, detail="Alert rule not found") + await require_scope(user, store, rule.scope_type, rule.scope_id, write=True) + require_notification_access(user, rule, rule) + if not await store.delete_rule(rule_id, existing=rule): + raise HTTPException(status_code=409, detail="Alert rule changed while being deleted. Refresh and try again") + return Response(status_code=204) + + +@router.get("/alerts", response_model=UsageAlertList) +async def list_usage_alerts( + user: User, + repository: Repository, + scope_type: UsageAlertScope | None = None, + scope_id: ScopeId = None, + metric: UsageAlertMetric | None = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: Limit = 50, + offset: Offset = 0, +) -> UsageAlertList: + store: Final = require_repository(repository) + await require_scope(user, store, scope_type, scope_id, write=False) + if start_date is not None and end_date is not None and start_date > end_date: + raise HTTPException(status_code=422, detail="start_date must not be after end_date") + return await store.list_alerts(scope_type, scope_id, metric, status, start_date, end_date, limit, offset) + + +@router.get("/alerts/{alert_id}", response_model=UsageAlert) +async def get_usage_alert(alert_id: str, user: User, repository: Repository) -> UsageAlert: + store: Final = require_repository(repository) + alert: Final = await store.get_alert(alert_id) + if alert is None: + raise HTTPException(status_code=404, detail="Usage alert not found") + await require_scope(user, store, alert.scope_type, alert.scope_id, write=False) + return alert + + +@router.patch("/alerts/{alert_id}", response_model=UsageAlert) +async def update_usage_alert(alert_id: str, data: UsageAlertUpdate, user: User, repository: Repository) -> UsageAlert: + store: Final = require_repository(repository) + require_write_access(user, f"/usage/alerts/{alert_id}") + alert: Final = await store.get_alert(alert_id) + if alert is None: + raise HTTPException(status_code=404, detail="Usage alert not found") + await require_scope(user, store, alert.scope_type, alert.scope_id, write=True) + updated: Final = await store.set_status(alert_id, data.status) + if updated is None: + raise HTTPException(status_code=404, detail="Usage alert not found") + return updated diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index 8a91c7b0a0c..a210ba6eb69 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -1,5 +1,7 @@ from __future__ import annotations +import hashlib +import json import logging from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass @@ -28,6 +30,11 @@ _SHARED_SCHEDULE_PROVIDER: Final[Mapping[str, str]] = MappingProxyType({"azure_a LoadedCatalog: TypeAlias = Callable[[], Mapping[str, Mapping[str, object]]] +def deprecation_alert_state_key(model: str, deprecation_date: date) -> str: + identity: Final = json.dumps([model, deprecation_date.isoformat()]) + return f"model_deprecation_alert:{hashlib.sha256(identity.encode()).hexdigest()}" + + @dataclass(frozen=True, slots=True) class _ResolvedDeprecation: deprecation_date: date diff --git a/litellm/proxy/deps/repositories.py b/litellm/proxy/deps/repositories.py index 323cf57dfec..1e65f6d6c9d 100644 --- a/litellm/proxy/deps/repositories.py +++ b/litellm/proxy/deps/repositories.py @@ -4,14 +4,19 @@ from fastapi import Depends from litellm.models.deployment_warning import DeploymentWarningIgnoreStore from litellm.models.gateway_activity import GatewayActivityStore +from litellm.models.issues import IssueActivityStore, IssueStore from litellm.models.plugin_upload import PluginUploadStore from litellm.models.skill_submission import SkillSubmissionStore +from litellm.models.usage_alerts import UsageAlertStore from litellm.models.user_banner import UserBannerStore from litellm.proxy.deps.database import DatabaseDep from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository from litellm.repositories.gateway_activity_repository import GatewayActivityRepository +from litellm.repositories.issue_activity import IssueActivityRepository +from litellm.repositories.issues import IssuesRepository from litellm.repositories.plugin_upload_repository import PluginUploadRepository from litellm.repositories.skill_submission_repository import SkillSubmissionRepository +from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.repositories.user_banner_repository import UserBannerRepository @@ -54,3 +59,24 @@ async def get_skill_submission_repository(database: DatabaseDep) -> SkillSubmiss SkillSubmissionRepositoryDep: TypeAlias = Annotated[ SkillSubmissionStore | None, Depends(get_skill_submission_repository) ] + + +async def get_usage_alerts_repository(database: DatabaseDep) -> UsageAlertStore | None: + return None if database is None else UsageAlertsRepository.from_prisma(database.prisma) + + +UsageAlertsRepositoryDep: TypeAlias = Annotated[UsageAlertStore | None, Depends(get_usage_alerts_repository)] + + +async def get_issues_repository(database: DatabaseDep) -> IssueStore | None: + return None if database is None else IssuesRepository.from_prisma(database.prisma) + + +IssuesRepositoryDep: TypeAlias = Annotated[IssueStore | None, Depends(get_issues_repository)] + + +async def get_issue_activity_repository(database: DatabaseDep) -> IssueActivityStore | None: + return None if database is None else IssueActivityRepository.from_prisma(database.prisma) + + +IssueActivityRepositoryDep: TypeAlias = Annotated[IssueActivityStore | None, Depends(get_issue_activity_repository)] diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 34a5cbde76f..3593322738e 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -1162,7 +1162,7 @@ async def get_daily_activity_aggregated( scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit ), _guardrail_spend_or_empty(repository, scope) - if include_guardrail_breakdown and not scope.model + if include_guardrail_breakdown and not scope.model and scope.provider is None and not scope.model_pairs else asyncio.sleep(0, result=()), ) records: Final = aggregated_rows.grouping_rows diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index e9b9dd81362..0b36c4f9136 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -141,6 +141,8 @@ def _key_activity_row(row: KeySpendRow, metadata: Mapping[str, KeyMetadataRow]) failed_requests=row.failed_requests, cache_read_input_tokens=row.cache_read_input_tokens, cache_creation_input_tokens=row.cache_creation_input_tokens, + total_response_time_ms=row.total_response_time_ms, + timed_requests=row.timed_requests, ), metadata=_key_metadata(row.api_key, metadata), ) @@ -175,6 +177,8 @@ def _user_activity_row(row: UserSpendRow, metadata: Mapping[str, UserMetadataRow api_requests=row.api_requests, successful_requests=row.successful_requests, failed_requests=row.failed_requests, + total_response_time_ms=row.total_response_time_ms, + timed_requests=row.timed_requests, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3297a298af4..ea764733a79 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -427,9 +427,11 @@ from litellm.messages.dispatch import messages_target from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import * from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling +from litellm.proxy.alerting_endpoints.issues import router as issues_router from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( router as model_deprecation_alerts_router, ) +from litellm.proxy.alerting_endpoints.usage_alerts import router as usage_alerts_router from litellm.proxy.analytics_endpoints.analytics_endpoints import ( router as analytics_router, ) @@ -22507,6 +22509,8 @@ app.include_router(router_settings_router) app.include_router(fallback_management_router) app.include_router(cache_settings_router) app.include_router(model_deprecation_alerts_router) +app.include_router(usage_alerts_router) +app.include_router(issues_router) app.include_router(coordination_redis_settings_router) app.include_router(user_agent_analytics_router) app.include_router(gateway_request_router) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 818f2dc4858..21ca3d46844 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2168,3 +2168,96 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +model LiteLLM_UsageAlertRule { + id String @id + scope_type String + scope_id String? + enabled Boolean @default(true) + record Json + deleted Boolean @default(false) + @@index([enabled, deleted, id]) + @@index([scope_type, scope_id, deleted]) +} + +model LiteLLM_UsageAlertIncident { + id String @id + rule_id String + scope_key String + scope_type String + scope_id String? + metric String + status String + day String + evaluation_day String + evaluation_revision String @default("") + breached Boolean + record Json + notified_channels String[] @default([]) + updated_at DateTime @default(now()) + @@index([rule_id, scope_key, evaluation_day]) + @@index([evaluation_day, breached, id]) + @@index([scope_type, scope_id, day]) + @@index([status, day]) + @@index([updated_at]) +} + +model LiteLLM_UsageAlertCheckpoint { + id String @id + completed Boolean @default(false) + owner String? + expires_at DateTime @default(now()) + @@index([expires_at]) +} + +model LiteLLM_Issue { + id String @id + cohort_key String + kind String + family String + scope_type String + scope_id String? + status String + active Boolean @default(true) + ack_key String? + record Json + first_seen DateTime + last_seen DateTime + updated_at DateTime @default(now()) + + @@index([status, last_seen, id]) + @@index([kind, family, last_seen, id]) + @@index([scope_type, scope_id, last_seen, id]) + @@index([cohort_key, active]) +} + +model LiteLLM_IssueSignal { + issue_id String + source String + source_id String + record Json + + @@id([issue_id, source, source_id]) + @@index([source, source_id]) +} + +model LiteLLM_IssueDelivery { + id String @id + issue_id String + record Json + sent Boolean @default(false) + next_attempt DateTime @default(now()) + updated_at DateTime @default(now()) + + @@index([sent, next_attempt, id]) + @@index([issue_id]) +} + +model LiteLLM_IssueDeliveryRule { + delivery_id String + rule_id String + source_id String + rule_revision String + + @@id([delivery_id, rule_id, source_id]) +} diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py index 36c6ab7f711..5cf5dafcce5 100644 --- a/litellm/repositories/config_repository.py +++ b/litellm/repositories/config_repository.py @@ -30,7 +30,7 @@ class _ConfigRow(Protocol): class _ConfigTable(Protocol): async def find_unique(self, *, where: Mapping[str, str]) -> _ConfigRow | None: ... - async def find_many(self) -> Sequence[_ConfigRow]: ... + async def find_many(self, *, where: Mapping[str, object] | None = None) -> Sequence[_ConfigRow]: ... async def upsert(self, *, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]]) -> _ConfigRow: ... @@ -83,6 +83,18 @@ class ConfigRepository: param_value = _decoded_json(param_value) return ConfigParam(param_name=param_name, param_value=param_value) + async def get_params(self, param_names: Sequence[str]) -> tuple[ConfigParam, ...]: + if not param_names: + return () + records: Final = await self._config_table.find_many(where={"param_name": {"in": tuple(param_names)}}) + return tuple( + ConfigParam( + record.param_name, + _decoded_json(record.param_value) if isinstance(record.param_value, str) else record.param_value, + ) + for record in records + ) + async def set_param(self, param_name: str, param_value: object) -> ConfigParam: """Set a config parameter in the database.""" value_json: Final = json.dumps(param_value) if not isinstance(param_value, str) else param_value diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 96a15e09ee4..e2e29b9b3ba 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -216,8 +216,10 @@ class DailyActivityRepository: rows: Final = _KEY_SPEND_ADAPTER.validate_python(await self._query(query)) return tuple(row.api_key for row in rows) - async def key_page(self, scope: DailyActivityScope, *, offset: int, limit: int) -> KeyPage: - query: Final = build_key_page_sql(scope, offset=offset, limit=limit) + async def key_page( + self, scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False + ) -> KeyPage: + query: Final = build_key_page_sql(scope, offset=offset, limit=limit, bounded_identifiers=bounded_identifiers) result: Final = await self._query(query) total_api_keys_value: Final = result[0].get("total_api_keys") if result else 0 total_api_keys: Final = ( @@ -226,8 +228,10 @@ class DailyActivityRepository: rows: Final = _KEY_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("api_key") is not None)) return KeyPage(rows=rows, total_api_keys=total_api_keys) - async def user_page(self, scope: DailyActivityScope, *, offset: int, limit: int) -> UserPage: - query: Final = build_user_page_sql(scope, offset=offset, limit=limit) + async def user_page( + self, scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False + ) -> UserPage: + query: Final = build_user_page_sql(scope, offset=offset, limit=limit, bounded_identifiers=bounded_identifiers) result: Final = await self._query(query) total_users_value: Final = result[0].get("total_users") if result else 0 total_users: Final = ( @@ -335,6 +339,8 @@ class DailyActivityRepository: ) async def daily_rows(self, scope: DailyActivityScope, *, page: int, page_size: int) -> DailyRowsPage: + if scope.provider is not None or scope.model_pairs: + raise ValueError("Provider cohorts require the aggregated Activity API") table: Final = _daily_rows_table(self._prisma_client, scope.table) adjusted_start, adjusted_end = adjust_dates_for_timezone( scope.start_date, diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 83ea31e461d..25fb3304f4e 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -1,3 +1,4 @@ +import json from collections.abc import Mapping from dataclasses import dataclass, replace from datetime import date, datetime, timedelta, timezone @@ -75,6 +76,8 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu exclusion_index: Final = entity_index + int(has_entity_array) model_index: Final = exclusion_index + int(bool(scope.exclude_entity_ids)) api_keys_index: Final = model_index + int(bool(scope.model)) + provider_index: Final = api_keys_index + int(bool(scope.api_keys)) + pairs_index: Final = provider_index + int(scope.provider is not None) conditions: Final = ( f"date >= ${start_index}", f"date <= ${start_index + 1}", @@ -85,6 +88,21 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu if has_entity_array else () ), + *( + (f"COALESCE(NULLIF(custom_llm_provider, ''), 'unknown') = ${provider_index}",) + if scope.provider is not None + else () + ), + *( + ( + f"EXISTS (SELECT 1 FROM jsonb_to_recordset(${pairs_index}::jsonb) " + f"AS cohort(match_provider text, match_model text) WHERE " + f"cohort.match_provider=COALESCE(NULLIF(custom_llm_provider, ''), 'unknown') " + f"AND cohort.match_model={_MODEL_EXPR})", + ) + if scope.model_pairs + else () + ), *( ( f'("{scope.entity_id_field}" IS NULL ' @@ -113,6 +131,16 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu *((list(scope.exclude_entity_ids),) if scope.exclude_entity_ids else ()), *((scope.model,) if scope.model else ()), *((list(scope.api_keys),) if scope.api_keys else ()), + *((scope.provider,) if scope.provider is not None else ()), + *( + ( + json.dumps( + tuple({"match_provider": provider, "match_model": model} for provider, model in scope.model_pairs) + ), + ) + if scope.model_pairs + else () + ), ) return " AND ".join(conditions), params @@ -378,10 +406,14 @@ def _key_spend_select() -> str: COALESCE(SUM(successful_requests), 0)::bigint AS successful_requests, COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests, COALESCE(SUM(cache_read_input_tokens), 0)::bigint AS cache_read_input_tokens, - COALESCE(SUM(cache_creation_input_tokens), 0)::bigint AS cache_creation_input_tokens""" + COALESCE(SUM(cache_creation_input_tokens), 0)::bigint AS cache_creation_input_tokens, + COALESCE(SUM(total_response_time_ms), 0)::bigint AS total_response_time_ms, + COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests""" -def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery: +def build_key_page_sql( + scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False +) -> SqlQuery: if not 1 <= limit <= constants.USAGE_KEY_PAGE_MAX: raise ValueError(f"limit must be between 1 and {constants.USAGE_KEY_PAGE_MAX}") if offset < 0: @@ -390,11 +422,17 @@ def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> sentinel_param: Final = len(where_params) + 1 limit_param: Final = sentinel_param + 1 offset_param: Final = limit_param + 1 + identity_filter: Final = ( + "AND CASE WHEN octet_length(api_key)<=4096 THEN " + "length(api_key)<=CASE WHEN api_key LIKE 'raw:%' THEN 1020 ELSE 1024 END ELSE false END" + if bounded_identifiers + else "" + ) sql: Final = f""" WITH ranked AS ( SELECT api_key,{_key_spend_select()}, SUM(spend::numeric) AS rank_spend FROM "{PRISMA_TO_PG_TABLE[scope.table]}" - WHERE {where_clause} AND api_key <> ${sentinel_param} + WHERE {where_clause} AND api_key <> ${sentinel_param} {identity_filter} GROUP BY api_key ) SELECT (SELECT COUNT(*) FROM ranked)::bigint AS total_api_keys, page.* @@ -416,10 +454,14 @@ def _user_spend_select() -> str: (COALESCE(SUM(prompt_tokens), 0) + COALESCE(SUM(completion_tokens), 0))::bigint AS total_tokens, COALESCE(SUM(api_requests), 0)::bigint AS api_requests, COALESCE(SUM(successful_requests), 0)::bigint AS successful_requests, - COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests""" + COALESCE(SUM(failed_requests), 0)::bigint AS failed_requests, + COALESCE(SUM(total_response_time_ms), 0)::bigint AS total_response_time_ms, + COALESCE(SUM(timed_requests), 0)::bigint AS timed_requests""" -def build_user_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> SqlQuery: +def build_user_page_sql( + scope: DailyActivityScope, *, offset: int, limit: int, bounded_identifiers: bool = False +) -> SqlQuery: if not 1 <= limit <= constants.USAGE_USER_PAGE_MAX: raise ValueError(f"limit must be between 1 and {constants.USAGE_USER_PAGE_MAX}") if offset < 0: @@ -428,11 +470,17 @@ def build_user_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) - limit_param: Final = len(where_params) + 1 offset_param: Final = limit_param + 1 group_expr: Final = f"NULLIF(\"{scope.entity_id_field}\", '')" + identity_filter: Final = ( + f"AND ({group_expr} IS NULL OR CASE WHEN octet_length({group_expr})<=4096 " + f"THEN length({group_expr})<=1024 ELSE false END)" + if bounded_identifiers + else "" + ) sql: Final = f""" WITH ranked AS ( SELECT {group_expr} AS user_id,{_user_spend_select()}, SUM(spend::numeric) AS rank_spend FROM "{PRISMA_TO_PG_TABLE[scope.table]}" - WHERE {where_clause} + WHERE {where_clause} {identity_filter} GROUP BY {group_expr} ) SELECT (SELECT COUNT(*) FROM ranked)::bigint AS total_users, page.* diff --git a/litellm/repositories/issue_activity.py b/litellm/repositories/issue_activity.py new file mode 100644 index 00000000000..51ec62d69a8 --- /dev/null +++ b/litellm/repositories/issue_activity.py @@ -0,0 +1,223 @@ +"""Issue investigations reuse the same aggregate filters as entity Activity.""" + +from collections.abc import Mapping +from dataclasses import asdict, replace +from datetime import date +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # generated Prisma structural database interface + +from pydantic import BaseModel, TypeAdapter + +from litellm.constants import PTU_SENTINEL_API_KEY +from litellm.repositories.analytics_budget import AnalyticsDatabase, BoundedAnalyticsDatabase +from litellm.repositories.bounded_daily_activity import BoundedDailyActivityRepository +from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket +from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase, DailyActivityRepository +from litellm.repositories.daily_activity_sql import PRISMA_TO_PG_TABLE, build_where_clause +from litellm.types.proxy.issues import ( + IssueActivityResponse, + IssueCohort, + IssueImpact, +) +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailyActivityKeyPageResponse, + DailyActivityUserPageResponse, + KeyMetadata, + KeySpendActivityRow, + KeySpendMetrics, + UserActivityRow, +) +from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +class _Counts(BaseModel, frozen=True): + keys: int + entities: int + models: int + requests: int + + +class _KeyLabel(BaseModel, frozen=True): + api_key: str + metadata: KeyMetadata + + +class _UserLabel(BaseModel, frozen=True): + user_id: str + user_alias: str | None + user_email: str | None + + +def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyActivityScope: + if start > end or (end - start).days > 365: + raise ValueError("Choose an ordered date range of at most 366 days") + return DailyActivityScope( + table=DailyActivityTable.TEAM if cohort.scope_type == "team" else DailyActivityTable.USER, + entity_id_field="team_id" if cohort.scope_type == "team" else "user_id", + entity_ids=(cohort.scope_id,) if cohort.scope_type in ("team", "user") and cohort.scope_id else None, + exclude_entity_ids=(), + api_keys=(api_key_from_bucket(cohort.scope_id),) if cohort.scope_type == "key" and cohort.scope_id else None, + start_date=start.isoformat(), + end_date=end.isoformat(), + model=cohort.scope_id if cohort.scope_type == "model" else None, + timezone_offset_minutes=0, + include_current_utc_day=True, + provider=cohort.scope_id if cohort.scope_type == "provider" else None, + model_pairs=tuple((pair.provider, pair.model) for pair in cohort.model_pairs), + ) + + +class IssueActivityRepository: + def __init__(self, activity: DailyActivityRepository, database: DailyActivityQueryDatabase) -> None: + self.activity = activity + self.database = database + + @classmethod + def from_prisma(cls, prisma: "PrismaClient") -> "IssueActivityRepository": + from litellm.proxy.management_endpoints.common_daily_activity import daily_activity_repository + + database: Final = BoundedAnalyticsDatabase(cast(AnalyticsDatabase, prisma.db)) # cast-ok: generated Prisma tx + return cls(daily_activity_repository(prisma, query_database=database), database) + + async def daily_activity( + self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 + ) -> IssueActivityResponse: + return await BoundedDailyActivityRepository(self.database).aggregate( + issue_activity_scope(cohort, start, end), model_limit=model_limit if cohort.scope_type == "key" else None + ) + + async def keys( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityKeyPageResponse: + if not 1 <= limit <= 100 or offset < 0: + raise ValueError("Read between one and 100 keys with a nonnegative offset") + page: Final = await self.activity.key_page( + issue_activity_scope(cohort, start, end), limit=limit, offset=offset, bounded_identifiers=True + ) + metadata: Final = await self._key_labels(tuple(row.api_key for row in page.rows if row.api_key)) + return DailyActivityKeyPageResponse( + api_keys=[ + KeySpendActivityRow( + api_key=api_key_bucket(row.api_key), + metrics=KeySpendMetrics.model_validate(asdict(row)), + metadata=metadata.get(row.api_key, KeyMetadata()), + ) + for row in page.rows + ], + total_api_keys=page.total_api_keys, + limit=limit, + offset=offset, + ) + + async def users( + self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int + ) -> DailyActivityUserPageResponse: + if not 1 <= limit <= 100 or offset < 0: + raise ValueError("Read between one and 100 users with a nonnegative offset") + if cohort.scope_type == "team": + raise ValueError("Historical team aggregates do not retain user membership") + page: Final = await self.activity.user_page( + issue_activity_scope(cohort, start, end), limit=limit, offset=offset, bounded_identifiers=True + ) + metadata: Final = await self._user_labels(tuple(row.user_id for row in page.rows if row.user_id)) + return DailyActivityUserPageResponse( + users=tuple( + UserActivityRow.model_validate( + { + **asdict(row), + **( + metadata[row.user_id].model_dump() + if row.user_id is not None and row.user_id in metadata + else {} + ), + } + ) + for row in page.rows + ), + total_users=page.total_users, + limit=limit, + offset=offset, + ) + + async def _key_labels(self, ids: tuple[str, ...]) -> Mapping[str, KeyMetadata]: + if len(ids) > 100: + raise ValueError("Read at most 100 key labels") + if not ids: + return MappingProxyType({}) + rows: Final = TypeAdapter(tuple[_KeyLabel, ...]).validate_python( + await self.database.query_raw( + """SELECT w.api_key,jsonb_build_object( + 'key_alias',LEFT(COALESCE(k.key_alias,d.key_alias),256), + 'team_id',CASE WHEN octet_length(COALESCE(k.team_id,d.team_id))<=4096 + THEN CASE WHEN length(COALESCE(k.team_id,d.team_id))<=1024 + THEN COALESCE(k.team_id,d.team_id) END END, + 'user_id',CASE WHEN octet_length(COALESCE(k.user_id,d.user_id))<=4096 + THEN CASE WHEN length(COALESCE(k.user_id,d.user_id))<=1024 + THEN COALESCE(k.user_id,d.user_id) END END, + 'user_email',CASE WHEN octet_length(u.user_email)<=1280 + THEN CASE WHEN length(u.user_email)<=320 THEN u.user_email END END, + 'key_exists',CASE WHEN k.token IS NOT NULL THEN true WHEN d.id IS NOT NULL THEN false END + ) AS metadata + FROM unnest($1::text[]) w(api_key) + LEFT JOIN "LiteLLM_VerificationToken" k ON k.token=w.api_key + LEFT JOIN LATERAL ( + SELECT id,key_alias,team_id,user_id FROM "LiteLLM_DeletedVerificationToken" + WHERE token=w.api_key AND k.token IS NULL ORDER BY deleted_at DESC,id DESC LIMIT 1 + ) d ON true + LEFT JOIN "LiteLLM_UserTable" u ON u.user_id=COALESCE(k.user_id,d.user_id) + LIMIT 100""", + list(ids), + ) + ) + return MappingProxyType({row.api_key: row.metadata for row in rows}) + + async def _user_labels(self, ids: tuple[str, ...]) -> Mapping[str, _UserLabel]: + if len(ids) > 100: + raise ValueError("Read at most 100 user labels") + if not ids: + return MappingProxyType({}) + rows: Final = TypeAdapter(tuple[_UserLabel, ...]).validate_python( + await self.database.query_raw( + """SELECT user_id,LEFT(user_alias,256) AS user_alias, + CASE WHEN octet_length(user_email)<=1280 + THEN CASE WHEN length(user_email)<=320 THEN user_email END END AS user_email + FROM "LiteLLM_UserTable" WHERE user_id=ANY($1::text[]) LIMIT 100""", + list(ids), + ) + ) + return MappingProxyType({row.user_id: row for row in rows}) + + async def _counts(self, scope: DailyActivityScope) -> _Counts: + where, params = build_where_clause(scope) + rows: Final = TypeAdapter(tuple[_Counts, ...]).validate_python( + await self.database.query_raw( + f'''SELECT COUNT(DISTINCT NULLIF(api_key,'')) FILTER(WHERE api_key<>${len(params) + 1})::int AS keys, + COUNT(DISTINCT NULLIF("{scope.entity_id_field}",'')) + FILTER(WHERE "{scope.entity_id_field}" NOT IN ('default_user_id','litellm-dashboard'))::int AS entities, + COUNT(DISTINCT NULLIF(model,''))::int AS models, + COALESCE(SUM(api_requests),0)::bigint AS requests + FROM "{PRISMA_TO_PG_TABLE[scope.table]}" WHERE {where}''', + *params, + PTU_SENTINEL_API_KEY, + ) + ) + return rows[0] + + async def impact(self, cohort: IssueCohort, start: date, end: date) -> IssueImpact: + scope: Final = issue_activity_scope(cohort, start, end) + counts: Final = await self._counts(scope) + teams: Final = ( + await self._counts(replace(scope, table=DailyActivityTable.TEAM, entity_id_field="team_id")) + if cohort.scope_type not in ("user", "team") + else None + ) + return IssueImpact( + keys=counts.keys, + models=counts.models, + requests=counts.requests, + users=counts.entities if cohort.scope_type != "team" else None, + teams=counts.entities if cohort.scope_type == "team" else teams.entities if teams else None, + ) diff --git a/litellm/repositories/issue_families.py b/litellm/repositories/issue_families.py new file mode 100644 index 00000000000..698a066db8b --- /dev/null +++ b/litellm/repositories/issue_families.py @@ -0,0 +1,165 @@ +"""Bounded settings for the built-in performance policies.""" + +import json +from collections import Counter +from datetime import datetime, timezone +from itertools import chain, product +from typing import Final + +from pydantic import BaseModel, TypeAdapter + +from litellm.repositories.usage_alerts import AlertDatabase +from litellm.types.proxy.issues import ( + IssueFamilyList, + IssueFamilyPolicy, + IssueFamilySetting, + IssueFamilyUpdate, +) +from litellm.types.proxy.usage_alert_defaults import ( + DEFAULT_USAGE_POLICIES, + DEFAULT_USAGE_RULE_IDS, + USAGE_ALERT_SCOPES, + PerformanceIssueFamily, + default_usage_rule_id, +) +from litellm.types.proxy.usage_alerts import UsageAlertEvaluationCounts, UsageAlertRule + + +class _RuleRow(BaseModel, frozen=True): + record: UsageAlertRule + + +class _Count(BaseModel, frozen=True): + count: int + + +class _ProjectionState(BaseModel, frozen=True): + pending: bool + + +def _setting(family: PerformanceIssueFamily, rules: tuple[UsageAlertRule, ...]) -> IssueFamilySetting: + policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family) + groups: Final = tuple( + (policy, tuple(rule for rule in rules if rule.metric == policy.metric and rule.direction == policy.direction)) + for policy in policies + ) + values: Final = tuple( + IssueFamilyPolicy( + direction=policy.direction, + enabled=group[0].enabled if group and len({rule.enabled for rule in group}) == 1 else None, + multiplier=group[0].multiplier if group and len({rule.multiplier for rule in group}) == 1 else None, + min_value=group[0].min_value if group and len({rule.min_value for rule in group}) == 1 else None, + min_requests=group[0].min_requests if group and len({rule.min_requests for rule in group}) == 1 else None, + baseline_days=group[0].baseline_days + if group and len({rule.baseline_days for rule in group}) == 1 + else None, + min_history_days=group[0].min_history_days + if group and len({rule.min_history_days for rule in group}) == 1 + else None, + mixed=len( + { + (r.enabled, r.multiplier, r.min_value, r.min_requests, r.baseline_days, r.min_history_days) + for r in group + } + ) + > 1, + total_rules=len(group), + enabled_rules=sum(rule.enabled for rule in group), + ) + for policy, group in groups + ) + family_rules: Final = tuple(chain.from_iterable(group for _, group in groups)) + return IssueFamilySetting( + family=family, + enabled=family_rules[0].enabled if family_rules and len({r.enabled for r in family_rules}) == 1 else None, + total_rules=len(family_rules), + enabled_rules=sum(rule.enabled for rule in family_rules), + policies=values, + evaluation_status_counts=UsageAlertEvaluationCounts.model_validate( + Counter(rule.evaluation_status for rule in family_rules if rule.enabled) + ), + ) + + +async def family_settings(database: AlertDatabase) -> IssueFamilyList: + rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python( + await database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted', + list(DEFAULT_USAGE_RULE_IDS), + ) + ) + projection: Final = TypeAdapter(tuple[_ProjectionState, ...]).validate_python( + await database.query_raw( + """SELECT EXISTS(SELECT 1 FROM "LiteLLM_UsageAlertCheckpoint" + WHERE id LIKE 'performance-dirty:%' AND NOT completed LIMIT 1) AS pending""" + ) + ) + return IssueFamilyList( + data=tuple( + _setting(family, tuple(row.record for row in rows)) + for family in ("spend", "latency", "reliability", "traffic") + ), + projection_pending=projection[0].pending, + ) + + +async def update_family( + database: AlertDatabase, + family: PerformanceIssueFamily, + data: IssueFamilyUpdate, +) -> IssueFamilySetting | None: + policies: Final = tuple(policy for policy in DEFAULT_USAGE_POLICIES if policy.family == family) + if any(update.direction not in {policy.direction for policy in policies} for update in data.policies): + raise ValueError("This family does not support that direction") + ids: Final = tuple( + default_usage_rule_id(scope, policy.metric, policy.direction) + for policy, scope in product(policies, USAGE_ALERT_SCOPES) + ) + rows: Final = TypeAdapter(tuple[_RuleRow, ...]).validate_python( + await database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=ANY($1::text[]) AND NOT deleted', + list(ids), + ) + ) + timestamp: Final = datetime.now(timezone.utc).isoformat() + updates: Final = { + policy.direction: policy.model_dump(exclude_none=True, exclude={"direction"}) for policy in data.policies + } + records: Final = tuple( + UsageAlertRule.model_validate( + { + **row.record.model_dump(), + **({"enabled": data.enabled} if data.enabled is not None else {}), + **updates.get(row.record.direction, {}), + "updated_at": timestamp, + "evaluation_status": "pending", + "last_evaluated_day": None, + } + ) + for row in rows + ) + if not records: + return None + changed: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await database.query_raw( + """WITH incoming AS MATERIALIZED ( + SELECT x->'record' AS record,x->>'revision' AS revision FROM jsonb_array_elements($1::jsonb) x + ), eligible AS MATERIALIZED ( + SELECT r.id FROM "LiteLLM_UsageAlertRule" r JOIN incoming x ON r.id=x.record->>'id' + WHERE NOT r.deleted AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz + ORDER BY r.id FOR UPDATE OF r + ), changed AS ( + UPDATE "LiteLLM_UsageAlertRule" r SET record=x.record,enabled=(x.record->>'enabled')::boolean + FROM incoming x WHERE r.id=x.record->>'id' AND NOT r.deleted + AND (r.record->>'updated_at')::timestamptz=x.revision::timestamptz + AND (SELECT count(*) FROM eligible)=(SELECT count(*) FROM incoming) RETURNING r.id + ) SELECT count(*)::int AS count FROM changed""", + json.dumps( + tuple( + {"record": record.model_dump(mode="json"), "revision": row.record.updated_at.isoformat()} + for record, row in zip(records, rows, strict=True) + ) + ), + ) + ) + return _setting(family, records) if changed[0].count == len(records) else None diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py new file mode 100644 index 00000000000..7e4f2b436ec --- /dev/null +++ b/litellm/repositories/issues.py @@ -0,0 +1,680 @@ +import json +from collections.abc import Mapping +from datetime import date, datetime, timezone +from typing import ( + TYPE_CHECKING, + Final, + Literal, + cast, # noqa: TID251 # generated Prisma client implements query_raw +) + +from pydantic import BaseModel, TypeAdapter + +from litellm.models.team import LiteLLM_TeamTable +from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key +from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository +from litellm.types.proxy.issues import ( + Issue, + IssueCohort, + IssueDeliveryBinding, + IssueDeliveryInput, + IssueFamily, + IssueFamilyList, + IssueFamilySetting, + IssueFamilyUpdate, + IssueInput, + IssueKind, + IssueList, + IssuePendingDelivery, + IssueProjectionSignal, + IssueScope, + IssueSignal, + IssueSignalInput, + IssueSignalList, +) +from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS, PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +class _IssueRow(BaseModel, frozen=True): + record: Issue + + +class _SignalRow(BaseModel, frozen=True): + record: IssueSignal + + +class _Count(BaseModel, frozen=True): + count: int + + +class _Identifier(BaseModel, frozen=True): + id: str + + +_ACK: Final = "CASE WHEN jsonb_typeof(c.param_value)='string' THEN (c.param_value#>>'{}')::jsonb ELSE c.param_value END" +_ACK_CURRENT: Final = f"""(({_ACK})->>'last_seen_date' > i.record->'deprecation'->>'last_used_date' + OR (({_ACK})->>'last_seen_date' = i.record->'deprecation'->>'last_used_date' + AND COALESCE((({_ACK})->>'last_seen_requests')::bigint,0) + >= (i.record->'deprecation'->>'last_day_requests')::bigint))""" +_STATUS: Final = f"""CASE WHEN i.kind <> 'model_deprecation' THEN i.status + WHEN NOT i.active THEN CASE WHEN ({_ACK})->>'status'='dismissed' AND ({_ACK_CURRENT}) + THEN 'dismissed' ELSE 'resolved' END + WHEN COALESCE(({_ACK})->>'status','open')='open' + OR ({_ACK})->>'last_seen_date' IS NULL + OR i.record->'deprecation'->>'last_used_date' > ({_ACK})->>'last_seen_date' + OR (i.record->'deprecation'->>'last_used_date' = ({_ACK})->>'last_seen_date' + AND (i.record->'deprecation'->>'last_day_requests')::bigint + > COALESCE((({_ACK})->>'last_seen_requests')::bigint,0)) THEN 'open' + ELSE ({_ACK})->>'status' END""" +_PRIOR_STATUS: Final = _STATUS.replace(_ACK, _ACK.replace("c.param_value", "prior.param_value")) +_JOIN: Final = 'FROM "LiteLLM_Issue" i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key' +_RECORD: Final = f"""i.record || jsonb_build_object('status',({_STATUS}),'active',i.active, + 'first_seen',i.first_seen AT TIME ZONE 'UTC','last_seen',i.last_seen AT TIME ZONE 'UTC', + 'updated_at',i.updated_at AT TIME ZONE 'UTC') || CASE WHEN i.kind='model_deprecation' + THEN jsonb_build_object('deprecation',i.record->'deprecation' || + jsonb_build_object('state',COALESCE(({_ACK}),i.record->'deprecation'->'state'))) + ELSE '{{}}'::jsonb END""" + + +def _utc(value: datetime) -> str: + return value.astimezone(timezone.utc).replace(tzinfo=None).isoformat() + + +def _issue_payload(data: IssueInput, status: UsageAlertStatus | None) -> Mapping[str, object]: + initial: Final = Issue.model_validate( + { + **data.model_dump(), + "status": status or "open", + "first_seen": data.observed_at, + "last_seen": data.observed_at, + "updated_at": data.observed_at, + } + ) + return { + "record": initial.model_dump(mode="json"), + "ack_key": deprecation_alert_state_key(data.deprecation.usage_model, data.deprecation.model.deprecation_date) + if data.deprecation + else None, + } + + +class IssuesRepository: + def __init__(self, database: AlertDatabase) -> None: + self.database = database + + @classmethod + def from_prisma(cls, prisma: "PrismaClient") -> "IssuesRepository": + return cls(cast(AlertDatabase, prisma.writer_db)) # cast-ok: generated Prisma delegate implements query_raw + + async def upsert_issue( + self, data: IssueInput, reopen: bool = False, status: UsageAlertStatus | None = None + ) -> Issue: + rows: Final = await self.upsert_issues((data,), reopen=reopen, status=status) + return rows[0] + + async def upsert_issues( + self, + inputs: tuple[IssueInput, ...], + reopen: bool = False, + status: UsageAlertStatus | None = None, + ) -> tuple[Issue, ...]: + if not inputs: + return () + if len(inputs) > 500 or len(frozenset(item.id for item in inputs)) != len(inputs): + raise ValueError("Write at most 500 distinct issues per batch") + payload: Final = tuple(_issue_payload(item, status) for item in inputs) + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw( + f"""WITH incoming AS ( + SELECT x->'record' AS record,x->>'ack_key' AS ack_key FROM jsonb_array_elements($1::jsonb) x + ), changed AS ( + INSERT INTO "LiteLLM_Issue" AS old + (id,cohort_key,kind,family,scope_type,scope_id,status,ack_key,record,first_seen,last_seen,updated_at) + SELECT record->>'id',record->>'cohort_key',record->>'kind',record->>'family', + record->'cohort'->>'scope_type',record->'cohort'->>'scope_id',record->>'status',ack_key,record, + (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC', + (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC', + (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC' FROM incoming + ON CONFLICT(id) DO UPDATE SET + record=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen + THEN EXCLUDED.record ELSE old.record END, + last_seen=GREATEST(old.last_seen,EXCLUDED.last_seen), + active=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen + THEN true ELSE old.active END, + status=CASE WHEN $2 AND old.active AND EXCLUDED.last_seen > old.updated_at + THEN 'open' ELSE old.status END, + updated_at=GREATEST(old.updated_at,EXCLUDED.last_seen) + WHERE old.kind=EXCLUDED.kind AND old.cohort_key=EXCLUDED.cohort_key + AND old.scope_type=EXCLUDED.scope_type AND old.scope_id IS NOT DISTINCT FROM EXCLUDED.scope_id + RETURNING * + ) SELECT {_RECORD} AS record FROM changed i LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key""", + json.dumps(payload), + reopen, + ) + ) + if len(rows) != len(inputs): + raise ValueError("An issue ID already belongs to another cohort") + return tuple(row.record for row in rows) + + async def get_issue(self, issue_id: str) -> Issue | None: + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw(f"SELECT {_RECORD} AS record {_JOIN} WHERE i.id=$1", issue_id) + ) + return (await self._labels((rows[0].record,)))[0] if rows else None + + async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: + rows: Final = TypeAdapter(tuple[LiteLLM_TeamTable, ...]).validate_python( + await self.database.query_raw( + """SELECT t.team_id, + CASE WHEN m.role IS NULL THEN '[]'::jsonb + ELSE jsonb_build_array(jsonb_build_object('user_id',$2::text,'role',m.role)) END + AS members_with_roles, + CASE WHEN '/team/daily/activity'=ANY(t.team_member_permissions) + THEN ARRAY['/team/daily/activity'] ELSE ARRAY[]::text[] END AS team_member_permissions + FROM "LiteLLM_TeamTable" t LEFT JOIN LATERAL ( + SELECT member->>'role' AS role FROM jsonb_array_elements(t.members_with_roles) member + WHERE member->>'user_id'=$2 ORDER BY (member->>'role'='admin') DESC LIMIT 1 + ) m ON true WHERE t.team_id=$1""", + team_id, + user_id, + ) + ) + return rows[0] if rows else None + + async def list_issues( + self, + kind: IssueKind | None = None, + family: IssueFamily | None = None, + status: UsageAlertStatus | None = None, + scope_type: IssueScope | None = None, + scope_id: str | None = None, + limit: int = 50, + offset: int = 0, + ) -> IssueList: + where: Final = f"""($1::text IS NULL OR i.kind=$1) AND ($2::text IS NULL OR i.family=$2) + AND ($3::text IS NULL OR ({_STATUS})=$3) AND ($4::text IS NULL OR i.scope_type=$4) + AND ($5::text IS NULL OR i.scope_id=$5)""" + args: Final = (kind, family, status, scope_type, scope_id) + count: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw(f"SELECT count(*)::int AS count {_JOIN} WHERE {where}", *args) + ) + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw( + f"SELECT {_RECORD} AS record {_JOIN} WHERE {where} ORDER BY i.last_seen DESC,i.id LIMIT $6 OFFSET $7", + *args, + limit, + offset, + ) + ) + return IssueList( + data=await self._labels(tuple(row.record for row in rows)), total=count[0].count, limit=limit, offset=offset + ) + + async def _labels(self, issues: tuple[Issue, ...]) -> tuple[Issue, ...]: + if not issues: + return () + labels: Final = await UsageAlertsRepository(self.database).resource_labels( + tuple(issue.cohort for issue in issues) + ) + return tuple( + issue.model_copy(update={"scope_label": labels.get((issue.cohort.scope_type, issue.cohort.scope_id))}) + for issue in issues + ) + + async def find_active_issue(self, cohort_key: str) -> Issue | None: + rows: Final = await self.find_active_issues((cohort_key,)) + return rows[0] if rows else None + + async def find_active_issues(self, cohort_keys: tuple[str, ...]) -> tuple[Issue, ...]: + if len(cohort_keys) > 500: + raise ValueError("Read at most 500 issue cohorts per batch") + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw( + f"SELECT DISTINCT ON(i.cohort_key) {_RECORD} AS record {_JOIN} " + "WHERE i.cohort_key=ANY($1::text[]) AND i.active " + "ORDER BY i.cohort_key,i.first_seen DESC,i.id", + list(cohort_keys), + ) + ) + return tuple(row.record for row in rows) + + async def set_status( + self, + issue_id: str, + status: UsageAlertStatus, + expected_status: UsageAlertStatus, + actor: str | None = None, + authorized_scopes: tuple[IssueCohort, ...] | None = None, + ) -> Issue | None: + if authorized_scopes is not None and len(authorized_scopes) > 10: + raise ValueError("Authorize at most 10 alert source scopes") + changed: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( + await self.database.query_raw( + f"""WITH candidate AS MATERIALIZED ( + SELECT i.*,({_STATUS}) AS effective_status {_JOIN} WHERE i.id=$1 + AND NOT(i.kind='model_deprecation' AND NOT i.active AND $2='open') + AND ($5::jsonb IS NULL OR NOT EXISTS ( + SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id AND NOT EXISTS ( + SELECT 1 FROM jsonb_to_recordset($5::jsonb) AS x(scope_type text,scope_id text) + WHERE s.source='usage_alert' AND x.scope_type=s.record->'usage_alert'->>'scope_type' + AND x.scope_id IS NOT DISTINCT FROM s.record->'usage_alert'->>'scope_id'))) + ), ack AS ( + INSERT INTO "LiteLLM_Config" AS prior (param_name,param_value) + SELECT ack_key,jsonb_build_object('model',record->'deprecation'->>'usage_model', + 'deprecation_date',record->'deprecation'->'model'->>'deprecation_date', + 'status',$2::text,'last_seen_date',record->'deprecation'->>'last_used_date', + 'last_seen_requests',record->'deprecation'->'last_day_requests', + 'updated_at',CURRENT_TIMESTAMP,'updated_by',$4::text) + FROM candidate WHERE kind='model_deprecation' AND effective_status=$3 + ON CONFLICT(param_name) DO UPDATE SET param_value=EXCLUDED.param_value + WHERE EXISTS(SELECT 1 FROM candidate i WHERE i.ack_key=prior.param_name + AND ({_PRIOR_STATUS})=$3) + RETURNING param_name + ) UPDATE "LiteLLM_Issue" i SET status=$2, + updated_at=GREATEST(CURRENT_TIMESTAMP,i.updated_at+interval '1 millisecond') + FROM candidate c WHERE i.id=c.id AND (c.kind='model_deprecation' OR i.updated_at=c.updated_at) + AND c.effective_status=$3 AND + ((c.kind<>'model_deprecation' AND i.status=$3) OR i.ack_key IN(SELECT param_name FROM ack)) + RETURNING i.id""", + issue_id, + status, + expected_status, + actor, + json.dumps(tuple(scope.model_dump(mode="json") for scope in authorized_scopes)) + if authorized_scopes is not None + else None, + ) + ) + return await self.get_issue(issue_id) if changed else None + + async def status_scopes(self, issue_id: str) -> tuple[IssueCohort, ...]: + return TypeAdapter(tuple[IssueCohort, ...]).validate_python( + await self.database.query_raw( + """SELECT DISTINCT CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_type' + ELSE 'deployment' END AS scope_type, + CASE WHEN source='usage_alert' THEN record->'usage_alert'->>'scope_id' END AS scope_id + FROM "LiteLLM_IssueSignal" WHERE issue_id=$1 ORDER BY scope_type,scope_id LIMIT 11""", + issue_id, + ) + ) + + async def add_signals(self, issue_id: str, signals: tuple[IssueSignalInput, ...]) -> None: + await self.add_signal_batch( + tuple(IssueSignal.model_validate({**signal.model_dump(), "issue_id": issue_id}) for signal in signals) + ) + + async def add_signal_batch(self, signals: tuple[IssueSignal, ...]) -> None: + if not signals: + return + if len(signals) > 500: + raise ValueError("Write at most 500 issue signals per batch") + await self.database.query_raw( + """WITH incoming AS MATERIALIZED ( + SELECT DISTINCT ON(x->>'issue_id',x->>'source',x->>'source_id') x AS record + FROM jsonb_array_elements($1::jsonb) x + ), parents AS MATERIALIZED ( + SELECT i.id FROM "LiteLLM_Issue" i WHERE i.id IN(SELECT record->>'issue_id' FROM incoming) + ORDER BY i.id FOR UPDATE + ), added AS ( + INSERT INTO "LiteLLM_IssueSignal" (issue_id,source,source_id,record) + SELECT x.record->>'issue_id',x.record->>'source',x.record->>'source_id',x.record + FROM incoming x JOIN parents p ON p.id=x.record->>'issue_id' + ON CONFLICT(issue_id,source,source_id) DO NOTHING RETURNING issue_id,source,record + ) UPDATE "LiteLLM_Issue" i SET + updated_at=GREATEST(CURRENT_TIMESTAMP,i.updated_at+interval '1 millisecond'), + status=CASE WHEN i.kind='performance' AND i.active AND i.status<>'open' AND EXISTS ( + SELECT 1 FROM added a WHERE a.issue_id=i.id AND a.source='usage_alert' + AND (a.record->'usage_alert'->>'scope_type'<>i.scope_type + OR a.record->'usage_alert'->>'scope_id' IS DISTINCT FROM i.scope_id) + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" prior + WHERE prior.issue_id=i.id AND prior.source='usage_alert' + AND prior.record->'usage_alert'->>'scope_type'=a.record->'usage_alert'->>'scope_type' + AND prior.record->'usage_alert'->>'scope_id' + IS NOT DISTINCT FROM a.record->'usage_alert'->>'scope_id') + ) THEN 'open' ELSE i.status END + WHERE i.id IN(SELECT issue_id FROM added) RETURNING i.id""", + json.dumps(tuple(signal.model_dump(mode="json") for signal in signals)), + ) + + async def list_signals( + self, + issue_id: str, + limit: int = 50, + offset: int = 0, + visible_scope: IssueCohort | None = None, + ) -> IssueSignalList: + where: Final = """issue_id=$1 AND ($2::text IS NULL OR (source='usage_alert' + AND record->'usage_alert'->>'scope_type'=$2 + AND record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM $3::text))""" + args: Final = ( + issue_id, + visible_scope.scope_type if visible_scope else None, + visible_scope.scope_id if visible_scope else None, + ) + counts: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + f'SELECT count(*)::int AS count FROM "LiteLLM_IssueSignal" WHERE {where}', + *args, + ) + ) + rows: Final = TypeAdapter(tuple[_SignalRow, ...]).validate_python( + await self.database.query_raw( + f'SELECT record FROM "LiteLLM_IssueSignal" WHERE {where} ORDER BY source,source_id LIMIT $4 OFFSET $5', + *args, + limit, + offset, + ) + ) + return IssueSignalList( + data=tuple(row.record for row in rows), total=counts[0].count, limit=limit, offset=offset + ) + + async def active_signals( + self, + day: date, + limit: int = 500, + cursor: str | None = None, + scope_type: IssueScope | None = None, + metric: UsageAlertMetric | None = None, + scope_ids: tuple[str, ...] | None = None, + ) -> tuple[IssueProjectionSignal, ...]: + if not 1 <= limit <= 500: + raise ValueError("Read between one and 500 source signals") + if scope_ids == (): + return () + if scope_ids is not None and len(scope_ids) > 500: + raise ValueError("Filter at most 500 source scopes") + return TypeAdapter(tuple[IssueProjectionSignal, ...]).validate_python( + await self.database.query_raw( + """SELECT a.record AS alert,r.record AS rule,a.evaluation_day + FROM "LiteLLM_UsageAlertIncident" a JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id + WHERE a.evaluation_day=$1 AND a.breached AND r.enabled AND NOT r.deleted + AND NULLIF(a.evaluation_revision,'')::timestamptz=(r.record->>'updated_at')::timestamptz + AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric' + AND a.record->>'direction'=r.record->>'direction' + AND (r.scope_id IS NULL OR a.scope_id=r.scope_id) + AND ($2::text IS NULL OR a.id>$2) AND ($4::text IS NULL OR a.scope_type=$4) + AND ($5::text IS NULL OR a.metric=$5) + AND ($6::text[] IS NULL OR a.scope_id=ANY($6::text[])) ORDER BY a.id LIMIT $3""", + day.isoformat(), + cursor, + limit, + scope_type, + metric, + list(scope_ids) if scope_ids is not None else None, + ) + ) + + async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: + if issue_ids == (): + return 0 + if issue_ids is not None and len(issue_ids) > 10: + raise ValueError("Recover at most 10 resource alerts") + selection: Final = "i.id=ANY($3::text[])" if issue_ids is not None else "$3::text[] IS NULL" + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + f"""WITH primary_signals AS ( + SELECT i.id,a.id AS source_id,a.breached,a.evaluation_day,a.evaluation_revision, + max(a.evaluation_day) OVER (PARTITION BY i.id) AS latest_observation_day, + r.id AS rule_id,r.record AS rule_record, + COALESCE(r.enabled AND NOT r.deleted + AND r.scope_type=a.scope_type AND r.record->>'metric'=a.metric + AND r.record->>'direction'=a.record->>'direction' + AND (r.scope_id IS NULL OR r.scope_id=a.scope_id),false) AS eligible, + COALESCE(r.id=ANY($2::text[]) AND EXISTS ( + SELECT 1 FROM "LiteLLM_UsageAlertRule" custom WHERE custom.enabled AND NOT custom.deleted + AND COALESCE((custom.record->>'overrides_defaults')::boolean,false) + AND NOT(custom.id=ANY($2::text[])) AND custom.scope_type=r.scope_type + AND custom.record->>'metric'=r.record->>'metric' + AND custom.record->>'direction'=r.record->>'direction' + AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id)),false) AS suppressed + FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id + LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id + LEFT JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id + WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='usage_alert' + AND ({selection}) + AND s.record->'usage_alert'->>'scope_type'=i.scope_type + AND s.record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM i.scope_id + AND ((i.family='spend' AND s.record->'usage_alert'->>'metric'='spend') + OR (i.family='latency' AND s.record->'usage_alert'->>'metric'='latency') + OR (i.family='traffic' AND s.record->'usage_alert'->>'metric'='requests') + OR (i.family='reliability' AND s.record->'usage_alert'->>'metric' IN ('errors','error_rate'))) + ), observations AS ( + SELECT *,COALESCE(source_id IS NOT NULL AND NOT breached AND evaluation_day<=$1 + AND evaluation_day>=latest_observation_day + AND NULLIF(evaluation_revision,'')::timestamptz=(rule_record->>'updated_at')::timestamptz,false) AS healthy + FROM primary_signals + ), healthy AS ( + SELECT id FROM observations GROUP BY id + HAVING bool_or(eligible AND healthy) AND bool_and(NOT eligible OR suppressed OR healthy) + ), changed AS ( + UPDATE "LiteLLM_Issue" i SET active=false, + status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP + WHERE id IN(SELECT id FROM healthy) AND active RETURNING id + ) SELECT count(*)::int AS count FROM changed""", + day.isoformat(), + tuple(DEFAULT_USAGE_RULE_IDS), + issue_ids, + ) + ) + return rows[0].count + + async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: + if not 1 <= limit <= 500: + raise ValueError("Read between one and 500 provider issues") + rows: Final = TypeAdapter(tuple[_IssueRow, ...]).validate_python( + await self.database.query_raw( + f"""SELECT {_RECORD} AS record {_JOIN} + WHERE i.kind='performance' AND i.scope_type='provider' AND i.active + AND ($2::text IS NULL OR i.id>$2) ORDER BY i.id LIMIT $1""", + limit, + cursor, + ) + ) + return tuple(row.record for row in rows) + + async def recover_issue_ids(self, ids: tuple[str, ...], observed_at: datetime) -> int: + if len(ids) > 500 or observed_at.tzinfo is None: + raise ValueError("Recover at most 500 issues with a timezone-aware observation") + if not ids: + return 0 + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + """WITH changed AS ( + UPDATE "LiteLLM_Issue" SET active=false, + status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP + WHERE id=ANY($1::text[]) AND kind='performance' AND active AND last_seen<$2::timestamp(3) + RETURNING id + ) SELECT count(*)::int AS count FROM changed""", + list(ids), + _utc(observed_at), + ) + ) + return rows[0].count + + async def projection_completed(self, day: date, version: str) -> bool: + rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( + await self.database.query_raw( + 'SELECT id FROM "LiteLLM_UsageAlertCheckpoint" WHERE id=$1 AND completed', + f"issues:{version}:{day.isoformat()}", + ) + ) + return bool(rows) + + async def complete_projection(self, day: date, version: str) -> None: + await self.database.query_raw( + """INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed,expires_at) VALUES($1,true,CURRENT_TIMESTAMP) + ON CONFLICT(id) DO UPDATE SET completed=true,expires_at=CURRENT_TIMESTAMP RETURNING id""", + f"issues:{version}:{day.isoformat()}", + ) + + async def finish_deprecation_projection(self, observed_at: datetime) -> int: + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + """WITH changed AS ( + UPDATE "LiteLLM_Issue" SET active=false, + status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP + WHERE kind='model_deprecation' AND active AND last_seen<$1::timestamp(3) RETURNING id + ) SELECT count(*)::int AS count FROM changed""", + _utc(observed_at), + ) + ) + return rows[0].count + + async def family_settings(self) -> "IssueFamilyList": + from litellm.repositories.issue_families import family_settings + + return await family_settings(self.database) + + async def update_family( + self, family: "PerformanceIssueFamily", data: "IssueFamilyUpdate" + ) -> "IssueFamilySetting | None": + from litellm.repositories.issue_families import update_family + + return await update_family(self.database, family, data) + + async def queue_delivery( + self, + data: IssueDeliveryInput, + rule_id: str, + rule_revision: str, + source_id: str, + ) -> None: + await self.queue_deliveries( + (IssueDeliveryBinding(delivery=data, rule_id=rule_id, rule_revision=rule_revision, source_id=source_id),) + ) + + async def queue_deliveries(self, bindings: tuple[IssueDeliveryBinding, ...]) -> None: + if not bindings: + return + if len(bindings) > 500: + raise ValueError("Queue at most 500 delivery bindings per batch") + await self.database.query_raw( + """WITH incoming AS MATERIALIZED ( + SELECT x->'delivery' AS delivery,x->>'rule_id' AS rule_id, + x->>'rule_revision' AS revision,x->>'source_id' AS source_id + FROM jsonb_array_elements($1::jsonb) x + WHERE EXISTS(SELECT 1 FROM "LiteLLM_Issue" WHERE id=x->'delivery'->>'issue_id') + ), saved AS ( + INSERT INTO "LiteLLM_IssueDelivery" (id,issue_id,record) + SELECT DISTINCT ON(delivery->>'id') delivery->>'id',delivery->>'issue_id',delivery + FROM incoming ON CONFLICT(id) DO NOTHING RETURNING id + ) INSERT INTO "LiteLLM_IssueDeliveryRule" (delivery_id,rule_id,source_id,rule_revision) + SELECT DISTINCT ON(delivery->>'id',rule_id,source_id) delivery->>'id',rule_id,source_id,revision + FROM incoming WHERE delivery->>'id' IN(SELECT id FROM saved) + OR EXISTS(SELECT 1 FROM "LiteLLM_IssueDelivery" d WHERE d.id=delivery->>'id' + AND d.issue_id=delivery->>'issue_id') + ON CONFLICT(delivery_id,rule_id,source_id) DO UPDATE SET rule_revision=EXCLUDED.rule_revision + RETURNING delivery_id""", + json.dumps(tuple(binding.model_dump(mode="json") for binding in bindings)), + ) + + async def pending_deliveries(self, limit: int = 100) -> tuple[IssuePendingDelivery, ...]: + return await self._pending_deliveries(limit, None) + + async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: + rows: Final = await self._pending_deliveries(1, delivery_id) + return rows[0] if rows else None + + async def _pending_deliveries(self, limit: int, delivery_id: str | None) -> tuple[IssuePendingDelivery, ...]: + if not 1 <= limit <= 500: + raise ValueError("Read between one and 500 deliveries") + return TypeAdapter(tuple[IssuePendingDelivery, ...]).validate_python( + await self.database.query_raw( + f"""SELECT d.record || jsonb_build_object('next_attempt',d.next_attempt AT TIME ZONE 'UTC') AS delivery, + {_RECORD} AS issue,bound.rule,bound.alert + FROM "LiteLLM_IssueDelivery" d JOIN "LiteLLM_Issue" i ON i.id=d.issue_id + LEFT JOIN "LiteLLM_Config" c ON c.param_name=i.ack_key + JOIN LATERAL ( + SELECT r.record AS rule,a.record AS alert FROM "LiteLLM_IssueDeliveryRule" b + JOIN "LiteLLM_UsageAlertRule" r ON r.id=b.rule_id + JOIN "LiteLLM_UsageAlertIncident" a ON a.id=b.source_id AND a.rule_id=r.id + WHERE b.delivery_id=d.id AND r.enabled AND NOT r.deleted AND a.breached AND a.status='open' + AND (r.record->'notification_channels') ? (d.record->>'channel') + AND (d.record->>'channel'<>'email' OR + (jsonb_array_length(d.record->'email_recipients')>0 + AND (d.record->'email_recipients') <@ (r.record->'email_recipients'))) + AND (r.record->>'updated_at')::timestamptz=b.rule_revision::timestamptz + AND NULLIF(a.evaluation_revision,'')::timestamptz=b.rule_revision::timestamptz + AND a.scope_type=r.scope_type AND a.metric=r.record->>'metric' + AND a.record->>'direction'=r.record->>'direction' + AND (r.scope_id IS NULL OR a.scope_id=r.scope_id) + AND (NOT(r.id=ANY($3::text[])) OR NOT EXISTS ( + SELECT 1 FROM "LiteLLM_UsageAlertRule" custom WHERE custom.enabled AND NOT custom.deleted + AND COALESCE((custom.record->>'overrides_defaults')::boolean,false) + AND NOT(custom.id=ANY($3::text[])) AND custom.scope_type=r.scope_type + AND custom.record->>'metric'=r.record->>'metric' + AND custom.record->>'direction'=r.record->>'direction' + AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id))) + AND EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id + AND s.source='usage_alert' AND s.source_id=a.id) + ORDER BY b.rule_id,b.source_id LIMIT 1 + ) bound ON true + WHERE NOT d.sent AND d.next_attempt<=CURRENT_TIMESTAMP AND i.active AND ({_STATUS})='open' + AND ($2::text IS NULL OR d.id=$2) + ORDER BY d.next_attempt,d.id LIMIT $1""", + limit, + delivery_id, + tuple(DEFAULT_USAGE_RULE_IDS), + ) + ) + + async def mark_delivery_sent(self, delivery_id: str) -> None: + await self.database.query_raw( + """UPDATE "LiteLLM_IssueDelivery" SET sent=true,updated_at=CURRENT_TIMESTAMP, + record=record || jsonb_build_object('last_error',NULL,'attempts', + COALESCE((record->>'attempts')::int,0)+1) + WHERE id=$1 AND NOT sent RETURNING id""", + delivery_id, + ) + + async def defer_delivery( + self, + delivery_id: str, + error: Literal["delivery_failed", "transport_not_configured"], + ) -> None: + await self.database.query_raw( + """UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP+interval '1 hour', + updated_at=CURRENT_TIMESTAMP,record=record || jsonb_build_object('last_error',$2::text, + 'attempts',COALESCE((record->>'attempts')::int,0)+1) + WHERE id=$1 AND NOT sent RETURNING id""", + delivery_id, + error, + ) + + async def prune(self, retention_days: int = 90, limit: int = 500) -> None: + if not 1 <= limit <= 500 or retention_days < 1: + raise ValueError("Retention and cleanup batch must be bounded positive values") + await self.database.query_raw( + """WITH expired AS MATERIALIZED ( + SELECT id FROM "LiteLLM_Issue" WHERE status<>'open' + AND updated_at$1::int) + ORDER BY updated_at,id LIMIT $2 + ), signals AS ( + DELETE FROM "LiteLLM_IssueSignal" WHERE (issue_id,source,source_id) IN ( + SELECT issue_id,source,source_id FROM "LiteLLM_IssueSignal" + WHERE issue_id IN(SELECT id FROM expired) ORDER BY issue_id,source,source_id LIMIT $2) + RETURNING issue_id + ), bindings AS ( + DELETE FROM "LiteLLM_IssueDeliveryRule" WHERE (delivery_id,rule_id,source_id) IN ( + SELECT b.delivery_id,b.rule_id,b.source_id FROM "LiteLLM_IssueDeliveryRule" b + JOIN "LiteLLM_IssueDelivery" d ON d.id=b.delivery_id + WHERE d.issue_id IN(SELECT id FROM expired) ORDER BY b.delivery_id,b.rule_id,b.source_id LIMIT $2) + RETURNING delivery_id + ), deliveries AS ( + DELETE FROM "LiteLLM_IssueDelivery" d WHERE d.id IN ( + SELECT id FROM "LiteLLM_IssueDelivery" d WHERE issue_id IN(SELECT id FROM expired) + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueDeliveryRule" b WHERE b.delivery_id=d.id) + ORDER BY id LIMIT $2) RETURNING issue_id + ), removed AS ( + DELETE FROM "LiteLLM_Issue" i WHERE id IN(SELECT id FROM expired) + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueSignal" s WHERE s.issue_id=i.id) + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_IssueDelivery" d WHERE d.issue_id=i.id) RETURNING id + ) SELECT count(*)::int AS count FROM removed""", + retention_days, + limit, + ) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py new file mode 100644 index 00000000000..1732eb86658 --- /dev/null +++ b/litellm/repositories/usage_alerts.py @@ -0,0 +1,752 @@ +"""Bounded, writer-consistent storage for daily alert rules and episodes.""" + +import json +from collections.abc import Mapping +from datetime import date, datetime, timedelta, timezone +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Final, + Protocol, + cast, # noqa: TID251 # PrismaWrapper forwards generated client methods dynamically +) +from urllib.parse import urlencode +from uuid import NAMESPACE_URL, uuid4, uuid5 + +from pydantic import BaseModel, TypeAdapter + +from litellm.repositories.daily_activity_buckets import api_key_from_bucket +from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS +from litellm.types.proxy.usage_alerts import ( + UsageAlert, + UsageAlertChannel, + UsageAlertDefaultSummary, + UsageAlertEvaluationStatus, + UsageAlertGroup, + UsageAlertGroupList, + UsageAlertGroupMetric, + UsageAlertGroupUpdate, + UsageAlertGroupUpdateResult, + UsageAlertList, + UsageAlertMetric, + UsageAlertModelList, + UsageAlertModelOption, + UsageAlertNotification, + UsageAlertObservation, + UsageAlertRule, + UsageAlertRuleInput, + UsageAlertRuleKind, + UsageAlertRuleList, + UsageAlertScope, + UsageAlertStatus, +) + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +class AlertDatabase(Protocol): + async def query_raw(self, query: str, *args: object) -> object: ... + + +class ResourceLabelScope(Protocol): + @property + def scope_type(self) -> str: ... + + @property + def scope_id(self) -> str | None: ... + + +class _Record(BaseModel, frozen=True): + record: Mapping[str, object] + + +class _Count(BaseModel, frozen=True): + count: int + + +class _RuleCounts(_Count, frozen=True): + default_summary: UsageAlertDefaultSummary + + +class _GroupRow(BaseModel, frozen=True): + scope_type: UsageAlertScope + scope_id: str | None + day: date + incident_count: int + metrics: tuple[UsageAlertGroupMetric, ...] + + +class _GroupPage(BaseModel, frozen=True): + total: int + data: tuple[_GroupRow, ...] + + +class _ResourceLabel(BaseModel, frozen=True): + scope_type: str + scope_id: str | None + label: str | None + + +class _ModelIdentity(BaseModel, frozen=True): + model: str + + +class _Pending(_Record, frozen=True): + rule: Mapping[str, object] + notified_channels: tuple[UsageAlertChannel, ...] + + +class _Ownership(BaseModel, frozen=True): + user_id: str | None + team_id: str | None + + +_DEFAULT_RULE_IDS: Final = DEFAULT_USAGE_RULE_IDS + + +def _rule(record: Mapping[str, object]) -> UsageAlertRule: + return UsageAlertRule.model_validate({**record, "is_default": str(record.get("id")) in _DEFAULT_RULE_IDS}) + + +def activity_path( + scope: UsageAlertScope, entity_id: str | None, day: date, alert_id: str, baseline_days: int = 7 +) -> str: + routes: Final = { + "key": ("/api-keys", "key"), + "team": ("/teams", "team"), + "user": ("/users", "user"), + "model": ("/models-and-endpoints", "activity_model"), + "deployment": ("/usage", ""), + } + route, parameter = routes[scope] + if scope == "deployment": + return ( + route + + "?" + + urlencode( + { + "tab": "keys", + "from": (day - timedelta(days=baseline_days)).isoformat(), + "to": day.isoformat(), + "alert": alert_id, + } + ) + ) + query: Final = { + "activity_from": (day - timedelta(days=baseline_days)).isoformat(), + "activity_to": day.isoformat(), + "alert": alert_id, + } + if parameter and entity_id and not entity_id.startswith("sk-"): + query[parameter] = entity_id + query["detail_tab"] = "activity" + return route + "?" + urlencode(query) + + +def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now: datetime) -> Mapping[str, object]: + alert_id: Final = str( + uuid5( + NAMESPACE_URL, + json.dumps( + [ + "litellm:usage-alert", + rule.id, + rule.scope_type, + item.entity_id, + rule.metric, + rule.direction, + item.day.isoformat(), + rule.updated_at.isoformat(), + ] + ), + ) + ) + incident: Final = UsageAlert( + id=alert_id, + rule_id=rule.id, + rule_name=rule.name, + scope_type=rule.scope_type, + scope_id=item.entity_id, + metric=rule.metric, + direction=rule.direction, + status="open", + day=item.day, + baseline_start=item.day - timedelta(days=rule.baseline_days), + baseline_end=item.day - timedelta(days=1), + baseline_sample_days=item.baseline_sample_days, + observed_value=item.observed, + baseline_value=item.baseline, + threshold=item.threshold, + first_seen=now, + last_seen=now, + activity_path=activity_path(rule.scope_type, item.entity_id, item.day, alert_id, rule.baseline_days), + ) + return { + "scope_key": item.entity_id or "", + "breached": item.breached, + "day": item.day.isoformat(), + "record": incident.model_dump(mode="json"), + } + + +class UsageAlertsRepository: + def __init__(self, database: AlertDatabase) -> None: + self.database: Final = database + + @classmethod + def from_prisma(cls, prisma: "PrismaClient") -> "UsageAlertsRepository": + return cls( + cast(AlertDatabase, prisma.writer_db) # cast-ok: PrismaWrapper forwards query_raw to its generated client + ) + + async def get_rule(self, rule_id: str) -> UsageAlertRule | None: + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND NOT deleted', rule_id + ) + ) + return _rule(rows[0].record) if rows else None + + async def save_rule( + self, + data: UsageAlertRuleInput, + created_by: str | None, + existing: UsageAlertRule | None = None, + *, + overrides_defaults: bool = False, + ) -> UsageAlertRule | None: + now: Final = datetime.now(timezone.utc) + rule: Final = UsageAlertRule.model_validate( + { + **data.model_dump(), + "id": existing.id if existing else str(uuid4()), + "created_by": existing.created_by if existing else created_by, + "created_at": existing.created_at if existing else now, + "updated_at": now, + "overrides_defaults": overrides_defaults, + } + ) + rows: Final = await self.database.query_raw( + """INSERT INTO "LiteLLM_UsageAlertRule" (id,scope_type,scope_id,enabled,record) VALUES ($1,$2,$3,$4,$5::jsonb) + ON CONFLICT (id) DO UPDATE SET scope_type=EXCLUDED.scope_type,scope_id=EXCLUDED.scope_id,enabled=EXCLUDED.enabled,record=EXCLUDED.record + WHERE NOT "LiteLLM_UsageAlertRule".deleted + AND ("LiteLLM_UsageAlertRule".record->>'updated_at')::timestamptz=$6::text::timestamptz + AND "LiteLLM_UsageAlertRule".scope_type=$7 + AND "LiteLLM_UsageAlertRule".scope_id IS NOT DISTINCT FROM $8::text + RETURNING record""", + rule.id, + rule.scope_type, + rule.scope_id, + rule.enabled, + json.dumps({**rule.model_dump(mode="json"), "overrides_defaults": overrides_defaults}), + existing.updated_at.isoformat() if existing else None, + existing.scope_type if existing else None, + existing.scope_id if existing else None, + ) + saved: Final = TypeAdapter(tuple[_Record, ...]).validate_python(rows) + return _rule(saved[0].record) if saved else None + + async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: + return bool( + await self.database.query_raw( + """UPDATE "LiteLLM_UsageAlertRule" SET deleted=true,enabled=false + WHERE id=$1 AND NOT deleted AND ($2::text::timestamptz IS NULL OR ( + (record->>'updated_at')::timestamptz=$2::text::timestamptz + AND scope_type=$3 AND scope_id IS NOT DISTINCT FROM $4::text)) RETURNING id""", + rule_id, + existing.updated_at.isoformat() if existing else None, + existing.scope_type if existing else None, + existing.scope_id if existing else None, + ) + ) + + async def list_rules( + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + limit: int = 50, + offset: int = 0, + kind: UsageAlertRuleKind = "all", + ) -> UsageAlertRuleList: + args: Final = (scope_type, scope_id, json.dumps(tuple(sorted(_DEFAULT_RULE_IDS))), kind) + where: Final = "NOT deleted AND ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2)" + builtin: Final = "id IN (SELECT jsonb_array_elements_text($3::jsonb))" + count: Final = TypeAdapter(tuple[_RuleCounts, ...]).validate_python( + await self.database.query_raw( + f"""WITH scoped AS ( + SELECT enabled,record,({builtin}) AS builtin FROM "LiteLLM_UsageAlertRule" WHERE {where} + ) SELECT count(*) FILTER (WHERE $4='all' OR builtin=($4='default'))::int AS count, + jsonb_build_object('total',count(*) FILTER (WHERE builtin), + 'enabled',count(*) FILTER (WHERE builtin AND enabled), + 'evaluation_status_counts',jsonb_build_object( + 'pending',count(*) FILTER (WHERE builtin AND enabled AND COALESCE(record->>'evaluation_status','pending')='pending'), + 'complete',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='complete'), + 'insufficient_coverage',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='insufficient_coverage'), + 'insufficient_history',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='insufficient_history') + )) AS default_summary FROM scoped""", + *args, + ) + ) + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + f"SELECT record FROM \"LiteLLM_UsageAlertRule\" WHERE {where} AND ($4='all' OR ({builtin})=($4='default')) ORDER BY id LIMIT $5 OFFSET $6", + *args, + limit, + offset, + ) + ) + return UsageAlertRuleList( + data=tuple(_rule(row.record) for row in rows), + total=count[0].count, + limit=limit, + offset=offset, + default_summary=count[0].default_summary, + ) + + async def list_enabled_rules(self, limit: int = 100, offset: int = 0) -> tuple[UsageAlertRule, ...]: + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE enabled AND NOT deleted ORDER BY id LIMIT $1 OFFSET $2', + limit, + offset, + ) + ) + return tuple(UsageAlertRule.model_validate(row.record) for row in rows) + + async def get_alert(self, alert_id: str) -> UsageAlert | None: + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw('SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE id=$1', alert_id) + ) + return UsageAlert.model_validate(rows[0].record) if rows else None + + async def list_alerts( + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + metric: UsageAlertMetric | None = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: int = 50, + offset: int = 0, + ) -> UsageAlertList: + args: Final = ( + scope_type, + scope_id, + metric, + status, + start_date.isoformat() if start_date else None, + end_date.isoformat() if end_date else None, + ) + where: Final = "($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2) AND ($3::text IS NULL OR metric=$3) AND ($4::text IS NULL OR status=$4) AND ($5::text IS NULL OR day >= $5) AND ($6::text IS NULL OR day <= $6)" + count: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + f'SELECT count(*)::int AS count FROM "LiteLLM_UsageAlertIncident" WHERE {where}', *args + ) + ) + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + f'SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE {where} ORDER BY day DESC,id LIMIT $7 OFFSET $8', + *args, + limit, + offset, + ) + ) + return UsageAlertList( + data=tuple(UsageAlert.model_validate(row.record) for row in rows), + total=count[0].count, + limit=limit, + offset=offset, + ) + + async def set_status(self, alert_id: str, status: UsageAlertStatus) -> UsageAlert | None: + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + """UPDATE "LiteLLM_UsageAlertIncident" SET status=$2, record=jsonb_set(record,'{status}',to_jsonb($2::text)),updated_at=CURRENT_TIMESTAMP WHERE id=$1 RETURNING record""", + alert_id, + status, + ) + ) + return UsageAlert.model_validate(rows[0].record) if rows else None + + async def list_groups( + self, + scope_type: UsageAlertScope | None = None, + scope_id: str | None = None, + status: UsageAlertStatus | None = None, + start_date: date | None = None, + end_date: date | None = None, + limit: int = 50, + offset: int = 0, + ) -> UsageAlertGroupList: + rows: Final = TypeAdapter(tuple[_GroupPage, ...]).validate_python( + await self.database.query_raw( + """WITH groups AS MATERIALIZED ( + SELECT scope_type,scope_id,day,count(*)::int AS incident_count FROM "LiteLLM_UsageAlertIncident" + WHERE ($1::text IS NULL OR scope_type=$1) AND ($2::text IS NULL OR scope_id=$2) + AND ($3::text IS NULL OR status=$3) AND ($4::text IS NULL OR day >= $4) + AND ($5::text IS NULL OR day <= $5) + GROUP BY scope_type,scope_id,day + ), page AS MATERIALIZED ( + SELECT * FROM groups ORDER BY day DESC,scope_type,scope_id NULLS FIRST LIMIT $6 OFFSET $7 + ), selected AS ( + SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i + ON i.scope_type=p.scope_type AND i.scope_id=p.scope_id AND i.day=p.day + WHERE ($3::text IS NULL OR i.status=$3) + UNION ALL + SELECT i.* FROM page p JOIN "LiteLLM_UsageAlertIncident" i + ON i.scope_type=p.scope_type AND i.scope_id IS NULL AND i.day=p.day + WHERE p.scope_id IS NULL AND ($3::text IS NULL OR i.status=$3) + ), ranked AS ( + SELECT scope_type,scope_id,day,metric,record, + count(*) OVER (PARTITION BY scope_type,scope_id,day,metric)::int AS incident_count, + row_number() OVER (PARTITION BY scope_type,scope_id,day,metric + ORDER BY record->>'first_seen' DESC,id) AS rank + FROM selected + ), packed AS ( + SELECT p.scope_type,p.scope_id,p.day,p.incident_count, + jsonb_agg(jsonb_build_object('metric',r.metric,'incident_count',r.incident_count, + 'representative',r.record) ORDER BY CASE r.metric + WHEN 'requests' THEN 0 WHEN 'spend' THEN 1 WHEN 'error_rate' THEN 2 + WHEN 'errors' THEN 3 ELSE 4 END) AS metrics + FROM page p JOIN ranked r ON p.scope_type=r.scope_type + AND p.scope_id IS NOT DISTINCT FROM r.scope_id AND p.day=r.day AND r.rank=1 + GROUP BY p.scope_type,p.scope_id,p.day,p.incident_count + ) SELECT (SELECT count(*)::int FROM groups) AS total, + COALESCE((SELECT jsonb_agg(to_jsonb(packed) ORDER BY day DESC,scope_type,scope_id NULLS FIRST) + FROM packed),'[]'::jsonb) AS data""", + scope_type, + scope_id, + status, + start_date.isoformat() if start_date else None, + end_date.isoformat() if end_date else None, + limit, + offset, + ) + ) + page: Final = rows[0] + labels: Final[Mapping[tuple[str, str | None], str | None]] = ( + await self.resource_labels(page.data) if page.data else {} + ) + return UsageAlertGroupList( + data=tuple( + UsageAlertGroup( + id=str( + uuid5( + NAMESPACE_URL, + json.dumps( + ["litellm:performance-group", row.scope_type, row.scope_id, row.day.isoformat()] + ), + ) + ), + scope_type=row.scope_type, + scope_id=row.scope_id, + scope_label=labels.get((row.scope_type, row.scope_id)), + day=row.day, + incident_count=row.incident_count, + metrics=row.metrics, + activity_path=row.metrics[0].representative.activity_path, + ) + for row in page.data + ), + total=page.total, + limit=limit, + offset=offset, + ) + + async def resource_labels( + self, groups: tuple[ResourceLabelScope, ...] + ) -> Mapping[tuple[str, str | None], str | None]: + rows: Final = TypeAdapter(tuple[_ResourceLabel, ...]).validate_python( + await self.database.query_raw( + """WITH scopes AS ( + SELECT DISTINCT scope_type,scope_id,key_id FROM jsonb_to_recordset($1::jsonb) + AS x(scope_type text,scope_id text,key_id text) + ) SELECT s.scope_type,s.scope_id,LEFT(CASE s.scope_type + WHEN 'key' THEN COALESCE(NULLIF(k.key_alias,''),NULLIF(k.key_name,'')) + WHEN 'team' THEN NULLIF(t.team_alias,'') + WHEN 'user' THEN COALESCE(NULLIF(u.user_alias,''),NULLIF(u.user_email,'')) + WHEN 'model' THEN s.scope_id WHEN 'provider' THEN s.scope_id + WHEN 'deployment' THEN 'Gateway' END,256) AS label + FROM scopes s LEFT JOIN "LiteLLM_VerificationToken" k ON s.scope_type='key' AND k.token=s.key_id + LEFT JOIN "LiteLLM_TeamTable" t ON s.scope_type='team' AND t.team_id=s.scope_id + LEFT JOIN "LiteLLM_UserTable" u ON s.scope_type='user' AND u.user_id=s.scope_id""", + json.dumps( + tuple( + { + "scope_type": row.scope_type, + "scope_id": row.scope_id, + "key_id": api_key_from_bucket(row.scope_id) + if row.scope_type == "key" and row.scope_id + else None, + } + for row in groups + ) + ), + ) + ) + return MappingProxyType({(row.scope_type, row.scope_id): row.label for row in rows}) + + async def set_group_status(self, data: UsageAlertGroupUpdate) -> UsageAlertGroupUpdateResult: + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + """WITH changed AS ( + UPDATE "LiteLLM_UsageAlertIncident" SET status=$5, + record=jsonb_set(record,'{status}',to_jsonb($5::text)),updated_at=CURRENT_TIMESTAMP + WHERE scope_type=$1 AND scope_id IS NOT DISTINCT FROM $2::text AND day=$3 AND status=$4 + RETURNING id + ) SELECT count(*)::int AS count FROM changed""", + data.scope_type, + data.scope_id, + data.day.isoformat(), + data.expected_status, + data.status, + ) + ) + return UsageAlertGroupUpdateResult(updated_count=rows[0].count) + + async def list_reported_models(self, search: str | None, cursor: str | None, limit: int) -> UsageAlertModelList: + today: Final = datetime.now(timezone.utc).date() + rows: Final = TypeAdapter(tuple[_ModelIdentity, ...]).validate_python( + await self.database.query_raw( + """SELECT DISTINCT model FROM "LiteLLM_DailyUserSpend" + WHERE date >= $1 AND date <= $2 AND model IS NOT NULL AND model <> '' + AND ($3::text IS NULL OR strpos(lower(model),lower($3))>0) + AND ($4::text IS NULL OR model>$4) + ORDER BY model LIMIT $5""", + (today - timedelta(days=29)).isoformat(), + today.isoformat(), + search, + cursor, + limit + 1, + ) + ) + selected: Final = rows[:limit] + return UsageAlertModelList( + data=tuple(UsageAlertModelOption(scope_id=row.model, label=row.model) for row in selected), + limit=limit, + next_cursor=selected[-1].model if len(rows) > limit else None, + start_date=today - timedelta(days=29), + end_date=today, + ) + + async def key_ownership(self, key_id: str) -> tuple[str | None, str | None] | None: + rows: Final = TypeAdapter(tuple[_Ownership, ...]).validate_python( + await self.database.query_raw( + """SELECT user_id,team_id FROM "LiteLLM_VerificationToken" WHERE token=$1 + UNION ALL + (SELECT user_id,team_id FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1 + AND NOT EXISTS(SELECT 1 FROM "LiteLLM_VerificationToken" WHERE token=$1) + ORDER BY deleted_at DESC,id DESC LIMIT 1)""", + key_id, + ) + ) + return (rows[0].user_id, rows[0].team_id) if rows else None + + async def claim_job(self, owner: str, lease_seconds: int = 900) -> bool: + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + """INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,owner,expires_at) VALUES ('worker',$1,CURRENT_TIMESTAMP+($2::int * INTERVAL '1 second')) + ON CONFLICT(id) DO UPDATE SET owner=EXCLUDED.owner,expires_at=EXCLUDED.expires_at WHERE "LiteLLM_UsageAlertCheckpoint".expires_at None: + await self.database.query_raw( + "DELETE FROM \"LiteLLM_UsageAlertCheckpoint\" WHERE id='worker' AND owner=$1 RETURNING id", owner + ) + + @staticmethod + def _checkpoint(rule: UsageAlertRule, day: date) -> str: + return f"{rule.id}:{rule.updated_at.isoformat()}:{day.isoformat()}" + + async def is_day_complete(self, rule: UsageAlertRule, day: date) -> bool: + rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( + await self.database.query_raw( + 'SELECT 1 AS count FROM "LiteLLM_UsageAlertCheckpoint" WHERE id=$1 AND completed', + self._checkpoint(rule, day), + ) + ) + return bool(rows) + + async def finish_day(self, rule: UsageAlertRule, day: date) -> None: + await self.database.query_raw( + 'INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed) VALUES ($1,true) ON CONFLICT(id) DO NOTHING RETURNING id', + self._checkpoint(rule, day), + ) + + async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None: + for rule in rules: + await self._ensure_default_rule(rule) + + async def _ensure_default_rule(self, rule: UsageAlertRuleInput) -> None: + rule_id: Final = str( + uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}") + ) + now: Final = datetime.now(timezone.utc) + record: Final = UsageAlertRule.model_validate( + {**rule.model_dump(), "id": rule_id, "created_at": now, "updated_at": now, "created_by": None} + ) + await self.database.query_raw( + 'INSERT INTO "LiteLLM_UsageAlertRule" (id,scope_type,scope_id,enabled,record) VALUES ($1,$2,$3,$4,$5::jsonb) ON CONFLICT(id) DO NOTHING RETURNING id', + rule_id, + rule.scope_type, + rule.scope_id, + rule.enabled, + record.model_dump_json(), + ) + + async def list_breached_entity_ids(self, rule_id: str, limit: int = 500, offset: int = 0) -> tuple[str | None, ...]: + rows: Final = TypeAdapter(tuple[Mapping[str, str | None], ...]).validate_python( + await self.database.query_raw( + """SELECT DISTINCT i.scope_id FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id WHERE i.rule_id=$1 AND i.breached AND i.scope_type=r.scope_type AND i.metric=r.record->>'metric' AND i.record->>'direction'=r.record->>'direction' AND (r.scope_id IS NULL OR i.scope_id=r.scope_id) ORDER BY i.scope_id NULLS FIRST LIMIT $2 OFFSET $3""", + rule_id, + limit, + offset, + ) + ) + return tuple(row["scope_id"] for row in rows) + + async def record_evaluation_status( + self, rule: UsageAlertRule, day: date, status: UsageAlertEvaluationStatus + ) -> None: + await self.database.query_raw( + """UPDATE "LiteLLM_UsageAlertRule" SET record=record || jsonb_build_object('last_evaluated_day',$2::text,'evaluation_status',$3::text) WHERE id=$1 AND (record->>'updated_at')::timestamptz=$4::timestamptz AND NOT deleted RETURNING id""", + rule.id, + day.isoformat(), + status, + rule.updated_at.isoformat(), + ) + + async def record_observation( + self, + rule: UsageAlertRule, + entity_id: str | None, + day: date, + observed: float, + baseline: float, + threshold: float, + breached: bool, + ) -> UsageAlert | None: + observation: Final = UsageAlertObservation( + entity_id=entity_id, day=day, observed=observed, baseline=baseline, threshold=threshold, breached=breached + ) + changed: Final = await self.record_observations(rule, (observation,)) + if changed: + return changed[0] + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + """SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=$2 AND scope_type=$3 AND metric=$4 AND record->>'direction'=$5 ORDER BY evaluation_day DESC,id DESC LIMIT 1""", + rule.id, + entity_id or "", + rule.scope_type, + rule.metric, + rule.direction, + ) + ) + return UsageAlert.model_validate(rows[0].record) if rows else None + + async def record_observations( + self, rule: UsageAlertRule, observations: tuple[UsageAlertObservation, ...] + ) -> tuple[UsageAlert, ...]: + if not observations: + return () + if len(observations) > 500 or len({item.entity_id for item in observations}) != len(observations): + raise ValueError("An observation batch must contain at most 500 distinct entities") + now: Final = datetime.now(timezone.utc) + records: Final = tuple(_observation_payload(rule, item, now) for item in observations) + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + """ + WITH input AS (SELECT * FROM jsonb_to_recordset($2::jsonb) AS x(scope_key text, breached boolean, day text, record jsonb)), + prior AS ( + SELECT x.*,p.id AS prior_id,p.evaluation_day,p.evaluation_revision AS prior_revision, + p.breached AS prior_breached,p.record AS prior_record + FROM input x LEFT JOIN LATERAL ( + SELECT * FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=x.scope_key + AND scope_type=x.record->>'scope_type' AND scope_id IS NOT DISTINCT FROM x.record->>'scope_id' + AND metric=x.record->>'metric' AND record->>'direction'=x.record->>'direction' + ORDER BY evaluation_day DESC,id DESC LIMIT 1 + ) p ON true + WHERE EXISTS (SELECT 1 FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND enabled AND NOT deleted AND (record->>'updated_at')::timestamptz=$3::timestamptz) + ), updated AS ( + UPDATE "LiteLLM_UsageAlertIncident" i SET + evaluation_day=p.day, evaluation_revision=$3::text, breached=p.breached, + record=i.record || jsonb_build_object( + 'last_seen',CASE WHEN p.breached THEN p.record->'last_seen' ELSE i.record->'last_seen' END, + 'status',CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END), + status=CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END, + updated_at=CURRENT_TIMESTAMP + FROM prior p WHERE i.id=p.prior_id + AND (p.evaluation_day>'id',$1,p.scope_key,p.record->>'scope_type',p.record->>'scope_id',p.record->>'metric','open',p.day,p.day,$3,true,p.record + FROM prior p WHERE p.breached AND (p.prior_id IS NULL OR (NOT p.prior_breached AND + (p.evaluation_day tuple[UsageAlertNotification, ...]: + rows: Final = TypeAdapter(tuple[_Pending, ...]).validate_python( + await self.database.query_raw( + """SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id + WHERE i.status='open' AND r.enabled AND NOT r.deleted AND EXISTS ( + SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels))) + ORDER BY i.updated_at,i.id LIMIT $1""", + limit, + ) + ) + return tuple( + UsageAlertNotification( + incident=UsageAlert.model_validate(row.record), + rule=UsageAlertRule.model_validate(row.rule), + pending_channels=tuple( + channel + for channel in UsageAlertRule.model_validate(row.rule).notification_channels + if channel not in row.notified_channels + ), + ) + for row in rows + ) + + async def mark_notified(self, incident_id: str, channel: UsageAlertChannel) -> None: + await self.database.query_raw( + 'UPDATE "LiteLLM_UsageAlertIncident" SET notified_channels=array_append(notified_channels,$2::text) WHERE id=$1 AND NOT ($2::text=ANY(notified_channels)) RETURNING id', + incident_id, + channel, + ) + + async def prune(self, retention_days: int = 90, limit: int = 500) -> None: + await self.database.query_raw( + """DELETE FROM "LiteLLM_UsageAlertIncident" WHERE id IN (SELECT id FROM "LiteLLM_UsageAlertIncident" WHERE status<>'open' AND updated_at str | None: + rows: Final = TypeAdapter(tuple[Mapping[str, str], ...]).validate_python( + await self.database.query_raw( + """SELECT member->>'role' AS role FROM "LiteLLM_TeamTable", jsonb_array_elements(members_with_roles) member WHERE team_id=$1 AND member->>'user_id'=$2 LIMIT 1""", + team_id, + user_id, + ) + ) + return rows[0]["role"] if rows else None diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py new file mode 100644 index 00000000000..4ad8f50cf42 --- /dev/null +++ b/litellm/types/proxy/issues.py @@ -0,0 +1,280 @@ +from datetime import date, datetime +from typing import Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import Self + +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + BoundedDailyActivityResponse, +) +from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alerts import ( + UsageAlert, + UsageAlertChannel, + UsageAlertDirection, + UsageAlertEvaluationCounts, + UsageAlertMetric, + UsageAlertRule, + UsageAlertStatus, +) + +IssueActivityResponse: TypeAlias = BoundedDailyActivityResponse + +IssueKind: TypeAlias = Literal["performance", "model_deprecation"] +IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] +IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] +IssueGroupingReason: TypeAlias = Literal["same_resource", "dominant_key", "provider_cohort", "model_deprecation"] +IssueSignalGroupingReason: TypeAlias = Literal[ + "same_resource", "same_key_traffic", "dominant_key", "provider_cohort", "volume_explains_spend", "model_deprecation" +] + + +class IssueModelPair(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + provider: str = Field(min_length=1, max_length=256) + model: str = Field(min_length=1, max_length=1024) + + +class IssueCohort(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + scope_type: IssueScope + scope_id: str | None = Field(default=None, min_length=1, max_length=1024) + model_pairs: tuple[IssueModelPair, ...] = Field(default=(), max_length=100) + + @model_validator(mode="after") + def validate_scope(self) -> Self: + if (self.scope_type == "deployment") != (self.scope_id is None): + raise ValueError("Only a deployment cohort omits scope_id") + if self.scope_type == "key" and self.scope_id and self.scope_id.removeprefix("raw:").startswith("sk-"): + raise ValueError("Use a reporting key ID, never a secret key") + if len(frozenset((pair.provider, pair.model) for pair in self.model_pairs)) != len(self.model_pairs): + raise ValueError("model_pairs must be unique") + return self + + +class IssueComparison(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + metric: UsageAlertMetric + direction: UsageAlertDirection + observed_value: float = Field(ge=0) + baseline_value: float = Field(ge=0) + threshold: float | None = None + baseline_start: date + baseline_end: date + baseline_sample_days: int | None = Field(default=None, ge=1, le=30) + + @model_validator(mode="after") + def validate_comparison(self) -> Self: + if self.baseline_start > self.baseline_end: + raise ValueError("baseline_start must not follow baseline_end") + if self.metric == "error_rate" and (self.observed_value > 100 or self.baseline_value > 100): + raise ValueError("error_rate uses percentages from 0 to 100") + return self + + +class IssueImpact(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + keys: int | None = Field(default=None, ge=0) + users: int | None = Field(default=None, ge=0) + teams: int | None = Field(default=None, ge=0) + models: int | None = Field(default=None, ge=0) + requests: int | None = Field(default=None, ge=0) + measured_keys: int | None = Field(default=None, ge=0) + measured_models: int | None = Field(default=None, ge=0) + + +class IssueDeprecation(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + model: ModelDeprecationInfo + usage_model: str = Field(min_length=1, max_length=1024) + requests: int = Field(ge=0) + last_used_date: date + last_day_requests: int = Field(ge=0) + state: DeprecationAlertState + + +class IssueInput(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + id: str = Field(min_length=1, max_length=200) + cohort_key: str = Field(min_length=1, max_length=200) + kind: IssueKind + family: IssueFamily + title: str = Field(min_length=1, max_length=2048) + cohort: IssueCohort + observed_on: date + observed_at: datetime + range_start: date + range_end: date + grouping_reason: IssueGroupingReason + explanation: str = Field(default="", max_length=2000) + comparisons: tuple[IssueComparison, ...] = Field(default=(), max_length=10) + impact: IssueImpact = Field(default_factory=IssueImpact) + deprecation: IssueDeprecation | None = None + + @model_validator(mode="after") + def validate_issue(self) -> Self: + if self.range_start > self.range_end: + raise ValueError("range_start must not follow range_end") + if (self.kind == "model_deprecation") != (self.deprecation is not None): + raise ValueError("Model deprecation issues require deprecation details") + if (self.kind == "model_deprecation") != (self.family == "retirement"): + raise ValueError("Only model deprecation issues use the retirement family") + if self.observed_at.tzinfo is None: + raise ValueError("observed_at must include a timezone") + return self + + +class Issue(IssueInput, frozen=True): + status: UsageAlertStatus + first_seen: datetime + last_seen: datetime + updated_at: datetime + scope_label: str | None = None + active: bool = True + can_update_status: bool | None = Field( + default=None, description="Current reader's status permission, evaluated by the issue-detail endpoint" + ) + can_view_activity: bool | None = Field( + default=None, description="Current reader's permission to inspect Activity, keys, users and impact" + ) + + +class IssueList(BaseModel, frozen=True): + data: tuple[Issue, ...] + total: int + limit: int + offset: int + + +class IssueUpdate(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: UsageAlertStatus + expected_status: UsageAlertStatus + + +class IssueSignalInput(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + source: Literal["usage_alert", "model_deprecation"] + source_id: str = Field(min_length=1, max_length=200) + usage_alert: UsageAlert | None = None + deprecation: IssueDeprecation | None = None + grouping_reason: IssueSignalGroupingReason = "same_resource" + explanation: str = Field(default="", max_length=2000) + + @model_validator(mode="after") + def validate_signal(self) -> Self: + if self.source == "usage_alert": + if self.usage_alert is None or self.deprecation is not None or self.usage_alert.id != self.source_id: + raise ValueError("Usage signals require their matching source alert") + elif self.deprecation is None or self.usage_alert is not None: + raise ValueError("Deprecation signals require deprecation details") + return self + + +class IssueSignal(IssueSignalInput, frozen=True): + issue_id: str + + +class IssueSignalList(BaseModel, frozen=True): + data: tuple[IssueSignal, ...] + total: int + limit: int + offset: int + + +class IssueProjectionSignal(BaseModel, frozen=True): + alert: UsageAlert + rule: UsageAlertRule + evaluation_day: date + + +class IssueDeliveryInput(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + id: str = Field(min_length=1, max_length=200) + issue_id: str = Field(min_length=1, max_length=200) + channel: UsageAlertChannel + email_recipients: tuple[str, ...] = Field(default=(), max_length=20) + + +class IssueDeliveryBinding(BaseModel, frozen=True): + delivery: IssueDeliveryInput + rule_id: str + rule_revision: str + source_id: str + + +class IssueDelivery(IssueDeliveryInput, frozen=True): + attempts: int = 0 + next_attempt: datetime | None = None + last_error: Literal["delivery_failed", "transport_not_configured"] | None = None + + +class IssuePendingDelivery(BaseModel, frozen=True): + delivery: IssueDelivery + issue: Issue + rule: UsageAlertRule + alert: UsageAlert + + +class IssueFamilyPolicy(BaseModel, frozen=True): + direction: UsageAlertDirection + enabled: bool | None + multiplier: float | None + min_value: float | None + min_requests: int | None + baseline_days: int | None + min_history_days: int | None + mixed: bool + total_rules: int + enabled_rules: int + + +class IssueFamilySetting(BaseModel, frozen=True): + family: PerformanceIssueFamily + enabled: bool | None + total_rules: int + enabled_rules: int + policies: tuple[IssueFamilyPolicy, ...] + evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts) + + +class IssueFamilyList(BaseModel, frozen=True): + data: tuple[IssueFamilySetting, ...] + projection_pending: bool = False + + +class IssueFamilyPolicyUpdate(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + direction: UsageAlertDirection + enabled: bool | None = None + multiplier: float | None = Field(default=None, gt=1, le=1000) + min_value: float | None = Field(default=None, ge=0) + min_requests: int | None = Field(default=None, ge=0) + baseline_days: int | None = Field(default=None, ge=1, le=30) + min_history_days: int | None = Field(default=None, ge=1, le=30) + + +class IssueFamilyUpdate(BaseModel, frozen=True): + model_config = ConfigDict(frozen=True, extra="forbid") + + enabled: bool | None = None + policies: tuple[IssueFamilyPolicyUpdate, ...] = Field(default=(), max_length=2) + + @model_validator(mode="after") + def validate_update(self) -> Self: + if len(frozenset(policy.direction for policy in self.policies)) != len(self.policies): + raise ValueError("Each direction can occur once") + if self.enabled is None and not self.policies: + raise ValueError("Specify enabled or a policy update") + return self diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 8a30176700f..d3379336b3e 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -182,6 +182,8 @@ class KeySpendMetrics(LiteLLMBaseModel): failed_requests: int = 0 cache_read_input_tokens: int = 0 cache_creation_input_tokens: int = 0 + total_response_time_ms: int = 0 + timed_requests: int = 0 class KeySpendActivityRow(LiteLLMBaseModel): @@ -214,6 +216,8 @@ class UserActivityRow(LiteLLMBaseModel): api_requests: int = 0 successful_requests: int = 0 failed_requests: int = 0 + total_response_time_ms: int = 0 + timed_requests: int = 0 class DailyActivityUserPageResponse(LiteLLMBaseModel): diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index 37c49d737af..33f9b942595 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -112,3 +112,15 @@ class DeprecationTagUsage(LiteLLMBaseModel): api_key: str tag: str + + +class ModelDeprecationUsage(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model: str + requests: int = Field(ge=0) + last_used_date: date + last_day_requests: int = Field(ge=0) + keys: int = Field(ge=0) + users: int = Field(ge=0) + teams: int = Field(ge=0) diff --git a/litellm/types/proxy/usage_alert_defaults.py b/litellm/types/proxy/usage_alert_defaults.py new file mode 100644 index 00000000000..887932a5194 --- /dev/null +++ b/litellm/types/proxy/usage_alert_defaults.py @@ -0,0 +1,76 @@ +from dataclasses import dataclass +from itertools import product +from typing import Final, Literal, TypeAlias +from uuid import NAMESPACE_URL, uuid5 + +from litellm.types.proxy.usage_alerts import ( + UsageAlertChannel, + UsageAlertDirection, + UsageAlertMetric, + UsageAlertRule, + UsageAlertRuleInput, + UsageAlertScope, +) + +PerformanceIssueFamily: TypeAlias = Literal["spend", "latency", "reliability", "traffic"] +USAGE_ALERT_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("deployment", "key", "user", "team", "model") + + +@dataclass(frozen=True, slots=True) +class DefaultUsagePolicy: + family: PerformanceIssueFamily + metric: UsageAlertMetric + direction: UsageAlertDirection + multiplier: float + min_value: float + min_requests: int = 0 + min_history_days: int = 3 + + +DEFAULT_USAGE_POLICIES: Final = ( + DefaultUsagePolicy("spend", "spend", "increase", 3, 10), + DefaultUsagePolicy("latency", "latency", "increase", 3, 500, 100), + DefaultUsagePolicy("reliability", "error_rate", "increase", 3, 2, 500), + DefaultUsagePolicy("traffic", "requests", "increase", 10, 1000), + DefaultUsagePolicy("traffic", "requests", "decrease", 5, 0, 500, 7), +) + + +def default_usage_rule_id(scope: UsageAlertScope, metric: UsageAlertMetric, direction: UsageAlertDirection) -> str: + return str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{scope}:{metric}:{direction}")) + + +DEFAULT_USAGE_RULE_IDS: Final = frozenset( + default_usage_rule_id(scope, policy.metric, policy.direction) + for scope, policy in product(USAGE_ALERT_SCOPES, DEFAULT_USAGE_POLICIES) +) + + +def is_default_usage_rule(rule: UsageAlertRule) -> bool: + return rule.id in DEFAULT_USAGE_RULE_IDS and rule.id == default_usage_rule_id( + rule.scope_type, rule.metric, rule.direction + ) + + +def default_usage_rules( + channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False +) -> tuple[UsageAlertRuleInput, ...]: + return tuple( + UsageAlertRuleInput( + name=f"Daily {scope} {policy.metric.replace('_', ' ')} {policy.direction}", + scope_type=scope, + metric=policy.metric, + direction=policy.direction, + multiplier=policy.multiplier, + min_value=policy.min_value, + min_requests=policy.min_requests, + min_history_days=policy.min_history_days, + notification_channels=( + () + if legacy_user_spend_alerts + and (scope, policy.metric, policy.direction) == ("user", "spend", "increase") + else channels + ), + ) + for scope, policy in product(USAGE_ALERT_SCOPES, DEFAULT_USAGE_POLICIES) + ) diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py new file mode 100644 index 00000000000..4c65994befa --- /dev/null +++ b/litellm/types/proxy/usage_alerts.py @@ -0,0 +1,198 @@ +"""Daily usage alert rules and incidents shared by the API and evaluator.""" + +from datetime import date, datetime +from typing import Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import Self + +UsageAlertScope: TypeAlias = Literal["deployment", "key", "user", "team", "model"] +UsageAlertMetric: TypeAlias = Literal["spend", "requests", "latency", "errors", "error_rate"] +UsageAlertDirection: TypeAlias = Literal["increase", "decrease"] +UsageAlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"] +UsageAlertChannel: TypeAlias = Literal["email", "slack"] +UsageAlertEvaluationStatus: TypeAlias = Literal["pending", "complete", "insufficient_coverage", "insufficient_history"] +UsageAlertRuleKind: TypeAlias = Literal["all", "default", "custom"] + + +class UsageAlertRuleInput(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + name: str = Field(min_length=1, max_length=200, pattern=r"^[^\x00-\x1f\x7f]+$") + scope_type: UsageAlertScope + scope_id: str | None = Field(default=None, min_length=1, max_length=1024) + metric: UsageAlertMetric + direction: UsageAlertDirection + multiplier: float = Field(default=3, gt=1, le=1000) + baseline_days: int = Field(default=7, ge=1, le=30) + min_history_days: int = Field(default=3, ge=1, le=30) + min_requests: int = Field(default=100, ge=0) + min_value: float = Field(default=0, ge=0, description="Minimum absolute change in the metric's units") + enabled: bool = True + notification_channels: tuple[UsageAlertChannel, ...] = Field(default=(), max_length=2) + email_recipients: tuple[str, ...] = Field(default=(), max_length=20) + + @model_validator(mode="after") + def validate_rule(self) -> Self: + if self.min_history_days > self.baseline_days: + raise ValueError("min_history_days must not exceed baseline_days") + if self.scope_type == "deployment" and self.scope_id is not None: + raise ValueError("Deployment alerts cover the whole gateway; omit scope_id") + if ( + self.scope_type == "key" + and self.scope_id is not None + and self.scope_id.removeprefix("raw:").startswith("sk-") + ): + raise ValueError("Use the key's reporting ID, never the secret key") + if len(set(self.notification_channels)) != len(self.notification_channels): + raise ValueError("notification_channels must be unique") + if "email" in self.notification_channels and not self.email_recipients: + raise ValueError("Email notifications require at least one recipient") + if any( + "@" not in email or "\n" in email or "\r" in email or len(email) > 254 for email in self.email_recipients + ): + raise ValueError("Invalid email recipient") + return self + + +class UsageAlertRule(UsageAlertRuleInput): + overrides_defaults: bool = Field(default=False, exclude=True) + id: str + created_at: datetime + updated_at: datetime + created_by: str | None + last_evaluated_day: date | None = None + evaluation_status: UsageAlertEvaluationStatus = "pending" + is_default: bool = False + + +class UsageAlert(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + id: str + rule_id: str + rule_name: str + scope_type: UsageAlertScope + scope_id: str | None + metric: UsageAlertMetric + direction: UsageAlertDirection + status: UsageAlertStatus + day: date + baseline_start: date + baseline_end: date + baseline_sample_days: int | None = Field(default=None, ge=1, le=30) + observed_value: float + baseline_value: float + threshold: float + first_seen: datetime + last_seen: datetime + activity_path: str + + +class UsageAlertUpdate(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + status: UsageAlertStatus + + +class UsageAlertEvaluationCounts(BaseModel, frozen=True): + pending: int = 0 + complete: int = 0 + insufficient_coverage: int = 0 + insufficient_history: int = 0 + + +class UsageAlertDefaultSummary(BaseModel, frozen=True): + total: int = 0 + enabled: int = 0 + evaluation_status_counts: UsageAlertEvaluationCounts = Field(default_factory=UsageAlertEvaluationCounts) + + +class UsageAlertRuleList(BaseModel, frozen=True): + data: tuple[UsageAlertRule, ...] + total: int + limit: int + offset: int + default_summary: UsageAlertDefaultSummary = Field(default_factory=UsageAlertDefaultSummary) + + +class UsageAlertList(BaseModel, frozen=True): + data: tuple[UsageAlert, ...] + total: int + limit: int + offset: int + + +class UsageAlertGroupMetric(BaseModel, frozen=True): + metric: UsageAlertMetric + incident_count: int + representative: UsageAlert + + +class UsageAlertGroup(BaseModel, frozen=True): + id: str + scope_type: UsageAlertScope + scope_id: str | None + scope_label: str | None + day: date + incident_count: int + metrics: tuple[UsageAlertGroupMetric, ...] = Field(max_length=5) + activity_path: str + + +class UsageAlertGroupList(BaseModel, frozen=True): + data: tuple[UsageAlertGroup, ...] + total: int + limit: int + offset: int + + +class UsageAlertGroupUpdate(BaseModel, frozen=True): + model_config = ConfigDict(extra="forbid") + scope_type: UsageAlertScope + scope_id: str | None = Field(default=None, min_length=1, max_length=1024) + day: date + expected_status: UsageAlertStatus + status: UsageAlertStatus + + @model_validator(mode="after") + def validate_group(self) -> Self: + if (self.scope_type == "deployment") != (self.scope_id is None): + raise ValueError("Specify a resource ID except for deployment alerts") + if self.expected_status == self.status: + raise ValueError("The new status must differ from the expected status") + return self + + +class UsageAlertGroupUpdateResult(BaseModel, frozen=True): + updated_count: int + + +class UsageAlertModelOption(BaseModel, frozen=True): + scope_id: str + label: str + + +class UsageAlertModelList(BaseModel, frozen=True): + data: tuple[UsageAlertModelOption, ...] + limit: int + next_cursor: str | None + start_date: date + end_date: date + + +class UsageAlertNotification(BaseModel): + model_config = ConfigDict(frozen=True) + incident: UsageAlert + rule: UsageAlertRule + pending_channels: tuple[UsageAlertChannel, ...] + + +class UsageAlertObservation(BaseModel): + model_config = ConfigDict(frozen=True, allow_inf_nan=False) + entity_id: str | None + day: date + observed: float + baseline: float + baseline_sample_days: int | None = Field(default=None, ge=1, le=30) + threshold: float + breached: bool diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index b013ce43113..938ef0467ed 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -39,6 +39,8 @@ class DailyActivityScope: model: str | None timezone_offset_minutes: int | None include_current_utc_day: bool = False + provider: str | None = None + model_pairs: tuple[tuple[str, str], ...] = () def __post_init__(self) -> None: if self.entity_id_field not in _ENTITY_FIELDS[self.table]: @@ -57,6 +59,8 @@ class KeySpendRow: failed_requests: int cache_read_input_tokens: int cache_creation_input_tokens: int + total_response_time_ms: int = 0 + timed_requests: int = 0 @dataclass(frozen=True, slots=True) @@ -75,6 +79,8 @@ class UserSpendRow: api_requests: int successful_requests: int failed_requests: int + total_response_time_ms: int = 0 + timed_requests: int = 0 @dataclass(frozen=True, slots=True) diff --git a/schema.prisma b/schema.prisma index 818f2dc4858..21ca3d46844 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2168,3 +2168,96 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +model LiteLLM_UsageAlertRule { + id String @id + scope_type String + scope_id String? + enabled Boolean @default(true) + record Json + deleted Boolean @default(false) + @@index([enabled, deleted, id]) + @@index([scope_type, scope_id, deleted]) +} + +model LiteLLM_UsageAlertIncident { + id String @id + rule_id String + scope_key String + scope_type String + scope_id String? + metric String + status String + day String + evaluation_day String + evaluation_revision String @default("") + breached Boolean + record Json + notified_channels String[] @default([]) + updated_at DateTime @default(now()) + @@index([rule_id, scope_key, evaluation_day]) + @@index([evaluation_day, breached, id]) + @@index([scope_type, scope_id, day]) + @@index([status, day]) + @@index([updated_at]) +} + +model LiteLLM_UsageAlertCheckpoint { + id String @id + completed Boolean @default(false) + owner String? + expires_at DateTime @default(now()) + @@index([expires_at]) +} + +model LiteLLM_Issue { + id String @id + cohort_key String + kind String + family String + scope_type String + scope_id String? + status String + active Boolean @default(true) + ack_key String? + record Json + first_seen DateTime + last_seen DateTime + updated_at DateTime @default(now()) + + @@index([status, last_seen, id]) + @@index([kind, family, last_seen, id]) + @@index([scope_type, scope_id, last_seen, id]) + @@index([cohort_key, active]) +} + +model LiteLLM_IssueSignal { + issue_id String + source String + source_id String + record Json + + @@id([issue_id, source, source_id]) + @@index([source, source_id]) +} + +model LiteLLM_IssueDelivery { + id String @id + issue_id String + record Json + sent Boolean @default(false) + next_attempt DateTime @default(now()) + updated_at DateTime @default(now()) + + @@index([sent, next_attempt, id]) + @@index([issue_id]) +} + +model LiteLLM_IssueDeliveryRule { + delivery_id String + rule_id String + source_id String + rule_revision String + + @@id([delivery_id, rule_id, source_id]) +} diff --git a/scripts/check_openapi_docs.py b/scripts/check_openapi_docs.py index 219ef912952..c6c7224e35d 100755 --- a/scripts/check_openapi_docs.py +++ b/scripts/check_openapi_docs.py @@ -7,7 +7,7 @@ count them per rule the same delta-vs-base way as the other lint gates. The rule OA001 no request body schema (the handler reads ``request.body()`` and never told FastAPI what is in it, see ``inline_request_body`` in custom_openapi_spec.py for the fix) OA002 no description (the handler has no docstring and the route no ``description=``) -OA003 the 200 response has no schema (no ``response_model`` and no ``responses={200: ...}``) +OA003 the 200 or 201 response has no schema (no ``response_model`` or explicit response schema) """ from __future__ import annotations @@ -45,7 +45,8 @@ def _operation_codes(operation: Mapping[str, object]) -> Iterator[str]: description: Final = operation.get("description") if not isinstance(description, str) or not description.strip(): yield "OA002" - if not _content_schema(_as_mapping(operation.get("responses")).get("200")): + responses: Final = _as_mapping(operation.get("responses")) + if not _content_schema(responses.get("200", responses.get("201"))): yield "OA003" diff --git a/scripts/openapi_docs_gate.py b/scripts/openapi_docs_gate.py index de340bf84e7..7b4a7e19e57 100755 --- a/scripts/openapi_docs_gate.py +++ b/scripts/openapi_docs_gate.py @@ -5,7 +5,7 @@ Sibling of scripts/test_quality_gate.py, pointed at the proxy's OpenAPI document instead of a source tree. Each rule is counted over every POST operation the proxy serves at HEAD and at the merge-base with the branch this change merges into, and the gate fails only when a rule grew past the merge-base count, so a new endpoint has to -ship with a request schema, a description and a 200 schema while the undocumented +ship with a request schema, a description and a 200/201 schema while the undocumented routes that already exist are only ever allowed to shrink. The merge-base counts come from scripts/lint_base_counts.py: the disk cache, then the @@ -167,7 +167,7 @@ def cmd_check(base: str) -> None: print( "Give every new POST route a request body schema (a typed body parameter, or " "`openapi_extra={'requestBody': inline_request_body(Model, example)}` when the handler reads " - "`request.body()`), a docstring, and a 200 schema (`response_model=` or `responses={200: {'model': ...}}`), " + "`request.body()`), a docstring, and a 200/201 schema (`response_model=` or an explicit response schema), " "or document an equal number of existing routes; the ceiling is the merge-base count. Run " "`python scripts/check_openapi_docs.py` to see every finding." ) diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py new file mode 100644 index 00000000000..e5457867b07 --- /dev/null +++ b/tests/integration/database/test_issues.py @@ -0,0 +1,1367 @@ +import asyncio +import json +from collections.abc import AsyncGenerator, Mapping, Sequence +from contextlib import asynccontextmanager +from datetime import date, datetime, timedelta, timezone +from hashlib import sha256 +from typing import Final, cast # noqa: TID251 # Generated Prisma query_raw has narrower overloads +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from uuid import uuid4 + +import psycopg +import pytest +from integration._support.actors import ManagementDatabase +from integration._support.client import Gateway +from prisma import Prisma +from prisma.types import DatasourceOverride +from psycopg import sql +from pydantic import BaseModel, TypeAdapter, ValidationError + +from litellm.proxy.management_endpoints.common_daily_activity import daily_activity_repository +from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase +from litellm.repositories.daily_activity_sql import build_bounded_daily_activity_sql +from litellm.repositories.issue_activity import IssueActivityRepository +from litellm.repositories.issues import IssuesRepository +from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository +from litellm.types.proxy.issues import ( + IssueActivityResponse, + IssueCohort, + IssueComparison, + IssueDeliveryInput, + IssueFamilyPolicyUpdate, + IssueFamilyUpdate, + IssueInput, + IssueList, + IssueModelPair, + IssueSignalInput, + IssueSignalList, +) +from litellm.types.proxy.usage_alert_defaults import default_usage_rule_id, default_usage_rules +from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertObservation, UsageAlertRuleInput + + +def _issue(cohort: IssueCohort) -> IssueInput: + identity: Final = str(uuid4()) + day: Final = date.today() - timedelta(days=3) + return IssueInput( + id=identity, + cohort_key=identity, + kind="performance", + family="traffic", + title="Requests increased", + cohort=cohort, + observed_on=day, + observed_at=datetime.now(timezone.utc), + range_start=day - timedelta(days=7), + range_end=day, + grouping_reason="same_resource", + explanation="Requests increased for this resource.", + comparisons=( + IssueComparison( + metric="requests", + direction="increase", + observed_value=420, + baseline_value=120, + threshold=360, + baseline_start=day - timedelta(days=7), + baseline_end=day - timedelta(days=1), + baseline_sample_days=7, + ), + ), + ) + + +async def _query(db: ManagementDatabase, query: str, *args: object) -> object: + return await IssuesRepository.from_prisma(db.prisma).database.query_raw(query, *args) + + +async def _remove_issues(db: ManagementDatabase, ids: tuple[str, ...]) -> None: + await _query( + db, + 'DELETE FROM "LiteLLM_IssueDeliveryRule" WHERE delivery_id IN ' + '(SELECT id FROM "LiteLLM_IssueDelivery" WHERE issue_id=ANY($1::text[])) RETURNING delivery_id', + list(ids), + ) + await _query(db, 'DELETE FROM "LiteLLM_IssueDelivery" WHERE issue_id=ANY($1::text[]) RETURNING id', list(ids)) + await _query(db, 'DELETE FROM "LiteLLM_IssueSignal" WHERE issue_id=ANY($1::text[]) RETURNING source_id', list(ids)) + await _query(db, 'DELETE FROM "LiteLLM_Issue" WHERE id=ANY($1::text[]) RETURNING id', list(ids)) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gateway, db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + outsider: Final = scenario.user(user_role="internal_user") + viewer: Final = scenario.user(user_role="proxy_admin_viewer") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + info_key: Final = scenario.key(user_id=owner, key_type="default", allowed_routes=["info_routes"]) + outsider_key: Final = scenario.key(user_id=outsider, key_type="management") + viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") + inference_key: Final = scenario.key(user_id=owner, key_type="llm_api", key_alias="Example checkout") + reporting_id: Final = sha256(inference_key.encode()).hexdigest() + first: Final = _issue(IssueCohort(scope_type="key", scope_id=reporting_id)) + second: Final = _issue(first.cohort) + global_issue: Final = _issue(IssueCohort(scope_type="provider", scope_id="example-provider")) + rule: Final = await alerts.save_rule( + UsageAlertRuleInput( + name="Visibility fixture", + scope_type="key", + scope_id=reporting_id, + metric="requests", + direction="increase", + ), + owner, + ) + assert rule is not None + try: + await store.upsert_issues((first, second, global_issue)) + source: Final = ( + await alerts.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=reporting_id, + day=first.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + broader: Final = source.model_copy( + update={ + "id": str(uuid4()), + "scope_type": "deployment", + "scope_id": None, + "observed_value": 99999, + "baseline_value": 9999, + } + ) + await store.add_signals( + first.id, + ( + IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source), + IssueSignalInput( + source="usage_alert", + source_id=broader.id, + usage_alert=broader, + grouping_reason="dominant_key", + explanation="Parent contribution evidence.", + ), + ), + ) + path: Final = f"/issues/{first.id}" + own: Final = gateway.request("GET", path, key=owner_key) + assert own.status_code == 200 and own.json()["scope_label"] == "Example checkout" + assert own.json()["can_update_status"] is False, "Broader source scopes require their own write permission" + assert own.json()["can_view_activity"] is True + assert gateway.request("GET", path, key=viewer_key).json()["can_update_status"] is False + assert gateway.request("GET", path, key=info_key).json()["can_update_status"] is False + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=info_key + ).status_code + == 403 + ) + assert "99999" not in own.text and "9999" not in own.text + pages: Final = tuple( + gateway.request( + "GET", + "/issues", + key=owner_key, + params={"scope_type": "key", "scope_id": reporting_id, "limit": "1", "offset": str(offset)}, + ) + for offset in (0, 1, 2) + ) + assert all(page.status_code == 200 for page in pages) + for offset, parsed in enumerate(IssueList.model_validate(page.json()) for page in pages): + assert parsed.total == 2 and len(parsed.data) == (1 if offset < 2 else 0) + assert gateway.request("GET", "/issues", key=owner_key).status_code == 403 + for suffix in ("", "/signals", "/activity", "/keys", "/users", "/impact"): + assert gateway.request("GET", f"/issues/{global_issue.id}{suffix}", key=owner_key).status_code == 403 + assert gateway.request("GET", path + suffix, key=outsider_key).status_code == 403 + assert gateway.request("GET", path + suffix, key=inference_key).status_code == 403 + public_signals: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1"}) + assert public_signals.status_code == 200 + visible: Final = IssueSignalList.model_validate(public_signals.json()) + assert visible.total == 1 and tuple(row.source_id for row in visible.data) == (source.id,) + assert "99999" not in public_signals.text + end: Final = gateway.request("GET", path + "/signals", key=owner_key, params={"limit": "1", "offset": "1"}) + assert end.json()["total"] == 1 and end.json()["data"] == [] + admin: Final = gateway.request("GET", path + "/signals", key=viewer_key) + assert admin.status_code == 200 and admin.json()["total"] == 2 and "99999" in admin.text + assert ( + gateway.request( + "PATCH", path, {"status": "dismissed", "expected_status": "open"}, key=viewer_key + ).status_code + == 403 + ) + assert ( + gateway.request( + "PATCH", path, {"status": "dismissed", "expected_status": "open"}, key=owner_key + ).status_code + == 403 + ) + assert gateway.request("PATCH", path, {"status": "dismissed", "expected_status": "open"}).status_code == 200 + assert gateway.request("PATCH", path, {"status": "resolved", "expected_status": "open"}).status_code == 409 + assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["status"] == "open" + assert gateway.request("GET", f"/issues/{second.id}", key=owner_key).json()["can_update_status"] is True + assert ( + gateway.request( + "PATCH", f"/issues/{second.id}", {"status": "dismissed", "expected_status": "open"}, key=owner_key + ).status_code + == 200 + ) + finally: + await _remove_issues(db, (first.id, second.id, global_issue.id)) + await alerts.delete_rule(rule.id) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_detail_reports_team_status_permission(gateway: Gateway, db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + with gateway.scenario() as scenario: + admin: Final = scenario.user(user_role="internal_user") + member: Final = scenario.user(user_role="internal_user") + team: Final = scenario.team( + members_with_roles=[{"user_id": admin, "role": "admin"}, {"user_id": member, "role": "user"}] + ) + permitted_team: Final = scenario.team( + members_with_roles=[{"user_id": member, "role": "user"}], + team_member_permissions=["/team/daily/activity"], + ) + admin_key: Final = scenario.key(user_id=admin, key_type="management") + member_key: Final = scenario.key(user_id=member, key_type="management") + service_key: Final = scenario.key( + team_id=team, + user_id=None, + key_type="management", + metadata={"service_account_id": "example-activity-service"}, + ) + issue: Final = _issue(IssueCohort(scope_type="team", scope_id=team)) + permitted_issue: Final = _issue(IssueCohort(scope_type="team", scope_id=permitted_team)) + try: + await store.upsert_issues((issue, permitted_issue)) + path: Final = f"/issues/{issue.id}" + assert gateway.request("GET", path, key=admin_key).json()["can_update_status"] is True + assert gateway.request("GET", path, key=admin_key).json()["can_view_activity"] is True + assert gateway.request("GET", path, key=member_key).json()["can_update_status"] is False + assert gateway.request("GET", path, key=member_key).json()["can_view_activity"] is False + for suffix in ("/activity", "/keys", "/users", "/impact"): + assert gateway.request("GET", path + suffix, key=member_key).status_code == 403 + assert gateway.request("GET", path + suffix, key=service_key).status_code == 403 + for suffix in ("/activity", "/keys", "/impact"): + assert gateway.request("GET", path + suffix, key=admin_key).status_code == 200 + assert ( + gateway.request("GET", f"/issues/{permitted_issue.id}" + suffix, key=member_key).status_code == 200 + ) + assert ( + gateway.request("GET", f"/issues/{permitted_issue.id}", key=member_key).json()["can_view_activity"] + is True + ) + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=member_key + ).status_code + == 403 + ) + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=admin_key + ).status_code + == 200 + ) + finally: + await _remove_issues(db, (issue.id, permitted_issue.id)) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gateway, db: ManagementDatabase) -> None: + from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated + from litellm.repositories.issue_activity import issue_activity_scope + + repository: Final = IssueActivityRepository.from_prisma(db.prisma) + store: Final = IssuesRepository.from_prisma(db.prisma) + identity: Final = str(uuid4()) + key: Final = sha256(identity.encode()).hexdigest() + provider: Final = f"example-provider-{identity}" + day: Final = date.today() - timedelta(days=5) + model_count: Final = 137 + cohort: Final = IssueCohort(scope_type="key", scope_id=key) + issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day + timedelta(days=2)}) + await _query( + db, + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,model_group,custom_llm_provider,mcp_namespaced_tool_name,endpoint, + api_requests,spend,prompt_tokens,completion_tokens,successful_requests,failed_requests, + total_response_time_ms,timed_requests,updated_at) + SELECT $1||'-'||m||'-'||d,$1,($2::date+d::int)::text,$3, + CASE WHEN m=0 THEN 'Other' ELSE 'example-model-'||lpad(m::text,4,'0') END, + 'group-'||m,$4,'tool-'||m,'endpoint-'||m, + m+2,CASE WHEN m=0 AND d=0 THEN 10000 ELSE m+1 END,m+1,2,m+1,1,(m+1)*200,m+1,CURRENT_TIMESTAMP + FROM generate_series(0,$5::int-1) m CROSS JOIN generate_series(0,2) d RETURNING id""", + identity, + day.isoformat(), + key, + provider, + model_count, + ) + try: + await store.upsert_issue(issue) + scope: Final = issue_activity_scope(cohort, day, day + timedelta(days=2)) + original: Final = await get_daily_activity_aggregated(repository.activity, scope, api_key_limit=20) + response: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + assert response.status_code == 200 + bounded: Final = IssueActivityResponse.model_validate(response.json()) + assert bounded.metadata.model_dump(exclude={"api_key_limit"}) == original.metadata.model_dump( + exclude={"api_key_limit"} + ) + assert tuple((row.date, row.metrics) for row in bounded.results) == tuple( + (row.date, row.metrics) for row in original.results + ) + models: Final = bounded.model_breakdown + assert models is not None and models.total_models == model_count and len(models.models) == models.limit == 20 + assert models.models[0].model == "Other" and models.models[0].metrics.spend == 10002 + assert models.other is not None and models.other.model_count == model_count - models.limit + components: Final = (*tuple(row.metrics for row in models.models), models.other.metrics) + assert sum(row.spend for row in components) == bounded.metadata.total_spend + assert sum(row.api_requests for row in components) == bounded.metadata.total_api_requests + assert sum(row.total_response_time_ms for row in components) == bounded.metadata.total_response_time_ms + assert all(not row.breakdown.models and not row.breakdown.endpoints for row in bounded.results) + query: Final = build_bounded_daily_activity_sql(scope, model_limit=100) + raw: Final = TypeAdapter(tuple[object, ...]).validate_python( + await repository.database.query_raw(query.sql, *query.params) + ) + assert len(raw) == 3 + 1 + 100 + 1 + largest: Final = gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "100"}) + assert largest.status_code == 200 + large_models: Final = IssueActivityResponse.model_validate(largest.json()).model_breakdown + assert large_models is not None and len(large_models.models) == 100 + assert large_models.other is not None and large_models.other.model_count == model_count - 100 + assert gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "101"}).status_code == 422 + assert ( + gateway.request( + "GET", + f"/issues/{issue.id}/activity", + params={"start_date": day.isoformat(), "end_date": (day + timedelta(days=366)).isoformat()}, + ).status_code + == 422 + ) + non_key: Final = await repository.daily_activity( + IssueCohort(scope_type="provider", scope_id=provider), day, day + timedelta(days=2) + ) + assert non_key.model_breakdown is None and non_key.metadata == bounded.metadata + assert len(non_key.results) == 3 + schemas: Final = TypeAdapter(tuple[Mapping[str, str], ...]).validate_python( + await _query(db, "SELECT current_schema() AS name") + ) + assert gateway.database_url is not None + with psycopg.connect(gateway.database_url) as blocker: + blocker.execute( + sql.SQL("LOCK TABLE {} IN ACCESS EXCLUSIVE MODE").format( + sql.Identifier(schemas[0]["name"], "LiteLLM_DailyUserSpend") + ) + ) + busy: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + assert busy.status_code == 503 and busy.headers["Retry-After"] == "5" + assert gateway.request("GET", f"/issues/{issue.id}/activity").status_code == 200 + finally: + await _remove_issues(db, (issue.id,)) + await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1 RETURNING id', identity) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( + gateway: Gateway, db: ManagementDatabase +) -> None: + repository: Final = IssueActivityRepository.from_prisma(db.prisma) + run_id: Final = str(uuid4()) + day: Final = date.today() - timedelta(days=3) + provider: Final = f"example-provider-{run_id}" + other: Final = f"other-provider-{run_id}" + key: Final = sha256(run_id.encode()).hexdigest() + entries: Final = tuple( + { + "id": f"{run_id}-{index}", + "user_id": f"user-{run_id}", + "date": day.isoformat(), + "api_key": key, + "model": model, + "custom_llm_provider": name, + "api_requests": requests, + "spend": spend, + "successful_requests": requests, + "total_response_time_ms": requests * latency, + "timed_requests": requests, + } + for index, (name, model, requests, spend, latency) in enumerate( + ( + (provider, "example-model-a", 100, 10, 200), + (other, "example-model-a", 1000, 100, 900), + (provider, "example-model-b", 50, 5, 400), + ) + ) + ) + await _query( + db, + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,api_requests,spend,successful_requests,total_response_time_ms,timed_requests,updated_at) + SELECT id,user_id,date,api_key,model,custom_llm_provider,api_requests,spend,successful_requests,total_response_time_ms,timed_requests,CURRENT_TIMESTAMP + FROM jsonb_to_recordset($1::jsonb) AS x(id text,user_id text,date text,api_key text,model text, + custom_llm_provider text,api_requests int,spend float,successful_requests int,total_response_time_ms bigint,timed_requests int) + RETURNING id""", + json.dumps(entries), + ) + cohort: Final = IssueCohort( + scope_type="provider", + scope_id=provider, + model_pairs=(IssueModelPair(provider=provider, model="example-model-a"),), + ) + store: Final = IssuesRepository.from_prisma(db.prisma) + issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day}) + try: + await store.upsert_issue(issue) + activity_http: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + assert activity_http.status_code == 200 and activity_http.json()["metadata"]["total_api_requests"] == 100 + keys_http: Final = gateway.request("GET", f"/issues/{issue.id}/keys", params={"limit": "1"}) + assert keys_http.status_code == 200 and keys_http.json()["api_keys"][0]["metrics"]["api_requests"] == 100 + users_http: Final = gateway.request("GET", f"/issues/{issue.id}/users") + assert users_http.status_code == 200 and users_http.json()["total_users"] == 1 + impact_http: Final = gateway.request("GET", f"/issues/{issue.id}/impact") + assert impact_http.status_code == 200 and impact_http.json()["requests"] == 100 + daily: Final = await repository.daily_activity(cohort, day, day) + assert daily.metadata.total_api_requests == 100 and daily.metadata.total_spend == 10 + assert daily.metadata.total_response_time_ms == 20000 and daily.metadata.total_timed_requests == 100 + keys: Final = await repository.keys(cohort, day, day, 1, 0) + assert keys.total_api_keys == 1 and keys.api_keys[0].metrics.api_requests == 100 + assert ( + keys.api_keys[0].metrics.total_response_time_ms == 20000 and keys.api_keys[0].metrics.timed_requests == 100 + ) + users: Final = await repository.users(cohort, day, day, 1, 0) + assert users.total_users == 1 and users.users[0].api_requests == 100 and users.users[0].timed_requests == 100 + impact: Final = await repository.impact(cohort, day, day) + assert (impact.keys, impact.users, impact.models, impact.requests) == (1, 1, 1, 100) + pairs: Final = IssueCohort( + scope_type="deployment", + model_pairs=( + IssueModelPair(provider=provider, model="example-model-a"), + IssueModelPair(provider=other, model="example-model-b"), + ), + ) + assert (await repository.daily_activity(pairs, day, day)).metadata.total_api_requests == 100 + injected: Final = IssueCohort(scope_type="provider", scope_id=provider + "' OR true --") + assert (await repository.impact(injected, day, day)).requests == 0 + from litellm.repositories.issue_activity import issue_activity_scope + from litellm.types.repositories.daily_activity import ExportType + + exported: Final = tuple( + [ + row + async for row in repository.activity.export_rows( + issue_activity_scope(cohort, day, day), export_type=ExportType.DAILY + ) + ] + ) + assert len(exported) == 1 and exported[0].api_requests == 100 + finally: + await _remove_issues(db, (issue.id,)) + await _query( + db, + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id=ANY($1::text[]) RETURNING id', + [entry["id"] for entry in entries], + ) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + original: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + definition: Final = UsageAlertRuleInput( + name="Delivery fixture", + scope_type="user", + scope_id=original.cohort.scope_id, + metric="requests", + direction="increase", + notification_channels=("slack", "email"), + email_recipients=("alerts@example.com",), + ) + rule: Final = await alerts.save_rule(definition, "qa-admin") + assert rule is not None + observation: Final = UsageAlertObservation( + entity_id=rule.scope_id, day=original.observed_on, observed=420, baseline=120, threshold=360, breached=True + ) + delivery: Final = IssueDeliveryInput(id=str(uuid4()), issue_id=original.id, channel="slack") + invalid: Final = IssueDeliveryInput( + id=str(uuid4()), issue_id=original.id, channel="email", email_recipients=("outside@example.com",) + ) + next_issue_id: Final = str(uuid4()) + try: + source: Final = (await alerts.record_observations(rule, (observation,)))[0] + initial: Final = await store.upsert_issue(original) + assert tuple( + item.alert.id + for item in await store.active_signals( + original.observed_on, scope_type="user", metric="requests", scope_ids=(str(rule.scope_id),) + ) + ) == (source.id,) + assert await store.active_signals(original.observed_on, scope_ids=()) == () + assert await store.active_signals(original.observed_on, scope_ids=("other-example-user",)) == () + await store.add_signals( + initial.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),) + ) + await store.queue_delivery(delivery, rule.id, rule.updated_at.isoformat(), source.id) + await store.queue_delivery(invalid, rule.id, rule.updated_at.isoformat(), source.id) + pending: Final = await store.pending_delivery(delivery.id) + assert pending is not None and pending.alert.id == source.id and pending.issue.id == initial.id + assert await store.pending_delivery(invalid.id) is None + disabled: Final = await alerts.save_rule(definition.model_copy(update={"enabled": False}), "qa-admin", rule) + assert disabled is not None and await store.pending_delivery(delivery.id) is None + enabled: Final = await alerts.save_rule(definition, "qa-admin", disabled) + assert enabled is not None and await store.pending_delivery(delivery.id) is None + reevaluated: Final = await alerts.record_observations(enabled, (observation,)) + assert len(reevaluated) == 1 and reevaluated[0].id == source.id + await store.queue_delivery(delivery, enabled.id, enabled.updated_at.isoformat(), source.id) + assert await store.pending_delivery(delivery.id) is not None + await alerts.set_status(source.id, "dismissed") + assert await store.pending_delivery(delivery.id) is None + await alerts.set_status(source.id, "open") + assert await store.set_status(initial.id, "dismissed", "open") is not None + assert await store.pending_delivery(delivery.id) is None + later: Final = original.model_copy( + update={ + "observed_on": original.observed_on + timedelta(days=1), + "observed_at": original.observed_at + timedelta(minutes=1), + "title": "Later comparison", + "comparisons": (original.comparisons[0].model_copy(update={"observed_value": 999}),), + } + ) + continued: Final = await store.upsert_issue(later) + assert continued.status == "dismissed" and continued.comparisons == initial.comparisons + assert continued.title == initial.title and continued.observed_on == initial.observed_on + assert continued.last_seen > initial.last_seen + assert await store.set_status(initial.id, "open", "dismissed") is not None + await store.defer_delivery(delivery.id, "delivery_failed") + assert await store.pending_delivery(delivery.id) is None + await _query( + db, + 'UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP WHERE id=$1 RETURNING id', + delivery.id, + ) + retried: Final = await store.pending_delivery(delivery.id) + assert ( + retried is not None and retried.delivery.attempts == 1 and retried.delivery.last_error == "delivery_failed" + ) + await store.mark_delivery_sent(delivery.id) + await store.queue_delivery(delivery, enabled.id, enabled.updated_at.isoformat(), source.id) + assert await store.pending_delivery(delivery.id) is None + healthy: Final = observation.model_copy( + update={"day": observation.day + timedelta(days=1), "breached": False, "observed": 120} + ) + await alerts.record_observations(enabled, (healthy,)) + await store.recover_issues(observation.day + timedelta(days=2)) + recovered: Final = await store.get_issue(initial.id) + assert recovered is not None and not recovered.active and recovered.status == "resolved" + assert await store.find_active_issue(original.cohort_key) is None + repeated: Final = ( + await alerts.record_observations( + enabled, (observation.model_copy(update={"day": observation.day + timedelta(days=2)}),) + ) + )[0] + assert repeated.id != source.id + new_episode: Final = await store.upsert_issue( + original.model_copy( + update={ + "id": next_issue_id, + "observed_on": repeated.day, + "observed_at": later.observed_at + timedelta(minutes=1), + } + ) + ) + assert new_episode.active and new_episode.status == "open" and new_episode.id != initial.id + assert (await store.find_active_issue(original.cohort_key)) == new_episode + finally: + await _remove_issues(db, (original.id, next_issue_id)) + await alerts.delete_rule(rule.id) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) + + +class _Pid(BaseModel, frozen=True): + pid: int + + +class _Blocked(BaseModel, frozen=True): + blocked: bool + + +@asynccontextmanager +async def _family_database(db: ManagementDatabase) -> AsyncGenerator[Prisma]: + url: Final = db.gateway.database_url + assert url is not None + schema: Final = f"issues_{uuid4().hex}" + parsed: Final = urlsplit(url) + scoped: Final = urlunsplit(parsed._replace(query=urlencode({**dict(parse_qsl(parsed.query)), "schema": schema}))) + with psycopg.connect(url, autocommit=True) as setup: + setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + try: + for table in ("LiteLLM_UsageAlertRule", "LiteLLM_UsageAlertCheckpoint", "LiteLLM_Config"): + setup.execute( + sql.SQL("CREATE TABLE {} (LIKE {} INCLUDING ALL)").format( + sql.Identifier(schema, table), sql.Identifier(table) + ) + ) + database: Final = Prisma(datasource=DatasourceOverride(url=scoped)) + await database.connect() + try: + assert await database.query_raw("SELECT current_schema() AS name") == [{"name": schema}] + yield database + finally: + await database.disconnect() + finally: + setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_late_default_override_stops_queued_delivery_and_recovers_on_fresh_evidence( + db: ManagementDatabase, +) -> None: + async with _family_database(db) as database: + borrowed: Final = cast(AlertDatabase, database) # cast-ok: real generated Prisma query delegate + for table in ( + "LiteLLM_UsageAlertIncident", + "LiteLLM_Issue", + "LiteLLM_IssueSignal", + "LiteLLM_IssueDelivery", + "LiteLLM_IssueDeliveryRule", + "LiteLLM_VerificationToken", + "LiteLLM_TeamTable", + "LiteLLM_UserTable", + ): + await borrowed.query_raw(f'CREATE TABLE "{table}" (LIKE public."{table}" INCLUDING ALL)') + alerts: Final = UsageAlertsRepository(borrowed) + store: Final = IssuesRepository(borrowed) + await alerts.ensure_default_rules(default_usage_rules(("slack",))) + rule: Final = await alerts.get_rule(default_usage_rule_id("key", "requests", "increase")) + assert rule is not None + issues: Final = tuple(_issue(IssueCohort(scope_type="key", scope_id=key)) for key in ("key-a", "key-b")) + observations: Final = tuple( + UsageAlertObservation( + entity_id=issue.cohort.scope_id, + day=issue.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ) + for issue in issues + ) + sources: Final = await alerts.record_observations(rule, observations) + await store.upsert_issues(issues) + for issue in issues: + source: UsageAlert = next(source for source in sources if source.scope_id == issue.cohort.scope_id) + await store.add_signals( + issue.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),) + ) + await store.queue_delivery( + IssueDeliveryInput(id=issue.id, issue_id=issue.id, channel="slack"), + rule.id, + rule.updated_at.isoformat(), + source.id, + ) + assert await store.pending_delivery(issue.id) is not None + definition: Final = UsageAlertRuleInput( + name="Project A threshold", + scope_type="key", + scope_id="key-a", + metric="requests", + direction="increase", + multiplier=100, + ) + ordinary: Final = await alerts.save_rule(definition, "example-user") + assert ordinary is not None and await store.pending_delivery(issues[0].id) is not None + assert await alerts.save_rule(definition, "qa-admin", ordinary, overrides_defaults=True) is not None + pending: Final = await store.pending_delivery(issues[0].id) + unaffected: Final = await store.pending_delivery(issues[1].id) + await store.recover_issues(issues[0].observed_on + timedelta(days=1), (issues[0].id,)) + awaiting_evidence: Final = await store.get_issue(issues[0].id) + assert awaiting_evidence is not None and awaiting_evidence.active + await alerts.record_observations( + rule, + ( + observations[0].model_copy( + update={"day": observations[0].day + timedelta(days=1), "observed": 120, "breached": False} + ), + ), + ) + await store.recover_issues(issues[0].observed_on + timedelta(days=2), (issues[0].id,)) + recovered: Final = await store.get_issue(issues[0].id) + other: Final = await store.get_issue(issues[1].id) + assert recovered is not None and other is not None + assert (pending is None, unaffected is not None, recovered.active, recovered.status, other.active) == ( + True, + True, + False, + "resolved", + True, + ), f"overridden delivery eligible={pending is not None}; recovered issue active={recovered.active}" + + +@pytest.mark.asyncio(loop_scope="session") +async def test_family_settings_preserve_custom_deleted_rules_and_abort_concurrent_edits(db: ManagementDatabase) -> None: + async with _family_database(db) as database: + borrowed: Final = cast( # cast-ok: real generated Prisma client with narrower overloads + AlertDatabase, database + ) + alerts: Final = UsageAlertsRepository(borrowed) + store: Final = IssuesRepository(borrowed) + await alerts.ensure_default_rules(default_usage_rules()) + custom: Final = await alerts.save_rule( + UsageAlertRuleInput( + name="Custom spend", scope_type="key", scope_id="example-key", metric="spend", direction="increase" + ), + "qa-admin", + ) + assert custom is not None + removed_id: Final = default_usage_rule_id("model", "spend", "increase") + await alerts.delete_rule(removed_id) + result: Final = await store.update_family( + "spend", + IssueFamilyUpdate( + enabled=False, policies=(IssueFamilyPolicyUpdate(direction="increase", multiplier=5, min_value=25),) + ), + ) + assert result is not None and result.total_rules == 4 and result.enabled_rules == 0 + assert result.policies[0].multiplier == 5 and result.policies[0].min_value == 25 + assert await alerts.get_rule(removed_id) is None + assert await alerts.get_rule(custom.id) == custom + baseline: Final = await store.family_settings() + with pytest.raises(ValidationError, match="min_history_days"): + await store.update_family( + "spend", + IssueFamilyUpdate( + policies=(IssueFamilyPolicyUpdate(direction="increase", baseline_days=1, min_history_days=7),) + ), + ) + assert await store.family_settings() == baseline + ids: Final = tuple( + sorted(default_usage_rule_id(scope, "spend", "increase") for scope in ("deployment", "key", "team", "user")) + ) + async with asyncio.TaskGroup() as tasks: + async with database.tx(timeout=timedelta(seconds=30)) as transaction: + await transaction.query_raw('SELECT id FROM "LiteLLM_UsageAlertRule" WHERE id=$1 FOR UPDATE', ids[0]) + pid: Final = ( + TypeAdapter(tuple[_Pid, ...]) + .validate_python(await transaction.query_raw("SELECT pg_backend_pid() AS pid"))[0] + .pid + ) + update: Final = tasks.create_task(store.update_family("spend", IssueFamilyUpdate(enabled=True))) + async with asyncio.timeout(10): + while ( + not TypeAdapter(tuple[_Blocked, ...]) + .validate_python( + await database.query_raw( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE $1::int=ANY(pg_blocking_pids(pid))) AS blocked", + pid, + ) + )[0] + .blocked + ): + await asyncio.sleep(0.01) + await transaction.query_raw( + """UPDATE "LiteLLM_UsageAlertRule" SET record=record || + jsonb_build_object('name','Concurrent edit','updated_at',CURRENT_TIMESTAMP) + WHERE id=$1 RETURNING id""", + ids[0], + ) + assert update.result() is None + final: Final = await store.family_settings() + spend: Final = next(family for family in final.data if family.family == "spend") + assert spend.total_rules == 4 and spend.enabled_rules == 0 + edited: Final = await alerts.get_rule(ids[0]) + assert edited is not None and edited.name == "Concurrent edit" + assert await alerts.get_rule(custom.id) == custom + + +@pytest.mark.asyncio(loop_scope="session") +async def test_family_edit_requires_a_new_evaluation(db: ManagementDatabase) -> None: + async with _family_database(db) as database: + borrowed: Final = cast(AlertDatabase, database) # cast-ok: real generated Prisma query delegate + alerts: Final = UsageAlertsRepository(borrowed) + store: Final = IssuesRepository(borrowed) + await alerts.ensure_default_rules(default_usage_rules()) + rule: Final = await alerts.get_rule(default_usage_rule_id("key", "spend", "increase")) + assert rule is not None + await alerts.record_evaluation_status(rule, date.today() - timedelta(days=1), "complete") + result: Final = await store.update_family( + "spend", IssueFamilyUpdate(policies=(IssueFamilyPolicyUpdate(direction="increase", multiplier=5),)) + ) + saved: Final = await alerts.get_rule(rule.id) + assert result is not None and saved is not None + assert (result.evaluation_status_counts.complete, saved.evaluation_status, saved.last_evaluated_day) == ( + 0, + "pending", + None, + ), "Changed family policy must reset stale evaluation status" + + +@pytest.mark.asyncio(loop_scope="session") +async def test_deleted_key_owner_retains_historical_issue_access(gateway: Gateway, db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + outsider: Final = scenario.user(user_role="internal_user") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + outsider_key: Final = scenario.key(user_id=outsider, key_type="management") + reporting_id: Final = sha256(uuid4().bytes).hexdigest() + issue: Final = _issue(IssueCohort(scope_type="key", scope_id=reporting_id)) + await _query( + db, + 'INSERT INTO "LiteLLM_DeletedVerificationToken" (id,token,user_id,models) VALUES ($1,$1,$2,ARRAY[]::text[])', + reporting_id, + owner, + ) + try: + await store.upsert_issue(issue) + path: Final = f"/issues/{issue.id}" + detail: Final = gateway.request("GET", path, key=owner_key) + assert detail.status_code == 200, "Deleted key owner must retain historical issue access" + assert detail.json()["can_update_status"] is True + assert gateway.request("GET", path, key=outsider_key).status_code == 403 + assert ( + gateway.request( + "PATCH", path, {"status": "dismissed", "expected_status": "open"}, key=owner_key + ).status_code + == 200 + ) + await _query( + db, + 'INSERT INTO "LiteLLM_VerificationToken" (token,user_id,models) VALUES ($1,$2,ARRAY[]::text[])', + reporting_id, + outsider, + ) + assert gateway.request("GET", path, key=owner_key).status_code == 403 + assert gateway.request("GET", path, key=outsider_key).status_code == 200 + finally: + await _remove_issues(db, (issue.id,)) + await _query(db, 'DELETE FROM "LiteLLM_VerificationToken" WHERE token=$1', reporting_id) + await _query(db, 'DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', reporting_id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_status_rechecks_changed_scope_set_and_caps_sources(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + rule: Final = await alerts.save_rule( + UsageAlertRuleInput( + name="Scope guard", + scope_type="user", + scope_id=issue.cohort.scope_id, + metric="requests", + direction="increase", + ), + "qa-admin", + ) + assert rule is not None + try: + source: Final = ( + await alerts.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=rule.scope_id, + day=issue.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + await store.upsert_issue(issue) + await store.add_signals( + issue.id, (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),) + ) + authorized: Final = await store.status_scopes(issue.id) + assert authorized == (issue.cohort,) + assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=authorized) is not None + context: Final = source.model_copy(update={"id": str(uuid4()), "metric": "spend"}) + same_scope: Final = (IssueSignalInput(source="usage_alert", source_id=context.id, usage_alert=context),) + await store.add_signals(issue.id, same_scope) + unchanged: Final = await store.get_issue(issue.id) + assert unchanged is not None and unchanged.status == "dismissed" + await store.add_signals(issue.id, same_scope) + assert await store.get_issue(issue.id) == unchanged + broader: Final = source.model_copy(update={"id": str(uuid4()), "scope_type": "deployment", "scope_id": None}) + await store.add_signals( + issue.id, (IssueSignalInput(source="usage_alert", source_id=broader.id, usage_alert=broader),) + ) + assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=authorized) is None + current: Final = await store.get_issue(issue.id) + assert current is not None and current.status == "open" + all_scopes: Final = await store.status_scopes(issue.id) + assert await store.set_status(issue.id, "dismissed", "open", authorized_scopes=all_scopes) is not None + broader_context: Final = broader.model_copy(update={"id": str(uuid4()), "metric": "spend"}) + await store.add_signals( + issue.id, + (IssueSignalInput(source="usage_alert", source_id=broader_context.id, usage_alert=broader_context),), + ) + repeated_scope: Final = await store.get_issue(issue.id) + assert repeated_scope is not None and repeated_scope.status == "dismissed" + assert await store.set_status(issue.id, "open", "dismissed") is not None + racing: Final = source.model_copy(update={"id": str(uuid4()), "scope_type": "team", "scope_id": "racing-team"}) + async with asyncio.TaskGroup() as tasks: + async with db.prisma.writer_db.tx(timeout=timedelta(seconds=30)) as transaction: + await transaction.query_raw('SELECT id FROM "LiteLLM_Issue" WHERE id=$1 FOR UPDATE', issue.id) + pid: Final = ( + TypeAdapter(tuple[_Pid, ...]) + .validate_python(await transaction.query_raw("SELECT pg_backend_pid() AS pid"))[0] + .pid + ) + update: Final = tasks.create_task( + store.set_status(issue.id, "dismissed", "open", authorized_scopes=all_scopes) + ) + async with asyncio.timeout(10): + while ( + not TypeAdapter(tuple[_Blocked, ...]) + .validate_python( + await _query( + db, + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE $1::int=ANY(pg_blocking_pids(pid))) AS blocked", + pid, + ) + )[0] + .blocked + ): + await asyncio.sleep(0.01) + locked: Final = IssuesRepository( + cast(AlertDatabase, transaction) # cast-ok: generated Prisma transaction query delegate + ) + await locked.add_signals( + issue.id, (IssueSignalInput(source="usage_alert", source_id=racing.id, usage_alert=racing),) + ) + assert update.result() is None + raced: Final = await store.get_issue(issue.id) + assert raced is not None and raced.status == "open" + await store.add_signals( + issue.id, + tuple( + IssueSignalInput( + source="usage_alert", + source_id=f"scope-{index}", + usage_alert=source.model_copy(update={"id": f"scope-{index}", "scope_id": f"other-user-{index}"}), + ) + for index in range(20) + ), + ) + limited: Final = await store.status_scopes(issue.id) + assert len(limited) == 11 + with pytest.raises(ValueError, match="at most 10"): + await store.set_status(issue.id, "open", "dismissed", authorized_scopes=limited) + finally: + await _remove_issues(db, (issue.id,)) + await alerts.delete_rule(rule.id) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1', rule.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_resource_recovery_ignores_broader_and_traffic_context(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + fixture: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + issue: Final = fixture.model_copy( + update={"family": "spend", "comparisons": (fixture.comparisons[0].model_copy(update={"metric": "spend"}),)} + ) + definitions: Final = ( + UsageAlertRuleInput( + name="Primary cost", scope_type="user", scope_id=issue.cohort.scope_id, metric="spend", direction="increase" + ), + UsageAlertRuleInput( + name="Traffic context", + scope_type="user", + scope_id=issue.cohort.scope_id, + metric="requests", + direction="increase", + ), + UsageAlertRuleInput(name="Broader cost", scope_type="deployment", metric="spend", direction="increase"), + ) + rules: Final = tuple([await alerts.save_rule(definition, "qa-admin") for definition in definitions]) + assert all(rule is not None for rule in rules) + try: + sources: Final = tuple( + [ + ( + await alerts.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=rule.scope_id, + day=issue.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + for rule in rules + if rule is not None + ] + ) + await store.upsert_issue(issue) + await store.add_signals( + issue.id, + tuple( + IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source) for source in sources + ), + ) + primary: Final = rules[0] + assert primary is not None + await alerts.record_observations( + primary, + ( + UsageAlertObservation( + entity_id=primary.scope_id, + day=issue.observed_on + timedelta(days=1), + observed=120, + baseline=120, + threshold=360, + breached=False, + ), + ), + ) + await store.recover_issues(issue.observed_on + timedelta(days=2)) + recovered: Final = await store.get_issue(issue.id) + assert recovered is not None and recovered.status == "resolved" and not recovered.active + supporting: Final = tuple([await alerts.get_alert(source.id) for source in sources[1:]]) + assert all(source is not None and source.status == "open" for source in supporting) + finally: + await _remove_issues(db, (issue.id,)) + for rule in rules: + if rule is not None: + await alerts.delete_rule(rule.id) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_provider_recovery_pages_saved_cohorts_and_rejects_stale_observations(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + prefix: Final = f"fixture-{uuid4()}" + initial: Final = _issue(IssueCohort(scope_type="provider", scope_id="example-provider")) + template: Final = initial.model_copy(update={"observed_at": initial.observed_at.replace(microsecond=123789)}) + inputs: Final = tuple( + template.model_copy(update={"id": prefix + suffix, "cohort_key": prefix + suffix}) + for suffix in ("-a", "-b", "-c") + ) + try: + await store.upsert_issues(inputs) + first: Final = await store.active_provider_issues(1, prefix) + second: Final = await store.active_provider_issues(1, first[0].id) + assert first[0].id == inputs[0].id and second[0].id == inputs[1].id + assert await store.set_status(inputs[0].id, "dismissed", "open") is not None + assert await store.recover_issue_ids((inputs[0].id,), template.observed_at - timedelta(seconds=1)) == 0 + assert await store.recover_issue_ids((inputs[0].id, inputs[1].id), template.observed_at) == 0, ( + "A recovery at the last observed timestamp must not close the alert" + ) + assert ( + await store.recover_issue_ids( + (inputs[0].id, inputs[1].id), template.observed_at + timedelta(milliseconds=1) + ) + == 2 + ) + dismissed: Final = await store.get_issue(inputs[0].id) + resolved: Final = await store.get_issue(inputs[1].id) + assert dismissed is not None and not dismissed.active and dismissed.status == "dismissed" + assert resolved is not None and not resolved.active and resolved.status == "resolved" + await store.recover_issues(template.observed_on + timedelta(days=1)) + remaining: Final = await store.active_provider_issues(1, prefix) + assert remaining[0].id == inputs[2].id + finally: + await _remove_issues(db, tuple(issue.id for issue in inputs)) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_prune_removes_expired_children_in_bounded_passes( + db: ManagementDatabase, +) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + expired: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + open_issue: Final = _issue(expired.cohort) + recent: Final = _issue(expired.cohort) + rule: Final = await alerts.save_rule( + UsageAlertRuleInput( + name="Prune fixture", + scope_type="user", + scope_id=expired.cohort.scope_id, + metric="requests", + direction="increase", + ), + "qa-admin", + ) + assert rule is not None + deliveries: Final = tuple( + IssueDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") for _ in range(2) + ) + try: + await store.upsert_issues((expired, open_issue, recent)) + source: Final = ( + await alerts.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=rule.scope_id, + day=expired.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + await store.add_signals( + expired.id, + (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),), + ) + for delivery in deliveries: + await store.queue_delivery(delivery, rule.id, rule.updated_at.isoformat(), source.id) + assert await store.set_status(expired.id, "resolved", "open") is not None + assert await store.set_status(recent.id, "resolved", "open") is not None + await _query( + db, + "UPDATE \"LiteLLM_Issue\" SET updated_at=TIMESTAMP '2000-01-01' WHERE id=ANY($1::text[]) RETURNING id", + [expired.id, open_issue.id], + ) + await store.prune(retention_days=90, limit=1) + assert await store.get_issue(expired.id) is not None + for _ in range(7): + await store.prune(retention_days=90, limit=1) + assert await store.get_issue(expired.id) is None + assert await store.get_issue(open_issue.id) is not None + assert await store.get_issue(recent.id) is not None + assert (await store.list_signals(expired.id)).total == 0 + assert ( + await _query( + db, + 'SELECT id FROM "LiteLLM_IssueDelivery" WHERE issue_id=$1', + expired.id, + ) + == [] + ) + assert ( + await _query( + db, + 'SELECT delivery_id FROM "LiteLLM_IssueDeliveryRule" WHERE delivery_id=ANY($1::text[])', + [delivery.id for delivery in deliveries], + ) + == [] + ) + finally: + await _remove_issues(db, (expired.id, open_issue.id, recent.id)) + await alerts.delete_rule(rule.id) + await _query( + db, + 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', + rule.id, + ) + + +class _MeasuredActivityDatabase: + def __init__(self, database: DailyActivityQueryDatabase) -> None: + self.database = database + self.outputs: tuple[tuple[int, int], ...] = () + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]] | None: + result: Final = await self.database.query_raw(query, *args) + self.outputs = (*self.outputs, (len(result or ()), len(json.dumps(result, default=str)))) + return result + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_resource_metadata_bounds_labels_and_deleted_history( + gateway: Gateway, db: ManagementDatabase +) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + activity: Final = IssueActivityRepository.from_prisma(db.prisma) + measured: Final = _MeasuredActivityDatabase(activity.database) + repository: Final = IssueActivityRepository(daily_activity_repository(db.prisma, query_database=measured), measured) + identity: Final = uuid4().hex + reporting_key: Final = sha256(identity.encode()).hexdigest() + day: Final = date.today() - timedelta(days=3) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + issue: Final = _issue(IssueCohort(scope_type="provider", scope_id=identity)).model_copy( + update={"range_start": day, "range_end": day} + ) + try: + await _query( + db, + """UPDATE "LiteLLM_UserTable" SET user_alias=repeat('u',1024), + user_email=repeat('m',400)||'@example.com' WHERE user_id=$1""", + owner, + ) + await _query( + db, + """INSERT INTO "LiteLLM_DeletedVerificationToken" + (id,token,key_alias,user_id,models,metadata,deleted_at) + SELECT $1::text||'-'||n,$2::text,'archived-'||n||'-'||repeat('x',1024),$3::text, + ARRAY[]::text[],jsonb_build_object('padding',repeat('x',65536)), + CURRENT_TIMESTAMP+n*INTERVAL '1 second' FROM generate_series(1,400) n""", + identity, + reporting_key, + owner, + ) + await _query( + db, + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,api_requests,successful_requests,updated_at) + VALUES($1,$2,$3,$4,'example-model',$1,10,10,CURRENT_TIMESTAMP), + ($1||'-wide',repeat(chr(128640),8192),$3,repeat(chr(128640),8192), + 'example-model',$1,20,20,CURRENT_TIMESTAMP)""", + identity, + owner, + day.isoformat(), + reporting_key, + ) + await store.upsert_issue(issue) + keys: Final = await repository.keys(issue.cohort, day, day, 1, 0) + assert keys.total_api_keys == 1 and len(keys.api_keys) == 1 + key: Final = keys.api_keys[0] + assert key.api_key == reporting_key and key.metadata.key_exists is False + assert key.metadata.key_alias is not None and key.metadata.key_alias.startswith("archived-400-") + assert len(key.metadata.key_alias) == 256 and key.metadata.user_id == owner + assert key.metadata.user_email is None + assert measured.outputs[-1][0] == 1 and measured.outputs[-1][1] < 2048 + users: Final = await repository.users(issue.cohort, day, day, 1, 0) + assert users.total_users == 1 and len(users.users) == 1 + assert users.users[0].user_id == owner and users.users[0].user_alias == "u" * 256 + assert users.users[0].user_email is None + assert measured.outputs[-1][0] == 1 and measured.outputs[-1][1] < 2048 + assert all(size < 2048 for _, size in measured.outputs) + totals: Final = await repository.daily_activity(issue.cohort, day, day) + assert totals.metadata.total_api_requests == 30 + for endpoint in ("keys", "users"): + response: Final = gateway.request("GET", f"/issues/{issue.id}/{endpoint}", params={"limit": "1"}) + assert response.status_code == 200 and len(response.content) < 2048 + assert "padding" not in response.text + await _query( + db, + """UPDATE "LiteLLM_DeletedVerificationToken" + SET user_id=repeat(chr(128640),8192),team_id=repeat(chr(128640),8192) + WHERE id=$1||'-400'""", + identity, + ) + legacy: Final = await repository.keys(issue.cohort, day, day, 1, 0) + assert legacy.api_keys[0].metadata.user_id is None and legacy.api_keys[0].metadata.team_id is None + assert all(size < 2048 for _, size in measured.outputs[-2:]) + finally: + await _remove_issues(db, (issue.id,)) + await _query(db, "DELETE FROM \"LiteLLM_DailyUserSpend\" WHERE id IN ($1,$1||'-wide')", identity) + await _query(db, 'DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', reporting_key) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + definition: Final = UsageAlertRuleInput( + name="Primary traffic", + scope_type="user", + scope_id=issue.cohort.scope_id, + metric="requests", + direction="increase", + ) + first: Final = await alerts.save_rule(definition, "qa-admin") + second: Final = await alerts.save_rule(definition.model_copy(update={"name": "Another traffic policy"}), "qa-admin") + assert first is not None and second is not None + breach: Final = UsageAlertObservation( + entity_id=issue.cohort.scope_id, + day=issue.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ) + try: + sources: Final = tuple([(await alerts.record_observations(rule, (breach,)))[0] for rule in (first, second)]) + await store.upsert_issue(issue) + await store.add_signals( + issue.id, + tuple( + IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source) for source in sources + ), + ) + await alerts.record_observations( + first, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=1), + "observed": 120, + "breached": False, + } + ), + ), + ) + await alerts.record_observations( + second, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=2), + } + ), + ), + ) + assert await alerts.save_rule(definition.model_copy(update={"enabled": False}), "qa-admin", second) + await store.recover_issues(issue.observed_on + timedelta(days=3)) + unresolved: Final = await store.get_issue(issue.id) + assert unresolved is not None and unresolved.active + await alerts.record_observations( + first, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=3), + "observed": 120, + "breached": False, + } + ), + ), + ) + await store.recover_issues(issue.observed_on + timedelta(days=3)) + recovered: Final = await store.get_issue(issue.id) + assert recovered is not None and not recovered.active + finally: + await _remove_issues(db, (issue.id,)) + for rule in (first, second): + await alerts.delete_rule(rule.id) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_family_settings_expose_projection_backlog_without_delayed_rules(db: ManagementDatabase) -> None: + async with _family_database(db) as database: + borrowed: Final = cast(AlertDatabase, database) # cast-ok: real generated Prisma query delegate + store: Final = IssuesRepository(borrowed) + assert not (await store.family_settings()).projection_pending + await borrowed.query_raw( + """INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed,owner) + VALUES ('performance-dirty:2026-11-04',false,'fixture') RETURNING id""" + ) + pending: Final = await store.family_settings() + assert pending.projection_pending and all(item.total_rules == 0 for item in pending.data) + await borrowed.query_raw( + """UPDATE "LiteLLM_UsageAlertCheckpoint" SET completed=true + WHERE id='performance-dirty:2026-11-04' RETURNING id""" + ) + assert not (await store.family_settings()).projection_pending diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py new file mode 100644 index 00000000000..77362be35d0 --- /dev/null +++ b/tests/integration/database/test_usage_alerts.py @@ -0,0 +1,693 @@ +from datetime import date, timedelta +from hashlib import sha256 +from typing import Final +from uuid import uuid4 + +import pytest +from integration._support.actors import ManagementDatabase +from integration._support.client import Gateway, string_value + +from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.types.proxy.usage_alerts import ( + UsageAlertGroupList, + UsageAlertMetric, + UsageAlertModelList, + UsageAlertObservation, + UsageAlertRule, + UsageAlertRuleInput, +) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_alert_resource_labels_bound_legacy_display_values(db: ManagementDatabase) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + identity: Final = "label-bound-" + uuid4().hex + scope: Final = UsageAlertRuleInput( + name="Display label", scope_type="user", scope_id=identity, metric="spend", direction="increase" + ) + await repository.database.query_raw( + 'INSERT INTO "LiteLLM_UserTable" (user_id,user_alias) VALUES ($1,repeat(chr(128640),65536))', identity + ) + try: + labels: Final = await repository.resource_labels((scope,)) + assert labels == {("user", identity): chr(128640) * 256} + stored: Final = await repository.database.query_raw( + 'SELECT length(user_alias) AS characters FROM "LiteLLM_UserTable" WHERE user_id=$1', identity + ) + assert stored[0]["characters"] == 65536 + finally: + await repository.database.query_raw('DELETE FROM "LiteLLM_UserTable" WHERE user_id=$1', identity) + + +def test_scoped_rules_use_real_management_auth_and_preserve_read_only_roles(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + outsider: Final = scenario.user(user_role="internal_user") + viewer: Final = scenario.user(user_role="proxy_admin_viewer") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + outsider_key: Final = scenario.key(user_id=outsider, key_type="management") + viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") + inference_key: Final = scenario.key(user_id=owner, key_type="llm_api") + config: Final = { + "name": "Daily key spend", + "scope_type": "key", + "scope_id": sha256(inference_key.encode()).hexdigest(), + "metric": "spend", + "direction": "increase", + "multiplier": 3, + } + created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + assert created.status_code == 201, created.text + rule_id: Final = string_value(created.json()["id"]) + path: Final = f"/usage/alert-rules/{rule_id}" + try: + assert gateway.request("GET", path, key=owner_key).status_code == 200 + assert gateway.request("GET", path, key=outsider_key).status_code == 403 + assert gateway.request("GET", path, key=inference_key).status_code == 403 + assert gateway.request("GET", path, key=viewer_key).status_code == 200 + assert gateway.request("PUT", path, config, key=viewer_key).status_code == 403 + assert gateway.request("PUT", path, config, key=outsider_key).status_code == 403 + assert ( + gateway.request("POST", "/usage/alert-rules", {**config, "scope_id": None}, key=owner_key).status_code + == 403 + ) + disabled: Final = gateway.request("PUT", path, {**config, "enabled": False}, key=owner_key) + assert disabled.status_code == 200 and disabled.json()["enabled"] is False + page: Final = gateway.request( + "GET", + "/usage/alert-rules", + key=owner_key, + params={"scope_type": "key", "scope_id": str(config["scope_id"]), "limit": "1"}, + ) + assert page.status_code == 200 and page.json()["total"] == 1 + assert page.json()["data"][0]["id"] == rule_id + finally: + assert gateway.request("DELETE", path, key=owner_key).status_code == 204 + assert gateway.request("GET", path, key=owner_key).status_code == 404 + + +def test_admin_edit_preserves_owner_rule_access(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + config: Final = { + "name": "Owner policy", + "scope_type": "user", + "scope_id": owner, + "metric": "spend", + "direction": "increase", + } + created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + assert created.status_code == 201, created.text + path: Final = f"/usage/alert-rules/{created.json()['id']}" + try: + edited: Final = gateway.request("PUT", path, {**config, "name": "Renamed by admin"}) + assert edited.status_code == 200, edited.text + owner_edit: Final = gateway.request("PUT", path, {**config, "multiplier": 5}, key=owner_key) + assert owner_edit.status_code == 200, "Admin edit must preserve the owner's rule access" + finally: + assert gateway.request("DELETE", path).status_code == 204 + + +def test_team_admin_can_manage_team_alerts_but_member_can_only_read(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + admin_id: Final = scenario.user(user_role="internal_user") + member_id: Final = scenario.user(user_role="internal_user") + team_id: Final = scenario.team( + members_with_roles=[{"user_id": admin_id, "role": "admin"}, {"user_id": member_id, "role": "user"}] + ) + admin_key: Final = scenario.key(user_id=admin_id, key_type="management") + member_key: Final = scenario.key(user_id=member_id, key_type="management") + config: Final = { + "name": "Daily team requests", + "scope_type": "team", + "scope_id": team_id, + "metric": "requests", + "direction": "decrease", + "multiplier": 2, + } + created: Final = gateway.request("POST", "/usage/alert-rules", config, key=admin_key) + assert created.status_code == 201, created.text + rule_id: Final = string_value(created.json()["id"]) + path: Final = f"/usage/alert-rules/{rule_id}" + try: + assert gateway.request("GET", path, key=member_key).status_code == 200 + assert gateway.request("PUT", path, config, key=member_key).status_code == 403 + assert gateway.request("DELETE", path, key=member_key).status_code == 403 + finally: + assert gateway.request("DELETE", path, key=admin_key).status_code == 204 + + +def test_info_only_credentials_cannot_mutate_alerts(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + admin: Final = scenario.user(user_role="proxy_admin") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + reader: Final = scenario.key(user_id=owner, key_type="default", allowed_routes=["info_routes"]) + admin_reader: Final = scenario.key(user_id=admin, key_type="default", allowed_routes=["info_routes"]) + config: Final = { + "name": "Read-only route grant", + "scope_type": "user", + "scope_id": owner, + "metric": "spend", + "direction": "increase", + } + created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + assert created.status_code == 201 + path: Final = f"/usage/alert-rules/{created.json()['id']}" + try: + assert gateway.request("GET", path, key=reader).status_code == 200 + assert gateway.request("POST", "/usage/alert-rules", config, key=reader).status_code == 403 + assert gateway.request("PUT", path, config, key=reader).status_code == 403 + assert gateway.request("DELETE", path, key=reader).status_code == 403 + assert ( + gateway.request("PATCH", "/issues/families/spend", {"enabled": False}, key=admin_reader).status_code + == 403 + ) + assert ( + gateway.request( + "PATCH", "/issues/unavailable", {"status": "resolved", "expected_status": "open"}, key=admin_reader + ).status_code + == 403 + ) + finally: + assert gateway.request("DELETE", path, key=owner_key).status_code == 204 + + +def test_only_admins_can_configure_operator_notifications(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + config: Final = { + "name": "Operator delivery", + "scope_type": "user", + "scope_id": owner, + "metric": "spend", + "direction": "increase", + } + for notification in ( + {"notification_channels": ["slack"]}, + {"notification_channels": ["email"], "email_recipients": ["ops@example.com"]}, + ): + assert ( + gateway.request("POST", "/usage/alert-rules", {**config, **notification}, key=owner_key).status_code + == 403 + ) + assert ( + gateway.request( + "POST", "/usage/alert-rules", {**config, "overrides_defaults": True}, key=owner_key + ).status_code + == 422 + ) + assert ( + gateway.request("POST", "/usage/alert-rules", {**config, "name": "Spend\r\nInjected: header"}).status_code + == 422 + ) + created: Final = gateway.request("POST", "/usage/alert-rules", {**config, "notification_channels": ["slack"]}) + assert created.status_code == 201 + assert "overrides_defaults" not in created.json() + path: Final = f"/usage/alert-rules/{created.json()['id']}" + try: + assert gateway.request("PUT", path, config, key=owner_key).status_code == 403 + assert gateway.request("DELETE", path, key=owner_key).status_code == 403 + finally: + assert gateway.request("DELETE", path).status_code == 204 + + +@pytest.mark.asyncio(loop_scope="session") +async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gateway, db: ManagementDatabase) -> None: + """Populate an explicit storage fixture, then exercise the real incident HTTP API.""" + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + outsider: Final = scenario.user(user_role="internal_user") + viewer: Final = scenario.user(user_role="proxy_admin_viewer") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + outsider_key: Final = scenario.key(user_id=outsider, key_type="management") + viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") + created: Final = gateway.request( + "POST", + "/usage/alert-rules", + { + "name": "Incident API fixture", + "scope_type": "user", + "scope_id": owner, + "metric": "requests", + "direction": "increase", + "multiplier": 3, + }, + key=owner_key, + ) + assert created.status_code == 201 + rule: Final = UsageAlertRule.model_validate(created.json()) + try: + incident: Final = ( + await repository.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=owner, + day=date(2026, 11, 4), + observed=420, + baseline=120, + baseline_sample_days=7, + threshold=360, + breached=True, + ), + ), + ) + )[0] + path: Final = f"/usage/alerts/{incident.id}" + params: Final = {"scope_type": "user", "scope_id": owner, "limit": "1"} + page: Final = gateway.request("GET", "/usage/alerts", key=owner_key, params=params) + assert page.status_code == 200 and page.json()["data"][0]["id"] == incident.id + detail: Final = gateway.request("GET", path, key=owner_key) + assert detail.status_code == 200 + assert detail.json()["observed_value"] == 420 + assert detail.json()["baseline_value"] == 120 + assert detail.json()["threshold"] == 360 + assert detail.json()["baseline_sample_days"] == 7 + assert "detail_tab=activity" in detail.json()["activity_path"] + assert gateway.request("GET", "/usage/alerts", key=outsider_key, params=params).status_code == 403 + assert gateway.request("GET", path, key=outsider_key).status_code == 403 + assert gateway.request("PATCH", path, {"status": "dismissed"}, key=outsider_key).status_code == 403 + assert gateway.request("GET", path, key=viewer_key).status_code == 200 + assert gateway.request("PATCH", path, {"status": "dismissed"}, key=viewer_key).status_code == 403 + dismissed: Final = gateway.request("PATCH", path, {"status": "dismissed"}, key=owner_key) + assert dismissed.status_code == 200 and dismissed.json()["status"] == "dismissed" + assert gateway.request("GET", path, key=owner_key).json()["status"] == "dismissed" + finally: + assert gateway.request("DELETE", f"/usage/alert-rules/{rule.id}", key=owner_key).status_code == 204 + + +@pytest.mark.asyncio(loop_scope="session") +async def test_ongoing_episode_preserves_its_original_evidence_and_activity_link(db: ManagementDatabase) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + definition: Final = UsageAlertRuleInput( + name="Original episode", scope_type="user", scope_id="episode-example", metric="requests", direction="increase" + ) + rule: Final = await repository.save_rule(definition, "qa-admin") + assert rule is not None + initial: Final = UsageAlertObservation( + entity_id=rule.scope_id, + day=date(2026, 11, 2), + observed=420, + baseline=120, + baseline_sample_days=7, + threshold=360, + breached=True, + ) + try: + original: Final = (await repository.record_observations(rule, (initial,)))[0] + edited: Final = await repository.save_rule( + definition.model_copy(update={"name": "Edited rule", "multiplier": 4}), "qa-admin", rule + ) + assert edited is not None + later: Final = initial.model_copy( + update={ + "day": date(2026, 11, 3), + "observed": 600, + "baseline": 100, + "threshold": 400, + "baseline_sample_days": 4, + } + ) + ongoing: Final = (await repository.record_observations(edited, (later,)))[0] + assert ongoing.last_seen >= original.last_seen + assert ongoing.model_copy(update={"last_seen": original.last_seen}) == original + stored: Final = await repository.get_alert(original.id) + assert stored == ongoing + healthy: Final = later.model_copy(update={"day": date(2026, 11, 4), "breached": False}) + resolved: Final = (await repository.record_observations(edited, (healthy,)))[0] + assert resolved.model_copy(update={"status": "open", "last_seen": original.last_seen}) == original + repeated: Final = later.model_copy(update={"day": date(2026, 11, 5)}) + new_episode: Final = (await repository.record_observations(edited, (repeated,)))[0] + assert new_episode.id != original.id + assert new_episode.day == repeated.day and new_episode.observed_value == 600 + assert new_episode.threshold == 400 and new_episode.rule_name == "Edited rule" + assert new_episode.activity_path != original.activity_path + finally: + await repository.delete_rule(rule.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_rule_scope_changes_reject_stale_updates_and_deletes(db: ManagementDatabase) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + definition: Final = UsageAlertRuleInput( + name="Concurrent scope edit", scope_type="user", scope_id="original-owner", metric="spend", direction="increase" + ) + original: Final = await repository.save_rule(definition, "qa-admin") + assert original is not None + try: + moved: Final = await repository.save_rule( + definition.model_copy(update={"scope_id": "different-owner"}), "qa-admin", existing=original + ) + assert moved is not None + assert await repository.save_rule(definition, "original-owner", existing=original) is None + assert not await repository.delete_rule(original.id, existing=original) + assert await repository.get_rule(original.id) == moved + current: Final = await repository.save_rule( + definition.model_copy(update={"scope_id": moved.scope_id, "name": "Current authorized edit"}), + "qa-admin", + existing=moved, + ) + assert current is not None + assert not await repository.delete_rule(original.id, existing=moved) + assert await repository.delete_rule(original.id, existing=current) + assert await repository.save_rule(definition, "qa-admin", existing=current) is None + assert await repository.get_rule(original.id) is None + finally: + await repository.delete_rule(original.id) + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.parametrize( + "changed_field,new_value", + [("metric", "latency"), ("direction", "decrease"), ("scope_type", "user"), ("scope_id", "semantic-other")], +) +async def test_rule_identity_edits_preserve_old_incident_and_filter_consistency( + db: ManagementDatabase, changed_field: str, new_value: str +) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + definition: Final = UsageAlertRuleInput( + name="Semantic identity", + scope_type="team", + scope_id="semantic-example", + metric="spend", + direction="increase", + notification_channels=("slack",), + ) + rule: Final = await repository.save_rule(definition, "qa-admin") + assert rule is not None + initial: Final = UsageAlertObservation( + entity_id=rule.scope_id, day=date(2026, 11, 4), observed=40, baseline=5, threshold=15, breached=True + ) + try: + original: Final = (await repository.record_observations(rule, (initial,)))[0] + edited: Final = await repository.save_rule( + UsageAlertRuleInput.model_validate({**definition.model_dump(), changed_field: new_value}), "qa-admin", rule + ) + assert edited is not None + observation: Final = initial.model_copy(update={"entity_id": edited.scope_id}) + replacement: Final = (await repository.record_observations(edited, (observation,)))[0] + assert replacement.id != original.id + assert await repository.get_alert(original.id) == original + before_page: Final = await repository.list_alerts( + scope_type=original.scope_type, scope_id=original.scope_id, metric=original.metric + ) + after_page: Final = await repository.list_alerts( + scope_type=replacement.scope_type, scope_id=replacement.scope_id, metric=replacement.metric + ) + assert original.id in {item.id for item in before_page.data} + assert replacement.id in {item.id for item in after_page.data} + assert all( + item.scope_type == original.scope_type and item.metric == original.metric for item in before_page.data + ) + healthy: Final = observation.model_copy(update={"day": date(2026, 11, 5), "breached": False}) + await repository.record_observations(edited, (healthy,)) + assert (await repository.get_alert(original.id)) == original + resolved: Final = await repository.get_alert(replacement.id) + assert resolved is not None and resolved.status == "resolved" + finally: + await repository.delete_rule(rule.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_grouped_http_pages_preserve_all_metrics_history_and_scoped_actions( + gateway: Gateway, db: ManagementDatabase +) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + day: Final = date(2026, 11, 4) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user", user_alias="Checkout operator") + outsider: Final = scenario.user(user_role="internal_user") + viewer: Final = scenario.user(user_role="proxy_admin_viewer") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + outsider_key: Final = scenario.key(user_id=outsider, key_type="management") + viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") + inference_key: Final = scenario.key(user_id=owner, key_type="llm_api") + metrics: Final[tuple[UsageAlertMetric, ...]] = ( + *(("requests",) * 27), + "spend", + "errors", + "latency", + "error_rate", + ) + rules: Final = tuple( + [ + await repository.save_rule( + UsageAlertRuleInput( + name=f"Group rule {index}", + scope_type="user", + scope_id=owner, + metric=metric, + direction="increase", + baseline_days=7 if index % 2 else 14, + ), + owner, + ) + for index, metric in enumerate(metrics) + ] + ) + assert all(rule is not None for rule in rules) + incidents: Final = tuple( + [ + ( + await repository.record_observations( + rule, + ( + UsageAlertObservation( + entity_id=owner, + day=day, + observed=14 if rule.metric == "error_rate" else 420, + baseline=4 if rule.metric == "error_rate" else 0 if index == 0 else 120, + threshold=12 if rule.metric == "error_rate" else 100 if index == 0 else 360, + breached=True, + ), + ), + ) + )[0] + for index, rule in enumerate(rules) + if rule is not None + ] + ) + old_rule: Final = await repository.save_rule( + UsageAlertRuleInput( + name="Earlier day", scope_type="user", scope_id=owner, metric="requests", direction="increase" + ), + owner, + ) + assert old_rule is not None + old: Final = ( + await repository.record_observations( + old_rule, + ( + UsageAlertObservation( + entity_id=owner, + day=day - timedelta(days=1), + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + other_rule: Final = await repository.save_rule( + UsageAlertRuleInput( + name="Other resource", scope_type="user", scope_id=outsider, metric="requests", direction="increase" + ), + outsider, + ) + assert other_rule is not None + other: Final = ( + await repository.record_observations( + other_rule, + ( + UsageAlertObservation( + entity_id=outsider, + day=day, + observed=420, + baseline=120, + threshold=360, + breached=True, + ), + ), + ) + )[0] + params: Final = {"scope_type": "user", "scope_id": owner, "status": "open", "limit": "1"} + try: + response: Final = gateway.request("GET", "/usage/alert-groups", key=owner_key, params=params) + assert response.status_code == 200, response.text + page: Final = UsageAlertGroupList.model_validate(response.json()) + assert page.total == 2 and len(page.data) == 1 + group: Final = page.data[0] + assert group.day == day and group.scope_label == "Checkout operator" + assert group.incident_count == 31 and len(group.metrics) == 5 + assert {metric.metric: metric.incident_count for metric in group.metrics} == { + "requests": 27, + "spend": 1, + "errors": 1, + "latency": 1, + "error_rate": 1, + } + assert all(metric.representative.id in {incident.id for incident in incidents} for metric in group.metrics) + assert group.activity_path in {incident.activity_path for incident in incidents} + second: Final = UsageAlertGroupList.model_validate( + gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "1"}).json() + ) + assert second.total == 2 and second.data[0].day == old.day + empty: Final = UsageAlertGroupList.model_validate( + gateway.request("GET", "/usage/alert-groups", key=owner_key, params={**params, "offset": "2"}).json() + ) + assert empty.total == 2 and empty.data == () + history: Final = gateway.request( + "GET", + "/usage/alerts", + key=owner_key, + params={**params, "start_date": day.isoformat(), "end_date": day.isoformat(), "metric": "requests"}, + ) + assert history.status_code == 200 and history.json()["total"] == 27 + await repository.set_status(incidents[0].id, "dismissed") + body: Final = { + "scope_type": "user", + "scope_id": owner, + "day": day.isoformat(), + "expected_status": "open", + "status": "resolved", + } + for forbidden in (outsider_key, inference_key): + assert gateway.request("GET", "/usage/alert-groups", key=forbidden, params=params).status_code == 403 + for forbidden in (outsider_key, inference_key, viewer_key): + assert gateway.request("PATCH", "/usage/alert-groups", body, key=forbidden).status_code == 403 + assert gateway.request("GET", "/usage/alert-groups", key=viewer_key, params=params).status_code == 200 + updated: Final = gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key) + assert updated.status_code == 200 and updated.json()["updated_count"] == 30 + assert gateway.request("PATCH", "/usage/alert-groups", body, key=owner_key).json()["updated_count"] == 0 + assert await repository.get_alert(old.id) == old + assert await repository.get_alert(other.id) == other + dismissed: Final = await repository.get_alert(incidents[0].id) + assert dismissed is not None and dismissed.status == "dismissed" and dismissed.baseline_value == 0 + resolved: Final = await repository.get_alert(incidents[-1].id) + assert resolved == incidents[-1].model_copy(update={"status": "resolved"}) + assert ( + gateway.request( + "GET", "/usage/alert-groups", params={"scope_type": "user", "scope_id": "x' OR 1=1 --"} + ).json()["total"] + == 0 + ) + assert ( + gateway.request( + "GET", "/usage/alert-groups", params={"start_date": "2026-11-05", "end_date": "2026-11-04"} + ).status_code + == 422 + ) + finally: + for rule in (*rules, old_rule, other_rule): + if rule is not None: + await repository.delete_rule(rule.id) + deleted: Final = await repository.list_groups(scope_type="user", scope_id=owner) + assert deleted.total == 2 and all(group.scope_label is None and group.scope_id == owner for group in deleted.data) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_default_rule_summary_and_kind_filters_do_not_require_loading_every_rule( + gateway: Gateway, db: ManagementDatabase +) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + default: Final = UsageAlertRuleInput( + name="Built-in daily model requests", scope_type="model", metric="requests", direction="increase" + ) + await repository.ensure_default_rules((default,)) + custom: Final = await repository.save_rule( + default.model_copy(update={"scope_id": f"model-{uuid4().hex}"}), "qa-admin" + ) + assert custom is not None + try: + all_rules: Final = await repository.list_rules(scope_type="model", limit=1) + builtins: Final = await repository.list_rules(scope_type="model", limit=1, kind="default") + customs: Final = await repository.list_rules(scope_type="model", limit=1, kind="custom") + assert builtins.total >= 1 and all(rule.is_default for rule in builtins.data) + assert customs.total >= 1 and all(not rule.is_default for rule in customs.data) + assert all_rules.total == builtins.total + customs.total + assert all_rules.default_summary == builtins.default_summary == customs.default_summary + assert builtins.default_summary.total == builtins.total + assert ( + sum(builtins.default_summary.evaluation_status_counts.model_dump().values()) + == builtins.default_summary.enabled + ) + assert (await repository.list_rules(scope_type="model", kind="default", offset=builtins.total)).data == () + stored: Final = await repository.get_rule(builtins.data[0].id) + assert stored is not None and stored.is_default + filtered: Final = gateway.request( + "GET", "/usage/alert-rules", params={"scope_type": "model", "kind": "custom", "limit": "1"} + ) + assert filtered.status_code == 200 and filtered.json()["total"] == customs.total + assert all(not rule["is_default"] for rule in filtered.json()["data"]) + configuration: Final = stored.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields)) + path: Final = f"/usage/alert-rules/{stored.id}" + for change in ( + {"scope_type": "key"}, + {"scope_id": "model-example"}, + {"metric": "requests" if stored.metric != "requests" else "spend"}, + {"direction": "increase" if stored.direction == "decrease" else "decrease"}, + ): + assert gateway.request("PUT", path, {**configuration, **change}).status_code == 422 + assert await repository.get_rule(stored.id) == stored + try: + updated: Final = gateway.request("PUT", path, {**configuration, "min_value": stored.min_value + 1}) + assert updated.status_code == 200 and updated.json()["is_default"] is True + assert updated.json()["min_value"] == stored.min_value + 1 + finally: + assert gateway.request("PUT", path, configuration).status_code == 200 + custom_input: Final = custom.model_dump(mode="json", include=frozenset(UsageAlertRuleInput.model_fields)) + assert ( + gateway.request("PUT", f"/usage/alert-rules/{custom.id}", {**custom_input, "metric": "latency"}).status_code + == 200 + ) + finally: + await repository.delete_rule(custom.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_reported_model_search_preserves_exact_identity_and_cursor( + gateway: Gateway, db: ManagementDatabase +) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + prefix: Final = f"reported-{uuid4().hex}" + initial: Final = await repository.list_reported_models(prefix, None, 1) + assert initial.data == () + models: Final = (f"{prefix}/first", f"{prefix}/literal%quote'", f"{prefix}/second") + try: + for index, model in enumerate((*models, models[0])): + await db.prisma.writer_db.execute_raw( + 'INSERT INTO "LiteLLM_DailyUserSpend" (id,date,api_key,model,updated_at) VALUES ($1,$2,$3,$4,CURRENT_TIMESTAMP)', + f"{prefix}-{index}", + initial.end_date.isoformat(), + prefix, + model, + ) + await db.prisma.writer_db.execute_raw( + 'INSERT INTO "LiteLLM_DailyUserSpend" (id,date,api_key,model,updated_at) VALUES ($1,$2,$3,$4,CURRENT_TIMESTAMP)', + f"{prefix}-old", + (initial.start_date - timedelta(days=1)).isoformat(), + prefix, + f"{prefix}/old", + ) + response: Final = gateway.request("GET", "/usage/alert-models", params={"search": prefix, "limit": "1"}) + assert response.status_code == 200, response.text + first: Final = UsageAlertModelList.model_validate(response.json()) + assert first.data[0].scope_id == models[0] and first.next_cursor == models[0] + second: Final = await repository.list_reported_models(prefix, first.next_cursor, 1) + third: Final = await repository.list_reported_models(prefix, second.next_cursor, 1) + assert second.data[0].scope_id == models[1] and third.data[0].scope_id == models[2] + assert third.next_cursor is None + literal: Final = await repository.list_reported_models(f"{prefix}/literal%quote'", None, 20) + assert tuple(option.scope_id for option in literal.data) == (models[1],) + assert literal.data[0].label == models[1] + with gateway.scenario() as scenario: + user: Final = scenario.user(user_role="internal_user") + key: Final = scenario.key(user_id=user, key_type="management") + assert gateway.request("GET", "/usage/alert-models", key=key).status_code == 403 + finally: + await db.prisma.writer_db.execute_raw('DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', prefix) diff --git a/tests/unit/proxy/alerting_endpoints/__init__.py b/tests/unit/proxy/alerting_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py new file mode 100644 index 00000000000..1270b6809a9 --- /dev/null +++ b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py @@ -0,0 +1,87 @@ +from datetime import date +from typing import Final +from urllib.parse import parse_qs, urlsplit + +import pytest +from fastapi import HTTPException + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.alerting_endpoints.usage_alerts import require_scope, require_write_access +from litellm.repositories.usage_alerts import UsageAlertsRepository, activity_path +from litellm.types.proxy.auth.user_roles import LitellmUserRoles + + +class NoDatabase: + async def query_raw(self, query: str, *args: object) -> object: + raise AssertionError("Unauthorized callers must not reach the database") + + +@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER]) +@pytest.mark.parametrize("key_type", [None, "default", "management"]) +def test_info_grants_do_not_authorize_mutations(role: LitellmUserRoles, key_type: str | None) -> None: + user: Final = UserAPIKeyAuth(user_role=role, key_type=key_type, allowed_routes=["info_routes", "openai_routes"]) + with pytest.raises(HTTPException) as error: + require_write_access(user, "/usage/alert-rules/example") + assert error.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "role,key_type,write", + [ + (LitellmUserRoles.PROXY_ADMIN, "llm_api", False), + (LitellmUserRoles.PROXY_ADMIN, "read_only", True), + (LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, None, True), + (LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, None, True), + (LitellmUserRoles.INTERNAL_USER, None, False), + (LitellmUserRoles.CUSTOMER, None, False), + ], +) +async def test_global_access_rejects_inference_keys_and_unprivileged_roles( + role: LitellmUserRoles, key_type: str | None, write: bool +) -> None: + user: Final = UserAPIKeyAuth(user_id="qa-user", user_role=role, key_type=key_type) + with pytest.raises(HTTPException) as error: + await require_scope(user, UsageAlertsRepository(NoDatabase()), "deployment", None, write=write) + assert error.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_internal_user_can_read_and_manage_only_their_user_scope() -> None: + user: Final = UserAPIKeyAuth(user_id="qa-user", user_role=LitellmUserRoles.INTERNAL_USER) + repository: Final = UsageAlertsRepository(NoDatabase()) + await require_scope(user, repository, "user", "qa-user", write=True) + with pytest.raises(HTTPException) as error: + await require_scope(user, repository, "user", "other-user", write=False) + assert error.value.status_code == 403 + + +def test_activity_link_escapes_entity_identity_and_includes_baseline() -> None: + path: Final = activity_path("model", "provider/model?foo=bar&other=1", date(2026, 11, 4), "qa-alert", 7) + parsed: Final = urlsplit(path) + query: Final = parse_qs(parsed.query) + assert parsed.path == "/models-and-endpoints" + assert query["activity_model"] == ["provider/model?foo=bar&other=1"] + assert query["activity_from"] == ["2026-10-28"] + assert query["activity_to"] == ["2026-11-04"] + assert query["alert"] == ["qa-alert"] + assert query["detail_tab"] == ["activity"] + + +def test_global_activity_link_uses_the_usage_page_date_contract() -> None: + query: Final = parse_qs(urlsplit(activity_path("deployment", None, date(2026, 11, 4), "qa-alert")).query) + assert query["tab"] == ["keys"] + assert "range" not in query + assert query["from"] == ["2026-10-28"] + assert query["to"] == ["2026-11-04"] + + +@pytest.mark.asyncio +async def test_no_database_returns_an_actionable_service_error() -> None: + from litellm.proxy.alerting_endpoints.usage_alerts import list_usage_alerts + + user: Final = UserAPIKeyAuth(user_id="qa-admin", user_role=LitellmUserRoles.PROXY_ADMIN) + with pytest.raises(HTTPException) as error: + await list_usage_alerts(user, None) + assert error.value.status_code == 503 + assert error.value.detail == "Usage alerts require a database connection" diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index 4ad17272fad..672006743b3 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -424,6 +424,8 @@ def _user_spend_row(user_id: str, rows: Sequence[_Activity]) -> UserSpendRow: api_requests=sum(row.api_requests for row in matching), successful_requests=sum(row.successful_requests for row in matching), failed_requests=sum(row.failed_requests for row in matching), + total_response_time_ms=sum(row.total_response_time_ms for row in matching), + timed_requests=sum(row.timed_requests for row in matching), ) @@ -448,6 +450,8 @@ def _key_spend_row(api_key: str, rows: Sequence[_Activity]) -> KeySpendRow: api_requests=sum(row.api_requests for row in matching), successful_requests=sum(row.successful_requests for row in matching), failed_requests=sum(row.failed_requests for row in matching), + total_response_time_ms=sum(row.total_response_time_ms for row in matching), + timed_requests=sum(row.timed_requests for row in matching), cache_read_input_tokens=sum(row.cache_read_input_tokens for row in matching), cache_creation_input_tokens=sum(row.cache_creation_input_tokens for row in matching), ) @@ -873,6 +877,8 @@ def test_key_page_routes_map_ranked_rows_and_totals( "api_requests": 1, "successful_requests": 1, "failed_requests": 0, + "total_response_time_ms": 100, + "timed_requests": 1, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 1, }, @@ -894,6 +900,8 @@ def test_key_page_routes_map_ranked_rows_and_totals( "api_requests": 2, "successful_requests": 2, "failed_requests": 0, + "total_response_time_ms": 200, + "timed_requests": 2, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 2, }, @@ -1008,6 +1016,8 @@ def test_user_page_route_ranks_users_by_spend_and_attaches_metadata( "api_requests": 1, "successful_requests": 1, "failed_requests": 0, + "total_response_time_ms": 100, + "timed_requests": 1, }, { "user_id": "user-a", @@ -1020,6 +1030,8 @@ def test_user_page_route_ranks_users_by_spend_and_attaches_metadata( "api_requests": 6, "successful_requests": 6, "failed_requests": 0, + "total_response_time_ms": 600, + "timed_requests": 6, }, ], "total_users": 2, @@ -1380,6 +1392,8 @@ def test_model_top_routes_rank_keys_and_include_metadata( api_requests=1, successful_requests=1, failed_requests=0, + total_response_time_ms=100, + timed_requests=1, cache_read_input_tokens=0, cache_creation_input_tokens=1, ) @@ -1675,6 +1689,8 @@ def test_user_cache_leakage_route_returns_cache_keys_and_metadata( api_requests=1, successful_requests=1, failed_requests=0, + total_response_time_ms=100, + timed_requests=1, cache_read_input_tokens=20, cache_creation_input_tokens=1, ) diff --git a/tests/unit/repositories/test_config_repository.py b/tests/unit/repositories/test_config_repository.py new file mode 100644 index 00000000000..6c37a78b6ec --- /dev/null +++ b/tests/unit/repositories/test_config_repository.py @@ -0,0 +1,27 @@ +from types import SimpleNamespace +from typing import Final +from unittest.mock import AsyncMock + +import pytest + +from litellm.repositories.config_repository import ConfigRepository + + +@pytest.mark.asyncio +async def test_batch_reads_decode_legacy_json_and_current_objects_from_writer() -> None: + find_many: Final = AsyncMock( + return_value=( + SimpleNamespace(param_name="first", param_value='{"status":"resolved"}'), + SimpleNamespace(param_name="second", param_value={"status": "dismissed"}), + ) + ) + prisma: Final = SimpleNamespace(writer_db=SimpleNamespace(litellm_config=SimpleNamespace(find_many=find_many))) + repository: Final = ConfigRepository(prisma, use_writer=True) + rows: Final = await repository.get_params(("first", "second", "absent")) + assert {row.param_name: row.param_value for row in rows} == { + "first": {"status": "resolved"}, + "second": {"status": "dismissed"}, + } + find_many.assert_awaited_once_with(where={"param_name": {"in": ("first", "second", "absent")}}) + assert await repository.get_params(()) == () + assert find_many.await_count == 1 diff --git a/tests/unit/repositories/test_issue_families.py b/tests/unit/repositories/test_issue_families.py new file mode 100644 index 00000000000..a22159e9b5a --- /dev/null +++ b/tests/unit/repositories/test_issue_families.py @@ -0,0 +1,63 @@ +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Final, get_args + +import pytest + +from litellm.repositories.issue_families import family_settings +from litellm.types.proxy.usage_alerts import ( + UsageAlertEvaluationStatus, + UsageAlertMetric, + UsageAlertRule, + UsageAlertScope, +) + + +@dataclass +class RuleRows: + rules: tuple[UsageAlertRule, ...] + calls: int = 0 + + async def query_raw(self, query: str, *args: object) -> object: + self.calls += 1 + if "SELECT EXISTS" in query: + return ({"pending": False},) + return tuple({"record": rule.model_dump(mode="json")} for rule in self.rules) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", get_args(UsageAlertEvaluationStatus)) +async def test_family_counts_only_its_enabled_rules_without_an_extra_query(status: UsageAlertEvaluationStatus) -> None: + timestamp: Final = datetime(2026, 11, 5, tzinfo=timezone.utc) + inputs: Final[tuple[tuple[UsageAlertScope, UsageAlertMetric, bool], ...]] = ( + ("key", "spend", True), + ("team", "spend", True), + ("user", "spend", False), + ("model", "latency", True), + ) + rules: Final = tuple( + UsageAlertRule( + id=f"test-{scope}-{metric}", + name="Daily monitoring", + scope_type=scope, + metric=metric, + direction="increase", + enabled=enabled, + evaluation_status=status, + created_at=timestamp, + updated_at=timestamp, + created_by="test-admin", + ) + for scope, metric, enabled in inputs + ) + + database: Final = RuleRows(rules) + result: Final = next(setting for setting in (await family_settings(database)).data if setting.family == "spend") + + assert database.calls == 2 # existing rule rows plus the separate projection-backlog read + assert result.enabled_rules == 2 + assert result.total_rules == 3 + assert result.evaluation_status_counts.model_dump() == { + **dict.fromkeys(get_args(UsageAlertEvaluationStatus), 0), + status: 2, + } diff --git a/tests/unit/test_check_openapi_docs.py b/tests/unit/test_check_openapi_docs.py index c6ea0cd936e..7d5f09c522b 100644 --- a/tests/unit/test_check_openapi_docs.py +++ b/tests/unit/test_check_openapi_docs.py @@ -40,6 +40,23 @@ def test_each_missing_piece_is_its_own_rule() -> None: assert codes == {"OA001", "OA002", "OA003"} +def test_created_response_schema_satisfies_the_success_contract() -> None: + operation: Final = {**_DOCUMENTED, "responses": {"201": _RESPONSES["200"]}} + assert checker.violations({"paths": {"/v1/thing": {"post": operation}}}) == () + + +@pytest.mark.parametrize("responses", [ + {"201": {"description": "Created"}}, + {"400": _RESPONSES["200"]}, + {"200": {}, "201": _RESPONSES["200"]}, +]) +def test_success_response_must_still_have_a_schema(responses: Mapping[str, object]) -> None: + operation: Final = {**_DOCUMENTED, "responses": responses} + assert checker.violations({"paths": {"/v1/thing": {"post": operation}}}) == ( + checker.Violation("POST", "/v1/thing", "OA003"), + ) + + @pytest.mark.parametrize("body", [{}, {"content": {"multipart/form-data": {}}}]) def test_only_the_missing_piece_is_reported(body: Mapping[str, object]) -> None: no_body: Final = {**_DOCUMENTED, "requestBody": body} diff --git a/tests/unit/types/proxy/test_usage_alerts.py b/tests/unit/types/proxy/test_usage_alerts.py new file mode 100644 index 00000000000..3ee4618d619 --- /dev/null +++ b/tests/unit/types/proxy/test_usage_alerts.py @@ -0,0 +1,64 @@ +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.types.proxy.usage_alerts import UsageAlertGroupUpdate, UsageAlertRuleInput + +RULE: Final = { + "name": "Daily spend", + "scope_type": "team", + "scope_id": "qa-team", + "metric": "spend", + "direction": "increase", +} + + +@pytest.mark.parametrize( + "changes", + [ + {"multiplier": 1}, + {"multiplier": float("inf")}, + {"min_value": float("nan")}, + {"baseline_days": 2, "min_history_days": 3}, + {"baseline_days": 31}, + {"scope_type": "deployment"}, + {"scope_type": "key", "scope_id": "sk-test-placeholder"}, + {"notification_channels": ["email"]}, + {"notification_channels": ["slack", "slack"]}, + {"email_recipients": ["ops@example.com\r\nBcc: other@example.com"]}, + {"name": "Spend\r\nBcc: other@example.com"}, + {"name": "Spend\x00notification"}, + {"overrides_defaults": True}, + ], +) +def test_invalid_rules_cannot_be_stored(changes: dict[str, object]) -> None: + with pytest.raises(ValidationError): + UsageAlertRuleInput.model_validate({**RULE, **changes}) + + +def test_zero_request_floor_and_daily_drop_multiplier_are_supported() -> None: + rule: Final = UsageAlertRuleInput.model_validate( + {**RULE, "metric": "requests", "direction": "decrease", "multiplier": 2, "min_requests": 0} + ) + assert rule.direction == "decrease" + assert rule.multiplier == 2 + assert rule.min_requests == 0 + + +@pytest.mark.parametrize( + "changes", + [{"scope_id": None}, {"scope_type": "deployment"}, {"status": "open"}, {"status": "unknown"}], +) +def test_group_updates_require_exact_resource_and_a_different_valid_status(changes: dict[str, object]) -> None: + with pytest.raises(ValidationError): + UsageAlertGroupUpdate.model_validate( + { + "scope_type": "team", + "scope_id": "team-example", + "day": "2026-11-04", + "expected_status": "open", + "status": "resolved", + **changes, + } + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index b6c3067ed10..60d3f878e55 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8369,6 +8369,160 @@ export interface paths { patch?: never; trace?: never; }; + "/issues": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Issues */ + get: operations["list_issues_issues_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/families": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Issue Families */ + get: operations["list_issue_families_issues_families_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/families/{family}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Issue Family */ + patch: operations["update_issue_family_issues_families__family__patch"]; + trace?: never; + }; + "/issues/{issue_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue */ + get: operations["get_issue_issues__issue_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Issue */ + patch: operations["update_issue_issues__issue_id__patch"]; + trace?: never; + }; + "/issues/{issue_id}/activity": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Activity */ + get: operations["get_issue_activity_issues__issue_id__activity_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/impact": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Impact */ + get: operations["get_issue_impact_issues__issue_id__impact_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/keys": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Keys */ + get: operations["get_issue_keys_issues__issue_id__keys_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/signals": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Issue Signals */ + get: operations["list_issue_signals_issues__issue_id__signals_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/issues/{issue_id}/users": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Issue Users */ + get: operations["get_issue_users_issues__issue_id__users_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/jwt/key/mapping/delete": { parameters: { query?: never; @@ -19088,6 +19242,116 @@ export interface paths { patch?: never; trace?: never; }; + "/usage/alert-groups": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Usage Alert Groups */ + get: operations["list_usage_alert_groups_usage_alert_groups_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Usage Alert Group */ + patch: operations["update_usage_alert_group_usage_alert_groups_patch"]; + trace?: never; + }; + "/usage/alert-models": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Usage Alert Models */ + get: operations["list_usage_alert_models_usage_alert_models_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/usage/alert-rules": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Usage Alert Rules */ + get: operations["list_usage_alert_rules_usage_alert_rules_get"]; + put?: never; + /** + * Create Usage Alert Rule + * @description Create a daily usage alert rule for a scope the caller can manage. + */ + post: operations["create_usage_alert_rule_usage_alert_rules_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/usage/alert-rules/{rule_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Usage Alert Rule */ + get: operations["get_usage_alert_rule_usage_alert_rules__rule_id__get"]; + /** Update Usage Alert Rule */ + put: operations["update_usage_alert_rule_usage_alert_rules__rule_id__put"]; + post?: never; + /** Delete Usage Alert Rule */ + delete: operations["delete_usage_alert_rule_usage_alert_rules__rule_id__delete"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/usage/alerts": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Usage Alerts */ + get: operations["list_usage_alerts_usage_alerts_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/usage/alerts/{alert_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Usage Alert */ + get: operations["get_usage_alert_usage_alerts__alert_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Usage Alert */ + patch: operations["update_usage_alert_usage_alerts__alert_id__patch"]; + trace?: never; + }; "/user/available_roles": { parameters: { query?: never; @@ -34939,6 +35203,329 @@ export interface components { /** Is Accepted */ is_accepted: boolean; }; + /** Issue */ + Issue: { + /** + * Active + * @default true + */ + active: boolean; + /** + * Can Update Status + * @description Current reader's status permission, evaluated by the issue-detail endpoint + */ + can_update_status?: boolean | null; + /** + * Can View Activity + * @description Current reader's permission to inspect Activity, keys, users and impact + */ + can_view_activity?: boolean | null; + cohort: components["schemas"]["IssueCohort"]; + /** Cohort Key */ + cohort_key: string; + /** + * Comparisons + * @default [] + */ + comparisons: components["schemas"]["IssueComparison"][]; + deprecation?: components["schemas"]["IssueDeprecation"] | null; + /** + * Explanation + * @default + */ + explanation: string; + /** + * Family + * @enum {string} + */ + family: "retirement" | "latency" | "reliability" | "spend" | "traffic"; + /** + * First Seen + * Format: date-time + */ + first_seen: string; + /** + * Grouping Reason + * @enum {string} + */ + grouping_reason: "same_resource" | "dominant_key" | "provider_cohort" | "model_deprecation"; + /** Id */ + id: string; + impact?: components["schemas"]["IssueImpact"]; + /** + * Kind + * @enum {string} + */ + kind: "performance" | "model_deprecation"; + /** + * Last Seen + * Format: date-time + */ + last_seen: string; + /** + * Observed At + * Format: date-time + */ + observed_at: string; + /** + * Observed On + * Format: date + */ + observed_on: string; + /** + * Range End + * Format: date + */ + range_end: string; + /** + * Range Start + * Format: date + */ + range_start: string; + /** Scope Label */ + scope_label?: string | null; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + /** Title */ + title: string; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; + /** IssueCohort */ + IssueCohort: { + /** + * Model Pairs + * @default [] + */ + model_pairs: components["schemas"]["IssueModelPair"][]; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model" | "provider"; + }; + /** IssueComparison */ + IssueComparison: { + /** + * Baseline End + * Format: date + */ + baseline_end: string; + /** Baseline Sample Days */ + baseline_sample_days?: number | null; + /** + * Baseline Start + * Format: date + */ + baseline_start: string; + /** Baseline Value */ + baseline_value: number; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + /** Observed Value */ + observed_value: number; + /** Threshold */ + threshold?: number | null; + }; + /** IssueDeprecation */ + IssueDeprecation: { + /** Last Day Requests */ + last_day_requests: number; + /** + * Last Used Date + * Format: date + */ + last_used_date: string; + model: components["schemas"]["ModelDeprecationInfo"]; + /** Requests */ + requests: number; + state: components["schemas"]["DeprecationAlertState"]; + /** Usage Model */ + usage_model: string; + }; + /** IssueFamilyList */ + IssueFamilyList: { + /** Data */ + data: components["schemas"]["IssueFamilySetting"][]; + /** + * Projection Pending + * @default false + */ + projection_pending: boolean; + }; + /** IssueFamilyPolicy */ + IssueFamilyPolicy: { + /** Baseline Days */ + baseline_days: number | null; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** Enabled */ + enabled: boolean | null; + /** Enabled Rules */ + enabled_rules: number; + /** Min History Days */ + min_history_days: number | null; + /** Min Requests */ + min_requests: number | null; + /** Min Value */ + min_value: number | null; + /** Mixed */ + mixed: boolean; + /** Multiplier */ + multiplier: number | null; + /** Total Rules */ + total_rules: number; + }; + /** IssueFamilyPolicyUpdate */ + IssueFamilyPolicyUpdate: { + /** Baseline Days */ + baseline_days?: number | null; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** Enabled */ + enabled?: boolean | null; + /** Min History Days */ + min_history_days?: number | null; + /** Min Requests */ + min_requests?: number | null; + /** Min Value */ + min_value?: number | null; + /** Multiplier */ + multiplier?: number | null; + }; + /** IssueFamilySetting */ + IssueFamilySetting: { + /** Enabled */ + enabled: boolean | null; + /** Enabled Rules */ + enabled_rules: number; + evaluation_status_counts?: components["schemas"]["UsageAlertEvaluationCounts"]; + /** + * Family + * @enum {string} + */ + family: "spend" | "latency" | "reliability" | "traffic"; + /** Policies */ + policies: components["schemas"]["IssueFamilyPolicy"][]; + /** Total Rules */ + total_rules: number; + }; + /** IssueFamilyUpdate */ + IssueFamilyUpdate: { + /** Enabled */ + enabled?: boolean | null; + /** + * Policies + * @default [] + */ + policies: components["schemas"]["IssueFamilyPolicyUpdate"][]; + }; + /** IssueImpact */ + IssueImpact: { + /** Keys */ + keys?: number | null; + /** Measured Keys */ + measured_keys?: number | null; + /** Measured Models */ + measured_models?: number | null; + /** Models */ + models?: number | null; + /** Requests */ + requests?: number | null; + /** Teams */ + teams?: number | null; + /** Users */ + users?: number | null; + }; + /** IssueList */ + IssueList: { + /** Data */ + data: components["schemas"]["Issue"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** IssueModelPair */ + IssueModelPair: { + /** Model */ + model: string; + /** Provider */ + provider: string; + }; + /** IssueSignal */ + IssueSignal: { + deprecation?: components["schemas"]["IssueDeprecation"] | null; + /** + * Explanation + * @default + */ + explanation: string; + /** + * Grouping Reason + * @default same_resource + * @enum {string} + */ + grouping_reason: "same_resource" | "same_key_traffic" | "dominant_key" | "provider_cohort" | "volume_explains_spend" | "model_deprecation"; + /** Issue Id */ + issue_id: string; + /** + * Source + * @enum {string} + */ + source: "usage_alert" | "model_deprecation"; + /** Source Id */ + source_id: string; + usage_alert?: components["schemas"]["UsageAlert"] | null; + }; + /** IssueSignalList */ + IssueSignalList: { + /** Data */ + data: components["schemas"]["IssueSignal"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** IssueUpdate */ + IssueUpdate: { + /** + * Expected Status + * @enum {string} + */ + expected_status: "open" | "resolved" | "dismissed"; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + }; /** * ItemReference * @description An internal identifier for an item to reference. @@ -35097,6 +35684,16 @@ export interface components { * @default 0 */ successful_requests: number; + /** + * Timed Requests + * @default 0 + */ + timed_requests: number; + /** + * Total Response Time Ms + * @default 0 + */ + total_response_time_ms: number; /** * Total Tokens * @default 0 @@ -51057,6 +51654,396 @@ export interface components { */ version: string | null; }; + /** UsageAlert */ + UsageAlert: { + /** Activity Path */ + activity_path: string; + /** + * Baseline End + * Format: date + */ + baseline_end: string; + /** Baseline Sample Days */ + baseline_sample_days?: number | null; + /** + * Baseline Start + * Format: date + */ + baseline_start: string; + /** Baseline Value */ + baseline_value: number; + /** + * Day + * Format: date + */ + day: string; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * First Seen + * Format: date-time + */ + first_seen: string; + /** Id */ + id: string; + /** + * Last Seen + * Format: date-time + */ + last_seen: string; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + /** Observed Value */ + observed_value: number; + /** Rule Id */ + rule_id: string; + /** Rule Name */ + rule_name: string; + /** Scope Id */ + scope_id: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + /** Threshold */ + threshold: number; + }; + /** UsageAlertDefaultSummary */ + UsageAlertDefaultSummary: { + /** + * Enabled + * @default 0 + */ + enabled: number; + evaluation_status_counts?: components["schemas"]["UsageAlertEvaluationCounts"]; + /** + * Total + * @default 0 + */ + total: number; + }; + /** UsageAlertEvaluationCounts */ + UsageAlertEvaluationCounts: { + /** + * Complete + * @default 0 + */ + complete: number; + /** + * Insufficient Coverage + * @default 0 + */ + insufficient_coverage: number; + /** + * Insufficient History + * @default 0 + */ + insufficient_history: number; + /** + * Pending + * @default 0 + */ + pending: number; + }; + /** UsageAlertGroup */ + UsageAlertGroup: { + /** Activity Path */ + activity_path: string; + /** + * Day + * Format: date + */ + day: string; + /** Id */ + id: string; + /** Incident Count */ + incident_count: number; + /** Metrics */ + metrics: components["schemas"]["UsageAlertGroupMetric"][]; + /** Scope Id */ + scope_id: string | null; + /** Scope Label */ + scope_label: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + }; + /** UsageAlertGroupList */ + UsageAlertGroupList: { + /** Data */ + data: components["schemas"]["UsageAlertGroup"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** UsageAlertGroupMetric */ + UsageAlertGroupMetric: { + /** Incident Count */ + incident_count: number; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + representative: components["schemas"]["UsageAlert"]; + }; + /** UsageAlertGroupUpdate */ + UsageAlertGroupUpdate: { + /** + * Day + * Format: date + */ + day: string; + /** + * Expected Status + * @enum {string} + */ + expected_status: "open" | "resolved" | "dismissed"; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + }; + /** UsageAlertGroupUpdateResult */ + UsageAlertGroupUpdateResult: { + /** Updated Count */ + updated_count: number; + }; + /** UsageAlertList */ + UsageAlertList: { + /** Data */ + data: components["schemas"]["UsageAlert"][]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** UsageAlertModelList */ + UsageAlertModelList: { + /** Data */ + data: components["schemas"]["UsageAlertModelOption"][]; + /** + * End Date + * Format: date + */ + end_date: string; + /** Limit */ + limit: number; + /** Next Cursor */ + next_cursor: string | null; + /** + * Start Date + * Format: date + */ + start_date: string; + }; + /** UsageAlertModelOption */ + UsageAlertModelOption: { + /** Label */ + label: string; + /** Scope Id */ + scope_id: string; + }; + /** UsageAlertRule */ + UsageAlertRule: { + /** + * Baseline Days + * @default 7 + */ + baseline_days: number; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** Created By */ + created_by: string | null; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * Email Recipients + * @default [] + */ + email_recipients: string[]; + /** + * Enabled + * @default true + */ + enabled: boolean; + /** + * Evaluation Status + * @default pending + * @enum {string} + */ + evaluation_status: "pending" | "complete" | "insufficient_coverage" | "insufficient_history"; + /** Id */ + id: string; + /** + * Is Default + * @default false + */ + is_default: boolean; + /** Last Evaluated Day */ + last_evaluated_day?: string | null; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + /** + * Min History Days + * @default 3 + */ + min_history_days: number; + /** + * Min Requests + * @default 100 + */ + min_requests: number; + /** + * Min Value + * @description Minimum absolute change in the metric's units + * @default 0 + */ + min_value: number; + /** + * Multiplier + * @default 3 + */ + multiplier: number; + /** Name */ + name: string; + /** + * Notification Channels + * @default [] + */ + notification_channels: ("email" | "slack")[]; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; + /** UsageAlertRuleInput */ + UsageAlertRuleInput: { + /** + * Baseline Days + * @default 7 + */ + baseline_days: number; + /** + * Direction + * @enum {string} + */ + direction: "increase" | "decrease"; + /** + * Email Recipients + * @default [] + */ + email_recipients: string[]; + /** + * Enabled + * @default true + */ + enabled: boolean; + /** + * Metric + * @enum {string} + */ + metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + /** + * Min History Days + * @default 3 + */ + min_history_days: number; + /** + * Min Requests + * @default 100 + */ + min_requests: number; + /** + * Min Value + * @description Minimum absolute change in the metric's units + * @default 0 + */ + min_value: number; + /** + * Multiplier + * @default 3 + */ + multiplier: number; + /** Name */ + name: string; + /** + * Notification Channels + * @default [] + */ + notification_channels: ("email" | "slack")[]; + /** Scope Id */ + scope_id?: string | null; + /** + * Scope Type + * @enum {string} + */ + scope_type: "deployment" | "key" | "user" | "team" | "model"; + }; + /** UsageAlertRuleList */ + UsageAlertRuleList: { + /** Data */ + data: components["schemas"]["UsageAlertRule"][]; + default_summary?: components["schemas"]["UsageAlertDefaultSummary"]; + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Total */ + total: number; + }; + /** UsageAlertUpdate */ + UsageAlertUpdate: { + /** + * Status + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + }; /** UsageChartPoint */ UsageChartPoint: { /** Blocked */ @@ -51575,6 +52562,16 @@ export interface components { * @default 0 */ successful_requests: number; + /** + * Timed Requests + * @default 0 + */ + timed_requests: number; + /** + * Total Response Time Ms + * @default 0 + */ + total_response_time_ms: number; /** * Total Tokens * @default 0 @@ -65275,6 +66272,339 @@ export interface operations { }; }; }; + list_issues_issues_get: { + parameters: { + query?: { + kind?: ("performance" | "model_deprecation") | null; + family?: ("retirement" | "latency" | "reliability" | "spend" | "traffic") | null; + status?: ("open" | "resolved" | "dismissed") | null; + scope_type?: ("deployment" | "key" | "user" | "team" | "model" | "provider") | null; + scope_id?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_issue_families_issues_families_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueFamilyList"]; + }; + }; + }; + }; + update_issue_family_issues_families__family__patch: { + parameters: { + query?: never; + header?: never; + path: { + family: "spend" | "latency" | "reliability" | "traffic"; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["IssueFamilyUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueFamilySetting"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_issues__issue_id__get: { + parameters: { + query?: never; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Issue"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_issue_issues__issue_id__patch: { + parameters: { + query?: never; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["IssueUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Issue"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_activity_issues__issue_id__activity_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + model_limit?: number; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BoundedDailyActivityResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_impact_issues__issue_id__impact_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueImpact"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_keys_issues__issue_id__keys_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DailyActivityKeyPageResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_issue_signals_issues__issue_id__signals_get: { + parameters: { + query?: { + limit?: number; + offset?: number; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IssueSignalList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_issue_users_issues__issue_id__users_get: { + parameters: { + query?: { + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path: { + issue_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DailyActivityUserPageResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; delete_jwt_key_mapping_jwt_key_mapping_delete_post: { parameters: { query?: never; @@ -79432,6 +80762,376 @@ export interface operations { }; }; }; + list_usage_alert_groups_usage_alert_groups_get: { + parameters: { + query?: { + scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; + scope_id?: string | null; + status?: ("open" | "resolved" | "dismissed") | null; + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertGroupList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_usage_alert_group_usage_alert_groups_patch: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UsageAlertGroupUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertGroupUpdateResult"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_usage_alert_models_usage_alert_models_get: { + parameters: { + query?: { + search?: string | null; + cursor?: string | null; + limit?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertModelList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_usage_alert_rules_usage_alert_rules_get: { + parameters: { + query?: { + scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; + scope_id?: string | null; + limit?: number; + offset?: number; + kind?: "all" | "default" | "custom"; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertRuleList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + create_usage_alert_rule_usage_alert_rules_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UsageAlertRuleInput"]; + }; + }; + responses: { + /** @description Successful Response */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertRule"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_usage_alert_rule_usage_alert_rules__rule_id__get: { + parameters: { + query?: never; + header?: never; + path: { + rule_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertRule"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_usage_alert_rule_usage_alert_rules__rule_id__put: { + parameters: { + query?: never; + header?: never; + path: { + rule_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UsageAlertRuleInput"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertRule"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + delete_usage_alert_rule_usage_alert_rules__rule_id__delete: { + parameters: { + query?: never; + header?: never; + path: { + rule_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + list_usage_alerts_usage_alerts_get: { + parameters: { + query?: { + scope_type?: ("deployment" | "key" | "user" | "team" | "model") | null; + scope_id?: string | null; + metric?: ("spend" | "requests" | "latency" | "errors" | "error_rate") | null; + status?: ("open" | "resolved" | "dismissed") | null; + start_date?: string | null; + end_date?: string | null; + limit?: number; + offset?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlertList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_usage_alert_usage_alerts__alert_id__get: { + parameters: { + query?: never; + header?: never; + path: { + alert_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlert"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_usage_alert_usage_alerts__alert_id__patch: { + parameters: { + query?: never; + header?: never; + path: { + alert_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UsageAlertUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UsageAlert"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; ui_get_available_role_user_available_roles_get: { parameters: { query?: never;