diff --git a/docs/features/bounded-activity.md b/docs/features/bounded-activity.md new file mode 100644 index 00000000000..879b78c4aef --- /dev/null +++ b/docs/features/bounded-activity.md @@ -0,0 +1,23 @@ +# Bounded Activity summaries + +Entity Activity tabs use a bounded summary of existing daily activity records. The summary preserves stored daily and range totals while limiting the model breakdown returned to the browser. + +Use `bounded=true` on `GET /user/daily/activity/aggregated` or `GET /team/daily/activity/aggregated`. Existing callers that omit this parameter keep the original response and behavior. Existing user, team, API-key and model filters and permissions still apply; ordinary team members retain their existing self-only view. + +Provide `start_date` and `end_date` in `YYYY-MM-DD` format. The inclusive range may contain at most 366 days. The response contains: + +- `results`: exact metrics for each recorded day, without per-day dimension dictionaries. +- `metadata`: exact totals across the full matching range. +- `model_breakdown`: null unless `include_model_breakdown=true`. + +With a model breakdown, `model_limit` selects the highest-spend models across the full range. It defaults to 20 and accepts 1–100. `total_models` counts all matching model identities. The separate `other` object contains the combined metrics and count of every remaining model. It is null when no models remain and is never used as a model identity. A real model named `Other` remains a distinct model. Identifiers longer than 1,024 characters are included in the `other` aggregate, preserving their totals without truncating an identity or returning an oversized name. + +For example, request a key's bounded summary with: + +```text +GET /user/daily/activity/aggregated?bounded=true&include_model_breakdown=true&model_limit=20&api_key=example-reporting-key-id&start_date=2026-11-01&end_date=2026-11-04 +``` + +Use a reporting key ID, not a secret API key. The model contributions plus `other` sum to the complete range totals. Derived PTU-hours and guardrail or model-group breakdowns are not part of this summary; use the existing Usage view for those breakdowns. + +The SQL query limits result rows before the database driver materializes them. At the maximum range and model limit it returns at most 468 rows; without models it returns at most 367. Queries also have bounded concurrent admission, statement time, lock wait and working memory. A busy or over-budget request returns `503` with `Retry-After: 5`. Retry later or narrow the date range. These limits bound the summary response and query resources; query cost still depends on the number of matching stored activity rows. diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index cfe853da626..b7ca3ed4844 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -1,6 +1,6 @@ -# Issues and daily Performance alerts +# Alerts and daily Performance alerts -The Issues inbox combines model retirements with daily Performance alerts. Open an issue to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause. +The Alerts page combines model retirements with daily Performance alerts. Open an issue to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause. Performance comparisons cover completed UTC days. Retirement issues show recent model usage and the provider's retirement schedule, including today's usage when present. @@ -63,7 +63,9 @@ curl "$LITELLM_API_BASE/issues/$ISSUE_ID/signals?limit=50" \ Replace `issue-example` with an ID returned by `/issues`. Activity uses the issue's saved date range by default. Optional `start_date` and `end_date` are inclusive UTC dates and can span at most 366 days. `/keys` and `/users` provide paginated Activity rows with `limit` and `offset`. Their latency totals are `total_response_time_ms` and `timed_requests`; divide the former by the latter only when timed requests are present. -`/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown. +Alert Activity returns exact daily totals in `results` and exact range totals in `metadata`. It does not return the general Usage API's per-day dimension dictionaries. Key issues also return `model_breakdown`, with up to 20 models by range spend by default. Set `model_limit` between 1 and 100 to change that limit. `total_models` counts every matching model; `other` contains the combined metrics and `model_count` of all remaining models, or null when none remain. It is an aggregate, not a model identifier. Adding the displayed models and `other` recovers the full totals. Model identifiers longer than 1,024 characters are included in `other` without truncation. Non-key issues return null for `model_breakdown`. Activity queries use a bounded execution budget. A busy or over-budget query returns `503` with `Retry-After: 5`; retry later or choose a shorter date range. + +`/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Resource pages fetch labels only for the selected page, using the latest deleted-key label when needed. They do not load arbitrary key or user metadata. Display aliases are limited to 256 characters; email labels longer than 320 characters are omitted. Resource IDs remain unchanged. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown. Supporting signals have their own saved comparisons and relationship explanations. Their baseline windows can differ, so do not derive a combined error rate or unit cost by adding their displayed values. Administrators and admin viewers can inspect broader supporting signals. A scoped reader receives only signals for the exact key, user, or team they can access. That restriction applies before the signal count and pagination; broader totals are not disclosed. @@ -80,6 +82,8 @@ curl -X PATCH "$LITELLM_API_BASE/issues/$ISSUE_ID" \ A stale status returns `409`; reload before retrying. Performance acknowledgements do not change other issues that share a supporting observation. Retirement acknowledgements use the same state as the existing model-deprecation API. An inactive retirement cannot be manually reopened. +The issue-detail response includes `can_update_status` and `can_view_activity` for the current reader. Clients should show status actions only when `can_update_status` is true and request Activity or resource breakdowns only when `can_view_activity` is true. These fields are null in responses where permission has not been evaluated, such as inbox lists. Every endpoint still checks authorization on each request. Team Activity requires team-admin access or the explicit `/team/daily/activity` permission; ordinary membership can read saved alert evidence but does not grant other members’ activity or key details. + ## Configure default families `GET /issues/families` returns the four Performance families, their enabled counts, and common policy values. A null common value means the underlying scopes have different settings. This is separate from custom rules and the model-retirement warning window. @@ -152,7 +156,7 @@ curl "$LITELLM_API_BASE/usage/alert-rules" \ }' ``` -The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value below half the baseline. `min_value` is the minimum absolute difference in the metric's own units. +The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value at or below half the baseline. `min_value` is the minimum absolute difference in the metric's own units. | Metric | Unit | | --- | --- | @@ -185,6 +189,6 @@ Rules can specify `notification_channels: ["slack", "email"]` and `email_recipie Gateway administrators can manage every scope. Admin viewers can read every scope. Internal users can manage their own user and key scopes; team administrators can manage their team's scope and keys. Team members can read their team's incidents. Read-only users cannot change rules or incident state, and inference-only keys cannot access these endpoints. -Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` indicates that the gateway has no database connection. +Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` can indicate an unavailable database or exhausted analytics capacity; a `Retry-After` header indicates that the client should retry later. -The dashboard's Issues inbox is available to administrators and admin viewers. Other authorized roles can use the scoped API. +The dashboard's Alerts page is available to administrators and admin viewers. Other authorized roles can use the scoped API. diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 3a08c094b3d..56098d4bd33 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2110,7 +2110,6 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } -// Daily alerts consume existing aggregate tables; no request-path writes. model LiteLLM_UsageAlertRule { id String @id scope_type String diff --git a/litellm/models/issues.py b/litellm/models/issues.py index cd3f9767638..6357072453a 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -1,8 +1,10 @@ from datetime import date, datetime from typing import Literal, Protocol +from litellm.models.team import LiteLLM_TeamTable from litellm.types.proxy.issues import ( Issue, + IssueActivityResponse, IssueCohort, IssueDeliveryBinding, IssueDeliveryInput, @@ -24,7 +26,6 @@ from litellm.types.proxy.issues import ( from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, DailyActivityUserPageResponse, - SpendAnalyticsPaginatedResponse, ) from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus @@ -33,6 +34,8 @@ from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus class IssueStore(Protocol): async def get_issue(self, issue_id: str) -> Issue | None: ... + async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: ... + async def list_issues( self, kind: IssueKind | None = None, @@ -88,7 +91,7 @@ class IssueStore(Protocol): scope_ids: tuple[str, ...] | None = None, ) -> tuple[IssueProjectionSignal, ...]: ... - async def recover_issues(self, day: date) -> int: ... + async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: ... async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: ... @@ -130,7 +133,9 @@ class IssueStore(Protocol): class IssueActivityStore(Protocol): - async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse: ... + async def daily_activity( + self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 + ) -> IssueActivityResponse: ... async def keys( self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int diff --git a/litellm/models/usage_alerts.py b/litellm/models/usage_alerts.py index 8858835eafc..63e2e05b024 100644 --- a/litellm/models/usage_alerts.py +++ b/litellm/models/usage_alerts.py @@ -22,10 +22,15 @@ class UsageAlertStore(Protocol): async def get_rule(self, rule_id: str) -> UsageAlertRule | None: ... async def save_rule( - self, data: UsageAlertRuleInput, created_by: str | None, existing: UsageAlertRule | None = None + self, + data: UsageAlertRuleInput, + created_by: str | None, + existing: UsageAlertRule | None = None, + *, + overrides_defaults: bool = False, ) -> UsageAlertRule | None: ... - async def delete_rule(self, rule_id: str) -> None: ... + async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: ... async def list_rules( self, diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index a450688de51..2a63986a2ff 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -3740,6 +3740,38 @@ "title": "AgentSkill", "type": "object" }, + "BoundedDailyActivityResponse": { + "properties": { + "metadata": { + "$ref": "#/components/schemas/DailySpendMetadata" + }, + "model_breakdown": { + "anyOf": [ + { + "$ref": "#/components/schemas/DailyActivityModelBreakdown" + }, + { + "type": "null" + } + ], + "description": "When requested, includes the highest-spend models across the full date range and an explicit aggregate of all remaining models. Daily totals always cover the complete activity scope. Other per-day dimension breakdowns are omitted." + }, + "results": { + "items": { + "$ref": "#/components/schemas/DailySpendData" + }, + "maxItems": 366, + "title": "Results", + "type": "array" + } + }, + "required": [ + "results", + "metadata" + ], + "title": "BoundedDailyActivityResponse", + "type": "object" + }, "BreakdownMetrics": { "description": "Breakdown of spend by different dimensions", "properties": { @@ -3851,6 +3883,82 @@ "title": "DailyActivityKeySearchResponse", "type": "object" }, + "DailyActivityModelBreakdown": { + "properties": { + "limit": { + "maximum": 100.0, + "minimum": 1.0, + "title": "Limit", + "type": "integer" + }, + "models": { + "items": { + "$ref": "#/components/schemas/DailyActivityModelContribution" + }, + "maxItems": 100, + "title": "Models", + "type": "array" + }, + "other": { + "anyOf": [ + { + "$ref": "#/components/schemas/DailyActivityOtherModels" + }, + { + "type": "null" + } + ] + }, + "total_models": { + "minimum": 0.0, + "title": "Total Models", + "type": "integer" + } + }, + "required": [ + "limit", + "total_models", + "models" + ], + "title": "DailyActivityModelBreakdown", + "type": "object" + }, + "DailyActivityModelContribution": { + "properties": { + "metrics": { + "$ref": "#/components/schemas/SpendMetrics" + }, + "model": { + "maxLength": 1024, + "title": "Model", + "type": "string" + } + }, + "required": [ + "model", + "metrics" + ], + "title": "DailyActivityModelContribution", + "type": "object" + }, + "DailyActivityOtherModels": { + "properties": { + "metrics": { + "$ref": "#/components/schemas/SpendMetrics" + }, + "model_count": { + "minimum": 1.0, + "title": "Model Count", + "type": "integer" + } + }, + "required": [ + "model_count", + "metrics" + ], + "title": "DailyActivityOtherModels", + "type": "object" + }, "DailySpendData": { "properties": { "breakdown": { @@ -5135,6 +5243,38 @@ "type": "integer" } }, + { + "in": "query", + "name": "bounded", + "required": false, + "schema": { + "default": false, + "title": "Bounded", + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_model_breakdown", + "required": false, + "schema": { + "default": false, + "title": "Include Model Breakdown", + "type": "boolean" + } + }, + { + "in": "query", + "name": "model_limit", + "required": false, + "schema": { + "default": 20, + "maximum": 100, + "minimum": 1, + "title": "Model Limit", + "type": "integer" + } + }, { "in": "query", "name": "agent_ids", @@ -5253,7 +5393,15 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SpendAnalyticsPaginatedResponse" + "anyOf": [ + { + "$ref": "#/components/schemas/SpendAnalyticsPaginatedResponse" + }, + { + "$ref": "#/components/schemas/BoundedDailyActivityResponse" + } + ], + "title": "Response Get Agent Daily Activity Aggregated Agent Daily Activity Aggregated Get" } } }, diff --git a/litellm/proxy/alerting_endpoints/issues.py b/litellm/proxy/alerting_endpoints/issues.py index cca5c730bb5..b87599a5d56 100644 --- a/litellm/proxy/alerting_endpoints/issues.py +++ b/litellm/proxy/alerting_endpoints/issues.py @@ -1,5 +1,7 @@ """Unified issue inbox and investigations backed by existing daily aggregates.""" +from collections.abc import Generator +from contextlib import contextmanager from datetime import date from typing import Final @@ -13,11 +15,16 @@ from litellm.proxy.alerting_endpoints.usage_alerts import ( User, require_repository, require_scope, + require_write_access, ) from litellm.proxy.deps.repositories import IssueActivityRepositoryDep, IssuesRepositoryDep, UsageAlertsRepositoryDep +from litellm.proxy.management.teams.authz import is_team_admin +from litellm.proxy.management_endpoints.common_utils import team_member_has_permission +from litellm.repositories.analytics_budget import AnalyticsCapacityError from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.issues import ( Issue, + IssueActivityResponse, IssueFamily, IssueFamilyList, IssueFamilySetting, @@ -32,7 +39,6 @@ from litellm.types.proxy.issues import ( from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, DailyActivityUserPageResponse, - SpendAnalyticsPaginatedResponse, ) from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertStatus @@ -52,6 +58,14 @@ def require_activity(repository: IssueActivityStore | None) -> IssueActivityStor return repository +@contextmanager +def activity_budget() -> Generator[None]: + try: + yield + except AnalyticsCapacityError as exc: + raise HTTPException(status_code=503, detail=str(exc), headers={"Retry-After": "5"}) from exc + + async def authorized_issue( issue_id: str, user: User, @@ -73,6 +87,25 @@ async def authorized_issue( return issue +async def can_view_activity(issue: Issue, user: User, repository: IssueStore) -> bool: + if issue.cohort.scope_type != "team" or user.user_role in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + return True + if user.user_id is None or issue.cohort.scope_id is None: + return False + team: Final = await repository.team_activity_access(issue.cohort.scope_id, user.user_id) + return team is not None and ( + is_team_admin(user, team) or team_member_has_permission(user, team, "/team/daily/activity") + ) + + +async def require_activity_access(issue: Issue, user: User, repository: IssuesRepositoryDep) -> None: + if not await can_view_activity(issue, user, require_issues(repository)): + raise HTTPException(status_code=403, detail="Full team Activity permission is required for this investigation") + + def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]: start: Final = start_date or issue.range_start end: Final = end_date or issue.range_end @@ -118,6 +151,7 @@ async def update_issue_family( repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, ) -> IssueFamilySetting: + require_write_access(user, f"/issues/families/{family}") await require_scope(user, require_repository(alerts), "deployment", None, write=True) try: result: Final = await require_issues(repository).update_family(family, data) @@ -137,7 +171,22 @@ async def get_issue( repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, ) -> Issue: - return await authorized_issue(issue_id, user, repository, alerts) + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + activity_allowed: Final = await can_view_activity(issue, user, require_issues(repository)) + try: + require_write_access(user, f"/issues/{issue_id}") + await require_scope( + user, + require_repository(alerts), + "model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type, + issue.cohort.scope_id, + write=True, + ) + except HTTPException as exc: + if exc.status_code != 403: + raise + return issue.model_copy(update={"can_update_status": False, "can_view_activity": activity_allowed}) + return issue.model_copy(update={"can_update_status": True, "can_view_activity": activity_allowed}) @router.patch("/{issue_id}", response_model=Issue) @@ -148,6 +197,7 @@ async def update_issue( repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, ) -> Issue: + require_write_access(user, f"/issues/{issue_id}") issue: Final = await authorized_issue(issue_id, user, repository, alerts, write=True) if issue.kind == "model_deprecation" and not issue.active and data.status == "open": raise HTTPException(status_code=409, detail="This retirement issue reopens when the model receives new traffic") @@ -181,7 +231,7 @@ async def list_issue_signals( return await require_issues(repository).list_signals(issue_id, limit, offset, visible_scope) -@router.get("/{issue_id}/activity", response_model=SpendAnalyticsPaginatedResponse) +@router.get("/{issue_id}/activity", response_model=IssueActivityResponse) async def get_issue_activity( issue_id: str, user: User, @@ -190,10 +240,13 @@ async def get_issue_activity( activity: IssueActivityRepositoryDep, start_date: date | None = None, end_date: date | None = None, -) -> SpendAnalyticsPaginatedResponse: + model_limit: Limit = 20, +) -> IssueActivityResponse: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).daily_activity(issue.cohort, start, end) + with activity_budget(): + return await require_activity(activity).daily_activity(issue.cohort, start, end, model_limit) @router.get("/{issue_id}/keys", response_model=DailyActivityKeyPageResponse) @@ -209,8 +262,10 @@ async def get_issue_keys( offset: Offset = 0, ) -> DailyActivityKeyPageResponse: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).keys(issue.cohort, start, end, limit, offset) + with activity_budget(): + return await require_activity(activity).keys(issue.cohort, start, end, limit, offset) @router.get("/{issue_id}/users", response_model=DailyActivityUserPageResponse) @@ -226,10 +281,12 @@ async def get_issue_users( offset: Offset = 0, ) -> DailyActivityUserPageResponse: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) if issue.cohort.scope_type == "team": raise HTTPException(status_code=422, detail="Historical team aggregates do not retain user membership") start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).users(issue.cohort, start, end, limit, offset) + with activity_budget(): + return await require_activity(activity).users(issue.cohort, start, end, limit, offset) @router.get("/{issue_id}/impact", response_model=IssueImpact) @@ -243,5 +300,7 @@ async def get_issue_impact( end_date: date | None = None, ) -> IssueImpact: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).impact(issue.cohort, start, end) + with activity_budget(): + return await require_activity(activity).impact(issue.cohort, start, end) diff --git a/litellm/proxy/alerting_endpoints/usage_alerts.py b/litellm/proxy/alerting_endpoints/usage_alerts.py index e6b1b22cc00..80d818f2dc7 100644 --- a/litellm/proxy/alerting_endpoints/usage_alerts.py +++ b/litellm/proxy/alerting_endpoints/usage_alerts.py @@ -4,9 +4,11 @@ from datetime import date from typing import Annotated, Final, TypeAlias from fastapi import APIRouter, Depends, HTTPException, Query, Response +from pydantic import TypeAdapter from litellm.models.usage_alerts import UsageAlertStore -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.deps.repositories import UsageAlertsRepositoryDep from litellm.repositories.daily_activity_buckets import api_key_from_bucket @@ -44,6 +46,36 @@ Offset: TypeAlias = Annotated[int, Query(ge=0, le=100000)] ScopeId: TypeAlias = Annotated[str | None, Query(min_length=1, max_length=1024)] +def _allows_write(grant: str, route: str) -> bool: + if grant == "info_routes": + return False + paths: Final = ( + TypeAdapter(tuple[str, ...]).validate_python(LiteLLMRoutes[grant].value) + if grant in LiteLLMRoutes.__members__ + else (grant,) + ) + return RouteChecks.check_route_access(route, paths) + + +def require_write_access(user: UserAPIKeyAuth, route: str) -> None: + if not user.allowed_routes: + return + grants: Final = TypeAdapter(tuple[str, ...]).validate_python(user.allowed_routes) + if not any(_allows_write(grant, route) for grant in grants): + raise HTTPException(status_code=403, detail="This credential does not grant alert write access") + + +def require_notification_access( + user: UserAPIKeyAuth, data: UsageAlertRuleInput, existing: UsageAlertRule | None = None +) -> None: + if user.user_role == LitellmUserRoles.PROXY_ADMIN: + return + if data.notification_channels or data.email_recipients: + raise HTTPException(status_code=403, detail="Only proxy admins can configure notification delivery") + if existing is not None and (existing.notification_channels or existing.overrides_defaults): + raise HTTPException(status_code=403, detail="Only proxy admins can change operator-managed alert rules") + + async def require_scope( user: UserAPIKeyAuth, repository: UsageAlertStore, @@ -129,6 +161,7 @@ async def update_usage_alert_group( data: UsageAlertGroupUpdate, user: User, repository: Repository ) -> UsageAlertGroupUpdateResult: store: Final = require_repository(repository) + require_write_access(user, "/usage/alert-groups") await require_scope(user, store, data.scope_type, data.scope_id, write=True) return await store.set_group_status(data) @@ -150,8 +183,12 @@ async def list_usage_alert_models( async def create_usage_alert_rule(data: UsageAlertRuleInput, user: User, repository: Repository) -> UsageAlertRule: """Create a daily usage alert rule for a scope the caller can manage.""" store: Final = require_repository(repository) + require_write_access(user, "/usage/alert-rules") await require_scope(user, store, data.scope_type, data.scope_id, write=True) - saved: Final = await store.save_rule(data, user.user_id) + require_notification_access(user, data) + saved: Final = await store.save_rule( + data, user.user_id, overrides_defaults=user.user_role == LitellmUserRoles.PROXY_ADMIN + ) if saved is None: raise HTTPException(status_code=409, detail="Alert rule could not be saved") return saved @@ -172,6 +209,7 @@ async def update_usage_alert_rule( rule_id: str, data: UsageAlertRuleInput, user: User, repository: Repository ) -> UsageAlertRule: store: Final = require_repository(repository) + require_write_access(user, f"/usage/alert-rules/{rule_id}") rule: Final = await store.get_rule(rule_id) if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") @@ -186,20 +224,26 @@ async def update_usage_alert_rule( status_code=422, detail="Create a custom rule to change a built-in rule's scope, metric, or direction" ) await require_scope(user, store, data.scope_type, data.scope_id, write=True) - saved: Final = await store.save_rule(data, user.user_id, existing=rule) + require_notification_access(user, data, rule) + saved: Final = await store.save_rule( + data, user.user_id, existing=rule, overrides_defaults=user.user_role == LitellmUserRoles.PROXY_ADMIN + ) if saved is None: - raise HTTPException(status_code=404, detail="Alert rule not found") + raise HTTPException(status_code=409, detail="Alert rule changed while being updated. Refresh and try again") return saved @router.delete("/alert-rules/{rule_id}", status_code=204) async def delete_usage_alert_rule(rule_id: str, user: User, repository: Repository) -> Response: store: Final = require_repository(repository) + require_write_access(user, f"/usage/alert-rules/{rule_id}") rule: Final = await store.get_rule(rule_id) if rule is None: raise HTTPException(status_code=404, detail="Alert rule not found") await require_scope(user, store, rule.scope_type, rule.scope_id, write=True) - await store.delete_rule(rule_id) + require_notification_access(user, rule, rule) + if not await store.delete_rule(rule_id, existing=rule): + raise HTTPException(status_code=409, detail="Alert rule changed while being deleted. Refresh and try again") return Response(status_code=204) @@ -236,6 +280,7 @@ async def get_usage_alert(alert_id: str, user: User, repository: Repository) -> @router.patch("/alerts/{alert_id}", response_model=UsageAlert) async def update_usage_alert(alert_id: str, data: UsageAlertUpdate, user: User, repository: Repository) -> UsageAlert: store: Final = require_repository(repository) + require_write_access(user, f"/usage/alerts/{alert_id}") alert: Final = await store.get_alert(alert_id) if alert is None: raise HTTPException(status_code=404, detail="Usage alert not found") diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 1fd3b8bae12..777da689add 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -29,7 +29,7 @@ from litellm.repositories.daily_activity_buckets import ( model_group_bucket, raw_api_key_filter, ) -from litellm.repositories.daily_activity_repository import DailyActivityRepository +from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase, DailyActivityRepository from litellm.types.proxy.management_endpoints.common_daily_activity import ( BreakdownMetrics, DailySpendData, @@ -572,8 +572,12 @@ class _ProxyDailyActivityReads: ) -def daily_activity_repository(prisma_client: PrismaClient) -> DailyActivityRepository: - return DailyActivityRepository(prisma_client, proxy_reads=_ProxyDailyActivityReads(prisma_client)) +def daily_activity_repository( + prisma_client: PrismaClient, *, query_database: DailyActivityQueryDatabase | None = None +) -> DailyActivityRepository: + return DailyActivityRepository( + prisma_client, proxy_reads=_ProxyDailyActivityReads(prisma_client), query_database=query_database + ) def daily_activity_scope( diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 6cf44e10109..58c6013276b 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -39,10 +39,13 @@ from litellm.proxy.management_endpoints.ptu_consumption import single_team_id, w from litellm.proxy.management_endpoints.team_endpoints import aggregated_date_range_error from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.proxy.utils import PrismaClient, get_prisma_client_or_throw +from litellm.repositories.analytics_budget import AnalyticsCapacityError +from litellm.repositories.bounded_daily_activity import BoundedDailyActivityRepository from litellm.repositories.daily_activity_buckets import api_key_bucket from litellm.repositories.daily_activity_repository import DailyActivityRepository from litellm.router import Router from litellm.types.proxy.management_endpoints.common_daily_activity import ( + BoundedDailyActivityResponse, CacheLeakageKeysResponse, DailyActivityKeyPageResponse, DailyActivityKeySearchResponse, @@ -272,8 +275,13 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver, api_key_limit: Annotated[ int, Query(ge=1, le=constants.USAGE_TOP_API_KEYS_MAX) ] = constants.USAGE_TOP_API_KEYS_DEFAULT, - ) -> SpendAnalyticsPaginatedResponse: + bounded: bool = False, + include_model_breakdown: bool = False, + model_limit: Annotated[int, Query(ge=1, le=100)] = 20, + ) -> SpendAnalyticsPaginatedResponse | BoundedDailyActivityResponse: try: + if bounded and resolver.entity not in ("user", "team"): + raise HTTPException(status_code=422, detail="Bounded Activity is supported for user and team scopes") resolved: ResolvedScope = await _resolved_scope( resolver, entity_query, @@ -281,6 +289,13 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver, prisma_client, user_aggregated=resolver.entity == "user", ) + if bounded: + try: + return await BoundedDailyActivityRepository.from_prisma(prisma_client).aggregate( + resolved.scope, model_limit=model_limit if include_model_breakdown else None + ) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc activity: Final = await get_daily_activity_aggregated( repository, resolved.scope, @@ -294,6 +309,8 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver, return with_ptu_consumption(activity, llm_router, single_team_id(resolved.scope.entity_ids)) except HTTPException: raise + except AnalyticsCapacityError as exc: + raise HTTPException(status_code=503, detail=str(exc), headers={"Retry-After": "5"}) from exc except Exception as exc: verbose_proxy_logger.exception("Daily activity aggregation failed: %s", exc) raise HTTPException(status_code=500, detail={"error": f"Failed to fetch analytics: {exc}"}) @@ -305,7 +322,7 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver, name=resolver.operation_names["aggregated"], tags=list(resolver.tags), dependencies=(Depends(user_api_key_auth),), - response_model=SpendAnalyticsPaginatedResponse, + response_model=SpendAnalyticsPaginatedResponse | BoundedDailyActivityResponse, include_in_schema=prefix != "/end_user", ) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 3a08c094b3d..56098d4bd33 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2110,7 +2110,6 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } -// Daily alerts consume existing aggregate tables; no request-path writes. model LiteLLM_UsageAlertRule { id String @id scope_type String diff --git a/litellm/repositories/analytics_budget.py b/litellm/repositories/analytics_budget.py new file mode 100644 index 00000000000..e310d1801a7 --- /dev/null +++ b/litellm/repositories/analytics_budget.py @@ -0,0 +1,109 @@ +import asyncio +from collections.abc import Generator, Mapping, Sequence +from contextlib import AbstractAsyncContextManager, contextmanager +from contextvars import ContextVar +from datetime import timedelta +from threading import BoundedSemaphore +from typing import Final, Protocol + +from prisma.errors import RawQueryError, TransactionExpiredError, TransactionNotStartedError +from pydantic import BaseModel + + +class AnalyticsTransaction(Protocol): + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... + + +class AnalyticsDatabase(Protocol): + def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AbstractAsyncContextManager[AnalyticsTransaction]: ... + + +class AnalyticsCapacityError(RuntimeError): + pass + + +class _Admission(BaseModel, frozen=True): + acquired: bool + + +class _PostgresError(BaseModel, frozen=True): + code: str = "" + + +_FOREGROUND: Final = BoundedSemaphore(2) +_BACKGROUND: Final = BoundedSemaphore(1) +_DEADLINE: Final = ContextVar[float | None]("analytics_deadline", default=None) +_CONFIGURE: Final = """ +SELECT + set_config('statement_timeout', LEAST(NULLIF( + EXTRACT(epoch FROM current_setting('statement_timeout')::interval)*1000,0),$2::int)::text,true), + set_config('lock_timeout', LEAST(NULLIF( + EXTRACT(epoch FROM current_setting('lock_timeout')::interval)*1000,0),100,$2::int)::text,true), + set_config('work_mem', LEAST(pg_size_bytes(current_setting('work_mem')),1048576)::text || 'B',true), + set_config('max_parallel_workers_per_gather','0',true), + CASE WHEN $1::boolean THEN pg_try_advisory_xact_lock(1876539401,2) + WHEN pg_try_advisory_xact_lock(1876539401,0) THEN true + ELSE pg_try_advisory_xact_lock(1876539401,1) END AS acquired +""" + + +@contextmanager +def analytics_deadline(deadline: float) -> Generator[None, None, None]: + current: Final = _DEADLINE.get() + token: Final = _DEADLINE.set(deadline if current is None else min(current, deadline)) + try: + yield + finally: + _DEADLINE.reset(token) + + +def _remaining_seconds() -> float | None: + deadline: Final = _DEADLINE.get() + if deadline is None: + return None + remaining: Final = deadline - asyncio.get_running_loop().time() + if remaining <= 0: + raise AnalyticsCapacityError("Analytics work budget is exhausted") + return remaining + + +def _milliseconds(limit: int) -> int: + remaining: Final = _remaining_seconds() + return limit if remaining is None else min(limit, max(1, int(remaining * 1000))) + + +class BoundedAnalyticsDatabase: + def __init__(self, database: AnalyticsDatabase, *, background: bool = False) -> None: + self._database = database + self._background = background + self._admission = _BACKGROUND if background else _FOREGROUND + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + if not self._admission.acquire(blocking=False): + raise AnalyticsCapacityError("Analytics capacity is busy. Try again shortly") + try: + remaining: Final = _remaining_seconds() + return await asyncio.wait_for(self._query(query, args), timeout=remaining) + except RawQueryError as error: + if _PostgresError.model_validate(error.meta or {}).code not in ("57014", "55P03"): + raise + raise AnalyticsCapacityError("Analytics exceeded its query budget. Try again shortly") from error + except ( + TransactionExpiredError, + TransactionNotStartedError, + asyncio.TimeoutError, + ) as error: + raise AnalyticsCapacityError("Analytics exceeded its connection budget. Try again shortly") from error + finally: + self._admission.release() + + async def _query(self, query: str, args: tuple[object, ...]) -> Sequence[Mapping[str, object]]: + async with self._database.tx( + max_wait=timedelta(milliseconds=_milliseconds(250)), + timeout=timedelta(milliseconds=_milliseconds(6000)), + ) as transaction: + configuration: Final = await transaction.query_raw(_CONFIGURE, self._background, _milliseconds(5000)) + if not configuration or not _Admission.model_validate(configuration[0]).acquired: + raise AnalyticsCapacityError("Analytics capacity is busy. Try again shortly") + _remaining_seconds() + return await transaction.query_raw(query, *args) diff --git a/litellm/repositories/bounded_daily_activity.py b/litellm/repositories/bounded_daily_activity.py new file mode 100644 index 00000000000..c0832c8e2df --- /dev/null +++ b/litellm/repositories/bounded_daily_activity.py @@ -0,0 +1,99 @@ +"""Bounded daily totals and optional model contributions for Activity panels.""" + +from datetime import date +from typing import TYPE_CHECKING, Final, Literal, cast # noqa: TID251 # generated Prisma transaction interface + +from pydantic import BaseModel, TypeAdapter + +from litellm.repositories.analytics_budget import AnalyticsDatabase, BoundedAnalyticsDatabase +from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase +from litellm.repositories.daily_activity_sql import build_bounded_daily_activity_sql +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + BoundedDailyActivityResponse, + DailyActivityModelBreakdown, + DailyActivityModelContribution, + DailyActivityOtherModels, + DailySpendData, + DailySpendMetadata, + SpendMetrics, +) +from litellm.types.repositories.daily_activity import DailyActivityScope + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +class _ActivityRow(BaseModel, frozen=True): + kind: Literal["day", "total", "model", "other"] + date: date | None + model: str | None + model_count: int + total_api_keys: int | None + metrics: SpendMetrics + + +def _metadata(row: _ActivityRow) -> DailySpendMetadata: + metrics: Final = row.metrics + return DailySpendMetadata( + total_spend=metrics.spend, + total_flat_cost=metrics.flat_cost, + total_prompt_tokens=metrics.prompt_tokens, + total_completion_tokens=metrics.completion_tokens, + total_tokens=metrics.total_tokens, + total_api_requests=metrics.api_requests, + total_successful_requests=metrics.successful_requests, + total_failed_requests=metrics.failed_requests, + total_cache_read_input_tokens=metrics.cache_read_input_tokens, + total_cache_creation_input_tokens=metrics.cache_creation_input_tokens, + total_compression_saved_tokens=metrics.compression_saved_tokens, + total_compression_savings_spend=metrics.compression_savings_spend, + total_prompt_caching_savings_spend=metrics.prompt_caching_savings_spend, + total_gateway_injected_caching_savings_spend=metrics.gateway_injected_caching_savings_spend, + total_autorouter_savings_spend=metrics.autorouter_savings_spend, + total_response_time_ms=metrics.total_response_time_ms, + total_timed_requests=metrics.timed_requests, + total_api_keys=row.total_api_keys, + ) + + +class BoundedDailyActivityRepository: + def __init__(self, database: DailyActivityQueryDatabase) -> None: + self.database = database + + @classmethod + def from_prisma(cls, prisma: "PrismaClient") -> "BoundedDailyActivityRepository": + return cls(BoundedAnalyticsDatabase(cast(AnalyticsDatabase, prisma.db))) # cast-ok: generated Prisma tx + + async def aggregate( + self, scope: DailyActivityScope, *, model_limit: int | None = None + ) -> BoundedDailyActivityResponse: + if model_limit is not None and not 1 <= model_limit <= 100: + raise ValueError("model_limit must be between 1 and 100") + query: Final = build_bounded_daily_activity_sql(scope, model_limit=model_limit) + rows: Final = TypeAdapter(tuple[_ActivityRow, ...]).validate_python( + await self.database.query_raw(query.sql, *query.params) + ) + total: Final = next(row for row in rows if row.kind == "total") + other: Final = next((row for row in rows if row.kind == "other"), None) + days: Final = tuple( + DailySpendData(date=row.date, metrics=row.metrics) + for row in rows + if row.kind == "day" and row.date is not None + ) + models: Final = tuple( + DailyActivityModelContribution(model=row.model, metrics=row.metrics) + for row in rows + if row.kind == "model" and row.model is not None + ) + return BoundedDailyActivityResponse( + results=tuple(sorted(days, key=lambda day: day.date, reverse=True)), + metadata=_metadata(total), + model_breakdown=DailyActivityModelBreakdown( + limit=model_limit, + total_models=total.model_count, + models=tuple(sorted(models, key=lambda model: (-model.metrics.spend, model.model))), + other=DailyActivityOtherModels(model_count=other.model_count, metrics=other.metrics) if other else None, + ) + if model_limit is not None + else None, + ) diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 262eb9d7ec1..0b285261658 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -71,6 +71,10 @@ class _QueryRaw(Protocol): async def __call__(self, query: str, *values: object) -> Sequence[Mapping[str, object]] | None: ... +class DailyActivityQueryDatabase(Protocol): + async def query_raw(self, query: str, *values: object) -> Sequence[Mapping[str, object]] | None: ... + + class _DailyActivityDatabase(Protocol): query_raw: _QueryRaw litellm_usertable: TableActions[_UserTableRow] @@ -155,9 +159,16 @@ def _next_export_cursor(batch: tuple[ExportRow, ...], export_type: ExportType) - class DailyActivityRepository: - def __init__(self, prisma_client: DailyActivityDatabase, *, proxy_reads: DailyActivityProxyReads) -> None: + def __init__( + self, + prisma_client: DailyActivityDatabase, + *, + proxy_reads: DailyActivityProxyReads, + query_database: DailyActivityQueryDatabase | None = None, + ) -> None: self._prisma_client = prisma_client self._proxy_reads = proxy_reads + self._query_database = query_database if query_database is not None else prisma_client.db async def _query(self, query: SqlQuery) -> tuple[Mapping[str, object], ...]: from litellm.proxy.db.db_span import db_span @@ -166,7 +177,7 @@ class DailyActivityRepository: first_line: Final = query.sql.lstrip().splitlines()[0].lstrip("(").strip() verbose_proxy_logger.debug("DailyActivityRepository query: %s", first_line) async with db_span("daily_activity_query", sql_relation(query.sql)): - result: Sequence[Mapping[str, object]] | None = await self._prisma_client.db.query_raw( + result: Final[Sequence[Mapping[str, object]] | None] = await self._query_database.query_raw( query.sql, *query.params ) if result is None: diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index c480af53272..ff9e8cb1040 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -1,7 +1,7 @@ import json from collections.abc import Mapping from dataclasses import dataclass -from datetime import datetime, timedelta, timezone +from datetime import date, datetime, timedelta, timezone from itertools import count, islice from types import MappingProxyType from typing import Final @@ -187,6 +187,78 @@ def _top_api_keys_sql(pg_table: str, where_clause: str, *, sentinel_param: int, """ +def _bounded_metrics_json(alias: str) -> str: + return f"""(jsonb_strip_nulls(to_jsonb({alias}) - ARRAY[ + 'date','model','group_level','model_count','total_api_keys','ptu_flat_cost' + ]) || jsonb_build_object( + 'flat_cost',COALESCE({alias}.ptu_flat_cost,0), + 'total_tokens',COALESCE({alias}.prompt_tokens,0)+COALESCE({alias}.completion_tokens,0) + ))""" + + +def build_bounded_daily_activity_sql(scope: DailyActivityScope, *, model_limit: int | None) -> SqlQuery: + days: Final = (date.fromisoformat(scope.end_date) - date.fromisoformat(scope.start_date)).days + if not 0 <= days <= 365: + raise ValueError("Choose an ordered date range of at most 366 days") + if model_limit is not None and not 1 <= model_limit <= 100: + raise ValueError("model_limit must be between 1 and 100") + where, params = build_where_clause(scope) + sentinel: Final = len(params) + 1 + limit_param: Final = len(params) + 2 + metrics: Final = _rollup_metric_select(scope.table) + grouping: Final = ( + f"GROUPING SETS ((date),({_MODEL_EXPR}),())" if model_limit is not None else "GROUPING SETS ((date),())" + ) + model: Final = _MODEL_EXPR if model_limit is not None else "NULL::text" + level: Final = f"GROUPING(date,{_MODEL_EXPR})" if model_limit is not None else "GROUPING(date)" + total_level: Final = 3 if model_limit is not None else 1 + day_level: Final = 1 if model_limit is not None else 0 + model_count: Final = "(SELECT count(*) FROM rollups WHERE group_level=2)" if model_limit is not None else "0" + model_ctes: Final = ( + f""", top_models AS MATERIALIZED ( + SELECT * FROM rollups WHERE group_level=2 AND length(model)<=1024 + ORDER BY spend DESC NULLS LAST,model LIMIT ${limit_param} + ), other_models AS ( + SELECT count(*)::bigint AS model_count,{metrics} + FROM rollups r WHERE group_level=2 + AND NOT EXISTS(SELECT 1 FROM top_models t WHERE t.model=r.model) + HAVING count(*)>0 + )""" + if model_limit is not None + else "" + ) + model_rows: Final = ( + f"""UNION ALL SELECT 'model',NULL::text,r.model,1::bigint,r.total_api_keys,{_bounded_metrics_json("r")} + FROM top_models r + UNION ALL SELECT 'other',NULL::text,NULL::text,r.model_count,NULL::bigint,{_bounded_metrics_json("r")} + FROM other_models r""" + if model_limit is not None + else "" + ) + sql: Final = f"""WITH rollups AS MATERIALIZED ( + SELECT date,{model} AS model,{level} AS group_level, + count(DISTINCT api_key) FILTER(WHERE api_key<>${sentinel})::bigint AS total_api_keys, + {metrics} + FROM "{PRISMA_TO_PG_TABLE[scope.table]}" WHERE {where} + GROUP BY {grouping} + ){model_ctes} + SELECT 'day' AS kind,date,NULL::text AS model,0::bigint AS model_count,total_api_keys, + {_bounded_metrics_json("r")} AS metrics FROM rollups r WHERE group_level={day_level} + UNION ALL SELECT 'total',NULL::text,NULL::text,{model_count},total_api_keys,{_bounded_metrics_json("r")} + FROM rollups r WHERE group_level={total_level} + {model_rows} + LIMIT ${limit_param + int(model_limit is not None)}""" + return SqlQuery( + sql=sql, + params=( + *params, + PTU_SENTINEL_API_KEY, + *((model_limit,) if model_limit is not None else ()), + 367 if model_limit is None else 368 + model_limit, + ), + ) + + def build_aggregated_sql(scope: DailyActivityScope, *, api_key_limit: int) -> SqlQuery: pg_table: Final = PRISMA_TO_PG_TABLE[scope.table] where_clause, where_params = build_where_clause(scope) diff --git a/litellm/repositories/issue_activity.py b/litellm/repositories/issue_activity.py index fed59a82cbc..a6b9bf96fe7 100644 --- a/litellm/repositories/issue_activity.py +++ b/litellm/repositories/issue_activity.py @@ -1,24 +1,30 @@ """Issue investigations reuse the same aggregate filters as entity Activity.""" +from collections.abc import Mapping from dataclasses import asdict, replace from datetime import date +from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # generated Prisma structural database interface from pydantic import BaseModel, TypeAdapter from litellm.constants import PTU_SENTINEL_API_KEY +from litellm.repositories.analytics_budget import AnalyticsDatabase, BoundedAnalyticsDatabase +from litellm.repositories.bounded_daily_activity import BoundedDailyActivityRepository from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket -from litellm.repositories.daily_activity_repository import DailyActivityRepository +from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase, DailyActivityRepository from litellm.repositories.daily_activity_sql import PRISMA_TO_PG_TABLE, build_where_clause -from litellm.repositories.usage_alerts import AlertDatabase -from litellm.types.proxy.issues import IssueCohort, IssueImpact +from litellm.types.proxy.issues import ( + IssueActivityResponse, + IssueCohort, + IssueImpact, +) from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, DailyActivityUserPageResponse, KeyMetadata, KeySpendActivityRow, KeySpendMetrics, - SpendAnalyticsPaginatedResponse, UserActivityRow, ) from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable @@ -34,6 +40,17 @@ class _Counts(BaseModel, frozen=True): requests: int +class _KeyLabel(BaseModel, frozen=True): + api_key: str + metadata: KeyMetadata + + +class _UserLabel(BaseModel, frozen=True): + user_id: str + user_alias: str | None + user_email: str | None + + def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyActivityScope: if start > end or (end - start).days > 365: raise ValueError("Choose an ordered date range of at most 366 days") @@ -54,7 +71,7 @@ def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyAc class IssueActivityRepository: - def __init__(self, activity: DailyActivityRepository, database: AlertDatabase) -> None: + def __init__(self, activity: DailyActivityRepository, database: DailyActivityQueryDatabase) -> None: self.activity = activity self.database = database @@ -62,31 +79,29 @@ class IssueActivityRepository: def from_prisma(cls, prisma: "PrismaClient") -> "IssueActivityRepository": from litellm.proxy.management_endpoints.common_daily_activity import daily_activity_repository - database: Final = cast(AlertDatabase, prisma.db) # cast-ok: generated Prisma query_raw - return cls(daily_activity_repository(prisma), database) + database: Final = BoundedAnalyticsDatabase(cast(AnalyticsDatabase, prisma.db)) # cast-ok: generated Prisma tx + return cls(daily_activity_repository(prisma, query_database=database), database) - async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse: - from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated - - return await get_daily_activity_aggregated( - self.activity, issue_activity_scope(cohort, start, end), api_key_limit=20 + async def daily_activity( + self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 + ) -> IssueActivityResponse: + return await BoundedDailyActivityRepository(self.database).aggregate( + issue_activity_scope(cohort, start, end), model_limit=model_limit if cohort.scope_type == "key" else None ) async def keys( self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityKeyPageResponse: + if not 1 <= limit <= 100 or offset < 0: + raise ValueError("Read between one and 100 keys with a nonnegative offset") page: Final = await self.activity.key_page(issue_activity_scope(cohort, start, end), limit=limit, offset=offset) - metadata: Final = await self.activity.key_metadata( - frozenset(row.api_key for row in page.rows if row.api_key), None - ) + metadata: Final = await self._key_labels(tuple(row.api_key for row in page.rows if row.api_key)) return DailyActivityKeyPageResponse( api_keys=[ KeySpendActivityRow( api_key=api_key_bucket(row.api_key), metrics=KeySpendMetrics.model_validate(asdict(row)), - metadata=KeyMetadata.model_validate(asdict(metadata[row.api_key])) - if row.api_key in metadata - else KeyMetadata(), + metadata=metadata.get(row.api_key, KeyMetadata()), ) for row in page.rows ], @@ -98,19 +113,23 @@ class IssueActivityRepository: async def users( self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int ) -> DailyActivityUserPageResponse: + if not 1 <= limit <= 100 or offset < 0: + raise ValueError("Read between one and 100 users with a nonnegative offset") if cohort.scope_type == "team": raise ValueError("Historical team aggregates do not retain user membership") page: Final = await self.activity.user_page( issue_activity_scope(cohort, start, end), limit=limit, offset=offset ) - metadata: Final = await self.activity.user_metadata(frozenset(row.user_id for row in page.rows if row.user_id)) + metadata: Final = await self._user_labels(tuple(row.user_id for row in page.rows if row.user_id)) return DailyActivityUserPageResponse( users=tuple( UserActivityRow.model_validate( { **asdict(row), **( - asdict(metadata[row.user_id]) if row.user_id is not None and row.user_id in metadata else {} + metadata[row.user_id].model_dump() + if row.user_id is not None and row.user_id in metadata + else {} ), } ) @@ -121,6 +140,47 @@ class IssueActivityRepository: offset=offset, ) + async def _key_labels(self, ids: tuple[str, ...]) -> Mapping[str, KeyMetadata]: + if len(ids) > 100: + raise ValueError("Read at most 100 key labels") + if not ids: + return MappingProxyType({}) + rows: Final = TypeAdapter(tuple[_KeyLabel, ...]).validate_python( + await self.database.query_raw( + """SELECT w.api_key,jsonb_build_object( + 'key_alias',LEFT(COALESCE(k.key_alias,d.key_alias),256), + 'team_id',COALESCE(k.team_id,d.team_id),'user_id',COALESCE(k.user_id,d.user_id), + 'user_email',CASE WHEN length(u.user_email)<=320 THEN u.user_email END, + 'key_exists',CASE WHEN k.token IS NOT NULL THEN true WHEN d.id IS NOT NULL THEN false END + ) AS metadata + FROM unnest($1::text[]) w(api_key) + LEFT JOIN "LiteLLM_VerificationToken" k ON k.token=w.api_key + LEFT JOIN LATERAL ( + SELECT id,key_alias,team_id,user_id FROM "LiteLLM_DeletedVerificationToken" + WHERE token=w.api_key AND k.token IS NULL ORDER BY deleted_at DESC,id DESC LIMIT 1 + ) d ON true + LEFT JOIN "LiteLLM_UserTable" u ON u.user_id=COALESCE(k.user_id,d.user_id) + LIMIT 100""", + list(ids), + ) + ) + return MappingProxyType({row.api_key: row.metadata for row in rows}) + + async def _user_labels(self, ids: tuple[str, ...]) -> Mapping[str, _UserLabel]: + if len(ids) > 100: + raise ValueError("Read at most 100 user labels") + if not ids: + return MappingProxyType({}) + rows: Final = TypeAdapter(tuple[_UserLabel, ...]).validate_python( + await self.database.query_raw( + """SELECT user_id,LEFT(user_alias,256) AS user_alias, + CASE WHEN length(user_email)<=320 THEN user_email END AS user_email + FROM "LiteLLM_UserTable" WHERE user_id=ANY($1::text[]) LIMIT 100""", + list(ids), + ) + ) + return MappingProxyType({row.user_id: row for row in rows}) + async def _counts(self, scope: DailyActivityScope) -> _Counts: where, params = build_where_clause(scope) rows: Final = TypeAdapter(tuple[_Counts, ...]).validate_python( diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index fda3bc33f89..855b6962f61 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -10,6 +10,7 @@ from typing import ( from pydantic import BaseModel, TypeAdapter +from litellm.models.team import LiteLLM_TeamTable from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( @@ -31,7 +32,7 @@ from litellm.types.proxy.issues import ( IssueSignalInput, IssueSignalList, ) -from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS, PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus if TYPE_CHECKING: @@ -165,6 +166,25 @@ class IssuesRepository: ) return (await self._labels((rows[0].record,)))[0] if rows else None + async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: + rows: Final = TypeAdapter(tuple[LiteLLM_TeamTable, ...]).validate_python( + await self.database.query_raw( + """SELECT t.team_id, + CASE WHEN m.role IS NULL THEN '[]'::jsonb + ELSE jsonb_build_array(jsonb_build_object('user_id',$2::text,'role',m.role)) END + AS members_with_roles, + CASE WHEN '/team/daily/activity'=ANY(t.team_member_permissions) + THEN ARRAY['/team/daily/activity'] ELSE ARRAY[]::text[] END AS team_member_permissions + FROM "LiteLLM_TeamTable" t LEFT JOIN LATERAL ( + SELECT member->>'role' AS role FROM jsonb_array_elements(t.members_with_roles) member + WHERE member->>'user_id'=$2 ORDER BY (member->>'role'='admin') DESC LIMIT 1 + ) m ON true WHERE t.team_id=$1""", + team_id, + user_id, + ) + ) + return rows[0] if rows else None + async def list_issues( self, kind: IssueKind | None = None, @@ -346,26 +366,56 @@ class IssuesRepository: ) ) - async def recover_issues(self, day: date) -> int: + async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: + if issue_ids == (): + return 0 + if issue_ids is not None and len(issue_ids) > 10: + raise ValueError("Recover at most 10 resource alerts") + selection: Final = "i.id=ANY($3::text[])" if issue_ids is not None else "$3::text[] IS NULL" rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( await self.database.query_raw( - """WITH healthy AS ( - SELECT i.id FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id + f"""WITH primary_signals AS ( + SELECT i.id,a.id AS source_id,a.breached,a.evaluation_day,a.evaluation_revision, + max(a.evaluation_day) OVER (PARTITION BY i.id) AS latest_observation_day, + r.id AS rule_id,r.record AS rule_record, + COALESCE(r.enabled AND NOT r.deleted + AND r.scope_type=a.scope_type AND r.record->>'metric'=a.metric + AND r.record->>'direction'=a.record->>'direction' + AND (r.scope_id IS NULL OR r.scope_id=a.scope_id) + AND (NOT(r.id=ANY($2::text[])) OR NOT EXISTS ( + SELECT 1 FROM "LiteLLM_UsageAlertRule" custom WHERE custom.enabled AND NOT custom.deleted + AND COALESCE((custom.record->>'overrides_defaults')::boolean,false) + AND NOT(custom.id=ANY($2::text[])) AND custom.scope_type=r.scope_type + AND custom.record->>'metric'=r.record->>'metric' + AND custom.record->>'direction'=r.record->>'direction' + AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id))),false) AS eligible + FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id + LEFT JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='usage_alert' + AND ({selection}) AND s.record->'usage_alert'->>'scope_type'=i.scope_type AND s.record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM i.scope_id AND ((i.family='spend' AND s.record->'usage_alert'->>'metric'='spend') OR (i.family='latency' AND s.record->'usage_alert'->>'metric'='latency') OR (i.family='traffic' AND s.record->'usage_alert'->>'metric'='requests') OR (i.family='reliability' AND s.record->'usage_alert'->>'metric' IN ('errors','error_rate'))) - GROUP BY i.id HAVING bool_and(a.id IS NOT NULL AND NOT a.breached AND a.evaluation_day<=$1) + ), observations AS ( + SELECT *,COALESCE(source_id IS NOT NULL AND NOT breached AND evaluation_day<=$1 + AND evaluation_day>=latest_observation_day + AND NULLIF(evaluation_revision,'')::timestamptz=(rule_record->>'updated_at')::timestamptz,false) AS healthy + FROM primary_signals + ), healthy AS ( + SELECT id FROM observations GROUP BY id + HAVING bool_or(eligible AND healthy) AND bool_and(NOT eligible OR healthy) ), changed AS ( UPDATE "LiteLLM_Issue" i SET active=false, status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP WHERE id IN(SELECT id FROM healthy) AND active RETURNING id ) SELECT count(*)::int AS count FROM changed""", day.isoformat(), + tuple(DEFAULT_USAGE_RULE_IDS), + issue_ids, ) ) return rows[0].count diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index be17170a129..775afe67aee 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -216,7 +216,12 @@ class UsageAlertsRepository: return _rule(rows[0].record) if rows else None async def save_rule( - self, data: UsageAlertRuleInput, created_by: str | None, existing: UsageAlertRule | None = None + self, + data: UsageAlertRuleInput, + created_by: str | None, + existing: UsageAlertRule | None = None, + *, + overrides_defaults: bool = False, ) -> UsageAlertRule | None: now: Final = datetime.now(timezone.utc) rule: Final = UsageAlertRule.model_validate( @@ -226,24 +231,41 @@ class UsageAlertsRepository: "created_by": existing.created_by if existing else created_by, "created_at": existing.created_at if existing else now, "updated_at": now, + "overrides_defaults": overrides_defaults, } ) rows: Final = await self.database.query_raw( """INSERT INTO "LiteLLM_UsageAlertRule" (id,scope_type,scope_id,enabled,record) VALUES ($1,$2,$3,$4,$5::jsonb) ON CONFLICT (id) DO UPDATE SET scope_type=EXCLUDED.scope_type,scope_id=EXCLUDED.scope_id,enabled=EXCLUDED.enabled,record=EXCLUDED.record - WHERE NOT "LiteLLM_UsageAlertRule".deleted RETURNING record""", + WHERE NOT "LiteLLM_UsageAlertRule".deleted + AND ("LiteLLM_UsageAlertRule".record->>'updated_at')::timestamptz=$6::text::timestamptz + AND "LiteLLM_UsageAlertRule".scope_type=$7 + AND "LiteLLM_UsageAlertRule".scope_id IS NOT DISTINCT FROM $8::text + RETURNING record""", rule.id, rule.scope_type, rule.scope_id, rule.enabled, - rule.model_dump_json(), + json.dumps({**rule.model_dump(mode="json"), "overrides_defaults": overrides_defaults}), + existing.updated_at.isoformat() if existing else None, + existing.scope_type if existing else None, + existing.scope_id if existing else None, ) saved: Final = TypeAdapter(tuple[_Record, ...]).validate_python(rows) return _rule(saved[0].record) if saved else None - async def delete_rule(self, rule_id: str) -> None: - await self.database.query_raw( - 'UPDATE "LiteLLM_UsageAlertRule" SET deleted=true,enabled=false WHERE id=$1 RETURNING id', rule_id + async def delete_rule(self, rule_id: str, *, existing: UsageAlertRule | None = None) -> bool: + return bool( + await self.database.query_raw( + """UPDATE "LiteLLM_UsageAlertRule" SET deleted=true,enabled=false + WHERE id=$1 AND NOT deleted AND ($2::text::timestamptz IS NULL OR ( + (record->>'updated_at')::timestamptz=$2::text::timestamptz + AND scope_type=$3 AND scope_id IS NOT DISTINCT FROM $4::text)) RETURNING id""", + rule_id, + existing.updated_at.isoformat() if existing else None, + existing.scope_type if existing else None, + existing.scope_id if existing else None, + ) ) async def list_rules( diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py index 1a4104726fb..42dd4355a53 100644 --- a/litellm/types/proxy/issues.py +++ b/litellm/types/proxy/issues.py @@ -3,6 +3,9 @@ from typing import Literal, Self, TypeAlias from pydantic import BaseModel, ConfigDict, Field, model_validator +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + BoundedDailyActivityResponse, +) from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import ( @@ -14,6 +17,8 @@ from litellm.types.proxy.usage_alerts import ( UsageAlertStatus, ) +IssueActivityResponse: TypeAlias = BoundedDailyActivityResponse + IssueKind: TypeAlias = Literal["performance", "model_deprecation"] IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] @@ -131,6 +136,12 @@ class Issue(IssueInput, frozen=True): updated_at: datetime scope_label: str | None = None active: bool = True + can_update_status: bool | None = Field( + default=None, description="Current reader's status permission, evaluated by the issue-detail endpoint" + ) + can_view_activity: bool | None = Field( + default=None, description="Current reader's permission to inspect Activity, keys, users and impact" + ) class IssueList(BaseModel, frozen=True): diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index c95df8d6840..d3379336b3e 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -2,7 +2,7 @@ from collections.abc import Mapping, Sequence from datetime import date from enum import Enum -from pydantic import ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field from typing_extensions import TypedDict from litellm.types.llms.base import LiteLLMBaseModel @@ -268,3 +268,31 @@ class LiteLLM_DailyUserSpend(LiteLLMBaseModel): class GroupedData(TypedDict): metrics: SpendMetrics breakdown: BreakdownMetrics + + +class DailyActivityModelContribution(BaseModel, frozen=True): + model: str = Field(max_length=1024) + metrics: SpendMetrics + + +class DailyActivityOtherModels(BaseModel, frozen=True): + model_count: int = Field(ge=1) + metrics: SpendMetrics + + +class DailyActivityModelBreakdown(BaseModel, frozen=True): + limit: int = Field(ge=1, le=100) + total_models: int = Field(ge=0) + models: tuple[DailyActivityModelContribution, ...] = Field(max_length=100) + other: DailyActivityOtherModels | None = None + + +class BoundedDailyActivityResponse(BaseModel, frozen=True): + results: tuple[DailySpendData, ...] = Field(max_length=366) + metadata: DailySpendMetadata + model_breakdown: DailyActivityModelBreakdown | None = Field( + default=None, + description="When requested, includes the highest-spend models across the full date range and an explicit " + "aggregate of all remaining models. Daily totals always cover the complete activity scope. " + "Other per-day dimension breakdowns are omitted.", + ) diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index cc967e03e66..d6f7e599683 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -17,7 +17,7 @@ UsageAlertRuleKind: TypeAlias = Literal["all", "default", "custom"] class UsageAlertRuleInput(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) - name: str = Field(min_length=1, max_length=200) + name: str = Field(min_length=1, max_length=200, pattern=r"^[^\x00-\x1f\x7f]+$") scope_type: UsageAlertScope scope_id: str | None = Field(default=None, min_length=1, max_length=1024) metric: UsageAlertMetric @@ -55,6 +55,7 @@ class UsageAlertRuleInput(BaseModel): class UsageAlertRule(UsageAlertRuleInput): + overrides_defaults: bool = Field(default=False, exclude=True) id: str created_at: datetime updated_at: datetime diff --git a/schema.prisma b/schema.prisma index 3a08c094b3d..56098d4bd33 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2110,7 +2110,6 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } -// Daily alerts consume existing aggregate tables; no request-path writes. model LiteLLM_UsageAlertRule { id String @id scope_type String diff --git a/tests/integration/database/test_analytics_budget.py b/tests/integration/database/test_analytics_budget.py new file mode 100644 index 00000000000..9f2a9b5c17a --- /dev/null +++ b/tests/integration/database/test_analytics_budget.py @@ -0,0 +1,82 @@ +import asyncio +from datetime import timedelta +from typing import Final, cast + +import pytest +from integration._support.actors import ManagementDatabase + +from litellm.repositories.analytics_budget import ( + AnalyticsCapacityError, + AnalyticsDatabase, + BoundedAnalyticsDatabase, + analytics_deadline, +) + +_SETTINGS: Final = """SELECT current_setting('statement_timeout') AS statement_timeout, + current_setting('lock_timeout') AS lock_timeout,current_setting('work_mem') AS work_mem, + current_setting('max_parallel_workers_per_gather') AS parallelism""" + + +@pytest.mark.asyncio(loop_scope="session") +async def test_analytics_limits_are_transaction_local_and_admission_is_shared(db: ManagementDatabase) -> None: + database: Final = cast(AnalyticsDatabase, db.prisma.writer_db) + bounded: Final = BoundedAnalyticsDatabase(database) + before: Final = await db.prisma.writer_db.query_raw(_SETTINGS) + limits: Final = await bounded.query_raw( + """SELECT EXTRACT(epoch FROM current_setting('statement_timeout')::interval) AS seconds, + EXTRACT(epoch FROM current_setting('lock_timeout')::interval) AS lock_seconds, + pg_size_bytes(current_setting('work_mem')) AS bytes, + current_setting('max_parallel_workers_per_gather')::int AS parallelism""" + ) + assert 0 < float(str(limits[0]["seconds"])) <= 5 + assert 0 < float(str(limits[0]["lock_seconds"])) <= 0.1 + assert int(str(limits[0]["bytes"])) <= 1048576 + assert limits[0]["parallelism"] == 0 + assert await db.prisma.writer_db.query_raw(_SETTINGS) == before + async with database.tx(max_wait=timedelta(seconds=1), timeout=timedelta(seconds=5)) as transaction: + await transaction.query_raw("SELECT true AS locked FROM pg_advisory_xact_lock(1876539401,0)") + await transaction.query_raw("SELECT true AS locked FROM pg_advisory_xact_lock(1876539401,1)") + with pytest.raises(AnalyticsCapacityError): + await bounded.query_raw("SELECT 1 AS result") + assert (await BoundedAnalyticsDatabase(database, background=True).query_raw("SELECT 1 AS result"))[0][ + "result" + ] == 1 + assert (await bounded.query_raw("SELECT 1 AS result"))[0]["result"] == 1 + + +@pytest.mark.asyncio(loop_scope="session") +async def test_expensive_analytics_are_cancelled_and_release_capacity(db: ManagementDatabase) -> None: + database: Final = cast(AnalyticsDatabase, db.prisma.writer_db) + bounded: Final = BoundedAnalyticsDatabase(database) + before: Final = await db.prisma.writer_db.query_raw(_SETTINGS) + with pytest.raises(AnalyticsCapacityError): + await bounded.query_raw("SELECT 1 AS result FROM pg_sleep(30) /* bounded_analytics_timeout */") + active: Final = await db.prisma.writer_db.query_raw( + """SELECT count(*)::int AS running FROM pg_stat_activity + WHERE pid<>pg_backend_pid() AND state='active' AND query LIKE $1""", + "%bounded_analytics_timeout%", + ) + assert active[0]["running"] == 0 + assert (await bounded.query_raw("SELECT 1 AS result"))[0]["result"] == 1 + assert await db.prisma.writer_db.query_raw(_SETTINGS) == before + + +@pytest.mark.asyncio(loop_scope="session") +async def test_work_deadline_cancels_inflight_sql_and_restores_capacity(db: ManagementDatabase) -> None: + database: Final = cast(AnalyticsDatabase, db.prisma.writer_db) + bounded: Final = BoundedAnalyticsDatabase(database, background=True) + loop: Final = asyncio.get_running_loop() + start: Final = loop.time() + with analytics_deadline(start + 0.25), analytics_deadline(start + 10): + with pytest.raises(AnalyticsCapacityError): + await bounded.query_raw("SELECT 1 FROM pg_sleep(30) /* bounded_analytics_deadline */") + assert loop.time() - start < 2 + active: Final = await db.prisma.writer_db.query_raw( + """SELECT count(*)::int AS running FROM pg_stat_activity + WHERE pid<>pg_backend_pid() AND state='active' AND query LIKE $1""", + "%bounded_analytics_deadline%", + ) + assert active[0]["running"] == 0 + with analytics_deadline(loop.time() - 1), pytest.raises(AnalyticsCapacityError): + await bounded.query_raw("SELECT 1 AS result") + assert (await bounded.query_raw("SELECT 1 AS result"))[0]["result"] == 1 diff --git a/tests/integration/database/test_bounded_daily_activity.py b/tests/integration/database/test_bounded_daily_activity.py new file mode 100644 index 00000000000..f014018c544 --- /dev/null +++ b/tests/integration/database/test_bounded_daily_activity.py @@ -0,0 +1,160 @@ +from collections.abc import Mapping +from datetime import date, timedelta +from hashlib import sha256 +from typing import Final +from uuid import uuid4 + +import psycopg +import pytest +from integration._support.actors import ManagementDatabase +from integration._support.client import Gateway +from psycopg import sql +from pydantic import TypeAdapter + +from litellm.repositories.bounded_daily_activity import BoundedDailyActivityRepository +from litellm.types.proxy.management_endpoints.common_daily_activity import BoundedDailyActivityResponse + + +@pytest.mark.asyncio(loop_scope="session") +async def test_bounded_activity_routes_keep_exact_totals_and_scoped_permissions( + gateway: Gateway, db: ManagementDatabase +) -> None: + repository: Final = BoundedDailyActivityRepository.from_prisma(db.prisma) + identity: Final = uuid4().hex + day: Final = date.today() - timedelta(days=3) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + member: Final = scenario.user(user_role="internal_user") + team: Final = scenario.team( + members_with_roles=[{"user_id": owner, "role": "admin"}, {"user_id": member, "role": "user"}] + ) + owner_key: Final = scenario.key(user_id=owner, team_id=team, key_type="management") + member_key: Final = scenario.key(user_id=member, team_id=team, key_type="management") + reporting_key: Final = sha256(owner_key.encode()).hexdigest() + member_reporting_key: Final = sha256(member_key.encode()).hexdigest() + for table, entity, column in ( + ("LiteLLM_DailyUserSpend", owner, "user_id"), + ("LiteLLM_DailyTeamSpend", team, "team_id"), + ): + await repository.database.query_raw( + f'''INSERT INTO "{table}" (id,{column},date,api_key,model,custom_llm_provider, + spend,api_requests,successful_requests,total_response_time_ms,timed_requests,updated_at) + SELECT $1::text||'-'||m||'-'||d,$2::text,($3::date+d)::text,$4::text, + CASE WHEN m=1 THEN 'Other' WHEN m=137 THEN repeat('x',1025) + ELSE 'example-model-'||m END,'example-provider', + CASE WHEN m=1 THEN 10000 ELSE m END,10,10,2000,10,CURRENT_TIMESTAMP + FROM generate_series(1,137) m CROSS JOIN generate_series(0,2) d''', + identity, + entity, + day.isoformat(), + reporting_key, + ) + await repository.database.query_raw( + """INSERT INTO "LiteLLM_DailyTeamSpend" + (id,team_id,date,api_key,model,api_requests,successful_requests,updated_at) + VALUES($1,$2,$3,$4,'member-model',7,7,CURRENT_TIMESTAMP)""", + identity + "-member", + team, + day.isoformat(), + member_reporting_key, + ) + query: Final = {"start_date": day.isoformat(), "end_date": (day + timedelta(days=2)).isoformat()} + await repository.database.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,api_requests,successful_requests,updated_at) + VALUES($1,$2,$3,$4,'today-model',11,11,CURRENT_TIMESTAMP)""", + identity + "-today", + owner, + date.today().isoformat(), + reporting_key, + ) + try: + path: Final = "/user/daily/activity/aggregated" + legacy: Final = gateway.request("GET", path, params={**query, "api_key": reporting_key}) + assert legacy.status_code == 200 and "model_breakdown" not in legacy.json() + result: Final = gateway.request( + "GET", + path, + params={**query, "api_key": reporting_key, "bounded": "true", "include_model_breakdown": "true"}, + key=owner_key, + ) + assert result.status_code == 200, result.text + bounded: Final = BoundedDailyActivityResponse.model_validate(result.json()) + assert bounded.metadata.model_dump(exclude={"api_key_limit"}) == { + key: value for key, value in legacy.json()["metadata"].items() if key != "api_key_limit" + } + models: Final = bounded.model_breakdown + assert models is not None and models.total_models == 137 and len(models.models) == 20 + assert models.models[0].model == "Other" and models.other is not None and models.other.model_count == 117 + assert all(len(item.model) <= 1024 for item in models.models) + assert ( + sum(item.metrics.spend for item in models.models) + models.other.metrics.spend + == bounded.metadata.total_spend + ) + assert len(result.content) < len(legacy.content) / 10 + assert bounded.metadata.total_api_requests == 4110 and len(bounded.results) == 3 + largest: Final = gateway.request( + "GET", + path, + params={ + **query, + "api_key": reporting_key, + "bounded": "true", + "include_model_breakdown": "true", + "model_limit": "100", + }, + ) + assert largest.status_code == 200 and len(largest.json()["model_breakdown"]["models"]) == 100 + assert ( + gateway.request("GET", path, params={**query, "bounded": "true", "model_limit": "101"}).status_code + == 422 + ) + assert ( + gateway.request( + "GET", + path, + params={**query, "bounded": "true", "end_date": (day + timedelta(days=366)).isoformat()}, + ).status_code + == 422 + ) + team_query: Final = {**query, "team_ids": team, "bounded": "true"} + team_path: Final = "/team/daily/activity/aggregated" + team_admin: Final = gateway.request("GET", team_path, params=team_query, key=owner_key) + team_member: Final = gateway.request("GET", team_path, params=team_query, key=member_key) + assert team_admin.status_code == team_member.status_code == 200 + assert team_admin.json()["metadata"]["total_api_requests"] == 4117 + assert team_member.json()["metadata"]["total_api_requests"] == 7 + assert team_admin.json()["model_breakdown"] is None + for include_today in ("true", "false"): + today: Final = { + "start_date": date.today().isoformat(), + "end_date": date.today().isoformat(), + "api_key": reporting_key, + "timezone": "0", + "include_current_utc_day": include_today, + } + legacy_today: Final = gateway.request("GET", path, params=today) + bounded_today: Final = gateway.request("GET", path, params={**today, "bounded": "true"}) + assert legacy_today.status_code == bounded_today.status_code == 200 + assert ( + legacy_today.json()["metadata"]["total_api_requests"] + == bounded_today.json()["metadata"]["total_api_requests"] + ) + if include_today == "true": + assert bounded_today.json()["metadata"]["total_api_requests"] == 11 + schemas: Final = TypeAdapter(tuple[Mapping[str, str], ...]).validate_python( + await repository.database.query_raw("SELECT current_schema() AS name") + ) + assert gateway.database_url is not None + with psycopg.connect(gateway.database_url) as blocker: + blocker.execute( + sql.SQL("LOCK TABLE {} IN ACCESS EXCLUSIVE MODE").format( + sql.Identifier(schemas[0]["name"], "LiteLLM_DailyUserSpend") + ) + ) + busy: Final = gateway.request("GET", path, params={**query, "bounded": "true"}) + assert busy.status_code == 503 and busy.headers["Retry-After"] == "5" + assert gateway.request("GET", path, params={**query, "bounded": "true"}).status_code == 200 + finally: + await repository.database.query_raw('DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1', owner) + await repository.database.query_raw('DELETE FROM "LiteLLM_DailyTeamSpend" WHERE team_id=$1', team) diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index 4fa58c18adb..98530a6f3a9 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Mapping, Sequence from contextlib import asynccontextmanager from datetime import date, datetime, timedelta, timezone from hashlib import sha256 @@ -17,10 +17,13 @@ from prisma.types import DatasourceOverride from psycopg import sql from pydantic import BaseModel, TypeAdapter, ValidationError +from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase +from litellm.repositories.daily_activity_sql import build_bounded_daily_activity_sql from litellm.repositories.issue_activity import IssueActivityRepository from litellm.repositories.issues import IssuesRepository from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( + IssueActivityResponse, IssueCohort, IssueComparison, IssueDeliveryInput, @@ -92,6 +95,7 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa outsider: Final = scenario.user(user_role="internal_user") viewer: Final = scenario.user(user_role="proxy_admin_viewer") owner_key: Final = scenario.key(user_id=owner, key_type="management") + info_key: Final = scenario.key(user_id=owner, key_type="default", allowed_routes=["info_routes"]) outsider_key: Final = scenario.key(user_id=outsider, key_type="management") viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") inference_key: Final = scenario.key(user_id=owner, key_type="llm_api", key_alias="Example checkout") @@ -152,6 +156,16 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa path: Final = f"/issues/{first.id}" own: Final = gateway.request("GET", path, key=owner_key) assert own.status_code == 200 and own.json()["scope_label"] == "Example checkout" + assert own.json()["can_update_status"] is True + assert own.json()["can_view_activity"] is True + assert gateway.request("GET", path, key=viewer_key).json()["can_update_status"] is False + assert gateway.request("GET", path, key=info_key).json()["can_update_status"] is False + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=info_key + ).status_code + == 403 + ) assert "99999" not in own.text and "9999" not in own.text pages: Final = tuple( gateway.request( @@ -204,6 +218,159 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_detail_reports_team_status_permission(gateway: Gateway, db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + with gateway.scenario() as scenario: + admin: Final = scenario.user(user_role="internal_user") + member: Final = scenario.user(user_role="internal_user") + team: Final = scenario.team( + members_with_roles=[{"user_id": admin, "role": "admin"}, {"user_id": member, "role": "user"}] + ) + permitted_team: Final = scenario.team( + members_with_roles=[{"user_id": member, "role": "user"}], + team_member_permissions=["/team/daily/activity"], + ) + admin_key: Final = scenario.key(user_id=admin, key_type="management") + member_key: Final = scenario.key(user_id=member, key_type="management") + service_key: Final = scenario.key( + team_id=team, + user_id=None, + key_type="management", + metadata={"service_account_id": "example-activity-service"}, + ) + issue: Final = _issue(IssueCohort(scope_type="team", scope_id=team)) + permitted_issue: Final = _issue(IssueCohort(scope_type="team", scope_id=permitted_team)) + try: + await store.upsert_issues((issue, permitted_issue)) + path: Final = f"/issues/{issue.id}" + assert gateway.request("GET", path, key=admin_key).json()["can_update_status"] is True + assert gateway.request("GET", path, key=admin_key).json()["can_view_activity"] is True + assert gateway.request("GET", path, key=member_key).json()["can_update_status"] is False + assert gateway.request("GET", path, key=member_key).json()["can_view_activity"] is False + for suffix in ("/activity", "/keys", "/users", "/impact"): + assert gateway.request("GET", path + suffix, key=member_key).status_code == 403 + assert gateway.request("GET", path + suffix, key=service_key).status_code == 403 + for suffix in ("/activity", "/keys", "/impact"): + assert gateway.request("GET", path + suffix, key=admin_key).status_code == 200 + assert ( + gateway.request("GET", f"/issues/{permitted_issue.id}" + suffix, key=member_key).status_code == 200 + ) + assert ( + gateway.request("GET", f"/issues/{permitted_issue.id}", key=member_key).json()["can_view_activity"] + is True + ) + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=member_key + ).status_code + == 403 + ) + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=admin_key + ).status_code + == 200 + ) + finally: + await _remove_issues(db, (issue.id, permitted_issue.id)) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gateway, db: ManagementDatabase) -> None: + from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated + from litellm.repositories.issue_activity import issue_activity_scope + + repository: Final = IssueActivityRepository.from_prisma(db.prisma) + store: Final = IssuesRepository.from_prisma(db.prisma) + identity: Final = str(uuid4()) + key: Final = sha256(identity.encode()).hexdigest() + provider: Final = f"example-provider-{identity}" + day: Final = date.today() - timedelta(days=5) + model_count: Final = 137 + cohort: Final = IssueCohort(scope_type="key", scope_id=key) + issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day + timedelta(days=2)}) + await _query( + db, + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,model_group,custom_llm_provider,mcp_namespaced_tool_name,endpoint, + api_requests,spend,prompt_tokens,completion_tokens,successful_requests,failed_requests, + total_response_time_ms,timed_requests,updated_at) + SELECT $1||'-'||m||'-'||d,$1,($2::date+d::int)::text,$3, + CASE WHEN m=0 THEN 'Other' ELSE 'example-model-'||lpad(m::text,4,'0') END, + 'group-'||m,$4,'tool-'||m,'endpoint-'||m, + m+2,CASE WHEN m=0 AND d=0 THEN 10000 ELSE m+1 END,m+1,2,m+1,1,(m+1)*200,m+1,CURRENT_TIMESTAMP + FROM generate_series(0,$5::int-1) m CROSS JOIN generate_series(0,2) d RETURNING id""", + identity, + day.isoformat(), + key, + provider, + model_count, + ) + try: + await store.upsert_issue(issue) + scope: Final = issue_activity_scope(cohort, day, day + timedelta(days=2)) + original: Final = await get_daily_activity_aggregated(repository.activity, scope, api_key_limit=20) + response: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + assert response.status_code == 200 + bounded: Final = IssueActivityResponse.model_validate(response.json()) + assert bounded.metadata.model_dump(exclude={"api_key_limit"}) == original.metadata.model_dump( + exclude={"api_key_limit"} + ) + assert tuple((row.date, row.metrics) for row in bounded.results) == tuple( + (row.date, row.metrics) for row in original.results + ) + models: Final = bounded.model_breakdown + assert models is not None and models.total_models == model_count and len(models.models) == models.limit == 20 + assert models.models[0].model == "Other" and models.models[0].metrics.spend == 10002 + assert models.other is not None and models.other.model_count == model_count - models.limit + components: Final = (*tuple(row.metrics for row in models.models), models.other.metrics) + assert sum(row.spend for row in components) == bounded.metadata.total_spend + assert sum(row.api_requests for row in components) == bounded.metadata.total_api_requests + assert sum(row.total_response_time_ms for row in components) == bounded.metadata.total_response_time_ms + assert all(not row.breakdown.models and not row.breakdown.endpoints for row in bounded.results) + query: Final = build_bounded_daily_activity_sql(scope, model_limit=100) + raw: Final = TypeAdapter(tuple[object, ...]).validate_python( + await repository.database.query_raw(query.sql, *query.params) + ) + assert len(raw) == 3 + 1 + 100 + 1 + largest: Final = gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "100"}) + assert largest.status_code == 200 + large_models: Final = IssueActivityResponse.model_validate(largest.json()).model_breakdown + assert large_models is not None and len(large_models.models) == 100 + assert large_models.other is not None and large_models.other.model_count == model_count - 100 + assert gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "101"}).status_code == 422 + assert ( + gateway.request( + "GET", + f"/issues/{issue.id}/activity", + params={"start_date": day.isoformat(), "end_date": (day + timedelta(days=366)).isoformat()}, + ).status_code + == 422 + ) + non_key: Final = await repository.daily_activity( + IssueCohort(scope_type="provider", scope_id=provider), day, day + timedelta(days=2) + ) + assert non_key.model_breakdown is None and non_key.metadata == bounded.metadata + assert len(non_key.results) == 3 + schemas: Final = TypeAdapter(tuple[Mapping[str, str], ...]).validate_python( + await _query(db, "SELECT current_schema() AS name") + ) + assert gateway.database_url is not None + with psycopg.connect(gateway.database_url) as blocker: + blocker.execute( + sql.SQL("LOCK TABLE {} IN ACCESS EXCLUSIVE MODE").format( + sql.Identifier(schemas[0]["name"], "LiteLLM_DailyUserSpend") + ) + ) + busy: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + assert busy.status_code == 503 and busy.headers["Retry-After"] == "5" + assert gateway.request("GET", f"/issues/{issue.id}/activity").status_code == 200 + finally: + await _remove_issues(db, (issue.id,)) + await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1 RETURNING id', identity) + + @pytest.mark.asyncio(loop_scope="session") async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( gateway: Gateway, db: ManagementDatabase @@ -332,12 +499,6 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma next_issue_id: Final = str(uuid4()) try: source: Final = (await alerts.record_observations(rule, (observation,)))[0] - await _query( - db, - 'UPDATE "LiteLLM_UsageAlertIncident" SET evaluation_revision=$2 WHERE id=$1 RETURNING id', - source.id, - rule.updated_at.isoformat(), - ) initial: Final = await store.upsert_issue(original) assert tuple( item.alert.id @@ -359,12 +520,8 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma assert disabled is not None and await store.pending_delivery(delivery.id) is None enabled: Final = await alerts.save_rule(definition, "qa-admin", disabled) assert enabled is not None and await store.pending_delivery(delivery.id) is None - await _query( - db, - 'UPDATE "LiteLLM_UsageAlertIncident" SET evaluation_revision=$2 WHERE id=$1 RETURNING id', - source.id, - enabled.updated_at.isoformat(), - ) + reevaluated: Final = await alerts.record_observations(enabled, (observation,)) + assert len(reevaluated) == 1 and reevaluated[0].id == source.id await store.queue_delivery(delivery, enabled.id, enabled.updated_at.isoformat(), source.id) assert await store.pending_delivery(delivery.id) is not None await alerts.set_status(source.id, "dismissed") @@ -654,9 +811,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - expired: Final = _issue( - IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}") - ) + expired: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) open_issue: Final = _issue(expired.cohort) recent: Final = _issue(expired.cohort) rule: Final = await alerts.save_rule( @@ -671,8 +826,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) assert rule is not None deliveries: Final = tuple( - IssueDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") - for _ in range(2) + IssueDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") for _ in range(2) ) try: await store.upsert_issues((expired, open_issue, recent)) @@ -693,16 +847,10 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( )[0] await store.add_signals( expired.id, - ( - IssueSignalInput( - source="usage_alert", source_id=source.id, usage_alert=source - ), - ), + (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),), ) for delivery in deliveries: - await store.queue_delivery( - delivery, rule.id, rule.updated_at.isoformat(), source.id - ) + await store.queue_delivery(delivery, rule.id, rule.updated_at.isoformat(), source.id) assert await store.set_status(expired.id, "resolved", "open") is not None assert await store.set_status(recent.id, "resolved", "open") is not None await _query( @@ -742,3 +890,162 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id, ) + + +class _MeasuredActivityDatabase: + def __init__(self, database: DailyActivityQueryDatabase) -> None: + self.database = database + self.outputs: tuple[tuple[int, int], ...] = () + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]] | None: + result: Final = await self.database.query_raw(query, *args) + self.outputs = (*self.outputs, (len(result or ()), len(json.dumps(result, default=str)))) + return result + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_resource_metadata_bounds_labels_and_deleted_history( + gateway: Gateway, db: ManagementDatabase +) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + activity: Final = IssueActivityRepository.from_prisma(db.prisma) + measured: Final = _MeasuredActivityDatabase(activity.database) + repository: Final = IssueActivityRepository(activity.activity, measured) + identity: Final = uuid4().hex + reporting_key: Final = sha256(identity.encode()).hexdigest() + day: Final = date.today() - timedelta(days=3) + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + issue: Final = _issue(IssueCohort(scope_type="user", scope_id=owner)).model_copy( + update={"range_start": day, "range_end": day} + ) + try: + await _query( + db, + """UPDATE "LiteLLM_UserTable" SET user_alias=repeat('u',1024), + user_email=repeat('m',400)||'@example.com' WHERE user_id=$1""", + owner, + ) + await _query( + db, + """INSERT INTO "LiteLLM_DeletedVerificationToken" + (id,token,key_alias,user_id,models,metadata,deleted_at) + SELECT $1::text||'-'||n,$2::text,'archived-'||n||'-'||repeat('x',1024),$3::text, + ARRAY[]::text[],jsonb_build_object('padding',repeat('x',65536)), + CURRENT_TIMESTAMP+n*INTERVAL '1 second' FROM generate_series(1,400) n""", + identity, + reporting_key, + owner, + ) + await _query( + db, + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,api_requests,successful_requests,updated_at) + VALUES($1,$2,$3,$4,'example-model',10,10,CURRENT_TIMESTAMP)""", + identity, + owner, + day.isoformat(), + reporting_key, + ) + await store.upsert_issue(issue) + keys: Final = await repository.keys(issue.cohort, day, day, 1, 0) + assert keys.total_api_keys == 1 and len(keys.api_keys) == 1 + key: Final = keys.api_keys[0] + assert key.api_key == reporting_key and key.metadata.key_exists is False + assert key.metadata.key_alias is not None and key.metadata.key_alias.startswith("archived-400-") + assert len(key.metadata.key_alias) == 256 and key.metadata.user_id == owner + assert key.metadata.user_email is None + assert measured.outputs[-1][0] == 1 and measured.outputs[-1][1] < 2048 + users: Final = await repository.users(issue.cohort, day, day, 1, 0) + assert users.total_users == 1 and len(users.users) == 1 + assert users.users[0].user_id == owner and users.users[0].user_alias == "u" * 256 + assert users.users[0].user_email is None + assert measured.outputs[-1][0] == 1 and measured.outputs[-1][1] < 2048 + for endpoint in ("keys", "users"): + response: Final = gateway.request("GET", f"/issues/{issue.id}/{endpoint}", params={"limit": "1"}) + assert response.status_code == 200 and len(response.content) < 2048 + assert "padding" not in response.text + finally: + await _remove_issues(db, (issue.id,)) + await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id=$1', identity) + await _query(db, 'DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', reporting_key) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + definition: Final = UsageAlertRuleInput( + name="Primary traffic", + scope_type="user", + scope_id=issue.cohort.scope_id, + metric="requests", + direction="increase", + ) + first: Final = await alerts.save_rule(definition, "qa-admin") + second: Final = await alerts.save_rule(definition.model_copy(update={"name": "Another traffic policy"}), "qa-admin") + assert first is not None and second is not None + breach: Final = UsageAlertObservation( + entity_id=issue.cohort.scope_id, + day=issue.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ) + try: + sources: Final = tuple([(await alerts.record_observations(rule, (breach,)))[0] for rule in (first, second)]) + await store.upsert_issue(issue) + await store.add_signals( + issue.id, + tuple( + IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source) for source in sources + ), + ) + await alerts.record_observations( + first, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=1), + "observed": 120, + "breached": False, + } + ), + ), + ) + await alerts.record_observations( + second, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=2), + } + ), + ), + ) + assert await alerts.save_rule(definition.model_copy(update={"enabled": False}), "qa-admin", second) + await store.recover_issues(issue.observed_on + timedelta(days=3)) + unresolved: Final = await store.get_issue(issue.id) + assert unresolved is not None and unresolved.active + await alerts.record_observations( + first, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=3), + "observed": 120, + "breached": False, + } + ), + ), + ) + await store.recover_issues(issue.observed_on + timedelta(days=3)) + recovered: Final = await store.get_issue(issue.id) + assert recovered is not None and not recovered.active + finally: + await _remove_issues(db, (issue.id,)) + for rule in (first, second): + await alerts.delete_rule(rule.id) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) diff --git a/tests/integration/database/test_usage_alerts.py b/tests/integration/database/test_usage_alerts.py index 51978152ed2..e5990d7eebe 100644 --- a/tests/integration/database/test_usage_alerts.py +++ b/tests/integration/database/test_usage_alerts.py @@ -94,6 +94,82 @@ def test_team_admin_can_manage_team_alerts_but_member_can_only_read(gateway: Gat assert gateway.request("DELETE", path, key=admin_key).status_code == 204 +def test_info_only_credentials_cannot_mutate_alerts(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + admin: Final = scenario.user(user_role="proxy_admin") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + reader: Final = scenario.key(user_id=owner, key_type="default", allowed_routes=["info_routes"]) + admin_reader: Final = scenario.key(user_id=admin, key_type="default", allowed_routes=["info_routes"]) + config: Final = { + "name": "Read-only route grant", + "scope_type": "user", + "scope_id": owner, + "metric": "spend", + "direction": "increase", + } + created: Final = gateway.request("POST", "/usage/alert-rules", config, key=owner_key) + assert created.status_code == 201 + path: Final = f"/usage/alert-rules/{created.json()['id']}" + try: + assert gateway.request("GET", path, key=reader).status_code == 200 + assert gateway.request("POST", "/usage/alert-rules", config, key=reader).status_code == 403 + assert gateway.request("PUT", path, config, key=reader).status_code == 403 + assert gateway.request("DELETE", path, key=reader).status_code == 403 + assert ( + gateway.request("PATCH", "/issues/families/spend", {"enabled": False}, key=admin_reader).status_code + == 403 + ) + assert ( + gateway.request( + "PATCH", "/issues/unavailable", {"status": "resolved", "expected_status": "open"}, key=admin_reader + ).status_code + == 403 + ) + finally: + assert gateway.request("DELETE", path, key=owner_key).status_code == 204 + + +def test_only_admins_can_configure_operator_notifications(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner: Final = scenario.user(user_role="internal_user") + owner_key: Final = scenario.key(user_id=owner, key_type="management") + config: Final = { + "name": "Operator delivery", + "scope_type": "user", + "scope_id": owner, + "metric": "spend", + "direction": "increase", + } + for notification in ( + {"notification_channels": ["slack"]}, + {"notification_channels": ["email"], "email_recipients": ["ops@example.com"]}, + ): + assert ( + gateway.request("POST", "/usage/alert-rules", {**config, **notification}, key=owner_key).status_code + == 403 + ) + assert ( + gateway.request( + "POST", "/usage/alert-rules", {**config, "overrides_defaults": True}, key=owner_key + ).status_code + == 422 + ) + assert ( + gateway.request("POST", "/usage/alert-rules", {**config, "name": "Spend\r\nInjected: header"}).status_code + == 422 + ) + created: Final = gateway.request("POST", "/usage/alert-rules", {**config, "notification_channels": ["slack"]}) + assert created.status_code == 201 + assert "overrides_defaults" not in created.json() + path: Final = f"/usage/alert-rules/{created.json()['id']}" + try: + assert gateway.request("PUT", path, config, key=owner_key).status_code == 403 + assert gateway.request("DELETE", path, key=owner_key).status_code == 403 + finally: + assert gateway.request("DELETE", path).status_code == 204 + + @pytest.mark.asyncio(loop_scope="session") async def test_incident_http_read_dismiss_and_scope_authorization(gateway: Gateway, db: ManagementDatabase) -> None: """Populate an explicit storage fixture, then exercise the real incident HTTP API.""" @@ -210,6 +286,36 @@ async def test_ongoing_episode_preserves_its_original_evidence_and_activity_link await repository.delete_rule(rule.id) +@pytest.mark.asyncio(loop_scope="session") +async def test_rule_scope_changes_reject_stale_updates_and_deletes(db: ManagementDatabase) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + definition: Final = UsageAlertRuleInput( + name="Concurrent scope edit", scope_type="user", scope_id="original-owner", metric="spend", direction="increase" + ) + original: Final = await repository.save_rule(definition, "qa-admin") + assert original is not None + try: + moved: Final = await repository.save_rule( + definition.model_copy(update={"scope_id": "different-owner"}), "qa-admin", existing=original + ) + assert moved is not None + assert await repository.save_rule(definition, "original-owner", existing=original) is None + assert not await repository.delete_rule(original.id, existing=original) + assert await repository.get_rule(original.id) == moved + current: Final = await repository.save_rule( + definition.model_copy(update={"scope_id": moved.scope_id, "name": "Current authorized edit"}), + "qa-admin", + existing=moved, + ) + assert current is not None + assert not await repository.delete_rule(original.id, existing=moved) + assert await repository.delete_rule(original.id, existing=current) + assert await repository.save_rule(definition, "qa-admin", existing=current) is None + assert await repository.get_rule(original.id) is None + finally: + await repository.delete_rule(original.id) + + @pytest.mark.asyncio(loop_scope="session") @pytest.mark.parametrize( "changed_field,new_value", diff --git a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py index e8221599f8b..9d8906eb4e0 100644 --- a/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py +++ b/tests/unit/proxy/alerting_endpoints/test_usage_alerts.py @@ -6,7 +6,7 @@ import pytest from fastapi import HTTPException from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.alerting_endpoints.usage_alerts import require_scope +from litellm.proxy.alerting_endpoints.usage_alerts import require_scope, require_write_access from litellm.repositories.usage_alerts import UsageAlertsRepository, activity_path from litellm.types.proxy.auth.user_roles import LitellmUserRoles @@ -16,6 +16,21 @@ class NoDatabase: raise AssertionError("Unauthorized callers must not reach the database") +@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER]) +@pytest.mark.parametrize("key_type", [None, "default", "management"]) +def test_info_grants_do_not_authorize_mutations(role: LitellmUserRoles, key_type: str | None) -> None: + user: Final = UserAPIKeyAuth(user_role=role, key_type=key_type, allowed_routes=["info_routes", "openai_routes"]) + with pytest.raises(HTTPException) as error: + require_write_access(user, "/usage/alert-rules/example") + assert error.value.status_code == 403 + + +@pytest.mark.parametrize("grant", ["/usage/alert-rules/example", "/usage/*", "management_routes"]) +def test_explicit_management_grants_still_authorize_mutations(grant: str) -> None: + user: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, allowed_routes=["info_routes", grant]) + require_write_access(user, "/usage/alert-rules/example") + + @pytest.mark.asyncio @pytest.mark.parametrize( "role,key_type,write", diff --git a/tests/unit/repositories/test_config_repository.py b/tests/unit/repositories/test_config_repository.py index 0856aebf0ff..6c37a78b6ec 100644 --- a/tests/unit/repositories/test_config_repository.py +++ b/tests/unit/repositories/test_config_repository.py @@ -1,4 +1,5 @@ from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock import pytest @@ -7,16 +8,16 @@ from litellm.repositories.config_repository import ConfigRepository @pytest.mark.asyncio -async def test_batch_reads_decode_legacy_json_and_current_objects_from_writer(): - find_many = AsyncMock( +async def test_batch_reads_decode_legacy_json_and_current_objects_from_writer() -> None: + find_many: Final = AsyncMock( return_value=( SimpleNamespace(param_name="first", param_value='{"status":"resolved"}'), SimpleNamespace(param_name="second", param_value={"status": "dismissed"}), ) ) - prisma = SimpleNamespace(writer_db=SimpleNamespace(litellm_config=SimpleNamespace(find_many=find_many))) - repository = ConfigRepository(prisma, use_writer=True) - rows = await repository.get_params(("first", "second", "absent")) + prisma: Final = SimpleNamespace(writer_db=SimpleNamespace(litellm_config=SimpleNamespace(find_many=find_many))) + repository: Final = ConfigRepository(prisma, use_writer=True) + rows: Final = await repository.get_params(("first", "second", "absent")) assert {row.param_name: row.param_value for row in rows} == { "first": {"status": "resolved"}, "second": {"status": "dismissed"}, diff --git a/tests/unit/types/proxy/test_usage_alerts.py b/tests/unit/types/proxy/test_usage_alerts.py index 57d7e133438..3ee4618d619 100644 --- a/tests/unit/types/proxy/test_usage_alerts.py +++ b/tests/unit/types/proxy/test_usage_alerts.py @@ -27,6 +27,9 @@ RULE: Final = { {"notification_channels": ["email"]}, {"notification_channels": ["slack", "slack"]}, {"email_recipients": ["ops@example.com\r\nBcc: other@example.com"]}, + {"name": "Spend\r\nBcc: other@example.com"}, + {"name": "Spend\x00notification"}, + {"overrides_defaults": True}, ], ) def test_invalid_rules_cannot_be_stored(changes: dict[str, object]) -> None: diff --git a/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.integration.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.integration.test.tsx index 46c179889c9..dbb31d34cb6 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.integration.test.tsx @@ -57,7 +57,8 @@ describe("entity activity", () => { expect(url.searchParams.get("start_date")).toBe("2026-10-01"); expect(url.searchParams.get("end_date")).toBe("2026-10-07"); expect(url.searchParams.get("timezone")).toBe("0"); - expect(url.searchParams.get("api_key_limit")).toBe("1"); + expect(url.searchParams.get("bounded")).toBe("true"); + expect(url.searchParams.get("include_model_breakdown")).toBe(scope === "key" ? "true" : null); expect(url.searchParams.has("alert")).toBe(false); }); @@ -68,6 +69,17 @@ describe("entity activity", () => { expect(urls()[0].searchParams.get("user_id")).toBe("viewer"); }); + it("loads a member's key through the authorized team scope without filtering to the viewer", async () => { + auth.userRole = "Internal User"; + renderWithProviders(, { searchParams: dates }); + await screen.findByText("No activity in this date range."); + expect(urls()).toHaveLength(1); + expect(urls()[0].pathname).toBe("/team/daily/activity/aggregated"); + expect(urls()[0].searchParams.get("team_ids")).toBe("team-1"); + expect(urls()[0].searchParams.get("api_key")).toBe("member-key"); + expect(urls()[0].searchParams.has("user_id")).toBe(false); + }); + it("does not query global model activity for a member", async () => { auth.userRole = "Internal User"; renderWithProviders(, { searchParams: dates }); diff --git a/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.tsx b/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.tsx index f0916847fae..d046599e868 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/EntityActivity.tsx @@ -9,16 +9,17 @@ import { useTranslation } from "react-i18next"; import { Button } from "@/components/ui/button"; import { hasProxyWideSpendView, spendScopeUserId } from "@/utils/roles"; import type { ActivityScope } from "@/utils/entityLinks"; -import { EMPTY_DAILY_ACTIVITY_METADATA, toDailyData, type DailyActivityRequest } from "./dailyActivityApi"; -import { keyDetailFromResponse, overallUsageMetrics, reportingKeyId } from "./keyActivityData"; +import type { DailyActivityRequest } from "./dailyActivityApi"; +import { boundedActivityMetrics } from "./keyActivityData"; import { activityPresets, defaultActivityDays, entityActivityParsers, localActivityDay } from "./entityActivityState"; interface Props { scope: ActivityScope; id: string; + teamId?: string; } -export default function EntityActivity({ scope, id }: Props) { +export default function EntityActivity({ scope, id, teamId }: Props) { const { t } = useTranslation(); const { accessToken, userId, userRole } = useAuthorized(); const [params, setParams] = useQueryStates(entityActivityParsers); @@ -28,9 +29,12 @@ export default function EntityActivity({ scope, id }: Props) { const invalidRange = from > to; const allowed = scope !== "model" || hasProxyWideSpendView(userRole); const currentUser = spendScopeUserId(userRole, userId); + const keyTeamId = scope === "key" && currentUser ? teamId : undefined; + const entity = scope === "team" || keyTeamId ? "team" : "user"; const request = useMemo(() => { let entityIds = currentUser ? [currentUser] : null; if (scope === "team" || scope === "user") entityIds = [id]; + if (keyTeamId) entityIds = [keyTeamId]; return { accessToken: accessToken ?? "", startTime: new Date(`${from}T00:00:00`), @@ -39,24 +43,23 @@ export default function EntityActivity({ scope, id }: Props) { entityIds, apiKey: scope === "key" ? id : null, model: scope === "model" ? id : null, - apiKeyLimit: 1, + bounded: true, + includeModelBreakdown: scope === "key", includeCurrentUtcDay: true, }; - }, [accessToken, currentUser, scope, id, from, to]); + }, [accessToken, currentUser, scope, id, keyTeamId, from, to]); const hasScope = Boolean(accessToken && id); const { data, loading, failed, reload } = useAggregatedDailyActivity({ - fetch: () => ENTITY_API[scope === "team" ? "team" : "user"].aggregated(request), + fetch: () => ENTITY_API[entity].aggregated(request), enabled: hasScope && allowed && !invalidRange, - deps: [accessToken, currentUser, scope, id, from, to], + deps: [accessToken, currentUser, scope, id, keyTeamId, from, to], }); const fetchTopApiKeys = useCallback( (model: string) => ENTITY_API.user.modelTopKeys(request, model, false), [request], ); - const metrics = - (scope === "key" ? keyDetailFromResponse(data, reportingKeyId(id), []) : undefined) ?? - overallUsageMetrics(toDailyData(data), data.metadata ?? EMPTY_DAILY_ACTIVITY_METADATA); + const { metrics, otherModels } = boundedActivityMetrics(data); if (!allowed) return

{t("entityActivity.modelRequiresGlobal")}

; @@ -87,6 +90,7 @@ export default function EntityActivity({ scope, id }: Props) { ); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.tsx index 6295cfbfc32..77521ed619c 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.tsx @@ -7,17 +7,20 @@ import { formatNumberWithCommas } from "@/utils/dataUtils"; import type { ColumnDef } from "@tanstack/react-table"; import React, { useState } from "react"; import { useTranslation } from "react-i18next"; -import { TopModelData } from "../types"; +import { OtherModelData, TopModelData } from "../types"; interface KeyModelUsageViewProps { topModels: TopModelData[]; + otherModels?: OtherModelData; } +type ModelUsageRow = TopModelData & { id: string }; + const VISIBLE_ROWS = 5; const COMPACT_TABLE_HEADER_HEIGHT = 33; const COMPACT_TABLE_ROW_HEIGHT = 32; -const columns: ColumnDef[] = [ +const columns: ColumnDef[] = [ { header: () => i18n.t("usage.keyModelUsage.columns.model"), accessorKey: "model", @@ -49,11 +52,22 @@ const columns: ColumnDef[] = [ }, ]; -const KeyModelUsageView: React.FC = ({ topModels }) => { +const KeyModelUsageView: React.FC = ({ topModels, otherModels }) => { const { t } = useTranslation(); const [viewMode, setViewMode] = useState<"chart" | "table">("table"); + const models = topModels.map((model) => ({ ...model, id: `model:${model.model}` })); + const rows: ModelUsageRow[] = otherModels + ? [ + ...models, + { + ...otherModels, + id: "other", + model: `Other (${otherModels.model_count.toLocaleString()} ${otherModels.model_count === 1 ? "model" : "models"})`, + }, + ] + : models; - if (topModels.length === 0) { + if (rows.length === 0) { return null; } @@ -82,8 +96,8 @@ const KeyModelUsageView: React.FC = ({ topModels }) => { {viewMode === "chart" ? (
({ key: m.model, spend: m.spend }))} + style={{ height: rows.length * 40 }} + data={rows.map((m) => ({ key: m.model, spend: m.spend }))} index="key" categories={["spend"]} colors={["cyan"]} @@ -97,8 +111,8 @@ const KeyModelUsageView: React.FC = ({ topModels }) => { ) : ( row.model} + data={rows} + getRowId={(row) => row.id} maxBodyHeight={COMPACT_TABLE_HEADER_HEIGHT + VISIBLE_ROWS * COMPACT_TABLE_ROW_HEIGHT} size="compact" /> diff --git a/ui/litellm-dashboard/src/components/UsagePage/dailyActivityApi.ts b/ui/litellm-dashboard/src/components/UsagePage/dailyActivityApi.ts index 5e9ad31ac15..70d5810c0c8 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/dailyActivityApi.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/dailyActivityApi.ts @@ -2,7 +2,9 @@ import type { components } from "@/lib/http/schema"; import type { BreakdownMetrics, DailyData, KeyMetadata, KeyMetricWithMetadata, MetricWithMetadata } from "./types"; export type DailyActivityEntity = "user" | "team" | "tag" | "organization" | "customer" | "agent"; -export type DailyActivityAggregatedResponse = components["schemas"]["SpendAnalyticsPaginatedResponse"]; +export type DailyActivityAggregatedResponse = + | components["schemas"]["SpendAnalyticsPaginatedResponse"] + | components["schemas"]["BoundedDailyActivityResponse"]; export type DailyActivityMetadata = components["schemas"]["DailySpendMetadata"]; export type ExportType = components["schemas"]["ExportType"]; export type ExportFormat = "csv" | "json"; @@ -26,6 +28,9 @@ export interface DailyActivityRequest { model?: string | null; includeCurrentUtcDay?: boolean; apiKeyLimit?: number; + bounded?: boolean; + includeModelBreakdown?: boolean; + modelLimit?: number; timezoneOffsetMinutes?: number; } diff --git a/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.test.ts b/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.test.ts index dea4d991dc3..501515e204c 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.test.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.test.ts @@ -8,6 +8,7 @@ import { toDailyData, } from "./dailyActivityApi"; import { + boundedActivityMetrics, keyDetailFromResponse, mergeKeyActivityPages, overallUsageMetrics, @@ -86,6 +87,40 @@ const pageRow = (api_key: string): KeySpendActivityRow => ({ }); describe("key activity data", () => { + it("keeps exact totals and an explicit remainder distinct from a real model named Other", () => { + const response: DailyActivityAggregatedResponse = { + ...aggregatedResponse, + model_breakdown: { + limit: 20, + total_models: 137, + models: Array.from({ length: 20 }, (_, index) => ({ + model: index === 0 ? "Other" : `model-${index}`, + metrics: completeMetrics, + })), + other: { + model_count: 117, + metrics: { ...completeMetrics, spend: 475, api_requests: 160, total_tokens: 560 }, + }, + }, + }; + const { metrics, otherModels } = boundedActivityMetrics(response); + expect(metrics.total_spend).toBe(500); + expect(metrics.total_requests).toBe(200); + expect(metrics.total_tokens).toBe(700); + expect(metrics.top_models).toHaveLength(20); + expect(metrics.top_models?.[0]).toMatchObject({ model: "Other", spend: 1.25, requests: 2 }); + const expectedRemainder = { model_count: 117, spend: 475, requests: 160, tokens: 560 }; + expect(otherModels).toMatchObject(expectedRemainder); + }); + + it("uses exact daily totals without a model summary for non-key activity", () => { + const { metrics, otherModels } = boundedActivityMetrics(aggregatedResponse); + expect(metrics.total_spend).toBe(500); + expect(metrics.daily_data).toHaveLength(1); + expect(metrics.top_models).toEqual([]); + expect(otherModels).toBeUndefined(); + }); + it.each([ ["", "unattributed"], ["unattributed", "raw:unattributed"], diff --git a/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.ts b/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.ts index 74a06a9be26..a60f3e42567 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.ts @@ -2,8 +2,9 @@ import type { Team } from "@/components/key_team_helpers/key_list"; import i18n from "@/i18n/i18n"; import { processActivityData } from "@/components/activity_metrics"; import type { DailyActivityAggregatedResponse, DailyActivityMetadata, KeySpendActivityRow } from "./dailyActivityApi"; -import { toDailyData } from "./dailyActivityApi"; -import type { ModelActivityData } from "./types"; +import { EMPTY_DAILY_ACTIVITY_METADATA, toDailyData } from "./dailyActivityApi"; +import type { ModelActivityData, OtherModelData } from "./types"; +import type { components } from "@/lib/http/schema"; export const reportingKeyId = (key: string): string => { if (!key) return "unattributed"; @@ -72,3 +73,26 @@ export const keyDetailFromResponse = ( apiKey: string, teams: Team[], ): ModelActivityData | undefined => processActivityData({ results: toDailyData(response) }, "api_keys", teams)[apiKey]; + +const modelTotals = (metrics: components["schemas"]["SpendMetrics"]) => ({ + spend: metrics.spend ?? 0, + requests: metrics.api_requests ?? 0, + successful_requests: metrics.successful_requests ?? 0, + failed_requests: metrics.failed_requests ?? 0, + tokens: metrics.total_tokens ?? 0, +}); + +export const boundedActivityMetrics = ( + response: DailyActivityAggregatedResponse, +): { metrics: ModelActivityData; otherModels?: OtherModelData } => { + const breakdown = "model_breakdown" in response ? response.model_breakdown : undefined; + return { + metrics: { + ...overallUsageMetrics(toDailyData(response), response.metadata ?? EMPTY_DAILY_ACTIVITY_METADATA), + top_models: breakdown?.models.map(({ model, metrics }) => ({ model, ...modelTotals(metrics) })) ?? [], + }, + otherModels: breakdown?.other + ? { ...modelTotals(breakdown.other.metrics), model_count: breakdown.other.model_count } + : undefined, + }; +}; diff --git a/ui/litellm-dashboard/src/components/UsagePage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts index 198c9315046..ee095788da3 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/types.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/types.ts @@ -64,6 +64,8 @@ export interface TopModelData { tokens: number; } +export type OtherModelData = Omit & { model_count: number }; + export interface ModelActivityData { label: string; key_metadata?: KeyMetadata; diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index f0b4b0cfe86..928d393606b 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -25,7 +25,7 @@ import { Team } from "./key_team_helpers/key_list"; import KeyModelUsageView from "./UsagePage/components/KeyModelUsageView"; import { keyActivityLabel } from "./UsagePage/keyActivityLabel"; import type { ModelTopKeysResponse } from "./UsagePage/dailyActivityApi"; -import { DailyData, KeyMetricWithMetadata, ModelActivityData, TopModelData } from "./UsagePage/types"; +import { DailyData, KeyMetricWithMetadata, ModelActivityData, OtherModelData, TopModelData } from "./UsagePage/types"; import { averageResponseTimeMs, formatResponseTime, valueFormatter } from "./UsagePage/utils/value_formatters"; interface ActivityMetricsProps { @@ -155,7 +155,7 @@ const ModelTopKeys = ({ fetchTopApiKeys(modelName) .then((response) => { if (cancelled) return; - setSettled({ + const nextState = { modelName, fetchTopApiKeys, rows: response.api_keys.map((row) => ({ @@ -168,12 +168,14 @@ const ModelTopKeys = ({ tokens: row.metrics.total_tokens, })), failed: false, - }); + }; + setSettled(nextState); }) .catch((error) => { if (cancelled) return; console.error(`Failed to fetch top keys for ${modelName}:`, error); - setSettled({ modelName, fetchTopApiKeys, rows: [], failed: true }); + const failedState = { modelName, fetchTopApiKeys, rows: [], failed: true }; + setSettled(failedState); }); return () => { cancelled = true; @@ -255,13 +257,16 @@ export const ModelSection = ({ metrics, hidePromptCachingMetrics = false, fetchTopApiKeys, + otherModels, }: { modelName: string; metrics: ModelActivityData; hidePromptCachingMetrics?: boolean; fetchTopApiKeys?: (model: string) => Promise; + otherModels?: OtherModelData; }) => { const { t } = useTranslation(); + const hasModelUsage = Boolean(metrics.top_models?.length || otherModels); return (
@@ -308,7 +313,7 @@ export const ModelSection = ({ {fetchTopApiKeys && } - {metrics.top_models && metrics.top_models.length > 0 && } + {hasModelUsage && } { }); describe("dailyActivityAggregatedCall", () => { + it("opts into bounded model contributions without changing legacy requests", async () => { + const mockFetch = captureFetch(); + const options = { apiKey: "key-hash", bounded: true, includeModelBreakdown: true, modelLimit: 20 }; + await dailyActivityAggregatedCall("user", req(options)); + await dailyActivityAggregatedCall("user", req()); + expect(requestedUrl(mockFetch).searchParams.get("bounded")).toBe("true"); + expect(requestedUrl(mockFetch).searchParams.get("include_model_breakdown")).toBe("true"); + expect(requestedUrl(mockFetch).searchParams.get("model_limit")).toBe("20"); + for (const key of ["bounded", "include_model_breakdown", "model_limit"]) { + expect(requestedUrl(mockFetch, 1).searchParams.has(key)).toBe(false); + } + }); + it("uses UTC for incident ranges while preserving the default timezone for other callers", async () => { const mockFetch = captureFetch(); await dailyActivityAggregatedCall("user", req({ timezoneOffsetMinutes: 0 })); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index a3e5da18eb1..e390b77a7da 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1390,7 +1390,12 @@ export const dailyActivityAggregatedCall = ( ): Promise => apiClient.get(`/${entity}/daily/activity/aggregated`, { accessToken: req.accessToken, - query: dailyActivityQuery(entity, req, req.apiKeyLimit === undefined ? {} : { api_key_limit: req.apiKeyLimit }), + query: dailyActivityQuery(entity, req, { + api_key_limit: req.apiKeyLimit, + bounded: req.bounded || undefined, + include_model_breakdown: (req.bounded && req.includeModelBreakdown) || undefined, + model_limit: req.bounded ? req.modelLimit : undefined, + }), }); export const subscriptionUsageCall = ( diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx index c353a3e56be..e3e5a083223 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx @@ -504,6 +504,7 @@ describe("TeamVirtualKeysTable", () => { await screen.findByText("Key Info View"); expect(vi.mocked(KeyInfoView).mock.lastCall?.[0].keyData).toMatchObject({ organization_id: null }); + expect(vi.mocked(KeyInfoView).mock.lastCall?.[0].tabQueryKey).toBe("key_detail_tab"); }); describe("entity links out of the key rows", () => { diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx index 550709edebf..770a2ca65a4 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx @@ -547,6 +547,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi {selectedKey ? ( setSelectedKey(null)} keyData={selectedKey} teams={[currentTeam]} diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx index 8d9c2e7b7ab..cc56b123b16 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx @@ -259,6 +259,25 @@ describe("KeyInfoView", () => { expect(screen.queryByRole("tab", { name: "Auto-router usage" })).not.toBeInTheDocument(); }); + it("keeps the team tab when changing a nested key tab", async () => { + vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock); + const onUrlUpdate = vi.fn(); + renderWithProviders( + {}} + keyId="test-key-id" + teams={[]} + tabQueryKey="key_detail_tab" + />, + { searchParams: "detail_tab=virtual_keys", onUrlUpdate }, + ); + await userEvent.click(screen.getByRole("tab", { name: "Savings" })); + expect(screen.getByRole("tab", { name: "Savings" })).toHaveAttribute("aria-selected", "true"); + await waitFor(() => expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("key_detail_tab")).toBe("savings")); + expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("detail_tab")).toBe("virtual_keys"); + }); + describe("last updated", () => { const renderWithTimestamps = (overrides: Partial) => { vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock); diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 81603774d89..6598283bebd 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -76,6 +76,7 @@ interface KeyInfoViewProps { onDelete?: () => void; teams: any[] | null; backButtonText?: string; + tabQueryKey?: string; } // Premium fields (from LiteLLM_ManagementEndpoint_MetadataFields_Premium in @@ -107,13 +108,14 @@ export default function KeyInfoView({ onKeyDataUpdate, onDelete, backButtonText = i18n.t("keyDetail.keyInfoView.backToKeys"), + tabQueryKey = "detail_tab", }: KeyInfoViewProps) { const { t } = useTranslation(); const { accessToken, userId: userID, userRole, premiumUser } = useAuthorized(); const [activeTab, setActiveTab] = useUrlTab( ["overview", "activity", "savings", "auto-router-usage", "settings"], "overview", - "detail_tab", + tabQueryKey, ); const accessGroupNames = useModelAccessGroupNames(); const activityDateRange = useActivityDateRange(); @@ -770,7 +772,13 @@ export default function KeyInfoView({
- {activeTab === "activity" && } + {activeTab === "activity" && ( + + )} {/* Overview Panel */} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d8c9dde9b17..5589ecfc08f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -28639,6 +28639,14 @@ export interface components { /** Input Reference */ input_reference?: string | null; }; + /** BoundedDailyActivityResponse */ + BoundedDailyActivityResponse: { + metadata: components["schemas"]["DailySpendMetadata"]; + /** @description When requested, includes the highest-spend models across the full date range and an explicit aggregate of all remaining models. Daily totals always cover the complete activity scope. Other per-day dimension breakdowns are omitted. */ + model_breakdown?: components["schemas"]["DailyActivityModelBreakdown"] | null; + /** Results */ + results: components["schemas"]["DailySpendData"][]; + }; /** BoxAnswer */ BoxAnswer: { box: components["schemas"]["DecisionsBox"]; @@ -32394,6 +32402,28 @@ export interface components { /** Api Keys */ api_keys: components["schemas"]["KeyActivityRow"][]; }; + /** DailyActivityModelBreakdown */ + DailyActivityModelBreakdown: { + /** Limit */ + limit: number; + /** Models */ + models: components["schemas"]["DailyActivityModelContribution"][]; + other?: components["schemas"]["DailyActivityOtherModels"] | null; + /** Total Models */ + total_models: number; + }; + /** DailyActivityModelContribution */ + DailyActivityModelContribution: { + metrics: components["schemas"]["SpendMetrics"]; + /** Model */ + model: string; + }; + /** DailyActivityOtherModels */ + DailyActivityOtherModels: { + metrics: components["schemas"]["SpendMetrics"]; + /** Model Count */ + model_count: number; + }; /** DailyActivityUserPageResponse */ DailyActivityUserPageResponse: { /** Limit */ @@ -34871,6 +34901,16 @@ export interface components { * @default true */ active: boolean; + /** + * Can Update Status + * @description Current reader's status permission, evaluated by the issue-detail endpoint + */ + can_update_status?: boolean | null; + /** + * Can View Activity + * @description Current reader's permission to inspect Activity, keys, users and impact + */ + can_view_activity?: boolean | null; cohort: components["schemas"]["IssueCohort"]; /** Cohort Key */ cohort_key: string; @@ -54320,6 +54360,9 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + bounded?: boolean; + include_model_breakdown?: boolean; + model_limit?: number; agent_ids?: string | null; start_date?: string | null; end_date?: string | null; @@ -54340,7 +54383,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"] | components["schemas"]["BoundedDailyActivityResponse"]; }; }; /** @description Validation Error */ @@ -59971,6 +60014,9 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + bounded?: boolean; + include_model_breakdown?: boolean; + model_limit?: number; end_user_ids?: string | null; start_date?: string | null; end_date?: string | null; @@ -59991,7 +60037,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"] | components["schemas"]["BoundedDailyActivityResponse"]; }; }; /** @description Validation Error */ @@ -61022,6 +61068,9 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + bounded?: boolean; + include_model_breakdown?: boolean; + model_limit?: number; end_user_ids?: string | null; start_date?: string | null; end_date?: string | null; @@ -61042,7 +61091,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"] | components["schemas"]["BoundedDailyActivityResponse"]; }; }; /** @description Validation Error */ @@ -65654,6 +65703,7 @@ export interface operations { query?: { start_date?: string | null; end_date?: string | null; + model_limit?: number; }; header?: never; path: { @@ -65669,7 +65719,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + "application/json": components["schemas"]["BoundedDailyActivityResponse"]; }; }; /** @description Validation Error */ @@ -70714,6 +70764,9 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + bounded?: boolean; + include_model_breakdown?: boolean; + model_limit?: number; organization_ids?: string | null; start_date?: string | null; end_date?: string | null; @@ -70734,7 +70787,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"] | components["schemas"]["BoundedDailyActivityResponse"]; }; }; /** @description Validation Error */ @@ -76586,6 +76639,9 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + bounded?: boolean; + include_model_breakdown?: boolean; + model_limit?: number; start_date?: string | null; end_date?: string | null; model?: string | null; @@ -76605,7 +76661,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"] | components["schemas"]["BoundedDailyActivityResponse"]; }; }; /** @description Validation Error */ @@ -77276,6 +77332,9 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + bounded?: boolean; + include_model_breakdown?: boolean; + model_limit?: number; team_ids?: string | null; start_date?: string | null; end_date?: string | null; @@ -77296,7 +77355,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"] | components["schemas"]["BoundedDailyActivityResponse"]; }; }; /** @description Validation Error */ @@ -80069,6 +80128,9 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + bounded?: boolean; + include_model_breakdown?: boolean; + model_limit?: number; /** @description Start date in YYYY-MM-DD format */ start_date?: string | null; /** @description End date in YYYY-MM-DD format */ @@ -80096,7 +80158,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"] | components["schemas"]["BoundedDailyActivityResponse"]; }; }; /** @description Validation Error */