diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index c14aa564221..4407b9a4e90 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -1,6 +1,6 @@ -# Issues and daily Performance alerts +# Alerts and daily Performance alerts -The Issues inbox combines model retirements with daily Performance alerts. Open an issue to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause. +The Alerts page combines model retirements with daily Performance alerts. Open an issue to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause. Performance comparisons cover completed UTC days. Retirement issues show recent model usage and the provider's retirement schedule, including today's usage when present. @@ -55,6 +55,8 @@ curl "$LITELLM_API_BASE/issues/$ISSUE_ID/signals?limit=50" \ Replace `issue-example` with an ID returned by `/issues`. Activity uses the issue's saved date range by default. Optional `start_date` and `end_date` are inclusive UTC dates and can span at most 366 days. `/keys` and `/users` provide paginated Activity rows with `limit` and `offset`. Their latency totals are `total_response_time_ms` and `timed_requests`; divide the former by the latter only when timed requests are present. +Alert Activity returns exact daily totals in `results` and exact range totals in `metadata`. It does not return the general Usage API's per-day dimension dictionaries. Key issues also return `model_breakdown`, with up to 20 models by range spend by default. Set `model_limit` between 1 and 100 to change that limit. `total_models` counts every matching model; `other` contains the combined metrics and `model_count` of all remaining models, or null when none remain. It is an aggregate, not a model identifier. Adding the displayed models and `other` recovers the full totals. Model identifiers longer than 1,024 characters are included in `other` without truncation. Non-key issues return null for `model_breakdown`. Activity queries use a bounded execution budget. A busy or over-budget query returns `503` with `Retry-After: 5`; retry later or choose a shorter date range. + `/impact` reports distinct resource counts and requests within that same Activity range. These counts mean that resources used the cohort, not that each resource suffered a regression. Saved `impact.measured_keys` and `impact.measured_models`, when present, describe cohorts that met the detector's comparison criteria. Historical team aggregates do not retain user membership, so a team issue has no `/users` view and its user count is null. User issues similarly have no historical team count. Missing attribution remains unknown. Supporting signals have their own saved comparisons and relationship explanations. Their baseline windows can differ, so do not derive a combined error rate or unit cost by adding their displayed values. Administrators and admin viewers can inspect broader supporting signals. A scoped reader receives only signals for the exact key, user, or team they can access. That restriction applies before the signal count and pagination; broader totals are not disclosed. @@ -72,6 +74,8 @@ curl -X PATCH "$LITELLM_API_BASE/issues/$ISSUE_ID" \ A stale status returns `409`; reload before retrying. Performance acknowledgements do not change other issues that share a supporting observation. Retirement acknowledgements use the same state as the existing model-deprecation API. An inactive retirement cannot be manually reopened. +The issue-detail response includes `can_update_status` and `can_view_activity` for the current reader. Clients should show status actions only when `can_update_status` is true and request Activity or resource breakdowns only when `can_view_activity` is true. These fields are null in responses where permission has not been evaluated, such as inbox lists. Every endpoint still checks authorization on each request. Team Activity requires team-admin access or the explicit `/team/daily/activity` permission; ordinary membership can read saved alert evidence but does not grant other members’ activity or key details. + ## Configure default families `GET /issues/families` returns the four Performance families, their enabled counts, and common policy values. A null common value means the underlying scopes have different settings. This is separate from custom rules and the model-retirement warning window. @@ -123,7 +127,7 @@ The response contains `updated_count`. Only incidents in that exact group that s ## Create a rule -This example alerts when a team's daily spend is more than three times its baseline and the absolute change is at least $10. Replace `team-example` with the team's ID. +This example alerts when a team's daily spend is at least three times its baseline and the absolute change is at least $10. Replace `team-example` with the team's ID. ```bash curl "$LITELLM_API_BASE/usage/alert-rules" \ @@ -144,7 +148,7 @@ curl "$LITELLM_API_BASE/usage/alert-rules" \ }' ``` -The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value below half the baseline. `min_value` is the minimum absolute difference in the metric's own units. +The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value at or below half the baseline. `min_value` is the minimum absolute difference in the metric's own units. | Metric | Unit | | --- | --- | @@ -177,6 +181,6 @@ Rules can specify `notification_channels: ["slack", "email"]` and `email_recipie Gateway administrators can manage every scope. Admin viewers can read every scope. Internal users can manage their own user and key scopes; team administrators can manage their team's scope and keys. Team members can read their team's incidents. Read-only users cannot change rules or incident state, and inference-only keys cannot access these endpoints. -Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` indicates that the gateway has no database connection. +Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` can indicate an unavailable database or exhausted analytics capacity; a `Retry-After` header indicates that the client should retry later. -The dashboard's Issues inbox is available to administrators and admin viewers. Other authorized roles can use the scoped API. +The dashboard's Alerts page is available to administrators and admin viewers. Other authorized roles can use the scoped API. diff --git a/litellm/models/issues.py b/litellm/models/issues.py index cd3f9767638..61d3f45f839 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -1,8 +1,10 @@ from datetime import date, datetime from typing import Literal, Protocol +from litellm.models.team import LiteLLM_TeamTable from litellm.types.proxy.issues import ( Issue, + IssueActivityResponse, IssueCohort, IssueDeliveryBinding, IssueDeliveryInput, @@ -24,7 +26,6 @@ from litellm.types.proxy.issues import ( from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, DailyActivityUserPageResponse, - SpendAnalyticsPaginatedResponse, ) from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus @@ -33,6 +34,8 @@ from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus class IssueStore(Protocol): async def get_issue(self, issue_id: str) -> Issue | None: ... + async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: ... + async def list_issues( self, kind: IssueKind | None = None, @@ -130,7 +133,9 @@ class IssueStore(Protocol): class IssueActivityStore(Protocol): - async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse: ... + async def daily_activity( + self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 + ) -> IssueActivityResponse: ... async def keys( self, cohort: IssueCohort, start: date, end: date, limit: int, offset: int diff --git a/litellm/proxy/alerting_endpoints/issues.py b/litellm/proxy/alerting_endpoints/issues.py index cca5c730bb5..b87599a5d56 100644 --- a/litellm/proxy/alerting_endpoints/issues.py +++ b/litellm/proxy/alerting_endpoints/issues.py @@ -1,5 +1,7 @@ """Unified issue inbox and investigations backed by existing daily aggregates.""" +from collections.abc import Generator +from contextlib import contextmanager from datetime import date from typing import Final @@ -13,11 +15,16 @@ from litellm.proxy.alerting_endpoints.usage_alerts import ( User, require_repository, require_scope, + require_write_access, ) from litellm.proxy.deps.repositories import IssueActivityRepositoryDep, IssuesRepositoryDep, UsageAlertsRepositoryDep +from litellm.proxy.management.teams.authz import is_team_admin +from litellm.proxy.management_endpoints.common_utils import team_member_has_permission +from litellm.repositories.analytics_budget import AnalyticsCapacityError from litellm.types.proxy.auth.user_roles import LitellmUserRoles from litellm.types.proxy.issues import ( Issue, + IssueActivityResponse, IssueFamily, IssueFamilyList, IssueFamilySetting, @@ -32,7 +39,6 @@ from litellm.types.proxy.issues import ( from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, DailyActivityUserPageResponse, - SpendAnalyticsPaginatedResponse, ) from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertStatus @@ -52,6 +58,14 @@ def require_activity(repository: IssueActivityStore | None) -> IssueActivityStor return repository +@contextmanager +def activity_budget() -> Generator[None]: + try: + yield + except AnalyticsCapacityError as exc: + raise HTTPException(status_code=503, detail=str(exc), headers={"Retry-After": "5"}) from exc + + async def authorized_issue( issue_id: str, user: User, @@ -73,6 +87,25 @@ async def authorized_issue( return issue +async def can_view_activity(issue: Issue, user: User, repository: IssueStore) -> bool: + if issue.cohort.scope_type != "team" or user.user_role in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + return True + if user.user_id is None or issue.cohort.scope_id is None: + return False + team: Final = await repository.team_activity_access(issue.cohort.scope_id, user.user_id) + return team is not None and ( + is_team_admin(user, team) or team_member_has_permission(user, team, "/team/daily/activity") + ) + + +async def require_activity_access(issue: Issue, user: User, repository: IssuesRepositoryDep) -> None: + if not await can_view_activity(issue, user, require_issues(repository)): + raise HTTPException(status_code=403, detail="Full team Activity permission is required for this investigation") + + def activity_range(issue: Issue, start_date: date | None, end_date: date | None) -> tuple[date, date]: start: Final = start_date or issue.range_start end: Final = end_date or issue.range_end @@ -118,6 +151,7 @@ async def update_issue_family( repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, ) -> IssueFamilySetting: + require_write_access(user, f"/issues/families/{family}") await require_scope(user, require_repository(alerts), "deployment", None, write=True) try: result: Final = await require_issues(repository).update_family(family, data) @@ -137,7 +171,22 @@ async def get_issue( repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, ) -> Issue: - return await authorized_issue(issue_id, user, repository, alerts) + issue: Final = await authorized_issue(issue_id, user, repository, alerts) + activity_allowed: Final = await can_view_activity(issue, user, require_issues(repository)) + try: + require_write_access(user, f"/issues/{issue_id}") + await require_scope( + user, + require_repository(alerts), + "model" if issue.cohort.scope_type == "provider" else issue.cohort.scope_type, + issue.cohort.scope_id, + write=True, + ) + except HTTPException as exc: + if exc.status_code != 403: + raise + return issue.model_copy(update={"can_update_status": False, "can_view_activity": activity_allowed}) + return issue.model_copy(update={"can_update_status": True, "can_view_activity": activity_allowed}) @router.patch("/{issue_id}", response_model=Issue) @@ -148,6 +197,7 @@ async def update_issue( repository: IssuesRepositoryDep, alerts: UsageAlertsRepositoryDep, ) -> Issue: + require_write_access(user, f"/issues/{issue_id}") issue: Final = await authorized_issue(issue_id, user, repository, alerts, write=True) if issue.kind == "model_deprecation" and not issue.active and data.status == "open": raise HTTPException(status_code=409, detail="This retirement issue reopens when the model receives new traffic") @@ -181,7 +231,7 @@ async def list_issue_signals( return await require_issues(repository).list_signals(issue_id, limit, offset, visible_scope) -@router.get("/{issue_id}/activity", response_model=SpendAnalyticsPaginatedResponse) +@router.get("/{issue_id}/activity", response_model=IssueActivityResponse) async def get_issue_activity( issue_id: str, user: User, @@ -190,10 +240,13 @@ async def get_issue_activity( activity: IssueActivityRepositoryDep, start_date: date | None = None, end_date: date | None = None, -) -> SpendAnalyticsPaginatedResponse: + model_limit: Limit = 20, +) -> IssueActivityResponse: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).daily_activity(issue.cohort, start, end) + with activity_budget(): + return await require_activity(activity).daily_activity(issue.cohort, start, end, model_limit) @router.get("/{issue_id}/keys", response_model=DailyActivityKeyPageResponse) @@ -209,8 +262,10 @@ async def get_issue_keys( offset: Offset = 0, ) -> DailyActivityKeyPageResponse: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).keys(issue.cohort, start, end, limit, offset) + with activity_budget(): + return await require_activity(activity).keys(issue.cohort, start, end, limit, offset) @router.get("/{issue_id}/users", response_model=DailyActivityUserPageResponse) @@ -226,10 +281,12 @@ async def get_issue_users( offset: Offset = 0, ) -> DailyActivityUserPageResponse: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) if issue.cohort.scope_type == "team": raise HTTPException(status_code=422, detail="Historical team aggregates do not retain user membership") start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).users(issue.cohort, start, end, limit, offset) + with activity_budget(): + return await require_activity(activity).users(issue.cohort, start, end, limit, offset) @router.get("/{issue_id}/impact", response_model=IssueImpact) @@ -243,5 +300,7 @@ async def get_issue_impact( end_date: date | None = None, ) -> IssueImpact: issue: Final = await authorized_issue(issue_id, user, repository, alerts) + await require_activity_access(issue, user, repository) start, end = activity_range(issue, start_date, end_date) - return await require_activity(activity).impact(issue.cohort, start, end) + with activity_budget(): + return await require_activity(activity).impact(issue.cohort, start, end) diff --git a/litellm/repositories/issue_activity.py b/litellm/repositories/issue_activity.py index fed59a82cbc..6c85fa064b3 100644 --- a/litellm/repositories/issue_activity.py +++ b/litellm/repositories/issue_activity.py @@ -7,18 +7,22 @@ from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # generated Prism from pydantic import BaseModel, TypeAdapter from litellm.constants import PTU_SENTINEL_API_KEY +from litellm.repositories.analytics_budget import AnalyticsDatabase, BoundedAnalyticsDatabase +from litellm.repositories.bounded_daily_activity import BoundedDailyActivityRepository from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket -from litellm.repositories.daily_activity_repository import DailyActivityRepository +from litellm.repositories.daily_activity_repository import DailyActivityQueryDatabase, DailyActivityRepository from litellm.repositories.daily_activity_sql import PRISMA_TO_PG_TABLE, build_where_clause -from litellm.repositories.usage_alerts import AlertDatabase -from litellm.types.proxy.issues import IssueCohort, IssueImpact +from litellm.types.proxy.issues import ( + IssueActivityResponse, + IssueCohort, + IssueImpact, +) from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailyActivityKeyPageResponse, DailyActivityUserPageResponse, KeyMetadata, KeySpendActivityRow, KeySpendMetrics, - SpendAnalyticsPaginatedResponse, UserActivityRow, ) from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable @@ -54,7 +58,7 @@ def issue_activity_scope(cohort: IssueCohort, start: date, end: date) -> DailyAc class IssueActivityRepository: - def __init__(self, activity: DailyActivityRepository, database: AlertDatabase) -> None: + def __init__(self, activity: DailyActivityRepository, database: DailyActivityQueryDatabase) -> None: self.activity = activity self.database = database @@ -62,14 +66,14 @@ class IssueActivityRepository: def from_prisma(cls, prisma: "PrismaClient") -> "IssueActivityRepository": from litellm.proxy.management_endpoints.common_daily_activity import daily_activity_repository - database: Final = cast(AlertDatabase, prisma.db) # cast-ok: generated Prisma query_raw - return cls(daily_activity_repository(prisma), database) + database: Final = BoundedAnalyticsDatabase(cast(AnalyticsDatabase, prisma.db)) # cast-ok: generated Prisma tx + return cls(daily_activity_repository(prisma, query_database=database), database) - async def daily_activity(self, cohort: IssueCohort, start: date, end: date) -> SpendAnalyticsPaginatedResponse: - from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated - - return await get_daily_activity_aggregated( - self.activity, issue_activity_scope(cohort, start, end), api_key_limit=20 + async def daily_activity( + self, cohort: IssueCohort, start: date, end: date, model_limit: int = 20 + ) -> IssueActivityResponse: + return await BoundedDailyActivityRepository(self.database).aggregate( + issue_activity_scope(cohort, start, end), model_limit=model_limit if cohort.scope_type == "key" else None ) async def keys( diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index fda3bc33f89..0d68a957ebd 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -10,6 +10,7 @@ from typing import ( from pydantic import BaseModel, TypeAdapter +from litellm.models.team import LiteLLM_TeamTable from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( @@ -165,6 +166,25 @@ class IssuesRepository: ) return (await self._labels((rows[0].record,)))[0] if rows else None + async def team_activity_access(self, team_id: str, user_id: str) -> LiteLLM_TeamTable | None: + rows: Final = TypeAdapter(tuple[LiteLLM_TeamTable, ...]).validate_python( + await self.database.query_raw( + """SELECT t.team_id, + CASE WHEN m.role IS NULL THEN '[]'::jsonb + ELSE jsonb_build_array(jsonb_build_object('user_id',$2::text,'role',m.role)) END + AS members_with_roles, + CASE WHEN '/team/daily/activity'=ANY(t.team_member_permissions) + THEN ARRAY['/team/daily/activity'] ELSE ARRAY[]::text[] END AS team_member_permissions + FROM "LiteLLM_TeamTable" t LEFT JOIN LATERAL ( + SELECT member->>'role' AS role FROM jsonb_array_elements(t.members_with_roles) member + WHERE member->>'user_id'=$2 ORDER BY (member->>'role'='admin') DESC LIMIT 1 + ) m ON true WHERE t.team_id=$1""", + team_id, + user_id, + ) + ) + return rows[0] if rows else None + async def list_issues( self, kind: IssueKind | None = None, diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py index 1a4104726fb..42dd4355a53 100644 --- a/litellm/types/proxy/issues.py +++ b/litellm/types/proxy/issues.py @@ -3,6 +3,9 @@ from typing import Literal, Self, TypeAlias from pydantic import BaseModel, ConfigDict, Field, model_validator +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + BoundedDailyActivityResponse, +) from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily from litellm.types.proxy.usage_alerts import ( @@ -14,6 +17,8 @@ from litellm.types.proxy.usage_alerts import ( UsageAlertStatus, ) +IssueActivityResponse: TypeAlias = BoundedDailyActivityResponse + IssueKind: TypeAlias = Literal["performance", "model_deprecation"] IssueFamily: TypeAlias = Literal["retirement", "latency", "reliability", "spend", "traffic"] IssueScope: TypeAlias = Literal["deployment", "key", "user", "team", "model", "provider"] @@ -131,6 +136,12 @@ class Issue(IssueInput, frozen=True): updated_at: datetime scope_label: str | None = None active: bool = True + can_update_status: bool | None = Field( + default=None, description="Current reader's status permission, evaluated by the issue-detail endpoint" + ) + can_view_activity: bool | None = Field( + default=None, description="Current reader's permission to inspect Activity, keys, users and impact" + ) class IssueList(BaseModel, frozen=True): diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index 4fa58c18adb..d9631f3aee7 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Mapping from contextlib import asynccontextmanager from datetime import date, datetime, timedelta, timezone from hashlib import sha256 @@ -17,10 +17,12 @@ from prisma.types import DatasourceOverride from psycopg import sql from pydantic import BaseModel, TypeAdapter, ValidationError +from litellm.repositories.daily_activity_sql import build_bounded_daily_activity_sql from litellm.repositories.issue_activity import IssueActivityRepository from litellm.repositories.issues import IssuesRepository from litellm.repositories.usage_alerts import AlertDatabase, UsageAlertsRepository from litellm.types.proxy.issues import ( + IssueActivityResponse, IssueCohort, IssueComparison, IssueDeliveryInput, @@ -92,6 +94,7 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa outsider: Final = scenario.user(user_role="internal_user") viewer: Final = scenario.user(user_role="proxy_admin_viewer") owner_key: Final = scenario.key(user_id=owner, key_type="management") + info_key: Final = scenario.key(user_id=owner, key_type="default", allowed_routes=["info_routes"]) outsider_key: Final = scenario.key(user_id=outsider, key_type="management") viewer_key: Final = scenario.key(user_id=viewer, key_type="read_only") inference_key: Final = scenario.key(user_id=owner, key_type="llm_api", key_alias="Example checkout") @@ -152,6 +155,16 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa path: Final = f"/issues/{first.id}" own: Final = gateway.request("GET", path, key=owner_key) assert own.status_code == 200 and own.json()["scope_label"] == "Example checkout" + assert own.json()["can_update_status"] is True + assert own.json()["can_view_activity"] is True + assert gateway.request("GET", path, key=viewer_key).json()["can_update_status"] is False + assert gateway.request("GET", path, key=info_key).json()["can_update_status"] is False + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=info_key + ).status_code + == 403 + ) assert "99999" not in own.text and "9999" not in own.text pages: Final = tuple( gateway.request( @@ -204,6 +217,159 @@ async def test_issue_http_scope_pagination_and_source_visibility(gateway: Gatewa await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id) +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_detail_reports_team_status_permission(gateway: Gateway, db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + with gateway.scenario() as scenario: + admin: Final = scenario.user(user_role="internal_user") + member: Final = scenario.user(user_role="internal_user") + team: Final = scenario.team( + members_with_roles=[{"user_id": admin, "role": "admin"}, {"user_id": member, "role": "user"}] + ) + permitted_team: Final = scenario.team( + members_with_roles=[{"user_id": member, "role": "user"}], + team_member_permissions=["/team/daily/activity"], + ) + admin_key: Final = scenario.key(user_id=admin, key_type="management") + member_key: Final = scenario.key(user_id=member, key_type="management") + service_key: Final = scenario.key( + team_id=team, + user_id=None, + key_type="management", + metadata={"service_account_id": "example-activity-service"}, + ) + issue: Final = _issue(IssueCohort(scope_type="team", scope_id=team)) + permitted_issue: Final = _issue(IssueCohort(scope_type="team", scope_id=permitted_team)) + try: + await store.upsert_issues((issue, permitted_issue)) + path: Final = f"/issues/{issue.id}" + assert gateway.request("GET", path, key=admin_key).json()["can_update_status"] is True + assert gateway.request("GET", path, key=admin_key).json()["can_view_activity"] is True + assert gateway.request("GET", path, key=member_key).json()["can_update_status"] is False + assert gateway.request("GET", path, key=member_key).json()["can_view_activity"] is False + for suffix in ("/activity", "/keys", "/users", "/impact"): + assert gateway.request("GET", path + suffix, key=member_key).status_code == 403 + assert gateway.request("GET", path + suffix, key=service_key).status_code == 403 + for suffix in ("/activity", "/keys", "/impact"): + assert gateway.request("GET", path + suffix, key=admin_key).status_code == 200 + assert ( + gateway.request("GET", f"/issues/{permitted_issue.id}" + suffix, key=member_key).status_code == 200 + ) + assert ( + gateway.request("GET", f"/issues/{permitted_issue.id}", key=member_key).json()["can_view_activity"] + is True + ) + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=member_key + ).status_code + == 403 + ) + assert ( + gateway.request( + "PATCH", path, {"status": "resolved", "expected_status": "open"}, key=admin_key + ).status_code + == 200 + ) + finally: + await _remove_issues(db, (issue.id, permitted_issue.id)) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_issue_activity_bounds_models_without_losing_totals(gateway: Gateway, db: ManagementDatabase) -> None: + from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated + from litellm.repositories.issue_activity import issue_activity_scope + + repository: Final = IssueActivityRepository.from_prisma(db.prisma) + store: Final = IssuesRepository.from_prisma(db.prisma) + identity: Final = str(uuid4()) + key: Final = sha256(identity.encode()).hexdigest() + provider: Final = f"example-provider-{identity}" + day: Final = date.today() - timedelta(days=5) + model_count: Final = 137 + cohort: Final = IssueCohort(scope_type="key", scope_id=key) + issue: Final = _issue(cohort).model_copy(update={"range_start": day, "range_end": day + timedelta(days=2)}) + await _query( + db, + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,model_group,custom_llm_provider,mcp_namespaced_tool_name,endpoint, + api_requests,spend,prompt_tokens,completion_tokens,successful_requests,failed_requests, + total_response_time_ms,timed_requests,updated_at) + SELECT $1||'-'||m||'-'||d,$1,($2::date+d::int)::text,$3, + CASE WHEN m=0 THEN 'Other' ELSE 'example-model-'||lpad(m::text,4,'0') END, + 'group-'||m,$4,'tool-'||m,'endpoint-'||m, + m+2,CASE WHEN m=0 AND d=0 THEN 10000 ELSE m+1 END,m+1,2,m+1,1,(m+1)*200,m+1,CURRENT_TIMESTAMP + FROM generate_series(0,$5::int-1) m CROSS JOIN generate_series(0,2) d RETURNING id""", + identity, + day.isoformat(), + key, + provider, + model_count, + ) + try: + await store.upsert_issue(issue) + scope: Final = issue_activity_scope(cohort, day, day + timedelta(days=2)) + original: Final = await get_daily_activity_aggregated(repository.activity, scope, api_key_limit=20) + response: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + assert response.status_code == 200 + bounded: Final = IssueActivityResponse.model_validate(response.json()) + assert bounded.metadata.model_dump(exclude={"api_key_limit"}) == original.metadata.model_dump( + exclude={"api_key_limit"} + ) + assert tuple((row.date, row.metrics) for row in bounded.results) == tuple( + (row.date, row.metrics) for row in original.results + ) + models: Final = bounded.model_breakdown + assert models is not None and models.total_models == model_count and len(models.models) == models.limit == 20 + assert models.models[0].model == "Other" and models.models[0].metrics.spend == 10002 + assert models.other is not None and models.other.model_count == model_count - models.limit + components: Final = (*tuple(row.metrics for row in models.models), models.other.metrics) + assert sum(row.spend for row in components) == bounded.metadata.total_spend + assert sum(row.api_requests for row in components) == bounded.metadata.total_api_requests + assert sum(row.total_response_time_ms for row in components) == bounded.metadata.total_response_time_ms + assert all(not row.breakdown.models and not row.breakdown.endpoints for row in bounded.results) + query: Final = build_bounded_daily_activity_sql(scope, model_limit=100) + raw: Final = TypeAdapter(tuple[object, ...]).validate_python( + await repository.database.query_raw(query.sql, *query.params) + ) + assert len(raw) == 3 + 1 + 100 + 1 + largest: Final = gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "100"}) + assert largest.status_code == 200 + large_models: Final = IssueActivityResponse.model_validate(largest.json()).model_breakdown + assert large_models is not None and len(large_models.models) == 100 + assert large_models.other is not None and large_models.other.model_count == model_count - 100 + assert gateway.request("GET", f"/issues/{issue.id}/activity", params={"model_limit": "101"}).status_code == 422 + assert ( + gateway.request( + "GET", + f"/issues/{issue.id}/activity", + params={"start_date": day.isoformat(), "end_date": (day + timedelta(days=366)).isoformat()}, + ).status_code + == 422 + ) + non_key: Final = await repository.daily_activity( + IssueCohort(scope_type="provider", scope_id=provider), day, day + timedelta(days=2) + ) + assert non_key.model_breakdown is None and non_key.metadata == bounded.metadata + assert len(non_key.results) == 3 + schemas: Final = TypeAdapter(tuple[Mapping[str, str], ...]).validate_python( + await _query(db, "SELECT current_schema() AS name") + ) + assert gateway.database_url is not None + with psycopg.connect(gateway.database_url) as blocker: + blocker.execute( + sql.SQL("LOCK TABLE {} IN ACCESS EXCLUSIVE MODE").format( + sql.Identifier(schemas[0]["name"], "LiteLLM_DailyUserSpend") + ) + ) + busy: Final = gateway.request("GET", f"/issues/{issue.id}/activity") + assert busy.status_code == 503 and busy.headers["Retry-After"] == "5" + assert gateway.request("GET", f"/issues/{issue.id}/activity").status_code == 200 + finally: + await _remove_issues(db, (issue.id,)) + await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1 RETURNING id', identity) + + @pytest.mark.asyncio(loop_scope="session") async def test_issue_provider_pairs_filter_activity_pages_counts_and_exports( gateway: Gateway, db: ManagementDatabase @@ -332,12 +498,6 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma next_issue_id: Final = str(uuid4()) try: source: Final = (await alerts.record_observations(rule, (observation,)))[0] - await _query( - db, - 'UPDATE "LiteLLM_UsageAlertIncident" SET evaluation_revision=$2 WHERE id=$1 RETURNING id', - source.id, - rule.updated_at.isoformat(), - ) initial: Final = await store.upsert_issue(original) assert tuple( item.alert.id @@ -359,12 +519,8 @@ async def test_issue_delivery_guards_snapshot_freeze_and_delayed_recovery(db: Ma assert disabled is not None and await store.pending_delivery(delivery.id) is None enabled: Final = await alerts.save_rule(definition, "qa-admin", disabled) assert enabled is not None and await store.pending_delivery(delivery.id) is None - await _query( - db, - 'UPDATE "LiteLLM_UsageAlertIncident" SET evaluation_revision=$2 WHERE id=$1 RETURNING id', - source.id, - enabled.updated_at.isoformat(), - ) + reevaluated: Final = await alerts.record_observations(enabled, (observation,)) + assert len(reevaluated) == 1 and reevaluated[0].id == source.id await store.queue_delivery(delivery, enabled.id, enabled.updated_at.isoformat(), source.id) assert await store.pending_delivery(delivery.id) is not None await alerts.set_status(source.id, "dismissed") @@ -654,9 +810,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) -> None: store: Final = IssuesRepository.from_prisma(db.prisma) alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) - expired: Final = _issue( - IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}") - ) + expired: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) open_issue: Final = _issue(expired.cohort) recent: Final = _issue(expired.cohort) rule: Final = await alerts.save_rule( @@ -671,8 +825,7 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( ) assert rule is not None deliveries: Final = tuple( - IssueDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") - for _ in range(2) + IssueDeliveryInput(id=str(uuid4()), issue_id=expired.id, channel="slack") for _ in range(2) ) try: await store.upsert_issues((expired, open_issue, recent)) @@ -693,16 +846,10 @@ async def test_issue_prune_removes_expired_children_in_bounded_passes( )[0] await store.add_signals( expired.id, - ( - IssueSignalInput( - source="usage_alert", source_id=source.id, usage_alert=source - ), - ), + (IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source),), ) for delivery in deliveries: - await store.queue_delivery( - delivery, rule.id, rule.updated_at.isoformat(), source.id - ) + await store.queue_delivery(delivery, rule.id, rule.updated_at.isoformat(), source.id) assert await store.set_status(expired.id, "resolved", "open") is not None assert await store.set_status(recent.id, "resolved", "open") is not None await _query( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a0ee4b62418..8eb38be8bf9 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -34901,6 +34901,16 @@ export interface components { * @default true */ active: boolean; + /** + * Can Update Status + * @description Current reader's status permission, evaluated by the issue-detail endpoint + */ + can_update_status?: boolean | null; + /** + * Can View Activity + * @description Current reader's permission to inspect Activity, keys, users and impact + */ + can_view_activity?: boolean | null; cohort: components["schemas"]["IssueCohort"]; /** Cohort Key */ cohort_key: string; @@ -65677,6 +65687,7 @@ export interface operations { query?: { start_date?: string | null; end_date?: string | null; + model_limit?: number; }; header?: never; path: { @@ -65692,7 +65703,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + "application/json": components["schemas"]["BoundedDailyActivityResponse"]; }; }; /** @description Validation Error */