From 756fb4cada6e097b90a24fdc719ea12b2611f0bb Mon Sep 17 00:00:00 2001 From: mateo <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:01:34 +0000 Subject: [PATCH] fix(usage): tolerate entity metadata database failures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../common_daily_activity.py | 33 +++++++---- .../test_common_daily_activity.py | 57 +++++++++++++++++++ 2 files changed, 79 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index bbd6a1c9021..3f5f70e58b4 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -8,6 +8,7 @@ from types import MappingProxyType from typing import Final, Literal, NoReturn, Protocol from fastapi import HTTPException, status +from prisma.errors import PrismaError from typing_extensions import ReadOnly, TypedDict, assert_never from litellm import constants @@ -1117,7 +1118,11 @@ async def _user_entity_metadata( repository: DailyActivityRepository, entity_rows: Sequence[EntityRollupRow] ) -> Mapping[str, Mapping[str, object]]: user_ids: Final = frozenset(row.entity_id for row in entity_rows if row.entity_id) - users: Final = await repository.user_metadata(user_ids) + try: + users: Final = await repository.user_metadata(user_ids) + except PrismaError as exc: + verbose_proxy_logger.warning("Failed daily activity entity user metadata recovery: %s", exc) + return MappingProxyType({}) return MappingProxyType( { user_id: MappingProxyType({"user_email": user.user_email, "user_alias": user.user_alias}) @@ -1126,6 +1131,21 @@ async def _user_entity_metadata( ) +async def _entity_key_metadata( + repository: DailyActivityRepository, entity_rows: Sequence[EntityRollupRow] +) -> Mapping[str, KeyMetadataRow]: + api_keys: Final = frozenset( + row.api_key for row in entity_rows if row.api_key and row.api_key != PTU_SENTINEL_API_KEY + ) + if not api_keys: + return MappingProxyType({}) + try: + return await repository.key_metadata(api_keys, spend_logs_window(frozenset(row.date for row in entity_rows))) + except PrismaError as exc: + verbose_proxy_logger.warning("Failed daily activity entity key metadata recovery: %s", exc) + return MappingProxyType({}) + + async def get_daily_activity_aggregated( repository: DailyActivityRepository, scope: DailyActivityScope, @@ -1160,16 +1180,7 @@ async def get_daily_activity_aggregated( ) if aggregated_rows.entity_rows: entity_records: Final = aggregated_rows.entity_rows - entity_api_keys: Final = frozenset( - row.api_key for row in entity_records if row.api_key and row.api_key != PTU_SENTINEL_API_KEY - ) - entity_key_metadata: Final[Mapping[str, KeyMetadataRow]] = ( - await repository.key_metadata( - entity_api_keys, spend_logs_window(frozenset(row.date for row in entity_records)) - ) - if entity_api_keys - else MappingProxyType({}) - ) + entity_key_metadata: Final = await _entity_key_metadata(repository, entity_records) user_entity_metadata: Final[Mapping[str, Mapping[str, object]]] = ( await _user_entity_metadata(repository, entity_records) if scope.table is DailyActivityTable.USER diff --git a/tests/unit/proxy/management_endpoints/test_common_daily_activity.py b/tests/unit/proxy/management_endpoints/test_common_daily_activity.py index 5fe4cd2edcd..f49f42a3d83 100644 --- a/tests/unit/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/unit/proxy/management_endpoints/test_common_daily_activity.py @@ -1,4 +1,5 @@ from collections.abc import Mapping, Sequence +from dataclasses import asdict from datetime import date, datetime from types import SimpleNamespace from typing import Final @@ -6,6 +7,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException +from prisma.errors import PrismaError import litellm.proxy.management_endpoints.common_daily_activity as common_daily_activity_module from litellm.constants import USAGE_TOP_API_KEYS_DEFAULT @@ -32,12 +34,67 @@ from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_E from litellm.proxy.utils import PrismaClient, hash_token from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailySpendMetadata, + KeyMetadata, SpendAnalyticsPaginatedResponse, SpendMetrics, ) from litellm.types.repositories.daily_activity import GroupingSetsRow, KeyMetadataRow +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("key_failure", "user_failure", "empty"), + [(False, False, False), (True, False, False), (False, True, False), (True, True, False), (False, False, True)], +) +async def test_entity_metadata_outages_preserve_daily_spend_and_other_metadata( + key_failure: bool, user_failure: bool, empty: bool +) -> None: + prisma: Final = MagicMock() + key: Final = SimpleNamespace(token="key", key_alias="Alias", team_id=None, user_id=None, metadata=None) + user: Final = SimpleNamespace(user_id="user", user_email="user@example.com", user_alias="User") + main_rows: Final = tuple( + asdict(_grouping_row(level, api_key=api_key, spend=2.0)) + for level, api_key in ((127, None), (63, None), (31, "key")) + ) + entity_rows: Final = tuple( + {**asdict(_grouping_row(63, api_key=api_key, spend=2.0)), "entity_id": "user", "api_key_rolled": rolled} + for rolled, api_key in ((1, None), (0, "key")) + ) + prisma.db.query_raw = AsyncMock(side_effect=[main_rows, entity_rows]) + prisma.db.litellm_verificationtoken.find_many = AsyncMock( + side_effect=[[key], PrismaError("key metadata unavailable") if key_failure else ([] if empty else [key])] + ) + prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + prisma.db.litellm_usertable.find_many = AsyncMock( + side_effect=PrismaError("user metadata unavailable") if user_failure else None, + return_value=[] if empty else [user], + ) + + result: Final = await get_daily_activity_aggregated( + prisma_client=prisma, + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id=None, + start_date="2024-01-01", + end_date="2024-01-01", + model=None, + api_key=None, + include_entity_breakdown=True, + ) + + day: Final = result.results[0] + entity: Final = day.breakdown.entities["user"] + assert result.metadata.total_spend == day.metrics.spend == entity.metrics.spend == 2.0 + assert day.breakdown.api_keys["key"].metadata.key_alias == "Alias" + assert entity.metadata == ( + {} if user_failure or empty else {"user_email": "user@example.com", "user_alias": "User"} + ) + assert entity.api_key_breakdown["key"].metrics.spend == 2.0 + assert entity.api_key_breakdown["key"].metadata == KeyMetadata( + key_alias=None if key_failure or empty else "Alias", key_exists=not (key_failure or empty) + ) + + async def _run_aggregated_daily_activity( *, prisma_client: PrismaClient,