From cd9c39921bbe114c2652f993fd0e02a604c24a97 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 14 Sep 2026 04:55:39 +0000 Subject: [PATCH 01/12] feat(proxy): add POST /user/bulk_delete and POST /team/bulk_member_delete Batch user deletion that also removes each user from every team they belong to, and batch removal of many members from one team. Each touched team is rewritten once under the team advisory lock from a roster re-read under that lock Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 2 + litellm/proxy/auth/route_checks.py | 4 + .../internal_user_endpoints.py | 59 +++ .../management_endpoints/team_endpoints.py | 52 +++ .../management_helpers/bulk_user_deletion.py | 442 ++++++++++++++++++ .../internal_user_endpoints.py | 25 +- .../management_endpoints/team_endpoints.py | 27 +- .../endpointaudit/coverage_allowlist.txt | 2 + .../test_bulk_user_deletion.py | 434 +++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 227 ++++++++- 10 files changed, 1270 insertions(+), 4 deletions(-) create mode 100644 litellm/proxy/management_helpers/bulk_user_deletion.py create mode 100644 tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b315a2beac9..39fefd02b86 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -659,6 +659,7 @@ class LiteLLMRoutes(enum.Enum): "/user/update", "/user/bulk_update", "/user/delete", + "/user/bulk_delete", "/user/info", "/user/list", "/user/daily/activity", @@ -838,6 +839,7 @@ class LiteLLMRoutes(enum.Enum): self_managed_routes = [ "/team/member_add", "/team/member_delete", + "/team/bulk_member_delete", "/team/member_update", "/team/{team_id}/member/{user_id}/reset_spend", "/team/permissions_list", diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 953e3cf3e88..34fede7a967 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -25,9 +25,11 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset( # user "/user/new", "/user/delete", + "/user/bulk_delete", "/user/bulk_update", # team "/team/new", + "/team/bulk_member_delete", "/team/update", "/team/delete", "/team/block", @@ -756,8 +758,10 @@ class RouteChecks: [ "/user/new", "/user/delete", + "/user/bulk_delete", "/user/bulk_update", "/team/new", + "/team/bulk_member_delete", "/team/update", "/team/delete", "/model/new", diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index e3efda507f6..a2e084f265a 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -8,6 +8,7 @@ These are members of a Team on LiteLLM /user/update /user/bulk_update /user/delete +/user/bulk_delete /user/info /user/list """ @@ -55,6 +56,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, prepare_metadata_fields, ) +from litellm.proxy.management_helpers.bulk_user_deletion import bulk_delete_users from litellm.proxy.management_helpers.object_permission_utils import ( _set_object_permission, handle_update_object_permission_common, @@ -77,6 +79,8 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, ) from litellm.types.proxy.management_endpoints.internal_user_endpoints import ( + BulkDeleteUserRequest, + BulkDeleteUserResponse, BulkUpdateUserRequest, BulkUpdateUserResponse, UserListResponse, @@ -2496,6 +2500,61 @@ async def delete_user( return deleted_users +@router.post( + "/user/bulk_delete", + tags=["Internal User management"], # mutable-ok: FastAPI's `tags` param is typed as list[str], not Sequence + dependencies=(Depends(user_api_key_auth),), + response_model=BulkDeleteUserResponse, +) +@management_endpoint_wrapper +async def bulk_delete_user( + data: BulkDeleteUserRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI dependency injection + litellm_changed_by: str | None = Header( + None, + description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + ), +) -> BulkDeleteUserResponse: + """ + Delete up to 500 internal users in one request and remove each one from every team they belong to. + + Same authorization as `/user/delete`: proxy admins may delete anyone, org admins only users whose + organizations they all administer. Each team a deleted user was on is rewritten once under the team + lock, so the roster, the user's `teams` array and the `LiteLLM_TeamMembership` rows all agree afterwards. + Then the users' keys, invitation links, organization memberships and user rows are deleted. + + Rows fail independently: unknown, duplicate or out-of-scope ids are reported in `results` with + `success: false` and an `error`, and the other users are still deleted. + + Usage Example + + ```shell + curl -X POST "http://localhost:4000/user/bulk_delete" \\ + -H "Content-Type: application/json" \\ + -H "Authorization: Bearer sk-1234" \\ + -d '{"user_ids": ["user-1", "user-2"]}' + ``` + + Returns `results` (one entry per input id, in order, with `user_id`, `user_email`, `success`, + `teams_removed`, `error`), `total_requested`, `successful_deletions` and `failed_deletions`. + """ + from litellm.proxy.proxy_server import litellm_proxy_admin_name, prisma_client + + if prisma_client is None: + raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) + try: + return await bulk_delete_users( + data=data, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + litellm_proxy_admin_name=litellm_proxy_admin_name, + litellm_changed_by=litellm_changed_by, + ) + except Exception as e: # noqa: BLE001 # normalize every failure to the proxy exception contract + verbose_proxy_logger.exception("/user/bulk_delete: Exception occured") + raise handle_exception_on_proxy(e) + + async def add_internal_user_to_organization( user_id: str, organization_id: str, diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 0b7f69bbb7f..91e803d6f29 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -162,6 +162,8 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( from litellm.types.proxy.management_endpoints.team_endpoints import ( BulkTeamMemberAddRequest, BulkTeamMemberAddResponse, + BulkTeamMemberDeleteRequest, + BulkTeamMemberDeleteResponse, BulkUpdateTeamMemberPermissionsRequest, BulkUpdateTeamMemberPermissionsResponse, GetTeamMemberPermissionsResponse, @@ -3451,6 +3453,56 @@ async def team_member_delete( return existing_team_row +@router.post( + "/team/bulk_member_delete", + tags=["team management"], # mutable-ok: FastAPI's `tags` param is typed as list[str], not Sequence + dependencies=(Depends(user_api_key_auth),), + response_model=BulkTeamMemberDeleteResponse, +) +@management_endpoint_wrapper +async def bulk_team_member_delete( + data: BulkTeamMemberDeleteRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI dependency injection +) -> BulkTeamMemberDeleteResponse: + """ + Remove up to 500 members from one team in a single request. + + Same authorization as `/team/member_delete` (proxy admin, team admin, or org admin of the team's + organization). Each member is named by `user_id` or `user_email`. The team is rewritten once under + the team lock: the roster, every removed user's `teams` array, their `LiteLLM_TeamMembership` rows and + their team-scoped keys are all cleaned up together. Members that are not on the team are reported in + `results` with `success: false` and the rest are still removed. + + Example request: + ```bash + curl --location 'http://0.0.0.0:4000/team/bulk_member_delete' \\ + --header 'Authorization: Bearer sk-1234' \\ + --header 'Content-Type: application/json' \\ + --data '{ + "team_id": "team-1234", + "members": [{"user_id": "user1"}, {"user_email": "user2@example.com"}] + }' + ``` + + Returns `team_id`, `results` (one entry per input member, in order, with `user_id`, `user_email`, + `success`, `error`), `total_requested`, `successful_deletions` and `failed_deletions`. + """ + from litellm.proxy.management_helpers.bulk_user_deletion import bulk_remove_team_members + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) + try: + return await bulk_remove_team_members( + data=data, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + ) + except Exception as e: # noqa: BLE001 # normalize every failure to the proxy exception contract + verbose_proxy_logger.exception("/team/bulk_member_delete: Exception occured") + raise handle_exception_on_proxy(e) + + _MEMBER_BUDGET_PATCH_FIELDS: Final = { "max_budget_in_team": "max_budget", "tpm_limit": "tpm_limit", diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py new file mode 100644 index 00000000000..a2d3348a30e --- /dev/null +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -0,0 +1,442 @@ +"""Batched deletes behind `POST /user/bulk_delete` and `POST /team/bulk_member_delete`. + +Each team a batch touches is rewritten exactly once, under the same advisory lock +`/team/member_delete` takes and from a roster re-read under that lock, so a concurrent +member_add on the team is never overwritten from a stale read. +""" + +import asyncio +import json +from collections.abc import Awaitable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from fastapi import HTTPException +from typing_extensions import ReadOnly, TypedDict + +from litellm._logging import verbose_proxy_logger +from litellm.integrations.prometheus import PrometheusLogger +from litellm.proxy._types import ( + LiteLLM_TeamTable, + LitellmUserRoles, + Member, + MemberDeleteRequest, + UserAPIKeyAuth, +) +from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks +from litellm.proxy.management_endpoints.common_utils import ( + _is_user_org_admin_for_team, # pyright: ignore[reportPrivateUsage] # same check /team/member_delete uses + _is_user_team_admin, # pyright: ignore[reportPrivateUsage] # same check /team/member_delete uses +) +from litellm.proxy.management_endpoints.key_management_endpoints import ( + _persist_deleted_verification_tokens, # pyright: ignore[reportPrivateUsage] # same audit path /key/delete uses +) +from litellm.proxy.management_helpers.access_group_team_sync import TEAM_ADVISORY_LOCK_SQL +from litellm.proxy.utils import PrismaClient +from litellm.repositories.table_repositories import ( + InvitationLinkRepository, + OrganizationMembershipRepository, + TeamMembershipRepository, +) +from litellm.repositories.team_repository import TeamRepository +from litellm.repositories.user_repository import UserRepository +from litellm.repositories.verification_token_repository import VerificationTokenRepository +from litellm.types.proxy.management_endpoints.internal_user_endpoints import ( + BulkDeleteUserRequest, + BulkDeleteUserResponse, + UserDeleteResult, +) +from litellm.types.proxy.management_endpoints.team_endpoints import ( + BulkTeamMemberDeleteRequest, + BulkTeamMemberDeleteResponse, + TeamMemberDeleteResult, +) + +if TYPE_CHECKING: + from prisma import Prisma + from prisma import models as prisma_models + + from litellm.repositories.prisma_protocols import TableActions + +_TEAM_WRITE_CONCURRENCY: Final = 10 + + +class _ErrorDetail(TypedDict): + error: ReadOnly[str] + + +class _OrgAdminFilter(TypedDict): + user_id: ReadOnly[str] + user_role: ReadOnly[str] + + +class _RosterData(TypedDict): + members_with_roles: ReadOnly[str] + + +class _TeamsSet(TypedDict): + set: ReadOnly[tuple[str, ...]] + + +class _TeamsData(TypedDict): + teams: ReadOnly[_TeamsSet] + + +@dataclass(frozen=True, slots=True) +class _TeamRemoval: + """One team's rewrite. `removed` holds the user ids taken off the team (roster, `teams` array, or both); + `matched` holds the indexes into the requested members that named at least one of them.""" + + team: LiteLLM_TeamTable + removed: frozenset[str] + matched: frozenset[int] + + +def _http_error(status_code: int, message: str) -> HTTPException: + detail: Final[_ErrorDetail] = {"error": message} + return HTTPException(status_code=status_code, detail=detail) + + +def _in_filter(field: str, values: Iterable[str]) -> Mapping[str, object]: + return {field: {"in": sorted(values)}} # mutable-ok: Prisma query filters are dict-shaped + + +def _eq_filter(field: str, value: str) -> Mapping[str, object]: + return {field: value} # mutable-ok: Prisma query filters are dict-shaped + + +def _team_users_filter(team_id: str, user_ids: Iterable[str]) -> Mapping[str, object]: + return {"team_id": team_id, **_in_filter("user_id", user_ids)} # mutable-ok: Prisma query filters are dict-shaped + + +def _any_filter(*clauses: Mapping[str, object]) -> Mapping[str, object]: + return {"OR": clauses} # mutable-ok: Prisma query filters are dict-shaped + + +def _team_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamTable]": + return tx.litellm_teamtable # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do + + +def _user_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_UserTable]": + return tx.litellm_usertable # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do + + +def _membership_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamMembership]": + return tx.litellm_teammembership # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do + + +def _token_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_VerificationToken]": + return tx.litellm_verificationtoken # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do + + +def _addresses_member(member: Member, request: MemberDeleteRequest) -> bool: + return (request.user_id is not None and request.user_id == member.user_id) or ( + request.user_email is not None and request.user_email == member.user_email + ) + + +def _addresses_user(user: "prisma_models.LiteLLM_UserTable", request: MemberDeleteRequest) -> bool: + return (request.user_id is not None and request.user_id == user.user_id) or ( + request.user_email is not None and request.user_email == user.user_email + ) + + +def _error_message(exc: BaseException) -> str: + if isinstance(exc, HTTPException) and isinstance(exc.detail, dict): + return str(exc.detail.get("error", exc.detail)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # HTTPException.detail is untyped + if isinstance(exc, HTTPException): + return str(exc.detail) # pyright: ignore[reportUnknownArgumentType] # HTTPException.detail is untyped + return str(exc) or type(exc).__name__ + + +async def _bounded(awaitables: Iterable[Awaitable[object]]) -> tuple[object | BaseException, ...]: + semaphore: Final = asyncio.Semaphore(_TEAM_WRITE_CONCURRENCY) + + async def run(awaitable: Awaitable[object]) -> object: + async with semaphore: + return await awaitable + + return tuple(await asyncio.gather(*(run(a) for a in awaitables), return_exceptions=True)) + + +async def _remove_members_from_team( + prisma_client: PrismaClient, + team_id: str, + members: Sequence[MemberDeleteRequest], + user_api_key_dict: UserAPIKeyAuth, +) -> _TeamRemoval: + async with prisma_client.tx() as tx: + await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id) + roster: Final = await TeamRepository(prisma_client).get_members_with_roles_locked(tx, team_id) + if roster is None: + raise _http_error(400, f"Team id={team_id} does not exist in db") + + removed_members: Final = tuple(m for m in roster if any(_addresses_member(m, r) for r in members)) + kept_members: Final = tuple(m for m in roster if not any(_addresses_member(m, r) for r in members)) + removed_ids: Final = frozenset(m.user_id for m in removed_members if m.user_id is not None) + requested_ids: Final = frozenset(r.user_id for r in members if r.user_id is not None) + requested_emails: Final = frozenset(r.user_email for r in members if r.user_id is None and r.user_email) + user_rows: Final = await _user_tx_db(tx).find_many( + where=_any_filter( + _in_filter("user_id", removed_ids | requested_ids), + _in_filter("user_email", requested_emails), + ) + ) + stale_rows: Final = tuple(u for u in user_rows if team_id in u.teams) + cleanup_ids: Final = removed_ids | requested_ids | frozenset(u.user_id for u in stale_rows) + matched: Final = frozenset( + i + for i, r in enumerate(members) + if any(_addresses_member(m, r) for m in removed_members) or any(_addresses_user(u, r) for u in stale_rows) + ) + keys: Final = await _token_tx_db(tx).find_many(where=_team_users_filter(team_id, cleanup_ids)) + + if removed_members: + roster_data: Final[_RosterData] = { + "members_with_roles": json.dumps(tuple(m.model_dump() for m in kept_members)) + } + await _team_tx_db(tx).update(where=_eq_filter("team_id", team_id), data=roster_data) + for row in stale_rows: + teams_data: _TeamsData = {"teams": {"set": tuple(t for t in row.teams if t != team_id)}} + await _user_tx_db(tx).update(where=_eq_filter("user_id", row.user_id), data=teams_data) + await _membership_tx_db(tx).delete_many(where=_team_users_filter(team_id, cleanup_ids)) + if keys: + await _persist_deleted_verification_tokens( + keys=keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + tx=tx, + ) + await _token_tx_db(tx).delete_many(where=_team_users_filter(team_id, cleanup_ids)) + + return _TeamRemoval( + team=LiteLLM_TeamTable( + team_id=team_id, + members_with_roles=kept_members, # pyright: ignore[reportArgumentType] # pydantic coerces the tuple into the list field + ), + removed=removed_ids | frozenset(u.user_id for u in stale_rows), + matched=matched, + ) + + +def _emit_team_members_metric(team: LiteLLM_TeamTable) -> None: + prometheus_logger: Final = PrometheusLogger.get_instance() + if prometheus_logger is None: + return + try: + prometheus_logger.set_team_members_metric(team) + except Exception as e: + verbose_proxy_logger.debug("Prometheus: failed to emit team members metric: %s", str(e)) + + +async def bulk_remove_team_members( + data: BulkTeamMemberDeleteRequest, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: PrismaClient, +) -> BulkTeamMemberDeleteResponse: + team: Final = await TeamRepository(prisma_client).find_by_id(data.team_id) + if team is None: + raise _http_error(400, f"Team id={data.team_id} does not exist in db") + + if ( + user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value + and not _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team) + and not await _is_user_org_admin_for_team(user_api_key_dict=user_api_key_dict, team_obj=team) + ): + raise _http_error( + 403, + "Call not allowed. User not proxy admin OR team admin OR org admin for this team. " + f"route='/team/bulk_member_delete', team_id={data.team_id}", + ) + + removal: Final = await _remove_members_from_team(prisma_client, data.team_id, data.members, user_api_key_dict) + _emit_team_members_metric(removal.team) + + results: Final = tuple( + TeamMemberDeleteResult( + user_id=member.user_id, + user_email=member.user_email, + success=i in removal.matched, + error=None if i in removal.matched else "User not found in team", + ) + for i, member in enumerate(data.members) + ) + successful: Final = sum(1 for r in results if r.success) + return BulkTeamMemberDeleteResponse( + team_id=data.team_id, + results=results, + total_requested=len(results), + successful_deletions=successful, + failed_deletions=len(results) - successful, + ) + + +async def _caller_admin_org_ids(prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]: + if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value or not user_api_key_dict.user_id: + return frozenset() + where: Final[_OrgAdminFilter] = { + "user_id": user_api_key_dict.user_id, + "user_role": LitellmUserRoles.ORG_ADMIN.value, + } + memberships: Final = await OrganizationMembershipRepository(prisma_client).table.find_many(where=where) + return frozenset(m.organization_id for m in memberships if m.organization_id) + + +def _scope_error(user_id: str, target_org_ids: frozenset[str], caller_admin_org_ids: frozenset[str]) -> str | None: + if target_org_ids and target_org_ids <= caller_admin_org_ids: + return None + return ( + f"User {user_id} is not within your admin scope. " + "Only PROXY_ADMIN may delete users outside your administered organizations." + ) + + +async def _delete_user_rows( + prisma_client: PrismaClient, + users: Sequence["prisma_models.LiteLLM_UserTable"], + user_api_key_dict: UserAPIKeyAuth, + litellm_proxy_admin_name: str | None, + litellm_changed_by: str | None, +) -> None: + user_ids: Final = frozenset(u.user_id for u in users) + await _bounded( + UserManagementEventHooks.create_internal_user_audit_log( + user_id=u.user_id, + action="deleted", + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + before_value=u.model_dump_json(exclude_none=True), + ) + for u in users + ) + keys: Final = await VerificationTokenRepository(prisma_client).table.find_many( + where=_in_filter("user_id", user_ids) + ) + if keys: + await _persist_deleted_verification_tokens( + keys=keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + await VerificationTokenRepository(prisma_client).table.delete_many(where=_in_filter("user_id", user_ids)) + await InvitationLinkRepository(prisma_client).table.delete_many( + where=_any_filter( + _in_filter("user_id", user_ids), + _in_filter("created_by", user_ids), + _in_filter("updated_by", user_ids), + ) + ) + await OrganizationMembershipRepository(prisma_client).table.delete_many(where=_in_filter("user_id", user_ids)) + await TeamMembershipRepository(prisma_client).table.delete_many(where=_in_filter("user_id", user_ids)) + await UserRepository(prisma_client).table.delete_many(where=_in_filter("user_id", user_ids)) + + +async def bulk_delete_users( + data: BulkDeleteUserRequest, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: PrismaClient, + litellm_proxy_admin_name: str | None, + litellm_changed_by: str | None, +) -> BulkDeleteUserResponse: + caller_is_proxy_admin: Final = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value + caller_admin_org_ids: Final = await _caller_admin_org_ids(prisma_client, user_api_key_dict) + if not caller_is_proxy_admin and not caller_admin_org_ids: + raise _http_error(403, "Only PROXY_ADMIN or ORG_ADMIN users may delete users.") + + unique_ids: Final = frozenset(data.user_ids) + rows: Final = await UserRepository(prisma_client).table.find_many(where=_in_filter("user_id", unique_ids)) + rows_by_id: Final = MappingProxyType({row.user_id: row for row in rows}) + target_memberships: Final = ( + () + if caller_is_proxy_admin + else await OrganizationMembershipRepository(prisma_client).table.find_many( + where=_in_filter("user_id", unique_ids) + ) + ) + + def precheck_error(user_id: str) -> str | None: + if user_id not in rows_by_id: + return f"User id={user_id} not found" + if caller_is_proxy_admin: + return None + org_ids: Final = frozenset( + m.organization_id for m in target_memberships if m.user_id == user_id and m.organization_id + ) + return _scope_error(user_id, org_ids, caller_admin_org_ids) + + precheck_errors: Final = MappingProxyType({uid: precheck_error(uid) for uid in unique_ids}) + candidates: Final = tuple(rows_by_id[uid] for uid in sorted(unique_ids) if precheck_errors[uid] is None) + candidate_ids: Final = frozenset(u.user_id for u in candidates) + + memberships: Final = await TeamMembershipRepository(prisma_client).table.find_many( + where=_in_filter("user_id", candidate_ids) + ) + teams_of: Final = MappingProxyType( + { + u.user_id: frozenset(u.teams) | frozenset(m.team_id for m in memberships if m.user_id == u.user_id) + for u in candidates + } + ) + team_ids: Final = tuple(sorted(frozenset(t for u in candidates for t in teams_of[u.user_id]))) + members_by_team: Final = MappingProxyType( + { + tid: tuple( + MemberDeleteRequest(user_id=u.user_id, user_email=u.user_email) + for u in candidates + if tid in teams_of[u.user_id] + ) + for tid in team_ids + } + ) + outcomes: Final = await _bounded( + _remove_members_from_team(prisma_client, tid, members_by_team[tid], user_api_key_dict) for tid in team_ids + ) + removals: Final = MappingProxyType( + {tid: o for tid, o in zip(team_ids, outcomes, strict=True) if isinstance(o, _TeamRemoval)} + ) + team_failures: Final = MappingProxyType( + {tid: _error_message(o) for tid, o in zip(team_ids, outcomes, strict=True) if isinstance(o, BaseException)} + ) + for tid, err in team_failures.items(): + verbose_proxy_logger.error("/user/bulk_delete: failed to remove users from team %s: %s", tid, err) + for removal in removals.values(): + _emit_team_members_metric(removal.team) + + def team_errors(user_id: str) -> tuple[str, ...]: + return tuple( + f"Failed to remove from team {tid}: {err}" for tid, err in team_failures.items() if tid in teams_of[user_id] + ) + + deletable: Final = tuple(u for u in candidates if not team_errors(u.user_id)) + if deletable: + await _delete_user_rows( + prisma_client, deletable, user_api_key_dict, litellm_proxy_admin_name, litellm_changed_by + ) + + def result(index: int, user_id: str) -> UserDeleteResult: + if user_id in data.user_ids[:index]: + return UserDeleteResult(user_id=user_id, success=False, error=f"Duplicate user_id in request: {user_id}") + error: Final = precheck_errors[user_id] + if error is not None: + return UserDeleteResult(user_id=user_id, success=False, error=error) + errors: Final = team_errors(user_id) + return UserDeleteResult( + user_id=user_id, + user_email=rows_by_id[user_id].user_email, + success=not errors, + teams_removed=tuple(tid for tid in team_ids if tid in removals and user_id in removals[tid].removed), + error="; ".join(errors) or None, + ) + + results: Final = tuple(result(i, uid) for i, uid in enumerate(data.user_ids)) + successful: Final = sum(1 for r in results if r.success) + return BulkDeleteUserResponse( + results=results, + total_requested=len(results), + successful_deletions=successful, + failed_deletions=len(results) - successful, + ) diff --git a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py index 6973f1d1f12..7d9f2bd2a85 100644 --- a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py @@ -1,7 +1,7 @@ from collections.abc import Mapping from typing import Any, Final, Literal -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, Field, field_validator from typing_extensions import ReadOnly, TypedDict from litellm.proxy._types import ( @@ -10,6 +10,8 @@ from litellm.proxy._types import ( UpdateUserRequestNoUserIDorEmail, ) +MAX_BULK_DELETE_USERS: Final = 500 + class InsensitiveContains(TypedDict): contains: ReadOnly[str] @@ -83,3 +85,24 @@ class BulkUpdateUserResponse(BaseModel): total_requested: int successful_updates: int failed_updates: int + + +class BulkDeleteUserRequest(BaseModel): + user_ids: tuple[str, ...] = Field(min_length=1, max_length=MAX_BULK_DELETE_USERS) + + +class UserDeleteResult(BaseModel): + """Outcome for one row of `/user/bulk_delete`. `teams_removed` lists the teams the user was taken out of.""" + + user_id: str + user_email: str | None = None + success: bool + teams_removed: tuple[str, ...] = () + error: str | None = None + + +class BulkDeleteUserResponse(BaseModel): + results: tuple[UserDeleteResult, ...] + total_requested: int + successful_deletions: int + failed_deletions: int diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index a282430bb11..6952de9c1cc 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Any, Final, Literal from pydantic import BaseModel, ConfigDict, Field @@ -8,10 +8,13 @@ from litellm.proxy._types import ( LiteLLM_TeamMembership, LiteLLM_TeamTable, Member, + MemberDeleteRequest, ) TeamIdSearchMatch = Literal["exact", "prefix"] +MAX_BULK_TEAM_MEMBER_DELETES: Final = 500 + class GetTeamMemberPermissionsRequest(BaseModel): """Request to get the team member permissions for a team""" @@ -118,6 +121,28 @@ class BulkTeamMemberAddResponse(BaseModel): updated_team: dict[str, Any] | None = None +class BulkTeamMemberDeleteRequest(BaseModel): + team_id: str + members: tuple[MemberDeleteRequest, ...] = Field(min_length=1, max_length=MAX_BULK_TEAM_MEMBER_DELETES) + + +class TeamMemberDeleteResult(BaseModel): + """Outcome for one row of `/team/bulk_member_delete`.""" + + user_id: str | None = None + user_email: str | None = None + success: bool + error: str | None = None + + +class BulkTeamMemberDeleteResponse(BaseModel): + team_id: str + results: tuple[TeamMemberDeleteResult, ...] + total_requested: int + successful_deletions: int + failed_deletions: int + + class TeamMemberInfoResponse(LiteLLM_TeamMembership): """Response for GET /team/{team_id}/members/me — caller's own membership row.""" diff --git a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt index 6bc8947e89f..53a6bd4f8c6 100644 --- a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt +++ b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt @@ -80,10 +80,12 @@ POST /model/unblock POST /prompts/test POST /search_tools/test_connection POST /team/bulk_member_add +POST /team/bulk_member_delete POST /team/{team_id}/member/{user_id}/reset_spend POST /team/key/bulk_update POST /team/permissions_bulk_update POST /team/{team_id}/disable_logging +POST /user/bulk_delete POST /user/bulk_update # Alternate method or path for functionality the provider already manages elsewhere diff --git a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py new file mode 100644 index 00000000000..91ce7e56a40 --- /dev/null +++ b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py @@ -0,0 +1,434 @@ +import json +from collections.abc import Callable, Mapping, Sequence +from contextlib import asynccontextmanager +from typing import Final + +import pytest +from fastapi import HTTPException +from pydantic import BaseModel, ConfigDict, ValidationError + +from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, MemberDeleteRequest, UserAPIKeyAuth +from litellm.proxy.management_helpers.bulk_user_deletion import bulk_delete_users, bulk_remove_team_members +from litellm.types.proxy.management_endpoints.internal_user_endpoints import BulkDeleteUserRequest +from litellm.types.proxy.management_endpoints.team_endpoints import BulkTeamMemberDeleteRequest + +ADMIN: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin") +INTERNAL: Final = UserAPIKeyAuth(user_id="someone", user_role=LitellmUserRoles.INTERNAL_USER) +ORG_ADMIN: Final = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN) + + +class _UserRow(BaseModel): + model_config = ConfigDict(extra="allow") + + user_id: str + user_email: str | None = None + teams: list[str] = [] + + +class _Record(BaseModel): + """Attribute access like a Prisma row, over whatever columns the test seeded.""" + + model_config = ConfigDict(extra="allow") + + +def _in(where: Mapping[str, object], field: str) -> set[str] | None: + clause = where.get(field) + if isinstance(clause, dict) and "in" in clause: + return set(clause["in"]) + if isinstance(clause, str): + return {clause} + return None + + +def _matches(row: Mapping[str, object], where: Mapping[str, object]) -> bool: + if "OR" in where: + return any(_matches(row, clause) for clause in where["OR"]) + return all((wanted := _in(where, field)) is not None and row.get(field) in wanted for field in where) + + +class _Rows: + """A list-backed Prisma table supporting the `in`/equality/OR filters the helper issues.""" + + def __init__(self, rows: Sequence[Mapping[str, object]] = ()) -> None: + self.rows: list[dict[str, object]] = [dict(r) for r in rows] + + async def find_many(self, where: Mapping[str, object]) -> list[_Record]: + return [_Record.model_validate(r) for r in self.rows if _matches(r, where)] + + async def delete_many(self, where: Mapping[str, object]) -> int: + before = len(self.rows) + self.rows = [r for r in self.rows if not _matches(r, where)] + return before - len(self.rows) + + async def create_many(self, data: Sequence[Mapping[str, object]]) -> int: + self.rows.extend(dict(r) for r in data) + return len(data) + + +class _UserTable: + def __init__(self, users: Sequence[_UserRow]) -> None: + self.rows: dict[str, _UserRow] = {u.user_id: u for u in users} + + async def find_many(self, where: Mapping[str, object]) -> list[_UserRow]: + return [u for u in self.rows.values() if _matches(u.model_dump(), where)] + + async def update(self, where: Mapping[str, str], data: Mapping[str, Mapping[str, Sequence[str]]]) -> _UserRow: + row = self.rows[where["user_id"]] + updated = row.model_copy(update={"teams": list(data["teams"]["set"])}) + self.rows[row.user_id] = updated + return updated + + async def delete_many(self, where: Mapping[str, object]) -> int: + doomed = [uid for uid, u in self.rows.items() if _matches(u.model_dump(), where)] + for uid in doomed: + del self.rows[uid] + return len(doomed) + + +class _TeamTable: + def __init__(self, teams: Sequence[LiteLLM_TeamTable]) -> None: + self.rows: dict[str, LiteLLM_TeamTable] = {t.team_id: t for t in teams} + self.update_calls = 0 + + async def find_unique(self, where: Mapping[str, str]) -> LiteLLM_TeamTable | None: + return self.rows.get(where["team_id"]) + + async def update(self, where: Mapping[str, str], data: Mapping[str, str]) -> LiteLLM_TeamTable: + self.update_calls += 1 + team = self.rows[where["team_id"]] + team.members_with_roles = [Member(**m) for m in json.loads(data["members_with_roles"])] + return team + + +class _Db: + def __init__( + self, + users: Sequence[_UserRow], + teams: Sequence[LiteLLM_TeamTable], + memberships: Sequence[tuple[str, str]] = (), + tokens: Sequence[Mapping[str, object]] = (), + invitations: Sequence[Mapping[str, object]] = (), + org_memberships: Sequence[Mapping[str, object]] = (), + ) -> None: + self.litellm_usertable = _UserTable(users) + self.litellm_teamtable = _TeamTable(teams) + self.litellm_teammembership = _Rows([{"team_id": t, "user_id": u} for t, u in memberships]) + self.litellm_verificationtoken = _Rows(tokens) + self.litellm_deletedverificationtoken = _Rows() + self.litellm_invitationlink = _Rows(invitations) + self.litellm_organizationmembership = _Rows(org_memberships) + + +class _Tx: + def __init__(self, db: _Db, on_lock: Callable[[str], None], fail_locks: frozenset[str]) -> None: + self.litellm_teamtable = db.litellm_teamtable + self.litellm_usertable = db.litellm_usertable + self.litellm_teammembership = db.litellm_teammembership + self.litellm_verificationtoken = db.litellm_verificationtoken + self.litellm_deletedverificationtoken = db.litellm_deletedverificationtoken + self._on_lock = on_lock + self._fail_locks = fail_locks + self.locks: list[str] = [] + self.roster_reads: list[str] = [] + + async def query_raw(self, sql: str, *args: object) -> list[dict[str, object]]: + team_id = str(args[0]) + if "pg_advisory_xact_lock" in sql: + if team_id in self._fail_locks: + raise RuntimeError("lock timeout") + self.locks.append(team_id) + self._on_lock(team_id) + return [] + assert self.locks == [team_id], "roster must be read under this team's advisory lock" + self.roster_reads.append(team_id) + team = self.litellm_teamtable.rows.get(team_id) + if team is None: + return [] + return [{"members_with_roles": json.dumps([m.model_dump() for m in team.members_with_roles])}] + + +class _FakePrisma: + def __init__( + self, + users: Sequence[_UserRow] = (), + teams: Sequence[LiteLLM_TeamTable] = (), + memberships: Sequence[tuple[str, str]] = (), + tokens: Sequence[Mapping[str, object]] = (), + invitations: Sequence[Mapping[str, object]] = (), + org_memberships: Sequence[Mapping[str, object]] = (), + on_lock: Callable[[str], None] = lambda _: None, + fail_locks: frozenset[str] = frozenset(), + ) -> None: + self.db = _Db(users, teams, memberships, tokens, invitations, org_memberships) + self._on_lock = on_lock + self._fail_locks = fail_locks + self.locks: list[str] = [] + self.roster_reads: list[str] = [] + + @asynccontextmanager + async def tx(self): + tx = _Tx(self.db, self._on_lock, self._fail_locks) + yield tx + self.locks.extend(tx.locks) + self.roster_reads.extend(tx.roster_reads) + + +def _team(team_id: str, *members: str, org: str | None = None) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + organization_id=org, + members_with_roles=[Member(user_id=m, user_email=f"{m}@example.com", role="user") for m in members], + ) + + +def _user(user_id: str, *teams: str) -> _UserRow: + return _UserRow(user_id=user_id, user_email=f"{user_id}@example.com", teams=list(teams)) + + +def _roster(prisma: _FakePrisma, team_id: str) -> list[str | None]: + return [m.user_id for m in prisma.db.litellm_teamtable.rows[team_id].members_with_roles] + + +async def _delete(prisma: _FakePrisma, user_ids: Sequence[str], caller: UserAPIKeyAuth = ADMIN): + return await bulk_delete_users( + data=BulkDeleteUserRequest(user_ids=tuple(user_ids)), + user_api_key_dict=caller, + prisma_client=prisma, # pyright: ignore[reportArgumentType] # fake stands in for PrismaClient + litellm_proxy_admin_name="default_user_id", + litellm_changed_by=None, + ) + + +async def _remove(prisma: _FakePrisma, team_id: str, members: Sequence[Mapping[str, str]], caller=ADMIN): + return await bulk_remove_team_members( + data=BulkTeamMemberDeleteRequest(team_id=team_id, members=tuple(MemberDeleteRequest(**m) for m in members)), + user_api_key_dict=caller, + prisma_client=prisma, # pyright: ignore[reportArgumentType] # fake stands in for PrismaClient + ) + + +@pytest.mark.asyncio +async def test_bulk_delete_removes_users_from_every_team_and_store(): + prisma = _FakePrisma( + users=[_user("u1", "t1", "t2"), _user("u2", "t1"), _user("keep", "t1")], + teams=[_team("t1", "u1", "u2", "keep"), _team("t2", "u1", "other")], + memberships=[("t1", "u1"), ("t2", "u1"), ("t1", "u2"), ("t1", "keep")], + tokens=[{"token": "k1", "user_id": "u1", "team_id": "t1"}, {"token": "k2", "user_id": "keep"}], + invitations=[ + {"id": "i1", "user_id": "u2", "created_by": "admin", "updated_by": "admin"}, + {"id": "i2", "user_id": "keep", "created_by": "u1", "updated_by": "admin"}, + {"id": "i3", "user_id": "keep", "created_by": "admin", "updated_by": "admin"}, + ], + org_memberships=[{"user_id": "u1", "organization_id": "o1", "user_role": "internal_user"}], + ) + + response = await _delete(prisma, ["u1", "u2"]) + + assert (response.total_requested, response.successful_deletions, response.failed_deletions) == (2, 2, 0) + assert [(r.user_id, r.user_email, r.success, r.teams_removed) for r in response.results] == [ + ("u1", "u1@example.com", True, ("t1", "t2")), + ("u2", "u2@example.com", True, ("t1",)), + ] + assert _roster(prisma, "t1") == ["keep"] and _roster(prisma, "t2") == ["other"] + assert set(prisma.db.litellm_usertable.rows) == {"keep"} + assert prisma.db.litellm_teammembership.rows == [{"team_id": "t1", "user_id": "keep"}] + assert [t["token"] for t in prisma.db.litellm_verificationtoken.rows] == ["k2"] + assert [t["token"] for t in prisma.db.litellm_deletedverificationtoken.rows] == ["k1"] + assert [i["id"] for i in prisma.db.litellm_invitationlink.rows] == ["i3"] + assert prisma.db.litellm_organizationmembership.rows == [] + assert sorted(prisma.locks) == ["t1", "t2"] and sorted(prisma.roster_reads) == ["t1", "t2"] + + +@pytest.mark.asyncio +async def test_bulk_delete_finds_teams_through_membership_rows_when_user_teams_array_is_stale(): + prisma = _FakePrisma( + users=[_user("u1")], + teams=[_team("t1", "u1", "keep")], + memberships=[("t1", "u1")], + ) + + response = await _delete(prisma, ["u1"]) + + assert response.results[0].teams_removed == ("t1",) + assert _roster(prisma, "t1") == ["keep"] + assert prisma.db.litellm_teammembership.rows == [] + + +@pytest.mark.asyncio +async def test_bulk_delete_reads_roster_under_lock_so_a_concurrent_add_survives(): + team = _team("t1", "u1") + + def concurrent_member_add(team_id: str) -> None: + team.members_with_roles.append(Member(user_id="late", role="user")) + + prisma = _FakePrisma(users=[_user("u1", "t1")], teams=[team], on_lock=concurrent_member_add) + + response = await _delete(prisma, ["u1"]) + + assert response.results[0].success is True + assert _roster(prisma, "t1") == ["late"] + + +@pytest.mark.asyncio +async def test_bulk_delete_reports_missing_and_duplicate_ids_per_item_and_still_deletes_the_rest(): + prisma = _FakePrisma(users=[_user("u1")]) + + response = await _delete(prisma, ["u1", "ghost", "u1"]) + + assert (response.successful_deletions, response.failed_deletions) == (1, 2) + assert [(r.user_id, r.success, r.error) for r in response.results] == [ + ("u1", True, None), + ("ghost", False, "User id=ghost not found"), + ("u1", False, "Duplicate user_id in request: u1"), + ] + assert prisma.db.litellm_usertable.rows == {} + + +@pytest.mark.asyncio +async def test_bulk_delete_keeps_user_when_a_team_rewrite_fails_and_deletes_the_others(): + prisma = _FakePrisma( + users=[_user("u1", "bad", "good"), _user("u2", "good")], + teams=[_team("bad", "u1"), _team("good", "u1", "u2")], + fail_locks=frozenset({"bad"}), + ) + + response = await _delete(prisma, ["u1", "u2"]) + + assert [(r.user_id, r.success, r.teams_removed) for r in response.results] == [ + ("u1", False, ("good",)), + ("u2", True, ("good",)), + ] + assert response.results[0].error == "Failed to remove from team bad: lock timeout" + assert set(prisma.db.litellm_usertable.rows) == {"u1"} + assert _roster(prisma, "bad") == ["u1"] and _roster(prisma, "good") == [] + + +@pytest.mark.asyncio +async def test_bulk_delete_rejects_non_admin_callers_before_touching_the_db(): + prisma = _FakePrisma(users=[_user("u1")]) + + with pytest.raises(HTTPException) as exc: + await _delete(prisma, ["u1"], caller=INTERNAL) + + assert exc.value.status_code == 403 + assert set(prisma.db.litellm_usertable.rows) == {"u1"} + + +@pytest.mark.asyncio +async def test_org_admin_deletes_only_users_fully_inside_their_orgs(): + prisma = _FakePrisma( + users=[_user("inside"), _user("straddles"), _user("orgless")], + org_memberships=[ + {"user_id": "org-admin", "organization_id": "o1", "user_role": LitellmUserRoles.ORG_ADMIN.value}, + {"user_id": "inside", "organization_id": "o1", "user_role": "internal_user"}, + {"user_id": "straddles", "organization_id": "o1", "user_role": "internal_user"}, + {"user_id": "straddles", "organization_id": "o2", "user_role": "internal_user"}, + ], + ) + + response = await _delete(prisma, ["inside", "straddles", "orgless"], caller=ORG_ADMIN) + + assert [r.success for r in response.results] == [True, False, False] + assert all("not within your admin scope" in (r.error or "") for r in response.results[1:]) + assert set(prisma.db.litellm_usertable.rows) == {"straddles", "orgless"} + assert {(m["user_id"], m["organization_id"]) for m in prisma.db.litellm_organizationmembership.rows} == { + ("org-admin", "o1"), + ("straddles", "o1"), + ("straddles", "o2"), + } + + +@pytest.mark.asyncio +async def test_bulk_member_delete_removes_by_id_and_email_and_keeps_the_rest(): + prisma = _FakePrisma( + users=[_user("u1", "t1", "t2"), _user("u2", "t1"), _user("keep", "t1")], + teams=[_team("t1", "u1", "u2", "keep")], + memberships=[("t1", "u1"), ("t1", "u2"), ("t1", "keep")], + tokens=[ + {"token": "team-key", "user_id": "u1", "team_id": "t1"}, + {"token": "other-team-key", "user_id": "u1", "team_id": "t2"}, + {"token": "keep-key", "user_id": "keep", "team_id": "t1"}, + ], + ) + + response = await _remove(prisma, "t1", [{"user_id": "u1"}, {"user_email": "u2@example.com"}]) + + assert (response.team_id, response.successful_deletions, response.failed_deletions) == ("t1", 2, 0) + assert [(r.user_id, r.user_email, r.success) for r in response.results] == [ + ("u1", None, True), + (None, "u2@example.com", True), + ] + assert _roster(prisma, "t1") == ["keep"] + users = prisma.db.litellm_usertable.rows + assert users["u1"].teams == ["t2"] and users["u2"].teams == [] and users["keep"].teams == ["t1"] + assert prisma.db.litellm_teammembership.rows == [{"team_id": "t1", "user_id": "keep"}] + assert sorted(t["token"] for t in prisma.db.litellm_verificationtoken.rows) == ["keep-key", "other-team-key"] + assert [t["token"] for t in prisma.db.litellm_deletedverificationtoken.rows] == ["team-key"] + assert prisma.locks == ["t1"] and prisma.roster_reads == ["t1"] + + +@pytest.mark.asyncio +async def test_bulk_member_delete_reports_members_not_on_the_team_without_rewriting_the_roster(): + prisma = _FakePrisma(users=[_user("u1", "t1"), _user("elsewhere")], teams=[_team("t1", "u1")]) + + response = await _remove(prisma, "t1", [{"user_id": "elsewhere"}, {"user_email": "nobody@example.com"}]) + + assert [(r.success, r.error) for r in response.results] == [ + (False, "User not found in team"), + (False, "User not found in team"), + ] + assert (response.successful_deletions, response.failed_deletions) == (0, 2) + assert prisma.db.litellm_teamtable.update_calls == 0 + assert _roster(prisma, "t1") == ["u1"] + + +@pytest.mark.asyncio +async def test_bulk_member_delete_cleans_a_user_whose_teams_array_still_names_the_team(): + prisma = _FakePrisma(users=[_user("stale", "t1")], teams=[_team("t1", "other")], memberships=[("t1", "stale")]) + + response = await _remove(prisma, "t1", [{"user_id": "stale"}]) + + assert response.results[0].success is True + assert prisma.db.litellm_usertable.rows["stale"].teams == [] + assert prisma.db.litellm_teammembership.rows == [] + assert _roster(prisma, "t1") == ["other"] and prisma.db.litellm_teamtable.update_calls == 0 + + +@pytest.mark.asyncio +async def test_bulk_member_delete_rejects_unknown_team_and_unauthorized_callers(): + prisma = _FakePrisma(users=[_user("u1", "t1")], teams=[_team("t1", "u1")]) + + with pytest.raises(HTTPException) as missing: + await _remove(prisma, "nope", [{"user_id": "u1"}]) + with pytest.raises(HTTPException) as forbidden: + await _remove(prisma, "t1", [{"user_id": "u1"}], caller=INTERNAL) + + assert missing.value.status_code == 400 + assert forbidden.value.status_code == 403 + assert _roster(prisma, "t1") == ["u1"] and prisma.locks == [] + + +@pytest.mark.asyncio +async def test_team_admin_may_bulk_remove_members(): + team = _team("t1", "lead", "u1") + team.members_with_roles[0].role = "admin" + prisma = _FakePrisma(users=[_user("lead", "t1"), _user("u1", "t1")], teams=[team]) + + response = await _remove(prisma, "t1", [{"user_id": "u1"}], caller=UserAPIKeyAuth(user_id="lead")) + + assert response.results[0].success is True + assert _roster(prisma, "t1") == ["lead"] + + +def test_request_models_enforce_batch_bounds(): + with pytest.raises(ValidationError): + BulkDeleteUserRequest(user_ids=()) + with pytest.raises(ValidationError): + BulkDeleteUserRequest(user_ids=tuple(f"u{i}" for i in range(501))) + with pytest.raises(ValidationError): + BulkTeamMemberDeleteRequest(team_id="t1", members=()) + with pytest.raises(ValidationError): + BulkTeamMemberDeleteRequest( + team_id="t1", members=tuple(MemberDeleteRequest(user_id=f"u{i}") for i in range(501)) + ) + assert len(BulkDeleteUserRequest(user_ids=tuple(f"u{i}" for i in range(500))).user_ids) == 500 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7eadaa6c991..14aa5b081bd 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -15056,6 +15056,46 @@ export interface paths { patch?: never; trace?: never; }; + "/team/bulk_member_delete": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Bulk Team Member Delete + * @description Remove up to 500 members from one team in a single request. + * + * Same authorization as `/team/member_delete` (proxy admin, team admin, or org admin of the team's + * organization). Each member is named by `user_id` or `user_email`. The team is rewritten once under + * the team lock: the roster, every removed user's `teams` array, their `LiteLLM_TeamMembership` rows and + * their team-scoped keys are all cleaned up together. Members that are not on the team are reported in + * `results` with `success: false` and the rest are still removed. + * + * Example request: + * ```bash + * curl --location 'http://0.0.0.0:4000/team/bulk_member_delete' \ + * --header 'Authorization: Bearer sk-1234' \ + * --header 'Content-Type: application/json' \ + * --data '{ + * "team_id": "team-1234", + * "members": [{"user_id": "user1"}, {"user_email": "user2@example.com"}] + * }' + * ``` + * + * Returns `team_id`, `results` (one entry per input member, in order, with `user_id`, `user_email`, + * `success`, `error`), `total_requested`, `successful_deletions` and `failed_deletions`. + */ + post: operations["bulk_team_member_delete_team_bulk_member_delete_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/team/daily/activity": { parameters: { query?: never; @@ -16478,6 +16518,46 @@ export interface paths { patch?: never; trace?: never; }; + "/user/bulk_delete": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Bulk Delete User + * @description Delete up to 500 internal users in one request and remove each one from every team they belong to. + * + * Same authorization as `/user/delete`: proxy admins may delete anyone, org admins only users whose + * organizations they all administer. Each team a deleted user was on is rewritten once under the team + * lock, so the roster, the user's `teams` array and the `LiteLLM_TeamMembership` rows all agree afterwards. + * Then the users' keys, invitation links, organization memberships and user rows are deleted. + * + * Rows fail independently: unknown, duplicate or out-of-scope ids are reported in `results` with + * `success: false` and an `error`, and the other users are still deleted. + * + * Usage Example + * + * ```shell + * curl -X POST "http://localhost:4000/user/bulk_delete" \ + * -H "Content-Type: application/json" \ + * -H "Authorization: Bearer sk-1234" \ + * -d '{"user_ids": ["user-1", "user-2"]}' + * ``` + * + * Returns `results` (one entry per input id, in order, with `user_id`, `user_email`, `success`, + * `teams_removed`, `error`), `total_requested`, `successful_deletions` and `failed_deletions`. + */ + post: operations["bulk_delete_user_user_bulk_delete_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/user/bulk_update": { parameters: { query?: never; @@ -16781,7 +16861,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) @@ -16887,7 +16966,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) @@ -24505,6 +24583,22 @@ export interface components { /** Budgets */ budgets: string[]; }; + /** BulkDeleteUserRequest */ + BulkDeleteUserRequest: { + /** User Ids */ + user_ids: string[]; + }; + /** BulkDeleteUserResponse */ + BulkDeleteUserResponse: { + /** Failed Deletions */ + failed_deletions: number; + /** Results */ + results: components["schemas"]["UserDeleteResult"][]; + /** Successful Deletions */ + successful_deletions: number; + /** Total Requested */ + total_requested: number; + }; /** * BulkTeamMemberAddRequest * @description Request for bulk team member addition @@ -24542,6 +24636,26 @@ export interface components { [key: string]: unknown; } | null; }; + /** BulkTeamMemberDeleteRequest */ + BulkTeamMemberDeleteRequest: { + /** Members */ + members: components["schemas"]["MemberDeleteRequest"][]; + /** Team Id */ + team_id: string; + }; + /** BulkTeamMemberDeleteResponse */ + BulkTeamMemberDeleteResponse: { + /** Failed Deletions */ + failed_deletions: number; + /** Results */ + results: components["schemas"]["TeamMemberDeleteResult"][]; + /** Successful Deletions */ + successful_deletions: number; + /** Team Id */ + team_id: string; + /** Total Requested */ + total_requested: number; + }; /** * BulkUpdateKeyRequest * @description Request for bulk key updates @@ -31903,6 +32017,13 @@ export interface components { */ user_id?: string | null; }; + /** MemberDeleteRequest */ + MemberDeleteRequest: { + /** User Email */ + user_email?: string | null; + /** User Id */ + user_id?: string | null; + }; /** MemoryCreateRequest */ MemoryCreateRequest: { /** @@ -37166,6 +37287,20 @@ export interface components { /** User Id */ user_id?: string | null; }; + /** + * TeamMemberDeleteResult + * @description Outcome for one row of `/team/bulk_member_delete`. + */ + TeamMemberDeleteResult: { + /** Error */ + error?: string | null; + /** Success */ + success: boolean; + /** User Email */ + user_email?: string | null; + /** User Id */ + user_id?: string | null; + }; /** * TeamMemberInfoResponse * @description Response for GET /team/{team_id}/members/me — caller's own membership row. @@ -39345,6 +39480,25 @@ export interface components { */ severity: "info" | "warning" | "error"; }; + /** + * UserDeleteResult + * @description Outcome for one row of `/user/bulk_delete`. `teams_removed` lists the teams the user was taken out of. + */ + UserDeleteResult: { + /** Error */ + error?: string | null; + /** Success */ + success: boolean; + /** + * Teams Removed + * @default [] + */ + teams_removed: string[]; + /** User Email */ + user_email?: string | null; + /** User Id */ + user_id: string; + }; /** * UserHeaderMapping * @description Map an incoming HTTP header to a LiteLLM user role. @@ -58887,6 +59041,39 @@ export interface operations { }; }; }; + bulk_team_member_delete_team_bulk_member_delete_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BulkTeamMemberDeleteRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BulkTeamMemberDeleteResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_team_daily_activity_team_daily_activity_get: { parameters: { query?: { @@ -60631,6 +60818,42 @@ export interface operations { }; }; }; + bulk_delete_user_user_bulk_delete_post: { + parameters: { + query?: never; + header?: { + /** @description The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability */ + "litellm-changed-by"?: string | null; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BulkDeleteUserRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BulkDeleteUserResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; bulk_user_update_user_bulk_update_post: { parameters: { query?: never; From beaa96fc8f849262dd16ada53c272bb514c636be Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 14 Sep 2026 04:57:27 +0000 Subject: [PATCH 02/12] fix(proxy): log audit log failures in /user/bulk_delete instead of dropping them Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/management_helpers/bulk_user_deletion.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py index a2d3348a30e..f2ac8259fa5 100644 --- a/litellm/proxy/management_helpers/bulk_user_deletion.py +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -301,7 +301,7 @@ async def _delete_user_rows( litellm_changed_by: str | None, ) -> None: user_ids: Final = frozenset(u.user_id for u in users) - await _bounded( + audit_outcomes: Final = await _bounded( UserManagementEventHooks.create_internal_user_audit_log( user_id=u.user_id, action="deleted", @@ -312,6 +312,9 @@ async def _delete_user_rows( ) for u in users ) + for u, outcome in zip(users, audit_outcomes, strict=True): + if isinstance(outcome, BaseException): + verbose_proxy_logger.warning("Failed to create audit log for user %s: %s", u.user_id, outcome) keys: Final = await VerificationTokenRepository(prisma_client).table.find_many( where=_in_filter("user_id", user_ids) ) From 595aba3cb0d6a5b3815d4ab51811a33c50fc8aa5 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 14 Sep 2026 05:12:34 +0000 Subject: [PATCH 03/12] fix(proxy): evict deleted keys from the auth cache and make bulk user deletion transactional /user/bulk_delete now deletes the users' keys, invitation links, org and team memberships and user rows in one transaction and reports a rolled-back batch per row instead of leaving partial deletes behind. Both bulk endpoints evict the deleted keys (and deleted user objects) from the auth cache, so a deleted key stops authenticating immediately rather than at TTL expiry. /team/bulk_member_delete rejects member rows that carry both user_id and user_email, reports repeated rows as duplicates, and only cleans up keys and memberships of members it actually matched. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../internal_user_endpoints.py | 33 ++-- .../management_endpoints/team_endpoints.py | 27 +--- .../management_helpers/bulk_user_deletion.py | 150 ++++++++++++++---- .../management_endpoints/team_endpoints.py | 9 +- .../test_bulk_user_deletion.py | 150 +++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 47 ++---- 6 files changed, 303 insertions(+), 113 deletions(-) diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index a2e084f265a..6b11847f09d 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -2516,29 +2516,20 @@ async def bulk_delete_user( ), ) -> BulkDeleteUserResponse: """ - Delete up to 500 internal users in one request and remove each one from every team they belong to. + Delete up to 500 users and remove each one from every team they belong to. Same authorization as + `/user/delete`. Returns one result per user id, in order. - Same authorization as `/user/delete`: proxy admins may delete anyone, org admins only users whose - organizations they all administer. Each team a deleted user was on is rewritten once under the team - lock, so the roster, the user's `teams` array and the `LiteLLM_TeamMembership` rows all agree afterwards. - Then the users' keys, invitation links, organization memberships and user rows are deleted. - - Rows fail independently: unknown, duplicate or out-of-scope ids are reported in `results` with - `success: false` and an `error`, and the other users are still deleted. - - Usage Example - - ```shell - curl -X POST "http://localhost:4000/user/bulk_delete" \\ - -H "Content-Type: application/json" \\ - -H "Authorization: Bearer sk-1234" \\ - -d '{"user_ids": ["user-1", "user-2"]}' + ```bash + curl -X POST 'http://localhost:4000/user/bulk_delete' -H 'Authorization: Bearer sk-1234' \\ + -H 'Content-Type: application/json' -d '{"user_ids": ["user-1", "user-2"]}' ``` - - Returns `results` (one entry per input id, in order, with `user_id`, `user_email`, `success`, - `teams_removed`, `error`), `total_requested`, `successful_deletions` and `failed_deletions`. """ - from litellm.proxy.proxy_server import litellm_proxy_admin_name, prisma_client + from litellm.proxy.proxy_server import ( + litellm_proxy_admin_name, + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) if prisma_client is None: raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) @@ -2547,6 +2538,8 @@ async def bulk_delete_user( data=data, user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, litellm_proxy_admin_name=litellm_proxy_admin_name, litellm_changed_by=litellm_changed_by, ) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 91e803d6f29..8626c073e55 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3465,30 +3465,17 @@ async def bulk_team_member_delete( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI dependency injection ) -> BulkTeamMemberDeleteResponse: """ - Remove up to 500 members from one team in a single request. + Remove up to 500 members (each named by `user_id` or `user_email`) from one team. Same authorization + as `/team/member_delete`. Returns one result per member, in order. - Same authorization as `/team/member_delete` (proxy admin, team admin, or org admin of the team's - organization). Each member is named by `user_id` or `user_email`. The team is rewritten once under - the team lock: the roster, every removed user's `teams` array, their `LiteLLM_TeamMembership` rows and - their team-scoped keys are all cleaned up together. Members that are not on the team are reported in - `results` with `success: false` and the rest are still removed. - - Example request: ```bash - curl --location 'http://0.0.0.0:4000/team/bulk_member_delete' \\ - --header 'Authorization: Bearer sk-1234' \\ - --header 'Content-Type: application/json' \\ - --data '{ - "team_id": "team-1234", - "members": [{"user_id": "user1"}, {"user_email": "user2@example.com"}] - }' + curl -X POST 'http://0.0.0.0:4000/team/bulk_member_delete' -H 'Authorization: Bearer sk-1234' \\ + -H 'Content-Type: application/json' \\ + -d '{"team_id": "team-1234", "members": [{"user_id": "user1"}, {"user_email": "user2@example.com"}]}' ``` - - Returns `team_id`, `results` (one entry per input member, in order, with `user_id`, `user_email`, - `success`, `error`), `total_requested`, `successful_deletions` and `failed_deletions`. """ from litellm.proxy.management_helpers.bulk_user_deletion import bulk_remove_team_members - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache if prisma_client is None: raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) @@ -3497,6 +3484,8 @@ async def bulk_team_member_delete( data=data, user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, ) except Exception as e: # noqa: BLE001 # normalize every failure to the proxy exception contract verbose_proxy_logger.exception("/team/bulk_member_delete: Exception occured") diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py index f2ac8259fa5..4d3387aacb7 100644 --- a/litellm/proxy/management_helpers/bulk_user_deletion.py +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -24,6 +24,9 @@ from litellm.proxy._types import ( MemberDeleteRequest, UserAPIKeyAuth, ) +from litellm.proxy.auth.auth_checks import delete_cache_key_objects +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks from litellm.proxy.management_endpoints.common_utils import ( _is_user_org_admin_for_team, # pyright: ignore[reportPrivateUsage] # same check /team/member_delete uses @@ -33,15 +36,13 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( _persist_deleted_verification_tokens, # pyright: ignore[reportPrivateUsage] # same audit path /key/delete uses ) from litellm.proxy.management_helpers.access_group_team_sync import TEAM_ADVISORY_LOCK_SQL -from litellm.proxy.utils import PrismaClient +from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.table_repositories import ( - InvitationLinkRepository, OrganizationMembershipRepository, TeamMembershipRepository, ) from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository -from litellm.repositories.verification_token_repository import VerificationTokenRepository from litellm.types.proxy.management_endpoints.internal_user_endpoints import ( BulkDeleteUserRequest, BulkDeleteUserResponse, @@ -91,6 +92,7 @@ class _TeamRemoval: team: LiteLLM_TeamTable removed: frozenset[str] matched: frozenset[int] + deleted_key_tokens: tuple[str, ...] def _http_error(status_code: int, message: str) -> HTTPException: @@ -130,6 +132,14 @@ def _token_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_Verificati return tx.litellm_verificationtoken # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do +def _invitation_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_InvitationLink]": + return tx.litellm_invitationlink # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do + + +def _org_membership_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_OrganizationMembership]": + return tx.litellm_organizationmembership # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do + + def _addresses_member(member: Member, request: MemberDeleteRequest) -> bool: return (request.user_id is not None and request.user_id == member.user_id) or ( request.user_email is not None and request.user_email == member.user_email @@ -184,7 +194,7 @@ async def _remove_members_from_team( ) ) stale_rows: Final = tuple(u for u in user_rows if team_id in u.teams) - cleanup_ids: Final = removed_ids | requested_ids | frozenset(u.user_id for u in stale_rows) + cleanup_ids: Final = removed_ids | frozenset(u.user_id for u in stale_rows) matched: Final = frozenset( i for i, r in enumerate(members) @@ -216,8 +226,9 @@ async def _remove_members_from_team( team_id=team_id, members_with_roles=kept_members, # pyright: ignore[reportArgumentType] # pydantic coerces the tuple into the list field ), - removed=removed_ids | frozenset(u.user_id for u in stale_rows), + removed=cleanup_ids, matched=matched, + deleted_key_tokens=tuple(k.token for k in keys), ) @@ -231,10 +242,24 @@ def _emit_team_members_metric(team: LiteLLM_TeamTable) -> None: verbose_proxy_logger.debug("Prometheus: failed to emit team members metric: %s", str(e)) +def _duplicate_member_indexes(members: Sequence[MemberDeleteRequest]) -> frozenset[int]: + return frozenset( + i + for i, m in enumerate(members) + if any( + (m.user_id is not None and m.user_id == earlier.user_id) + or (m.user_email is not None and m.user_email == earlier.user_email) + for earlier in members[:i] + ) + ) + + async def bulk_remove_team_members( data: BulkTeamMemberDeleteRequest, user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging | None, ) -> BulkTeamMemberDeleteResponse: team: Final = await TeamRepository(prisma_client).find_by_id(data.team_id) if team is None: @@ -251,15 +276,30 @@ async def bulk_remove_team_members( f"route='/team/bulk_member_delete', team_id={data.team_id}", ) - removal: Final = await _remove_members_from_team(prisma_client, data.team_id, data.members, user_api_key_dict) + duplicates: Final = _duplicate_member_indexes(data.members) + kept_indexes: Final = tuple(i for i in range(len(data.members)) if i not in duplicates) + members: Final = tuple(data.members[i] for i in kept_indexes) + removal: Final = await _remove_members_from_team(prisma_client, data.team_id, members, user_api_key_dict) + await delete_cache_key_objects( + hashed_tokens=removal.deleted_key_tokens, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) _emit_team_members_metric(removal.team) + matched: Final = frozenset(kept_indexes[j] for j in removal.matched) + + def error(index: int) -> str | None: + if index in duplicates: + return "Duplicate member in request" + return None if index in matched else "User not found in team" + results: Final = tuple( TeamMemberDeleteResult( user_id=member.user_id, user_email=member.user_email, - success=i in removal.matched, - error=None if i in removal.matched else "User not found in team", + success=i in matched, + error=error(i), ) for i, member in enumerate(data.members) ) @@ -293,14 +333,60 @@ def _scope_error(user_id: str, target_org_ids: frozenset[str], caller_admin_org_ ) +async def _delete_user_rows_tx( + prisma_client: PrismaClient, + user_ids: frozenset[str], + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: str | None, +) -> tuple[str, ...]: + async with prisma_client.tx() as tx: + keys: Final = await _token_tx_db(tx).find_many(where=_in_filter("user_id", user_ids)) + if keys: + await _persist_deleted_verification_tokens( + keys=keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + tx=tx, + ) + await _token_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) + await _invitation_tx_db(tx).delete_many( + where=_any_filter( + _in_filter("user_id", user_ids), + _in_filter("created_by", user_ids), + _in_filter("updated_by", user_ids), + ) + ) + await _org_membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) + await _membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) + await _user_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) + return tuple(k.token for k in keys) + + async def _delete_user_rows( prisma_client: PrismaClient, users: Sequence["prisma_models.LiteLLM_UserTable"], user_api_key_dict: UserAPIKeyAuth, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging | None, litellm_proxy_admin_name: str | None, litellm_changed_by: str | None, -) -> None: +) -> str | None: + """Returns the error message when the transaction rolled back, in which case no row was touched.""" user_ids: Final = frozenset(u.user_id for u in users) + try: + deleted_key_tokens: Final = await _delete_user_rows_tx( + prisma_client, user_ids, user_api_key_dict, litellm_changed_by + ) + except Exception as e: # noqa: BLE001 # the rolled-back batch is reported per row, not as a request failure + verbose_proxy_logger.error("/user/bulk_delete: failed to delete users %s: %s", sorted(user_ids), e) + return _error_message(e) + await delete_cache_key_objects( + hashed_tokens=deleted_key_tokens, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + await evict_and_broadcast(cache_keys=sorted(user_ids), user_api_key_cache=user_api_key_cache) audit_outcomes: Final = await _bounded( UserManagementEventHooks.create_internal_user_audit_log( user_id=u.user_id, @@ -315,33 +401,15 @@ async def _delete_user_rows( for u, outcome in zip(users, audit_outcomes, strict=True): if isinstance(outcome, BaseException): verbose_proxy_logger.warning("Failed to create audit log for user %s: %s", u.user_id, outcome) - keys: Final = await VerificationTokenRepository(prisma_client).table.find_many( - where=_in_filter("user_id", user_ids) - ) - if keys: - await _persist_deleted_verification_tokens( - keys=keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken - prisma_client=prisma_client, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=litellm_changed_by, - ) - await VerificationTokenRepository(prisma_client).table.delete_many(where=_in_filter("user_id", user_ids)) - await InvitationLinkRepository(prisma_client).table.delete_many( - where=_any_filter( - _in_filter("user_id", user_ids), - _in_filter("created_by", user_ids), - _in_filter("updated_by", user_ids), - ) - ) - await OrganizationMembershipRepository(prisma_client).table.delete_many(where=_in_filter("user_id", user_ids)) - await TeamMembershipRepository(prisma_client).table.delete_many(where=_in_filter("user_id", user_ids)) - await UserRepository(prisma_client).table.delete_many(where=_in_filter("user_id", user_ids)) + return None async def bulk_delete_users( data: BulkDeleteUserRequest, user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging | None, litellm_proxy_admin_name: str | None, litellm_changed_by: str | None, ) -> BulkDeleteUserResponse: @@ -406,6 +474,11 @@ async def bulk_delete_users( ) for tid, err in team_failures.items(): verbose_proxy_logger.error("/user/bulk_delete: failed to remove users from team %s: %s", tid, err) + await delete_cache_key_objects( + hashed_tokens=tuple(t for removal in removals.values() for t in removal.deleted_key_tokens), + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) for removal in removals.values(): _emit_team_members_metric(removal.team) @@ -415,10 +488,19 @@ async def bulk_delete_users( ) deletable: Final = tuple(u for u in candidates if not team_errors(u.user_id)) - if deletable: + delete_error: Final = ( await _delete_user_rows( - prisma_client, deletable, user_api_key_dict, litellm_proxy_admin_name, litellm_changed_by + prisma_client, + deletable, + user_api_key_dict, + user_api_key_cache, + proxy_logging_obj, + litellm_proxy_admin_name, + litellm_changed_by, ) + if deletable + else None + ) def result(index: int, user_id: str) -> UserDeleteResult: if user_id in data.user_ids[:index]: @@ -426,7 +508,9 @@ async def bulk_delete_users( error: Final = precheck_errors[user_id] if error is not None: return UserDeleteResult(user_id=user_id, success=False, error=error) - errors: Final = team_errors(user_id) + errors: Final = team_errors(user_id) or ( + (f"Failed to delete user: {delete_error}",) if delete_error is not None else () + ) return UserDeleteResult( user_id=user_id, user_email=rows_by_id[user_id].user_email, diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 6952de9c1cc..1825813ae9c 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,6 +1,6 @@ from typing import Any, Final, Literal -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator from litellm.proxy._types import ( KeyManagementRoutes, @@ -125,6 +125,13 @@ class BulkTeamMemberDeleteRequest(BaseModel): team_id: str members: tuple[MemberDeleteRequest, ...] = Field(min_length=1, max_length=MAX_BULK_TEAM_MEMBER_DELETES) + @field_validator("members") + @classmethod + def one_identifier_per_member(cls, members: tuple[MemberDeleteRequest, ...]) -> tuple[MemberDeleteRequest, ...]: + if any(m.user_id is not None and m.user_email is not None for m in members): + raise ValueError("Each member must be identified by exactly one of user_id or user_email") + return members + class TeamMemberDeleteResult(BaseModel): """Outcome for one row of `/team/bulk_member_delete`.""" diff --git a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py index 91ce7e56a40..a8a09518326 100644 --- a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py +++ b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py @@ -1,3 +1,4 @@ +import copy import json from collections.abc import Callable, Mapping, Sequence from contextlib import asynccontextmanager @@ -8,6 +9,7 @@ from fastapi import HTTPException from pydantic import BaseModel, ConfigDict, ValidationError from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, MemberDeleteRequest, UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.management_helpers.bulk_user_deletion import bulk_delete_users, bulk_remove_team_members from litellm.types.proxy.management_endpoints.internal_user_endpoints import BulkDeleteUserRequest from litellm.types.proxy.management_endpoints.team_endpoints import BulkTeamMemberDeleteRequest @@ -126,6 +128,8 @@ class _Tx: self.litellm_teammembership = db.litellm_teammembership self.litellm_verificationtoken = db.litellm_verificationtoken self.litellm_deletedverificationtoken = db.litellm_deletedverificationtoken + self.litellm_invitationlink = db.litellm_invitationlink + self.litellm_organizationmembership = db.litellm_organizationmembership self._on_lock = on_lock self._fail_locks = fail_locks self.locks: list[str] = [] @@ -158,17 +162,26 @@ class _FakePrisma: org_memberships: Sequence[Mapping[str, object]] = (), on_lock: Callable[[str], None] = lambda _: None, fail_locks: frozenset[str] = frozenset(), + fail_user_delete: bool = False, ) -> None: self.db = _Db(users, teams, memberships, tokens, invitations, org_memberships) self._on_lock = on_lock self._fail_locks = fail_locks + self._fail_user_delete = fail_user_delete self.locks: list[str] = [] self.roster_reads: list[str] = [] @asynccontextmanager async def tx(self): + snapshot = copy.deepcopy(self.db) tx = _Tx(self.db, self._on_lock, self._fail_locks) - yield tx + try: + yield tx + if self._fail_user_delete and tx.locks == []: + raise RuntimeError("connection reset") + except BaseException: + self.db.__dict__.update(snapshot.__dict__) + raise self.locks.extend(tx.locks) self.roster_reads.extend(tx.roster_reads) @@ -189,21 +202,43 @@ def _roster(prisma: _FakePrisma, team_id: str) -> list[str | None]: return [m.user_id for m in prisma.db.litellm_teamtable.rows[team_id].members_with_roles] -async def _delete(prisma: _FakePrisma, user_ids: Sequence[str], caller: UserAPIKeyAuth = ADMIN): +def _cache_with(*hashed_tokens: str) -> UserApiKeyCache: + cache = UserApiKeyCache() + for token in hashed_tokens: + cache.set_cache(key=token, value=UserAPIKeyAuth(token=token)) + return cache + + +async def _delete( + prisma: _FakePrisma, + user_ids: Sequence[str], + caller: UserAPIKeyAuth = ADMIN, + cache: UserApiKeyCache | None = None, +): return await bulk_delete_users( data=BulkDeleteUserRequest(user_ids=tuple(user_ids)), user_api_key_dict=caller, prisma_client=prisma, # pyright: ignore[reportArgumentType] # fake stands in for PrismaClient + user_api_key_cache=cache or UserApiKeyCache(), + proxy_logging_obj=None, litellm_proxy_admin_name="default_user_id", litellm_changed_by=None, ) -async def _remove(prisma: _FakePrisma, team_id: str, members: Sequence[Mapping[str, str]], caller=ADMIN): +async def _remove( + prisma: _FakePrisma, + team_id: str, + members: Sequence[Mapping[str, str]], + caller: UserAPIKeyAuth = ADMIN, + cache: UserApiKeyCache | None = None, +): return await bulk_remove_team_members( data=BulkTeamMemberDeleteRequest(team_id=team_id, members=tuple(MemberDeleteRequest(**m) for m in members)), user_api_key_dict=caller, prisma_client=prisma, # pyright: ignore[reportArgumentType] # fake stands in for PrismaClient + user_api_key_cache=cache or UserApiKeyCache(), + proxy_logging_obj=None, ) @@ -303,6 +338,50 @@ async def test_bulk_delete_keeps_user_when_a_team_rewrite_fails_and_deletes_the_ assert _roster(prisma, "bad") == ["u1"] and _roster(prisma, "good") == [] +@pytest.mark.asyncio +async def test_bulk_delete_rolls_back_every_user_row_and_reports_it_per_row_when_the_delete_fails(): + prisma = _FakePrisma( + users=[_user("u1", "t1"), _user("u2")], + teams=[_team("t1", "u1")], + tokens=[{"token": "k1", "user_id": "u1"}], + fail_user_delete=True, + ) + cache = _cache_with("k1") + + response = await _delete(prisma, ["u1", "u2", "ghost"], cache=cache) + + assert [(r.user_id, r.success, r.error) for r in response.results] == [ + ("u1", False, "Failed to delete user: connection reset"), + ("u2", False, "Failed to delete user: connection reset"), + ("ghost", False, "User id=ghost not found"), + ] + assert set(prisma.db.litellm_usertable.rows) == {"u1", "u2"} + assert [t["token"] for t in prisma.db.litellm_verificationtoken.rows] == ["k1"] + assert prisma.db.litellm_deletedverificationtoken.rows == [] + assert cache.get_cache(key="k1") is not None + + +@pytest.mark.asyncio +async def test_bulk_delete_evicts_deleted_keys_and_users_from_the_auth_cache(): + prisma = _FakePrisma( + users=[_user("u1", "t1"), _user("keep", "t1")], + teams=[_team("t1", "u1", "keep")], + tokens=[ + {"token": "team-key", "user_id": "u1", "team_id": "t1"}, + {"token": "personal-key", "user_id": "u1"}, + {"token": "keep-key", "user_id": "keep", "team_id": "t1"}, + ], + ) + cache = _cache_with("team-key", "personal-key", "keep-key") + cache.set_cache(key="u1", value={"user_id": "u1"}) + + await _delete(prisma, ["u1"], cache=cache) + + assert cache.get_cache(key="team-key") is None and cache.get_cache(key="personal-key") is None + assert cache.get_cache(key="u1") is None + assert cache.get_cache(key="keep-key") is not None + + @pytest.mark.asyncio async def test_bulk_delete_rejects_non_admin_callers_before_touching_the_db(): prisma = _FakePrisma(users=[_user("u1")]) @@ -382,6 +461,58 @@ async def test_bulk_member_delete_reports_members_not_on_the_team_without_rewrit assert _roster(prisma, "t1") == ["u1"] +@pytest.mark.asyncio +async def test_bulk_member_delete_leaves_keys_and_memberships_of_unmatched_members_alone(): + prisma = _FakePrisma( + users=[_user("u1", "t1"), _user("elsewhere")], + teams=[_team("t1", "u1")], + memberships=[("t1", "elsewhere")], + tokens=[{"token": "orphan-key", "user_id": "elsewhere", "team_id": "t1"}], + ) + + response = await _remove(prisma, "t1", [{"user_id": "elsewhere"}]) + + assert response.results[0].success is False + assert prisma.db.litellm_teammembership.rows == [{"team_id": "t1", "user_id": "elsewhere"}] + assert [t["token"] for t in prisma.db.litellm_verificationtoken.rows] == ["orphan-key"] + + +@pytest.mark.asyncio +async def test_bulk_member_delete_reports_repeated_members_as_duplicates_and_removes_them_once(): + prisma = _FakePrisma(users=[_user("u1", "t1"), _user("u2", "t1")], teams=[_team("t1", "u1", "u2", "keep")]) + + response = await _remove( + prisma, "t1", [{"user_id": "u1"}, {"user_id": "u1"}, {"user_email": "u1@example.com"}, {"user_id": "u2"}] + ) + + assert [(r.success, r.error) for r in response.results] == [ + (True, None), + (False, "Duplicate member in request"), + (True, None), + (True, None), + ] + assert (response.successful_deletions, response.failed_deletions) == (3, 1) + assert _roster(prisma, "t1") == ["keep"] + + +@pytest.mark.asyncio +async def test_bulk_member_delete_evicts_the_removed_team_keys_from_the_auth_cache(): + prisma = _FakePrisma( + users=[_user("u1", "t1"), _user("keep", "t1")], + teams=[_team("t1", "u1", "keep")], + tokens=[ + {"token": "team-key", "user_id": "u1", "team_id": "t1"}, + {"token": "keep-key", "user_id": "keep", "team_id": "t1"}, + ], + ) + cache = _cache_with("team-key", "keep-key") + + await _remove(prisma, "t1", [{"user_id": "u1"}], cache=cache) + + assert cache.get_cache(key="team-key") is None + assert cache.get_cache(key="keep-key") is not None + + @pytest.mark.asyncio async def test_bulk_member_delete_cleans_a_user_whose_teams_array_still_names_the_team(): prisma = _FakePrisma(users=[_user("stale", "t1")], teams=[_team("t1", "other")], memberships=[("t1", "stale")]) @@ -432,3 +563,16 @@ def test_request_models_enforce_batch_bounds(): team_id="t1", members=tuple(MemberDeleteRequest(user_id=f"u{i}") for i in range(501)) ) assert len(BulkDeleteUserRequest(user_ids=tuple(f"u{i}" for i in range(500))).user_ids) == 500 + + +def test_bulk_member_delete_request_requires_exactly_one_identifier_per_member(): + with pytest.raises(ValidationError, match="exactly one of user_id or user_email"): + BulkTeamMemberDeleteRequest( + team_id="t1", members=(MemberDeleteRequest(user_id="u1", user_email="other@example.com"),) + ) + with pytest.raises(ValidationError): + BulkTeamMemberDeleteRequest.model_validate({"team_id": "t1", "members": [{}]}) + assert ( + BulkTeamMemberDeleteRequest(team_id="t1", members=(MemberDeleteRequest(user_id="u1"),)).members[0].user_id + == "u1" + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 14aa5b081bd..23c989a3220 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -15067,27 +15067,14 @@ export interface paths { put?: never; /** * Bulk Team Member Delete - * @description Remove up to 500 members from one team in a single request. + * @description Remove up to 500 members (each named by `user_id` or `user_email`) from one team. Same authorization + * as `/team/member_delete`. Returns one result per member, in order. * - * Same authorization as `/team/member_delete` (proxy admin, team admin, or org admin of the team's - * organization). Each member is named by `user_id` or `user_email`. The team is rewritten once under - * the team lock: the roster, every removed user's `teams` array, their `LiteLLM_TeamMembership` rows and - * their team-scoped keys are all cleaned up together. Members that are not on the team are reported in - * `results` with `success: false` and the rest are still removed. - * - * Example request: * ```bash - * curl --location 'http://0.0.0.0:4000/team/bulk_member_delete' \ - * --header 'Authorization: Bearer sk-1234' \ - * --header 'Content-Type: application/json' \ - * --data '{ - * "team_id": "team-1234", - * "members": [{"user_id": "user1"}, {"user_email": "user2@example.com"}] - * }' + * curl -X POST 'http://0.0.0.0:4000/team/bulk_member_delete' -H 'Authorization: Bearer sk-1234' \ + * -H 'Content-Type: application/json' \ + * -d '{"team_id": "team-1234", "members": [{"user_id": "user1"}, {"user_email": "user2@example.com"}]}' * ``` - * - * Returns `team_id`, `results` (one entry per input member, in order, with `user_id`, `user_email`, - * `success`, `error`), `total_requested`, `successful_deletions` and `failed_deletions`. */ post: operations["bulk_team_member_delete_team_bulk_member_delete_post"]; delete?: never; @@ -16529,27 +16516,13 @@ export interface paths { put?: never; /** * Bulk Delete User - * @description Delete up to 500 internal users in one request and remove each one from every team they belong to. + * @description Delete up to 500 users and remove each one from every team they belong to. Same authorization as + * `/user/delete`. Returns one result per user id, in order. * - * Same authorization as `/user/delete`: proxy admins may delete anyone, org admins only users whose - * organizations they all administer. Each team a deleted user was on is rewritten once under the team - * lock, so the roster, the user's `teams` array and the `LiteLLM_TeamMembership` rows all agree afterwards. - * Then the users' keys, invitation links, organization memberships and user rows are deleted. - * - * Rows fail independently: unknown, duplicate or out-of-scope ids are reported in `results` with - * `success: false` and an `error`, and the other users are still deleted. - * - * Usage Example - * - * ```shell - * curl -X POST "http://localhost:4000/user/bulk_delete" \ - * -H "Content-Type: application/json" \ - * -H "Authorization: Bearer sk-1234" \ - * -d '{"user_ids": ["user-1", "user-2"]}' + * ```bash + * curl -X POST 'http://localhost:4000/user/bulk_delete' -H 'Authorization: Bearer sk-1234' \ + * -H 'Content-Type: application/json' -d '{"user_ids": ["user-1", "user-2"]}' * ``` - * - * Returns `results` (one entry per input id, in order, with `user_id`, `user_email`, `success`, - * `teams_removed`, `error`), `total_requested`, `successful_deletions` and `failed_deletions`. */ post: operations["bulk_delete_user_user_bulk_delete_post"]; delete?: never; From 82872c9627a0109f2a42550c96db5c50e5f8cd87 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 14 Sep 2026 05:35:55 +0000 Subject: [PATCH 04/12] fix(proxy): run /user/bulk_delete team rewrites and user deletes in one transaction Lock affected teams in sorted order inside a single 60s transaction so a failure on any team rolls back every rewrite and every user row delete. PrismaClient.tx() gains an optional timeout for the larger batch. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_helpers/bulk_user_deletion.py | 260 ++++++++++-------- litellm/proxy/utils.py | 5 +- .../test_bulk_user_deletion.py | 53 ++-- 3 files changed, 178 insertions(+), 140 deletions(-) diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py index 4d3387aacb7..e7726652eca 100644 --- a/litellm/proxy/management_helpers/bulk_user_deletion.py +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -2,13 +2,16 @@ Each team a batch touches is rewritten exactly once, under the same advisory lock `/team/member_delete` takes and from a roster re-read under that lock, so a concurrent -member_add on the team is never overwritten from a stale read. +member_add on the team is never overwritten from a stale read. A user batch runs in one +transaction, taking its team locks in sorted order, so either every team rewrite and every +user row delete lands or none of them does. """ import asyncio import json from collections.abc import Awaitable, Iterable, Mapping, Sequence from dataclasses import dataclass +from datetime import timedelta from types import MappingProxyType from typing import TYPE_CHECKING, Final @@ -60,7 +63,8 @@ if TYPE_CHECKING: from litellm.repositories.prisma_protocols import TableActions -_TEAM_WRITE_CONCURRENCY: Final = 10 +_AUDIT_LOG_CONCURRENCY: Final = 10 +_USER_BATCH_TX_TIMEOUT: Final = timedelta(seconds=60) class _ErrorDetail(TypedDict): @@ -95,6 +99,12 @@ class _TeamRemoval: deleted_key_tokens: tuple[str, ...] +@dataclass(frozen=True, slots=True) +class _UserBatchDeletion: + removals: Mapping[str, _TeamRemoval] + deleted_key_tokens: tuple[str, ...] + + def _http_error(status_code: int, message: str) -> HTTPException: detail: Final[_ErrorDetail] = {"error": message} return HTTPException(status_code=status_code, detail=detail) @@ -161,7 +171,7 @@ def _error_message(exc: BaseException) -> str: async def _bounded(awaitables: Iterable[Awaitable[object]]) -> tuple[object | BaseException, ...]: - semaphore: Final = asyncio.Semaphore(_TEAM_WRITE_CONCURRENCY) + semaphore: Final = asyncio.Semaphore(_AUDIT_LOG_CONCURRENCY) async def run(awaitable: Awaitable[object]) -> object: async with semaphore: @@ -172,54 +182,54 @@ async def _bounded(awaitables: Iterable[Awaitable[object]]) -> tuple[object | Ba async def _remove_members_from_team( prisma_client: PrismaClient, + tx: "Prisma", team_id: str, members: Sequence[MemberDeleteRequest], user_api_key_dict: UserAPIKeyAuth, ) -> _TeamRemoval: - async with prisma_client.tx() as tx: - await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id) - roster: Final = await TeamRepository(prisma_client).get_members_with_roles_locked(tx, team_id) - if roster is None: - raise _http_error(400, f"Team id={team_id} does not exist in db") + await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id) + roster: Final = await TeamRepository(prisma_client).get_members_with_roles_locked(tx, team_id) + if roster is None: + raise _http_error(400, f"Team id={team_id} does not exist in db") - removed_members: Final = tuple(m for m in roster if any(_addresses_member(m, r) for r in members)) - kept_members: Final = tuple(m for m in roster if not any(_addresses_member(m, r) for r in members)) - removed_ids: Final = frozenset(m.user_id for m in removed_members if m.user_id is not None) - requested_ids: Final = frozenset(r.user_id for r in members if r.user_id is not None) - requested_emails: Final = frozenset(r.user_email for r in members if r.user_id is None and r.user_email) - user_rows: Final = await _user_tx_db(tx).find_many( - where=_any_filter( - _in_filter("user_id", removed_ids | requested_ids), - _in_filter("user_email", requested_emails), - ) + removed_members: Final = tuple(m for m in roster if any(_addresses_member(m, r) for r in members)) + kept_members: Final = tuple(m for m in roster if not any(_addresses_member(m, r) for r in members)) + removed_ids: Final = frozenset(m.user_id for m in removed_members if m.user_id is not None) + requested_ids: Final = frozenset(r.user_id for r in members if r.user_id is not None) + requested_emails: Final = frozenset(r.user_email for r in members if r.user_id is None and r.user_email) + user_rows: Final = await _user_tx_db(tx).find_many( + where=_any_filter( + _in_filter("user_id", removed_ids | requested_ids), + _in_filter("user_email", requested_emails), ) - stale_rows: Final = tuple(u for u in user_rows if team_id in u.teams) - cleanup_ids: Final = removed_ids | frozenset(u.user_id for u in stale_rows) - matched: Final = frozenset( - i - for i, r in enumerate(members) - if any(_addresses_member(m, r) for m in removed_members) or any(_addresses_user(u, r) for u in stale_rows) - ) - keys: Final = await _token_tx_db(tx).find_many(where=_team_users_filter(team_id, cleanup_ids)) + ) + stale_rows: Final = tuple(u for u in user_rows if team_id in u.teams) + cleanup_ids: Final = removed_ids | frozenset(u.user_id for u in stale_rows) + matched: Final = frozenset( + i + for i, r in enumerate(members) + if any(_addresses_member(m, r) for m in removed_members) or any(_addresses_user(u, r) for u in stale_rows) + ) + keys: Final = await _token_tx_db(tx).find_many(where=_team_users_filter(team_id, cleanup_ids)) - if removed_members: - roster_data: Final[_RosterData] = { - "members_with_roles": json.dumps(tuple(m.model_dump() for m in kept_members)) - } - await _team_tx_db(tx).update(where=_eq_filter("team_id", team_id), data=roster_data) - for row in stale_rows: - teams_data: _TeamsData = {"teams": {"set": tuple(t for t in row.teams if t != team_id)}} - await _user_tx_db(tx).update(where=_eq_filter("user_id", row.user_id), data=teams_data) - await _membership_tx_db(tx).delete_many(where=_team_users_filter(team_id, cleanup_ids)) - if keys: - await _persist_deleted_verification_tokens( - keys=keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken - prisma_client=prisma_client, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - tx=tx, - ) - await _token_tx_db(tx).delete_many(where=_team_users_filter(team_id, cleanup_ids)) + if removed_members: + roster_data: Final[_RosterData] = { + "members_with_roles": json.dumps(tuple(m.model_dump() for m in kept_members)) + } + await _team_tx_db(tx).update(where=_eq_filter("team_id", team_id), data=roster_data) + for row in stale_rows: + teams_data: _TeamsData = {"teams": {"set": tuple(t for t in row.teams if t != team_id)}} + await _user_tx_db(tx).update(where=_eq_filter("user_id", row.user_id), data=teams_data) + await _membership_tx_db(tx).delete_many(where=_team_users_filter(team_id, cleanup_ids)) + if keys: + await _persist_deleted_verification_tokens( + keys=keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + tx=tx, + ) + await _token_tx_db(tx).delete_many(where=_team_users_filter(team_id, cleanup_ids)) return _TeamRemoval( team=LiteLLM_TeamTable( @@ -279,7 +289,8 @@ async def bulk_remove_team_members( duplicates: Final = _duplicate_member_indexes(data.members) kept_indexes: Final = tuple(i for i in range(len(data.members)) if i not in duplicates) members: Final = tuple(data.members[i] for i in kept_indexes) - removal: Final = await _remove_members_from_team(prisma_client, data.team_id, members, user_api_key_dict) + async with prisma_client.tx() as tx: + removal: Final = await _remove_members_from_team(prisma_client, tx, data.team_id, members, user_api_key_dict) await delete_cache_key_objects( hashed_tokens=removal.deleted_key_tokens, user_api_key_cache=user_api_key_cache, @@ -333,60 +344,101 @@ def _scope_error(user_id: str, target_org_ids: frozenset[str], caller_admin_org_ ) -async def _delete_user_rows_tx( +async def _delete_user_rows( prisma_client: PrismaClient, + tx: "Prisma", user_ids: frozenset[str], user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: str | None, ) -> tuple[str, ...]: - async with prisma_client.tx() as tx: - keys: Final = await _token_tx_db(tx).find_many(where=_in_filter("user_id", user_ids)) - if keys: - await _persist_deleted_verification_tokens( - keys=keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken - prisma_client=prisma_client, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=litellm_changed_by, - tx=tx, - ) - await _token_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) - await _invitation_tx_db(tx).delete_many( - where=_any_filter( - _in_filter("user_id", user_ids), - _in_filter("created_by", user_ids), - _in_filter("updated_by", user_ids), - ) + keys: Final = await _token_tx_db(tx).find_many(where=_in_filter("user_id", user_ids)) + if keys: + await _persist_deleted_verification_tokens( + keys=keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + tx=tx, ) - await _org_membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) - await _membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) - await _user_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) + await _token_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) + await _invitation_tx_db(tx).delete_many( + where=_any_filter( + _in_filter("user_id", user_ids), + _in_filter("created_by", user_ids), + _in_filter("updated_by", user_ids), + ) + ) + await _org_membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) + await _membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) + await _user_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) return tuple(k.token for k in keys) -async def _delete_user_rows( +async def _delete_users_tx( prisma_client: PrismaClient, users: Sequence["prisma_models.LiteLLM_UserTable"], + teams_of: Mapping[str, frozenset[str]], + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: str | None, +) -> _UserBatchDeletion: + """Rewrites every team the users belong to and deletes their rows in one transaction, so a + failure anywhere rolls back the whole batch. Teams a user still names but which no longer exist + are skipped; the user row goes away regardless.""" + async with prisma_client.tx(timeout=_USER_BATCH_TX_TIMEOUT) as tx: + team_rows: Final = await _team_tx_db(tx).find_many( + where=_in_filter("team_id", frozenset(t for teams in teams_of.values() for t in teams)) + ) + team_ids: Final = tuple(sorted(t.team_id for t in team_rows)) + removals: Final = MappingProxyType( + { + tid: await _remove_members_from_team( + prisma_client, + tx, + tid, + tuple( + MemberDeleteRequest(user_id=u.user_id, user_email=u.user_email) + for u in users + if tid in teams_of[u.user_id] + ), + user_api_key_dict, + ) + for tid in team_ids + } + ) + deleted_key_tokens: Final = await _delete_user_rows( + prisma_client, tx, frozenset(u.user_id for u in users), user_api_key_dict, litellm_changed_by + ) + return _UserBatchDeletion( + removals=removals, + deleted_key_tokens=deleted_key_tokens + tuple(t for r in removals.values() for t in r.deleted_key_tokens), + ) + + +async def _delete_users( + prisma_client: PrismaClient, + users: Sequence["prisma_models.LiteLLM_UserTable"], + teams_of: Mapping[str, frozenset[str]], user_api_key_dict: UserAPIKeyAuth, user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging | None, litellm_proxy_admin_name: str | None, litellm_changed_by: str | None, -) -> str | None: +) -> _UserBatchDeletion | str: """Returns the error message when the transaction rolled back, in which case no row was touched.""" user_ids: Final = frozenset(u.user_id for u in users) try: - deleted_key_tokens: Final = await _delete_user_rows_tx( - prisma_client, user_ids, user_api_key_dict, litellm_changed_by - ) + deletion: Final = await _delete_users_tx(prisma_client, users, teams_of, user_api_key_dict, litellm_changed_by) except Exception as e: # noqa: BLE001 # the rolled-back batch is reported per row, not as a request failure verbose_proxy_logger.error("/user/bulk_delete: failed to delete users %s: %s", sorted(user_ids), e) return _error_message(e) await delete_cache_key_objects( - hashed_tokens=deleted_key_tokens, + hashed_tokens=deletion.deleted_key_tokens, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) await evict_and_broadcast(cache_keys=sorted(user_ids), user_api_key_cache=user_api_key_cache) + for removal in deletion.removals.values(): + _emit_team_members_metric(removal.team) audit_outcomes: Final = await _bounded( UserManagementEventHooks.create_internal_user_audit_log( user_id=u.user_id, @@ -401,7 +453,7 @@ async def _delete_user_rows( for u, outcome in zip(users, audit_outcomes, strict=True): if isinstance(outcome, BaseException): verbose_proxy_logger.warning("Failed to create audit log for user %s: %s", u.user_id, outcome) - return None + return deletion async def bulk_delete_users( @@ -452,54 +504,19 @@ async def bulk_delete_users( for u in candidates } ) - team_ids: Final = tuple(sorted(frozenset(t for u in candidates for t in teams_of[u.user_id]))) - members_by_team: Final = MappingProxyType( - { - tid: tuple( - MemberDeleteRequest(user_id=u.user_id, user_email=u.user_email) - for u in candidates - if tid in teams_of[u.user_id] - ) - for tid in team_ids - } - ) - outcomes: Final = await _bounded( - _remove_members_from_team(prisma_client, tid, members_by_team[tid], user_api_key_dict) for tid in team_ids - ) - removals: Final = MappingProxyType( - {tid: o for tid, o in zip(team_ids, outcomes, strict=True) if isinstance(o, _TeamRemoval)} - ) - team_failures: Final = MappingProxyType( - {tid: _error_message(o) for tid, o in zip(team_ids, outcomes, strict=True) if isinstance(o, BaseException)} - ) - for tid, err in team_failures.items(): - verbose_proxy_logger.error("/user/bulk_delete: failed to remove users from team %s: %s", tid, err) - await delete_cache_key_objects( - hashed_tokens=tuple(t for removal in removals.values() for t in removal.deleted_key_tokens), - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - for removal in removals.values(): - _emit_team_members_metric(removal.team) - - def team_errors(user_id: str) -> tuple[str, ...]: - return tuple( - f"Failed to remove from team {tid}: {err}" for tid, err in team_failures.items() if tid in teams_of[user_id] - ) - - deletable: Final = tuple(u for u in candidates if not team_errors(u.user_id)) - delete_error: Final = ( - await _delete_user_rows( + deletion: Final = ( + await _delete_users( prisma_client, - deletable, + candidates, + teams_of, user_api_key_dict, user_api_key_cache, proxy_logging_obj, litellm_proxy_admin_name, litellm_changed_by, ) - if deletable - else None + if candidates + else _UserBatchDeletion(removals=MappingProxyType({}), deleted_key_tokens=()) ) def result(index: int, user_id: str) -> UserDeleteResult: @@ -508,15 +525,18 @@ async def bulk_delete_users( error: Final = precheck_errors[user_id] if error is not None: return UserDeleteResult(user_id=user_id, success=False, error=error) - errors: Final = team_errors(user_id) or ( - (f"Failed to delete user: {delete_error}",) if delete_error is not None else () - ) + if isinstance(deletion, str): + return UserDeleteResult( + user_id=user_id, + user_email=rows_by_id[user_id].user_email, + success=False, + error=f"Failed to delete user: {deletion}", + ) return UserDeleteResult( user_id=user_id, user_email=rows_by_id[user_id].user_email, - success=not errors, - teams_removed=tuple(tid for tid in team_ids if tid in removals and user_id in removals[tid].removed), - error="; ".join(errors) or None, + success=True, + teams_removed=tuple(tid for tid, r in deletion.removals.items() if user_id in r.removed), ) results: Final = tuple(result(i, uid) for i, uid in enumerate(data.user_ids)) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c095586b6c9..a4875321a2a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3792,6 +3792,7 @@ def jsonify_object(data: dict) -> dict: # Bounded to prevent memory leaks from accumulated rotations. _deprecated_key_cache: Final[LimitedSizeOrderedDict] = LimitedSizeOrderedDict(max_size=1000) _DEPRECATED_KEY_CACHE_TTL_SECONDS: Final = 60 +_PRISMA_DEFAULT_TX_TIMEOUT: Final = timedelta(seconds=5) async def _lookup_deprecated_key( @@ -4170,13 +4171,13 @@ class PrismaClient: return self.db.read_target return self.db - def tx(self) -> "TransactionManager": + def tx(self, *, timeout: timedelta = _PRISMA_DEFAULT_TX_TIMEOUT) -> "TransactionManager": """Open an interactive transaction on the writer. Callers go through this instead of reaching into ``self.db`` so writer selection and read-replica routing stay encapsulated in the wrapper. """ - return cast("TransactionManager", self.db.tx()) # cast-ok: wrappers delegate tx via __getattr__ (untyped) + return cast("TransactionManager", self.db.tx(timeout=timeout)) # cast-ok: untyped __getattr__ delegate def get_request_status(self, payload: dict | SpendLogsPayload) -> Literal["success", "failure"]: """ diff --git a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py index a8a09518326..dd266880ce0 100644 --- a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py +++ b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py @@ -95,6 +95,9 @@ class _TeamTable: async def find_unique(self, where: Mapping[str, str]) -> LiteLLM_TeamTable | None: return self.rows.get(where["team_id"]) + async def find_many(self, where: Mapping[str, object]) -> list[LiteLLM_TeamTable]: + return [t for t in self.rows.values() if _matches({"team_id": t.team_id}, where)] + async def update(self, where: Mapping[str, str], data: Mapping[str, str]) -> LiteLLM_TeamTable: self.update_calls += 1 team = self.rows[where["team_id"]] @@ -143,7 +146,7 @@ class _Tx: self.locks.append(team_id) self._on_lock(team_id) return [] - assert self.locks == [team_id], "roster must be read under this team's advisory lock" + assert team_id in self.locks, "roster must be read under this team's advisory lock" self.roster_reads.append(team_id) team = self.litellm_teamtable.rows.get(team_id) if team is None: @@ -162,22 +165,22 @@ class _FakePrisma: org_memberships: Sequence[Mapping[str, object]] = (), on_lock: Callable[[str], None] = lambda _: None, fail_locks: frozenset[str] = frozenset(), - fail_user_delete: bool = False, + fail_commit: bool = False, ) -> None: self.db = _Db(users, teams, memberships, tokens, invitations, org_memberships) self._on_lock = on_lock self._fail_locks = fail_locks - self._fail_user_delete = fail_user_delete + self._fail_commit = fail_commit self.locks: list[str] = [] self.roster_reads: list[str] = [] @asynccontextmanager - async def tx(self): + async def tx(self, *, timeout: object = None): snapshot = copy.deepcopy(self.db) tx = _Tx(self.db, self._on_lock, self._fail_locks) try: yield tx - if self._fail_user_delete and tx.locks == []: + if self._fail_commit: raise RuntimeError("connection reset") except BaseException: self.db.__dict__.update(snapshot.__dict__) @@ -271,7 +274,7 @@ async def test_bulk_delete_removes_users_from_every_team_and_store(): assert [t["token"] for t in prisma.db.litellm_deletedverificationtoken.rows] == ["k1"] assert [i["id"] for i in prisma.db.litellm_invitationlink.rows] == ["i3"] assert prisma.db.litellm_organizationmembership.rows == [] - assert sorted(prisma.locks) == ["t1", "t2"] and sorted(prisma.roster_reads) == ["t1", "t2"] + assert prisma.locks == ["t1", "t2"] and prisma.roster_reads == ["t1", "t2"] @pytest.mark.asyncio @@ -320,22 +323,36 @@ async def test_bulk_delete_reports_missing_and_duplicate_ids_per_item_and_still_ @pytest.mark.asyncio -async def test_bulk_delete_keeps_user_when_a_team_rewrite_fails_and_deletes_the_others(): +async def test_bulk_delete_rolls_back_every_team_and_user_when_one_team_rewrite_fails(): prisma = _FakePrisma( - users=[_user("u1", "bad", "good"), _user("u2", "good")], - teams=[_team("bad", "u1"), _team("good", "u1", "u2")], - fail_locks=frozenset({"bad"}), + users=[_user("u1", "a-good", "z-bad"), _user("u2", "a-good")], + teams=[_team("a-good", "u1", "u2"), _team("z-bad", "u1")], + tokens=[{"token": "k1", "user_id": "u1", "team_id": "a-good"}], + fail_locks=frozenset({"z-bad"}), ) + cache = _cache_with("k1") - response = await _delete(prisma, ["u1", "u2"]) + response = await _delete(prisma, ["u1", "u2"], cache=cache) - assert [(r.user_id, r.success, r.teams_removed) for r in response.results] == [ - ("u1", False, ("good",)), - ("u2", True, ("good",)), + assert [(r.user_id, r.success, r.teams_removed, r.error) for r in response.results] == [ + ("u1", False, (), "Failed to delete user: lock timeout"), + ("u2", False, (), "Failed to delete user: lock timeout"), ] - assert response.results[0].error == "Failed to remove from team bad: lock timeout" - assert set(prisma.db.litellm_usertable.rows) == {"u1"} - assert _roster(prisma, "bad") == ["u1"] and _roster(prisma, "good") == [] + assert set(prisma.db.litellm_usertable.rows) == {"u1", "u2"} + assert _roster(prisma, "a-good") == ["u1", "u2"] and _roster(prisma, "z-bad") == ["u1"] + assert [t["token"] for t in prisma.db.litellm_verificationtoken.rows] == ["k1"] + assert cache.get_cache(key="k1") is not None + + +@pytest.mark.asyncio +async def test_bulk_delete_skips_teams_the_user_still_names_but_which_no_longer_exist(): + prisma = _FakePrisma(users=[_user("u1", "gone", "t1")], teams=[_team("t1", "u1", "keep")]) + + response = await _delete(prisma, ["u1"]) + + assert [(r.success, r.teams_removed) for r in response.results] == [(True, ("t1",))] + assert prisma.db.litellm_usertable.rows == {} and _roster(prisma, "t1") == ["keep"] + assert prisma.locks == ["t1"] @pytest.mark.asyncio @@ -344,7 +361,7 @@ async def test_bulk_delete_rolls_back_every_user_row_and_reports_it_per_row_when users=[_user("u1", "t1"), _user("u2")], teams=[_team("t1", "u1")], tokens=[{"token": "k1", "user_id": "u1"}], - fail_user_delete=True, + fail_commit=True, ) cache = _cache_with("k1") From 450f7deff843b92d774fa0fb4c679119295a2b31 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 14 Sep 2026 05:46:02 +0000 Subject: [PATCH 05/12] docs(proxy): shorten bulk delete endpoint descriptions to one sentence Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../internal_user_endpoints.py | 10 +--------- .../management_endpoints/team_endpoints.py | 11 +---------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 17 ++--------------- 3 files changed, 4 insertions(+), 34 deletions(-) diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 6b11847f09d..c9fc6720490 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -2515,15 +2515,7 @@ async def bulk_delete_user( description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", ), ) -> BulkDeleteUserResponse: - """ - Delete up to 500 users and remove each one from every team they belong to. Same authorization as - `/user/delete`. Returns one result per user id, in order. - - ```bash - curl -X POST 'http://localhost:4000/user/bulk_delete' -H 'Authorization: Bearer sk-1234' \\ - -H 'Content-Type: application/json' -d '{"user_ids": ["user-1", "user-2"]}' - ``` - """ + """Delete up to 500 users, removing each from every team; same authorization as `/user/delete`.""" from litellm.proxy.proxy_server import ( litellm_proxy_admin_name, prisma_client, diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 8626c073e55..b5e3978a2a8 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3464,16 +3464,7 @@ async def bulk_team_member_delete( data: BulkTeamMemberDeleteRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI dependency injection ) -> BulkTeamMemberDeleteResponse: - """ - Remove up to 500 members (each named by `user_id` or `user_email`) from one team. Same authorization - as `/team/member_delete`. Returns one result per member, in order. - - ```bash - curl -X POST 'http://0.0.0.0:4000/team/bulk_member_delete' -H 'Authorization: Bearer sk-1234' \\ - -H 'Content-Type: application/json' \\ - -d '{"team_id": "team-1234", "members": [{"user_id": "user1"}, {"user_email": "user2@example.com"}]}' - ``` - """ + """Remove up to 500 members from one team; same authorization as `/team/member_delete`.""" from litellm.proxy.management_helpers.bulk_user_deletion import bulk_remove_team_members from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 23c989a3220..58f64b64949 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -15067,14 +15067,7 @@ export interface paths { put?: never; /** * Bulk Team Member Delete - * @description Remove up to 500 members (each named by `user_id` or `user_email`) from one team. Same authorization - * as `/team/member_delete`. Returns one result per member, in order. - * - * ```bash - * curl -X POST 'http://0.0.0.0:4000/team/bulk_member_delete' -H 'Authorization: Bearer sk-1234' \ - * -H 'Content-Type: application/json' \ - * -d '{"team_id": "team-1234", "members": [{"user_id": "user1"}, {"user_email": "user2@example.com"}]}' - * ``` + * @description Remove up to 500 members from one team; same authorization as `/team/member_delete`. */ post: operations["bulk_team_member_delete_team_bulk_member_delete_post"]; delete?: never; @@ -16516,13 +16509,7 @@ export interface paths { put?: never; /** * Bulk Delete User - * @description Delete up to 500 users and remove each one from every team they belong to. Same authorization as - * `/user/delete`. Returns one result per user id, in order. - * - * ```bash - * curl -X POST 'http://localhost:4000/user/bulk_delete' -H 'Authorization: Bearer sk-1234' \ - * -H 'Content-Type: application/json' -d '{"user_ids": ["user-1", "user-2"]}' - * ``` + * @description Delete up to 500 users, removing each from every team; same authorization as `/user/delete`. */ post: operations["bulk_delete_user_user_bulk_delete_post"]; delete?: never; From d442d90411fbc55ead49e1f42dc379920b3033f7 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 14 Sep 2026 05:56:52 +0000 Subject: [PATCH 06/12] test(proxy): add /team/bulk_member_delete behavior-suite scenarios for route coverage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_team_bulk_member_delete.py | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 tests/proxy_behavior/management/test_team_bulk_member_delete.py diff --git a/tests/proxy_behavior/management/test_team_bulk_member_delete.py b/tests/proxy_behavior/management/test_team_bulk_member_delete.py new file mode 100644 index 00000000000..09a3cd54783 --- /dev/null +++ b/tests/proxy_behavior/management/test_team_bulk_member_delete.py @@ -0,0 +1,129 @@ +import pytest + +from .actors import Actor +from .conftest import create_scratch_team + +pytestmark = pytest.mark.asyncio(loop_scope="session") + +_MATRIX = [ + ("alpha/proxy_admin", Actor.PROXY_ADMIN, "alpha", 200), + ("alpha/org_admin", Actor.ORG_ADMIN, "alpha", 200), + ("alpha/team_admin", Actor.TEAM_ADMIN, "alpha", 200), + ("alpha/internal_user", Actor.INTERNAL_USER, "alpha", 403), + ("alpha/owner", Actor.OWNER, "alpha", 403), + ("alpha/unrelated_same_org", Actor.UNRELATED_SAME_ORG, "alpha", 403), + ("alpha/cross_org_user", Actor.CROSS_ORG_USER, "alpha", 403), + ("alpha/service_account", Actor.SERVICE_ACCOUNT, "alpha", 403), + ("alpha/org_b_admin", Actor.ORG_B_ADMIN, "alpha", 403), + ("beta/proxy_admin", Actor.PROXY_ADMIN, "beta", 200), + ("beta/org_admin", Actor.ORG_ADMIN, "beta", 403), + ("beta/team_admin", Actor.TEAM_ADMIN, "beta", 403), + ("beta/internal_user", Actor.INTERNAL_USER, "beta", 403), + ("beta/owner", Actor.OWNER, "beta", 403), + ("beta/unrelated_same_org", Actor.UNRELATED_SAME_ORG, "beta", 403), + ("beta/cross_org_user", Actor.CROSS_ORG_USER, "beta", 403), + ("beta/service_account", Actor.SERVICE_ACCOUNT, "beta", 403), + ("beta/org_b_admin", Actor.ORG_B_ADMIN, "beta", 200), +] + + +async def _seed_target(prisma, world, shape: str, team_id: str, victim_ids: list) -> None: + if shape == "alpha": + await create_scratch_team( + prisma, + team_id, + organization_id=world.org_a_id, + admin_user_ids=[world.keys[Actor.TEAM_ADMIN].user_id], + member_user_ids=victim_ids, + ) + elif shape == "beta": + await create_scratch_team( + prisma, + team_id, + organization_id=world.org_b_id, + member_user_ids=victim_ids, + ) + else: # pragma: no cover - guard + pytest.fail(f"unknown shape={shape}") + + +def _member_ids(row) -> list: + return [m["user_id"] for m in (row.members_with_roles or [])] + + +@pytest.mark.parametrize( + "actor,shape,expected_status", + [(a, sh, s) for (_id, a, sh, s) in _MATRIX], + ids=[s[0] for s in _MATRIX], +) +async def test_team_bulk_member_delete_authz_matrix( + actor: Actor, + shape: str, + expected_status: int, + proxy_client, + prisma, + scratch, + world, +): + victims = [scratch.tag("v1"), scratch.tag("v2")] + keep = scratch.tag("keep") + await _seed_target(prisma, world, shape, scratch.prefix, victims + [keep]) + caller = world.keys[actor] + + resp = await proxy_client.post( + "/team/bulk_member_delete", + headers={"Authorization": f"Bearer {caller.cleartext}"}, + json={"team_id": scratch.prefix, "members": [{"user_id": v} for v in victims]}, + ) + assert resp.status_code == expected_status, f"{actor.value} {shape}: {resp.status_code} {resp.text}" + + row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + assert row is not None + assert keep in _member_ids(row), "unrelated member removed" + if expected_status == 200: + assert [(r["user_id"], r["success"]) for r in resp.json()["results"]] == [(v, True) for v in victims] + assert not set(victims) & set(_member_ids(row)) + else: + assert set(victims) <= set(_member_ids(row)), "denied but members removed" + + +async def test_team_bulk_member_delete_reports_each_row_in_order(proxy_client, prisma, scratch, world): + victim = scratch.tag("victim") + keep = scratch.tag("keep") + stranger = scratch.tag("stranger") + await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=[victim, keep]) + + resp = await proxy_client.post( + "/team/bulk_member_delete", + headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"team_id": scratch.prefix, "members": [{"user_id": stranger}, {"user_id": victim}]}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert [(r["user_id"], r["success"]) for r in body["results"]] == [ + (stranger, False), + (victim, True), + ] + assert body["results"][0]["error"] == "User not found in team" + assert (body["successful_deletions"], body["failed_deletions"]) == (1, 1) + + row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + assert row is not None and _member_ids(row) == [keep] + + +async def test_team_bulk_member_delete_row_naming_both_identifiers_is_422(proxy_client, prisma, scratch, world): + victim = scratch.tag("victim") + await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=[victim]) + + resp = await proxy_client.post( + "/team/bulk_member_delete", + headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={ + "team_id": scratch.prefix, + "members": [{"user_id": victim, "user_email": f"{victim}@example.com"}], + }, + ) + assert resp.status_code == 422, resp.text + + row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + assert row is not None and victim in _member_ids(row) From dbb4de7bc2c1ea5288c8579073847b05282ce919 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 14 Sep 2026 05:59:19 +0000 Subject: [PATCH 07/12] fix(proxy): match bulk-deleted users on team rosters by user_id only and give /team/bulk_member_delete the 60s batch timeout Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_helpers/bulk_user_deletion.py | 12 +++---- .../test_bulk_user_deletion.py | 31 +++++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py index e7726652eca..d73901a3ea0 100644 --- a/litellm/proxy/management_helpers/bulk_user_deletion.py +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -64,7 +64,7 @@ if TYPE_CHECKING: from litellm.repositories.prisma_protocols import TableActions _AUDIT_LOG_CONCURRENCY: Final = 10 -_USER_BATCH_TX_TIMEOUT: Final = timedelta(seconds=60) +_BATCH_TX_TIMEOUT: Final = timedelta(seconds=60) class _ErrorDetail(TypedDict): @@ -289,7 +289,7 @@ async def bulk_remove_team_members( duplicates: Final = _duplicate_member_indexes(data.members) kept_indexes: Final = tuple(i for i in range(len(data.members)) if i not in duplicates) members: Final = tuple(data.members[i] for i in kept_indexes) - async with prisma_client.tx() as tx: + async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx: removal: Final = await _remove_members_from_team(prisma_client, tx, data.team_id, members, user_api_key_dict) await delete_cache_key_objects( hashed_tokens=removal.deleted_key_tokens, @@ -384,7 +384,7 @@ async def _delete_users_tx( """Rewrites every team the users belong to and deletes their rows in one transaction, so a failure anywhere rolls back the whole batch. Teams a user still names but which no longer exist are skipped; the user row goes away regardless.""" - async with prisma_client.tx(timeout=_USER_BATCH_TX_TIMEOUT) as tx: + async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx: team_rows: Final = await _team_tx_db(tx).find_many( where=_in_filter("team_id", frozenset(t for teams in teams_of.values() for t in teams)) ) @@ -395,11 +395,7 @@ async def _delete_users_tx( prisma_client, tx, tid, - tuple( - MemberDeleteRequest(user_id=u.user_id, user_email=u.user_email) - for u in users - if tid in teams_of[u.user_id] - ), + tuple(MemberDeleteRequest(user_id=u.user_id) for u in users if tid in teams_of[u.user_id]), user_api_key_dict, ) for tid in team_ids diff --git a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py index dd266880ce0..aac30184e1d 100644 --- a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py +++ b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py @@ -277,6 +277,37 @@ async def test_bulk_delete_removes_users_from_every_team_and_store(): assert prisma.locks == ["t1", "t2"] and prisma.roster_reads == ["t1", "t2"] +@pytest.mark.asyncio +async def test_bulk_delete_leaves_teammates_who_share_the_deleted_users_email_alone(): + twin = _UserRow(user_id="twin", user_email="u1@example.com", teams=["t1"]) + team = LiteLLM_TeamTable( + team_id="t1", + members_with_roles=[ + Member(user_id="u1", user_email="u1@example.com", role="user"), + Member(user_id="twin", user_email="u1@example.com", role="user"), + ], + ) + prisma = _FakePrisma( + users=[_user("u1", "t1"), twin], + teams=[team], + memberships=[("t1", "u1"), ("t1", "twin")], + tokens=[ + {"token": "k1", "user_id": "u1", "team_id": "t1"}, + {"token": "k-twin", "user_id": "twin", "team_id": "t1"}, + ], + ) + + response = await _delete(prisma, ["u1"]) + + assert [(r.success, r.teams_removed) for r in response.results] == [(True, ("t1",))] + assert _roster(prisma, "t1") == ["twin"] + assert set(prisma.db.litellm_usertable.rows) == {"twin"} and prisma.db.litellm_usertable.rows["twin"].teams == [ + "t1" + ] + assert prisma.db.litellm_teammembership.rows == [{"team_id": "t1", "user_id": "twin"}] + assert [t["token"] for t in prisma.db.litellm_verificationtoken.rows] == ["k-twin"] + + @pytest.mark.asyncio async def test_bulk_delete_finds_teams_through_membership_rows_when_user_teams_array_is_stale(): prisma = _FakePrisma( From c25498b66dfe9071ffa3f29784560561fb4ffb8d Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 14 Sep 2026 06:08:42 +0000 Subject: [PATCH 08/12] fix(proxy): remove a bulk-deleted user's email-only roster entries without touching same-email teammates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_helpers/bulk_user_deletion.py | 22 +++++++++++++------ .../test_bulk_user_deletion.py | 18 +++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py index d73901a3ea0..92e21a3803b 100644 --- a/litellm/proxy/management_helpers/bulk_user_deletion.py +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -150,16 +150,20 @@ def _org_membership_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_O return tx.litellm_organizationmembership # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do +def _same_email(email: str | None, request: MemberDeleteRequest) -> bool: + return request.user_email is not None and request.user_email == email + + def _addresses_member(member: Member, request: MemberDeleteRequest) -> bool: - return (request.user_id is not None and request.user_id == member.user_id) or ( - request.user_email is not None and request.user_email == member.user_email - ) + if request.user_id is None: + return _same_email(member.user_email, request) + return request.user_id == member.user_id or (member.user_id is None and _same_email(member.user_email, request)) def _addresses_user(user: "prisma_models.LiteLLM_UserTable", request: MemberDeleteRequest) -> bool: - return (request.user_id is not None and request.user_id == user.user_id) or ( - request.user_email is not None and request.user_email == user.user_email - ) + if request.user_id is None: + return _same_email(user.user_email, request) + return request.user_id == user.user_id def _error_message(exc: BaseException) -> str: @@ -395,7 +399,11 @@ async def _delete_users_tx( prisma_client, tx, tid, - tuple(MemberDeleteRequest(user_id=u.user_id) for u in users if tid in teams_of[u.user_id]), + tuple( + MemberDeleteRequest(user_id=u.user_id, user_email=u.user_email) + for u in users + if tid in teams_of[u.user_id] + ), user_api_key_dict, ) for tid in team_ids diff --git a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py index aac30184e1d..49aed00177a 100644 --- a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py +++ b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py @@ -308,6 +308,24 @@ async def test_bulk_delete_leaves_teammates_who_share_the_deleted_users_email_al assert [t["token"] for t in prisma.db.litellm_verificationtoken.rows] == ["k-twin"] +@pytest.mark.asyncio +async def test_bulk_delete_removes_the_deleted_users_email_only_roster_entry(): + team = LiteLLM_TeamTable( + team_id="t1", + members_with_roles=[ + Member(user_id=None, user_email="u1@example.com", role="user"), + Member(user_id="keep", user_email="keep@example.com", role="user"), + ], + ) + prisma = _FakePrisma(users=[_user("u1", "t1"), _user("keep", "t1")], teams=[team]) + + response = await _delete(prisma, ["u1"]) + + assert [(r.success, r.teams_removed) for r in response.results] == [(True, ("t1",))] + assert _roster(prisma, "t1") == ["keep"] + assert set(prisma.db.litellm_usertable.rows) == {"keep"} + + @pytest.mark.asyncio async def test_bulk_delete_finds_teams_through_membership_rows_when_user_teams_array_is_stale(): prisma = _FakePrisma( From ca04b03c2c4af8844199adc46315f043d06b92a3 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 14 Sep 2026 22:31:07 +0000 Subject: [PATCH 09/12] feat(proxy): move bulk user delete and team member delete under /management/v1 Replaces POST /user/bulk_delete and POST /team/bulk_member_delete with POST /management/v1/users/bulk_delete and POST /management/v1/teams/{team_id}/members/bulk_delete per the Management API modernization one-pager: {data} envelopes, application/problem+json errors with urn:litellm:error:* types, 422 on unknown body fields, 400 on unknown query params, 403 on authorization failures, 404 on unknown team. Route checks now match parametrized management/v1 paths so team-scoped callers reach the endpoint's own authorization and get a 403 problem instead of the generic 401. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 5 +- litellm/proxy/auth/route_checks.py | 20 +- .../internal_user_endpoints.py | 44 --- .../management_v1/__init__.py | 8 + .../management_v1/teams.py | 94 ++++++ .../management_v1/users.py | 101 ++++++ .../management_endpoints/team_endpoints.py | 32 -- .../management_helpers/bulk_user_deletion.py | 72 ++-- litellm/proxy/proxy_server.py | 25 +- .../internal_user_endpoints.py | 16 +- .../management_endpoints/management_v1.py | 6 + .../management_endpoints/team_endpoints.py | 37 ++- .../endpointaudit/coverage_allowlist.txt | 2 - .../test_team_bulk_member_delete.py | 76 ++++- .../management/test_users_bulk_delete.py | 137 ++++++++ .../test_bulk_user_deletion.py | 120 +++---- ui/litellm-dashboard/src/lib/http/schema.d.ts | 314 ++++++++++-------- 17 files changed, 735 insertions(+), 374 deletions(-) create mode 100644 litellm/proxy/management_endpoints/management_v1/teams.py create mode 100644 litellm/proxy/management_endpoints/management_v1/users.py create mode 100644 tests/proxy_behavior/management/test_users_bulk_delete.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 95ca0ad352b..9d6fff0fa37 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -659,7 +659,7 @@ class LiteLLMRoutes(enum.Enum): "/user/update", "/user/bulk_update", "/user/delete", - "/user/bulk_delete", + "/management/v1/users/bulk_delete", "/user/info", "/user/list", "/user/daily/activity", @@ -839,7 +839,7 @@ class LiteLLMRoutes(enum.Enum): self_managed_routes = [ "/team/member_add", "/team/member_delete", - "/team/bulk_member_delete", + "/management/v1/teams/{team_id}/members/bulk_delete", "/team/member_update", "/team/{team_id}/member/{user_id}/reset_spend", "/team/permissions_list", @@ -866,6 +866,7 @@ class LiteLLMRoutes(enum.Enum): "/organization/daily/activity", "/user/available_roles", # read-only role metadata; any authenticated user may read "/user/list", # org admins checked in endpoint; non-admins get 403 + "/management/v1/users/bulk_delete", # proxy admins delete anyone, org admins only their orgs' users; others 403 "/model/{model_id}/update", "/prompt/list", "/prompt/info", diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 38cabcd0944..4c64d4d7e23 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -1,5 +1,5 @@ import re -from collections.abc import Sequence +from collections.abc import Collection from typing import Final from fastapi import HTTPException, Request, status @@ -25,11 +25,11 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset( # user "/user/new", "/user/delete", - "/user/bulk_delete", + "/management/v1/users/bulk_delete", "/user/bulk_update", # team "/team/new", - "/team/bulk_member_delete", + "/management/v1/teams/{team_id}/members/bulk_delete", "/team/update", "/team/delete", "/team/block", @@ -589,7 +589,7 @@ class RouteChecks: return False @staticmethod - def check_route_access(route: str, allowed_routes: Sequence[str]) -> bool: + def check_route_access(route: str, allowed_routes: Collection[str]) -> bool: """ Check if a route has access by checking both exact matches and patterns @@ -761,10 +761,10 @@ class RouteChecks: [ "/user/new", "/user/delete", - "/user/bulk_delete", + "/management/v1/users/bulk_delete", "/user/bulk_update", "/team/new", - "/team/bulk_member_delete", + "/management/v1/teams/{team_id}/members/bulk_delete", "/team/update", "/team/delete", "/model/new", @@ -828,7 +828,7 @@ class RouteChecks: status_code=status.HTTP_403_FORBIDDEN, detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email and password can be updated", ) - elif route in _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES or ( + elif RouteChecks.check_route_access(route=route, allowed_routes=_PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES) or ( route.startswith("/key/") and route.endswith(_PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES) ): # Block write operations for PROXY_ADMIN_VIEW_ONLY @@ -863,9 +863,9 @@ class RouteChecks: # Hard-block known write routes regardless of HTTP method (defensive # — these are POSTs in practice, but pinning them here protects # against future GET-shaped writes). - if route in RouteChecks._ADMIN_VIEWER_BLOCKED_WRITE_ROUTES or ( - route.startswith("/key/") and route.endswith("/regenerate") - ): + if RouteChecks.check_route_access( + route=route, allowed_routes=RouteChecks._ADMIN_VIEWER_BLOCKED_WRITE_ROUTES + ) or (route.startswith("/key/") and route.endswith("/regenerate")): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route}", diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index c9fc6720490..e3efda507f6 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -8,7 +8,6 @@ These are members of a Team on LiteLLM /user/update /user/bulk_update /user/delete -/user/bulk_delete /user/info /user/list """ @@ -56,7 +55,6 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, prepare_metadata_fields, ) -from litellm.proxy.management_helpers.bulk_user_deletion import bulk_delete_users from litellm.proxy.management_helpers.object_permission_utils import ( _set_object_permission, handle_update_object_permission_common, @@ -79,8 +77,6 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, ) from litellm.types.proxy.management_endpoints.internal_user_endpoints import ( - BulkDeleteUserRequest, - BulkDeleteUserResponse, BulkUpdateUserRequest, BulkUpdateUserResponse, UserListResponse, @@ -2500,46 +2496,6 @@ async def delete_user( return deleted_users -@router.post( - "/user/bulk_delete", - tags=["Internal User management"], # mutable-ok: FastAPI's `tags` param is typed as list[str], not Sequence - dependencies=(Depends(user_api_key_auth),), - response_model=BulkDeleteUserResponse, -) -@management_endpoint_wrapper -async def bulk_delete_user( - data: BulkDeleteUserRequest, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI dependency injection - litellm_changed_by: str | None = Header( - None, - description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", - ), -) -> BulkDeleteUserResponse: - """Delete up to 500 users, removing each from every team; same authorization as `/user/delete`.""" - from litellm.proxy.proxy_server import ( - litellm_proxy_admin_name, - prisma_client, - proxy_logging_obj, - user_api_key_cache, - ) - - if prisma_client is None: - raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) - try: - return await bulk_delete_users( - data=data, - user_api_key_dict=user_api_key_dict, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - litellm_proxy_admin_name=litellm_proxy_admin_name, - litellm_changed_by=litellm_changed_by, - ) - except Exception as e: # noqa: BLE001 # normalize every failure to the proxy exception contract - verbose_proxy_logger.exception("/user/bulk_delete: Exception occured") - raise handle_exception_on_proxy(e) - - async def add_internal_user_to_organization( user_id: str, organization_id: str, diff --git a/litellm/proxy/management_endpoints/management_v1/__init__.py b/litellm/proxy/management_endpoints/management_v1/__init__.py index 342e6525cda..b29b7fe5dd5 100644 --- a/litellm/proxy/management_endpoints/management_v1/__init__.py +++ b/litellm/proxy/management_endpoints/management_v1/__init__.py @@ -10,9 +10,17 @@ from litellm.proxy.management_endpoints.management_v1.budgets import ( from litellm.proxy.management_endpoints.management_v1.spend_logs import ( router as spend_logs_router, ) +from litellm.proxy.management_endpoints.management_v1.teams import ( + router as teams_router, +) +from litellm.proxy.management_endpoints.management_v1.users import ( + router as users_router, +) router: Final = APIRouter() router.include_router(budgets_router) router.include_router(spend_logs_router) +router.include_router(teams_router) +router.include_router(users_router) __all__ = ["router"] diff --git a/litellm/proxy/management_endpoints/management_v1/teams.py b/litellm/proxy/management_endpoints/management_v1/teams.py new file mode 100644 index 00000000000..ba384bfb028 --- /dev/null +++ b/litellm/proxy/management_endpoints/management_v1/teams.py @@ -0,0 +1,94 @@ +"""`POST /management/v1/teams/{team_id}/members/bulk_delete`.""" + +from typing import Annotated, Final + +from fastapi import APIRouter, Depends + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem, reject_unknown_query_params +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX +from litellm.proxy.management_helpers.bulk_user_deletion import bulk_remove_team_members +from litellm.proxy.management_helpers.utils import ( + management_endpoint_wrapper, # pyright: ignore[reportUnknownVariableType] # legacy decorator is untyped +) +from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail +from litellm.types.proxy.management_endpoints.team_endpoints import ( + BulkTeamMemberDeleteRequest, + BulkTeamMemberDeleteResponse, +) + +router: Final = APIRouter(prefix=MANAGEMENT_V1_PREFIX) + + +@router.post( + "/teams/{team_id}/members/bulk_delete", + tags=["team management"], # mutable-ok: FastAPI types `tags` as list[str], not Sequence + dependencies=(Depends(user_api_key_auth), Depends(reject_unknown_query_params)), + response_model=BulkTeamMemberDeleteResponse, +) +@management_endpoint_wrapper +async def bulk_delete_team_members_action( + team_id: str, + data: BulkTeamMemberDeleteRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> BulkTeamMemberDeleteResponse: + """ + Remove up to 500 members from one team in one call. Same authorization as + `/team/member_delete`: proxy admins, the team's admins, and admins of the team's + organization. Each member is named by exactly one of `user_id` or `user_email`; + unknown body fields are a 422 and an unknown team is a 404. + + `data` holds one result per requested member, in request order. A row is + `success: false` with an `error` when it names nobody on the team or repeats an + earlier row. The roster is rewritten once, under the team's advisory lock, so a + concurrent member_add is never overwritten from a stale read. + + Example curl: + ``` + curl --location 'http://0.0.0.0:4000/management/v1/teams/team-1/members/bulk_delete' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{"members": [{"user_id": "user-1"}, {"user_email": "user-2@example.com"}]}' + ``` + """ + try: + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + if prisma_client is None: + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}database-not-connected", + title="Database not connected", + status=503, + detail=CommonProxyErrors.db_not_connected_error.value, + ) + ) + + results: Final = await bulk_remove_team_members( + team_id=team_id, + data=data, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + return BulkTeamMemberDeleteResponse(data=results) + + except ManagementProblem: + raise + except Exception as e: # noqa: BLE001 # a driver error answers as a problem document, not the OpenAI error shape + verbose_proxy_logger.exception( + "litellm.proxy.management_endpoints.management_v1.teams.bulk_delete_team_members_action(): " + "Exception occured - %s", + e, + ) + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}internal-server-error", + title="Internal server error", + status=500, + detail="Failed to remove team members.", + ) + ) diff --git a/litellm/proxy/management_endpoints/management_v1/users.py b/litellm/proxy/management_endpoints/management_v1/users.py new file mode 100644 index 00000000000..fa2a5cf3c3d --- /dev/null +++ b/litellm/proxy/management_endpoints/management_v1/users.py @@ -0,0 +1,101 @@ +"""`POST /management/v1/users/bulk_delete`.""" + +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, Header + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem, reject_unknown_query_params +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX +from litellm.proxy.management_helpers.bulk_user_deletion import bulk_delete_users +from litellm.proxy.management_helpers.utils import ( + management_endpoint_wrapper, # pyright: ignore[reportUnknownVariableType] # legacy decorator is untyped +) +from litellm.types.proxy.management_endpoints.internal_user_endpoints import ( + BulkDeleteUserRequest, + BulkDeleteUsersResponse, +) +from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail + +router: Final = APIRouter(prefix=MANAGEMENT_V1_PREFIX) + + +@router.post( + "/users/bulk_delete", + tags=["Internal User management"], # mutable-ok: FastAPI types `tags` as list[str], not Sequence + dependencies=(Depends(user_api_key_auth), Depends(reject_unknown_query_params)), + response_model=BulkDeleteUsersResponse, +) +@management_endpoint_wrapper +async def bulk_delete_users_action( + data: BulkDeleteUserRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + litellm_changed_by: Annotated[ + str | None, + Header(description="Who the caller is acting for; recorded on the audit log entries this call writes."), + ] = None, +) -> BulkDeleteUsersResponse: + """ + Delete up to 500 users in one call, taking each out of every team it belongs to. + Same authorization as `/user/delete`: proxy admins may delete anyone, org admins + only users inside organizations they administer. Unknown body fields are a 422. + + `data` holds one result per requested `user_id`, in request order. A row is + `success: false` with an `error` when the id is unknown, repeated in the request, + or outside the caller's scope. Rows that pass those checks are deleted together, + in one transaction, so either all of them go or none does. + + Example curl: + ``` + curl --location 'http://0.0.0.0:4000/management/v1/users/bulk_delete' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{"user_ids": ["user-1", "user-2"]}' + ``` + """ + try: + from litellm.proxy.proxy_server import ( + litellm_proxy_admin_name, + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + if prisma_client is None: + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}database-not-connected", + title="Database not connected", + status=503, + detail=CommonProxyErrors.db_not_connected_error.value, + ) + ) + + results: Final = await bulk_delete_users( + data=data, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + litellm_proxy_admin_name=litellm_proxy_admin_name, + litellm_changed_by=litellm_changed_by, + ) + return BulkDeleteUsersResponse(data=results) + + except ManagementProblem: + raise + except Exception as e: # noqa: BLE001 # a driver error answers as a problem document, not the OpenAI error shape + verbose_proxy_logger.exception( + "litellm.proxy.management_endpoints.management_v1.users.bulk_delete_users_action(): Exception occured - %s", + e, + ) + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}internal-server-error", + title="Internal server error", + status=500, + detail="Failed to delete users.", + ) + ) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index fbcde2bc032..6b16692f7ad 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -163,8 +163,6 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( from litellm.types.proxy.management_endpoints.team_endpoints import ( BulkTeamMemberAddRequest, BulkTeamMemberAddResponse, - BulkTeamMemberDeleteRequest, - BulkTeamMemberDeleteResponse, BulkUpdateTeamMemberPermissionsRequest, BulkUpdateTeamMemberPermissionsResponse, GetTeamMemberPermissionsResponse, @@ -3455,36 +3453,6 @@ async def team_member_delete( return existing_team_row -@router.post( - "/team/bulk_member_delete", - tags=["team management"], # mutable-ok: FastAPI's `tags` param is typed as list[str], not Sequence - dependencies=(Depends(user_api_key_auth),), - response_model=BulkTeamMemberDeleteResponse, -) -@management_endpoint_wrapper -async def bulk_team_member_delete( - data: BulkTeamMemberDeleteRequest, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI dependency injection -) -> BulkTeamMemberDeleteResponse: - """Remove up to 500 members from one team; same authorization as `/team/member_delete`.""" - from litellm.proxy.management_helpers.bulk_user_deletion import bulk_remove_team_members - from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache - - if prisma_client is None: - raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) - try: - return await bulk_remove_team_members( - data=data, - user_api_key_dict=user_api_key_dict, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - except Exception as e: # noqa: BLE001 # normalize every failure to the proxy exception contract - verbose_proxy_logger.exception("/team/bulk_member_delete: Exception occured") - raise handle_exception_on_proxy(e) - - _MEMBER_BUDGET_PATCH_FIELDS: Final = { "max_budget_in_team": "max_budget", "tpm_limit": "tpm_limit", diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py index 92e21a3803b..53527c42c25 100644 --- a/litellm/proxy/management_helpers/bulk_user_deletion.py +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -1,4 +1,5 @@ -"""Batched deletes behind `POST /user/bulk_delete` and `POST /team/bulk_member_delete`. +"""Batched deletes behind `POST /management/v1/users/bulk_delete` and +`POST /management/v1/teams/{team_id}/members/bulk_delete`. Each team a batch touches is rewritten exactly once, under the same advisory lock `/team/member_delete` takes and from a roster re-read under that lock, so a concurrent @@ -31,6 +32,7 @@ from litellm.proxy.auth.auth_checks import delete_cache_key_objects from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks +from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem from litellm.proxy.management_endpoints.common_utils import ( _is_user_org_admin_for_team, # pyright: ignore[reportPrivateUsage] # same check /team/member_delete uses _is_user_team_admin, # pyright: ignore[reportPrivateUsage] # same check /team/member_delete uses @@ -48,12 +50,11 @@ from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository from litellm.types.proxy.management_endpoints.internal_user_endpoints import ( BulkDeleteUserRequest, - BulkDeleteUserResponse, UserDeleteResult, ) +from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail from litellm.types.proxy.management_endpoints.team_endpoints import ( BulkTeamMemberDeleteRequest, - BulkTeamMemberDeleteResponse, TeamMemberDeleteResult, ) @@ -67,10 +68,6 @@ _AUDIT_LOG_CONCURRENCY: Final = 10 _BATCH_TX_TIMEOUT: Final = timedelta(seconds=60) -class _ErrorDetail(TypedDict): - error: ReadOnly[str] - - class _OrgAdminFilter(TypedDict): user_id: ReadOnly[str] user_role: ReadOnly[str] @@ -105,9 +102,21 @@ class _UserBatchDeletion: deleted_key_tokens: tuple[str, ...] -def _http_error(status_code: int, message: str) -> HTTPException: - detail: Final[_ErrorDetail] = {"error": message} - return HTTPException(status_code=status_code, detail=detail) +def _team_not_found(team_id: str) -> ManagementProblem: + return ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}team-not-found", + title="Team not found", + status=404, + detail=f"Team id={team_id} does not exist in db", + ) + ) + + +def _forbidden(detail: str) -> ManagementProblem: + return ManagementProblem( + ProblemDetail(type=f"{PROBLEM_TYPE_BASE}forbidden", title="Forbidden", status=403, detail=detail) + ) def _in_filter(field: str, values: Iterable[str]) -> Mapping[str, object]: @@ -167,6 +176,8 @@ def _addresses_user(user: "prisma_models.LiteLLM_UserTable", request: MemberDele def _error_message(exc: BaseException) -> str: + if isinstance(exc, ManagementProblem): + return exc.problem.detail if isinstance(exc, HTTPException) and isinstance(exc.detail, dict): return str(exc.detail.get("error", exc.detail)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # HTTPException.detail is untyped if isinstance(exc, HTTPException): @@ -194,7 +205,7 @@ async def _remove_members_from_team( await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id) roster: Final = await TeamRepository(prisma_client).get_members_with_roles_locked(tx, team_id) if roster is None: - raise _http_error(400, f"Team id={team_id} does not exist in db") + raise _team_not_found(team_id) removed_members: Final = tuple(m for m in roster if any(_addresses_member(m, r) for r in members)) kept_members: Final = tuple(m for m in roster if not any(_addresses_member(m, r) for r in members)) @@ -269,32 +280,32 @@ def _duplicate_member_indexes(members: Sequence[MemberDeleteRequest]) -> frozens async def bulk_remove_team_members( + team_id: str, data: BulkTeamMemberDeleteRequest, user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging | None, -) -> BulkTeamMemberDeleteResponse: - team: Final = await TeamRepository(prisma_client).find_by_id(data.team_id) +) -> tuple[TeamMemberDeleteResult, ...]: + team: Final = await TeamRepository(prisma_client).find_by_id(team_id) if team is None: - raise _http_error(400, f"Team id={data.team_id} does not exist in db") + raise _team_not_found(team_id) if ( user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value and not _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team) and not await _is_user_org_admin_for_team(user_api_key_dict=user_api_key_dict, team_obj=team) ): - raise _http_error( - 403, + raise _forbidden( "Call not allowed. User not proxy admin OR team admin OR org admin for this team. " - f"route='/team/bulk_member_delete', team_id={data.team_id}", + f"route='/management/v1/teams/{team_id}/members/bulk_delete'" ) duplicates: Final = _duplicate_member_indexes(data.members) kept_indexes: Final = tuple(i for i in range(len(data.members)) if i not in duplicates) members: Final = tuple(data.members[i] for i in kept_indexes) async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx: - removal: Final = await _remove_members_from_team(prisma_client, tx, data.team_id, members, user_api_key_dict) + removal: Final = await _remove_members_from_team(prisma_client, tx, team_id, members, user_api_key_dict) await delete_cache_key_objects( hashed_tokens=removal.deleted_key_tokens, user_api_key_cache=user_api_key_cache, @@ -309,7 +320,7 @@ async def bulk_remove_team_members( return "Duplicate member in request" return None if index in matched else "User not found in team" - results: Final = tuple( + return tuple( TeamMemberDeleteResult( user_id=member.user_id, user_email=member.user_email, @@ -318,14 +329,6 @@ async def bulk_remove_team_members( ) for i, member in enumerate(data.members) ) - successful: Final = sum(1 for r in results if r.success) - return BulkTeamMemberDeleteResponse( - team_id=data.team_id, - results=results, - total_requested=len(results), - successful_deletions=successful, - failed_deletions=len(results) - successful, - ) async def _caller_admin_org_ids(prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]: @@ -433,7 +436,7 @@ async def _delete_users( try: deletion: Final = await _delete_users_tx(prisma_client, users, teams_of, user_api_key_dict, litellm_changed_by) except Exception as e: # noqa: BLE001 # the rolled-back batch is reported per row, not as a request failure - verbose_proxy_logger.error("/user/bulk_delete: failed to delete users %s: %s", sorted(user_ids), e) + verbose_proxy_logger.error("users/bulk_delete: failed to delete users %s: %s", sorted(user_ids), e) return _error_message(e) await delete_cache_key_objects( hashed_tokens=deletion.deleted_key_tokens, @@ -468,11 +471,11 @@ async def bulk_delete_users( proxy_logging_obj: ProxyLogging | None, litellm_proxy_admin_name: str | None, litellm_changed_by: str | None, -) -> BulkDeleteUserResponse: +) -> tuple[UserDeleteResult, ...]: caller_is_proxy_admin: Final = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value caller_admin_org_ids: Final = await _caller_admin_org_ids(prisma_client, user_api_key_dict) if not caller_is_proxy_admin and not caller_admin_org_ids: - raise _http_error(403, "Only PROXY_ADMIN or ORG_ADMIN users may delete users.") + raise _forbidden("Only PROXY_ADMIN or ORG_ADMIN users may delete users.") unique_ids: Final = frozenset(data.user_ids) rows: Final = await UserRepository(prisma_client).table.find_many(where=_in_filter("user_id", unique_ids)) @@ -543,11 +546,4 @@ async def bulk_delete_users( teams_removed=tuple(tid for tid, r in deletion.removals.items() if user_id in r.removed), ) - results: Final = tuple(result(i, uid) for i, uid in enumerate(data.user_ids)) - successful: Final = sum(1 for r in results if r.success) - return BulkDeleteUserResponse( - results=results, - total_requested=len(results), - successful_deletions=successful, - failed_deletions=len(results) - successful, - ) + return tuple(result(i, uid) for i, uid in enumerate(data.user_ids)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1c931863a2f..88844c04e74 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1788,24 +1788,37 @@ class _ExceptionRow(TypedDict, total=False): class _ValidationErrorDetail(TypedDict): + type: str loc: tuple[int | str, ...] msg: str +def _is_length_error_of_rejected_items(error: _ValidationErrorDetail, errors: Sequence[_ValidationErrorDetail]) -> bool: + """pydantic counts only items that validated, so a bad item also trips the parent's min_length.""" + return error["type"] == "too_short" and any( + len(other["loc"]) > len(error["loc"]) and other["loc"][: len(error["loc"])] == error["loc"] for other in errors + ) + + @app.exception_handler(RequestValidationError) async def otel_request_validation_exception_handler(request: Request, exc: RequestValidationError): if request.url.path.startswith(MANAGEMENT_V1_PREFIX): - _close_dangling_otel_server_span(request, 400, exc=exc) - validation_errors: Final[Sequence[_ValidationErrorDetail]] = exc.errors() + raw_errors: Final[Sequence[_ValidationErrorDetail]] = exc.errors() + validation_errors: Final = tuple( + error for error in raw_errors if not _is_length_error_of_rejected_items(error, raw_errors) + ) + in_body: Final = any(error["loc"] and error["loc"][0] == "body" for error in validation_errors) + status: Final = 422 if in_body else 400 + _close_dangling_otel_server_span(request, status, exc=exc) return problem_response( ProblemDetail( - type=f"{PROBLEM_TYPE_BASE}invalid-query-parameter", - title="Invalid query parameter", - status=400, + type=f"{PROBLEM_TYPE_BASE}{'invalid-request-body' if in_body else 'invalid-query-parameter'}", + title="Invalid request body" if in_body else "Invalid query parameter", + status=status, detail="; ".join( f"{'.'.join(str(part) for part in error['loc'][1:])}: {error['msg']}" for error in validation_errors ) - or "The request query parameters are invalid.", + or "The request is invalid.", ) ) _close_dangling_otel_server_span(request, 422, exc=exc) diff --git a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py index 7d9f2bd2a85..2b41893663d 100644 --- a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py @@ -1,7 +1,7 @@ from collections.abc import Mapping from typing import Any, Final, Literal -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from typing_extensions import ReadOnly, TypedDict from litellm.proxy._types import ( @@ -9,6 +9,7 @@ from litellm.proxy._types import ( UpdateUserRequest, UpdateUserRequestNoUserIDorEmail, ) +from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse MAX_BULK_DELETE_USERS: Final = 500 @@ -88,11 +89,15 @@ class BulkUpdateUserResponse(BaseModel): class BulkDeleteUserRequest(BaseModel): + """Body of `POST /management/v1/users/bulk_delete`.""" + + model_config = ConfigDict(extra="forbid") + user_ids: tuple[str, ...] = Field(min_length=1, max_length=MAX_BULK_DELETE_USERS) class UserDeleteResult(BaseModel): - """Outcome for one row of `/user/bulk_delete`. `teams_removed` lists the teams the user was taken out of.""" + """Outcome for one requested user, in request order. `teams_removed` lists the teams the user left.""" user_id: str user_email: str | None = None @@ -101,8 +106,5 @@ class UserDeleteResult(BaseModel): error: str | None = None -class BulkDeleteUserResponse(BaseModel): - results: tuple[UserDeleteResult, ...] - total_requested: int - successful_deletions: int - failed_deletions: int +class BulkDeleteUsersResponse(ResourceResponse[tuple[UserDeleteResult, ...]]): + """`{data: [...]}` with one `UserDeleteResult` per requested user, in request order.""" diff --git a/litellm/types/proxy/management_endpoints/management_v1.py b/litellm/types/proxy/management_endpoints/management_v1.py index aa82138110d..c23d0ecfb54 100644 --- a/litellm/types/proxy/management_endpoints/management_v1.py +++ b/litellm/types/proxy/management_endpoints/management_v1.py @@ -65,6 +65,12 @@ class ListLinks(BaseModel): last: str +class ResourceResponse(BaseModel, Generic[TOut]): + """Envelope for a single resource or an action's result: `{data: ...}`, no `meta` or `links`.""" + + data: TOut + + class ListResponse(BaseModel, Generic[TOut]): """Rows stay flat: JSON:API's `{type, id, attributes}` wrapper is a deliberate deviation, so every dashboard column accessor would otherwise have to go through `.attributes`.""" diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 1825813ae9c..5f5be81ee4b 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,6 +1,6 @@ from typing import Any, Final, Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator from litellm.proxy._types import ( KeyManagementRoutes, @@ -10,6 +10,7 @@ from litellm.proxy._types import ( Member, MemberDeleteRequest, ) +from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse TeamIdSearchMatch = Literal["exact", "prefix"] @@ -121,20 +122,28 @@ class BulkTeamMemberAddResponse(BaseModel): updated_team: dict[str, Any] | None = None -class BulkTeamMemberDeleteRequest(BaseModel): - team_id: str - members: tuple[MemberDeleteRequest, ...] = Field(min_length=1, max_length=MAX_BULK_TEAM_MEMBER_DELETES) +class TeamMemberRef(MemberDeleteRequest): + """One member to remove, named by exactly one of `user_id` or `user_email`.""" - @field_validator("members") - @classmethod - def one_identifier_per_member(cls, members: tuple[MemberDeleteRequest, ...]) -> tuple[MemberDeleteRequest, ...]: - if any(m.user_id is not None and m.user_email is not None for m in members): + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="after") + def one_identifier(self) -> "TeamMemberRef": + if self.user_id is not None and self.user_email is not None: raise ValueError("Each member must be identified by exactly one of user_id or user_email") - return members + return self + + +class BulkTeamMemberDeleteRequest(BaseModel): + """Body of `POST /management/v1/teams/{team_id}/members/bulk_delete`.""" + + model_config = ConfigDict(extra="forbid") + + members: tuple[TeamMemberRef, ...] = Field(min_length=1, max_length=MAX_BULK_TEAM_MEMBER_DELETES) class TeamMemberDeleteResult(BaseModel): - """Outcome for one row of `/team/bulk_member_delete`.""" + """Outcome for one requested member, in request order.""" user_id: str | None = None user_email: str | None = None @@ -142,12 +151,8 @@ class TeamMemberDeleteResult(BaseModel): error: str | None = None -class BulkTeamMemberDeleteResponse(BaseModel): - team_id: str - results: tuple[TeamMemberDeleteResult, ...] - total_requested: int - successful_deletions: int - failed_deletions: int +class BulkTeamMemberDeleteResponse(ResourceResponse[tuple[TeamMemberDeleteResult, ...]]): + """`{data: [...]}` with one `TeamMemberDeleteResult` per requested member, in request order.""" class TeamMemberInfoResponse(LiteLLM_TeamMembership): diff --git a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt index 53a6bd4f8c6..6bc8947e89f 100644 --- a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt +++ b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt @@ -80,12 +80,10 @@ POST /model/unblock POST /prompts/test POST /search_tools/test_connection POST /team/bulk_member_add -POST /team/bulk_member_delete POST /team/{team_id}/member/{user_id}/reset_spend POST /team/key/bulk_update POST /team/permissions_bulk_update POST /team/{team_id}/disable_logging -POST /user/bulk_delete POST /user/bulk_update # Alternate method or path for functionality the provider already manages elsewhere diff --git a/tests/proxy_behavior/management/test_team_bulk_member_delete.py b/tests/proxy_behavior/management/test_team_bulk_member_delete.py index 09a3cd54783..5283497c6bf 100644 --- a/tests/proxy_behavior/management/test_team_bulk_member_delete.py +++ b/tests/proxy_behavior/management/test_team_bulk_member_delete.py @@ -71,17 +71,20 @@ async def test_team_bulk_member_delete_authz_matrix( caller = world.keys[actor] resp = await proxy_client.post( - "/team/bulk_member_delete", + f"/management/v1/teams/{scratch.prefix}/members/bulk_delete", headers={"Authorization": f"Bearer {caller.cleartext}"}, - json={"team_id": scratch.prefix, "members": [{"user_id": v} for v in victims]}, + json={"members": [{"user_id": v} for v in victims]}, ) assert resp.status_code == expected_status, f"{actor.value} {shape}: {resp.status_code} {resp.text}" + if expected_status == 403: + assert resp.headers["content-type"] == "application/problem+json" + assert resp.json()["type"] == "urn:litellm:error:forbidden" row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) assert row is not None assert keep in _member_ids(row), "unrelated member removed" if expected_status == 200: - assert [(r["user_id"], r["success"]) for r in resp.json()["results"]] == [(v, True) for v in victims] + assert [(r["user_id"], r["success"]) for r in resp.json()["data"]] == [(v, True) for v in victims] assert not set(victims) & set(_member_ids(row)) else: assert set(victims) <= set(_member_ids(row)), "denied but members removed" @@ -94,18 +97,18 @@ async def test_team_bulk_member_delete_reports_each_row_in_order(proxy_client, p await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=[victim, keep]) resp = await proxy_client.post( - "/team/bulk_member_delete", + f"/management/v1/teams/{scratch.prefix}/members/bulk_delete", headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={"team_id": scratch.prefix, "members": [{"user_id": stranger}, {"user_id": victim}]}, + json={"members": [{"user_id": stranger}, {"user_id": victim}]}, ) assert resp.status_code == 200, resp.text body = resp.json() - assert [(r["user_id"], r["success"]) for r in body["results"]] == [ + assert set(body) == {"data"} + assert [(r["user_id"], r["success"]) for r in body["data"]] == [ (stranger, False), (victim, True), ] - assert body["results"][0]["error"] == "User not found in team" - assert (body["successful_deletions"], body["failed_deletions"]) == (1, 1) + assert body["data"][0]["error"] == "User not found in team" row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) assert row is not None and _member_ids(row) == [keep] @@ -116,14 +119,61 @@ async def test_team_bulk_member_delete_row_naming_both_identifiers_is_422(proxy_ await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=[victim]) resp = await proxy_client.post( - "/team/bulk_member_delete", + f"/management/v1/teams/{scratch.prefix}/members/bulk_delete", headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={ - "team_id": scratch.prefix, - "members": [{"user_id": victim, "user_email": f"{victim}@example.com"}], - }, + json={"members": [{"user_id": victim, "user_email": f"{victim}@example.com"}]}, ) assert resp.status_code == 422, resp.text + assert resp.headers["content-type"] == "application/problem+json" + assert resp.json()["type"] == "urn:litellm:error:invalid-request-body" + assert ( + resp.json()["detail"] + == "members.0: Value error, Each member must be identified by exactly one of user_id or user_email" + ) row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) assert row is not None and victim in _member_ids(row) + + +async def test_team_bulk_member_delete_unknown_query_param_is_400(proxy_client, prisma, scratch, world): + victim = scratch.tag("victim") + await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=[victim]) + + resp = await proxy_client.post( + f"/management/v1/teams/{scratch.prefix}/members/bulk_delete?dry_run=1", + headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"members": [{"user_id": victim}]}, + ) + assert resp.status_code == 400, resp.text + assert resp.headers["content-type"] == "application/problem+json" + assert "dry_run" in resp.json()["detail"] + + row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + assert row is not None and victim in _member_ids(row) + + +async def test_team_bulk_member_delete_unknown_body_field_is_422(proxy_client, prisma, scratch, world): + victim = scratch.tag("victim") + await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=[victim]) + + resp = await proxy_client.post( + f"/management/v1/teams/{scratch.prefix}/members/bulk_delete", + headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"team_id": scratch.prefix, "members": [{"user_id": victim}]}, + ) + assert resp.status_code == 422, resp.text + assert "team_id" in resp.json()["detail"] + + row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + assert row is not None and victim in _member_ids(row) + + +async def test_team_bulk_member_delete_unknown_team_is_404_problem(proxy_client, scratch, world): + resp = await proxy_client.post( + f"/management/v1/teams/{scratch.tag('missing')}/members/bulk_delete", + headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"members": [{"user_id": scratch.tag("victim")}]}, + ) + assert resp.status_code == 404, resp.text + assert resp.headers["content-type"] == "application/problem+json" + assert resp.json()["type"] == "urn:litellm:error:team-not-found" diff --git a/tests/proxy_behavior/management/test_users_bulk_delete.py b/tests/proxy_behavior/management/test_users_bulk_delete.py new file mode 100644 index 00000000000..049234e737a --- /dev/null +++ b/tests/proxy_behavior/management/test_users_bulk_delete.py @@ -0,0 +1,137 @@ +import pytest + +from .actors import Actor +from .conftest import create_scratch_team, create_scratch_user + +pytestmark = pytest.mark.asyncio(loop_scope="session") + +_URL = "/management/v1/users/bulk_delete" + +# (id, actor, victims' org, expected status, whether the victims are gone afterwards) +_MATRIX = [ + ("org_a/proxy_admin", Actor.PROXY_ADMIN, "a", 200, True), + ("org_a/org_admin", Actor.ORG_ADMIN, "a", 200, True), + ("org_a/org_b_admin", Actor.ORG_B_ADMIN, "a", 200, False), + ("org_a/team_admin", Actor.TEAM_ADMIN, "a", 403, False), + ("org_a/internal_user", Actor.INTERNAL_USER, "a", 403, False), + ("org_a/owner", Actor.OWNER, "a", 403, False), + ("org_a/service_account", Actor.SERVICE_ACCOUNT, "a", 403, False), + ("no_org/proxy_admin", Actor.PROXY_ADMIN, None, 200, True), + ("no_org/org_admin", Actor.ORG_ADMIN, None, 200, False), +] + + +def _member_ids(row) -> list: + return [m["user_id"] for m in (row.members_with_roles or [])] + + +async def _seed_team_members(prisma, scratch, world, member_ids: list, org_id) -> None: + """Leave behind what /team/member_add would: roster entry, `teams` array, and org membership.""" + await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=member_ids) + await prisma.db.litellm_usertable.update_many( + where={"user_id": {"in": member_ids}}, data={"teams": {"set": [scratch.prefix]}} + ) + if org_id is None: + return + for uid in member_ids: + await prisma.db.litellm_organizationmembership.create( + data={"user_id": uid, "organization_id": org_id, "user_role": "internal_user"} + ) + + +@pytest.mark.parametrize( + "actor,org,expected_status,expect_deleted", + [(a, o, s, d) for (_id, a, o, s, d) in _MATRIX], + ids=[s[0] for s in _MATRIX], +) +async def test_users_bulk_delete_authz_matrix( + actor: Actor, + org, + expected_status: int, + expect_deleted: bool, + proxy_client, + prisma, + scratch, + world, +): + victims = [await create_scratch_user(prisma, scratch.prefix, suffix=s) for s in ("v1", "v2")] + keep = await create_scratch_user(prisma, scratch.prefix, suffix="keep") + await _seed_team_members(prisma, scratch, world, victims + [keep], world.org_a_id if org == "a" else None) + + resp = await proxy_client.post( + _URL, + headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, + json={"user_ids": victims}, + ) + assert resp.status_code == expected_status, f"{actor.value}: {resp.status_code} {resp.text}" + + team = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + assert team is not None and keep in _member_ids(team), "unrelated member removed" + remaining = {u.user_id for u in await prisma.db.litellm_usertable.find_many(where={"user_id": {"in": victims}})} + if expected_status == 403: + assert resp.headers["content-type"] == "application/problem+json" + assert resp.json()["type"] == "urn:litellm:error:forbidden" + assert remaining == set(victims), "denied but users deleted" + assert set(victims) <= set(_member_ids(team)), "denied but members removed" + return + + body = resp.json() + assert set(body) == {"data"} + rows = [(r["user_id"], r["success"], r["teams_removed"]) for r in body["data"]] + if expect_deleted: + assert rows == [(v, True, [scratch.prefix]) for v in victims] + assert remaining == set() + assert not set(victims) & set(_member_ids(team)) + return + assert rows == [(v, False, []) for v in victims] + assert all("not within your admin scope" in r["error"] for r in body["data"]) + assert remaining == set(victims), "out-of-scope rows reported failed but users deleted" + assert set(victims) <= set(_member_ids(team)) + + +async def test_users_bulk_delete_reports_each_row_in_order(proxy_client, prisma, scratch, world): + victim = await create_scratch_user(prisma, scratch.prefix, suffix="victim") + ghost = scratch.tag("ghost") + + resp = await proxy_client.post( + _URL, + headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"user_ids": [ghost, victim, victim]}, + ) + assert resp.status_code == 200, resp.text + assert [(r["user_id"], r["success"]) for r in resp.json()["data"]] == [ + (ghost, False), + (victim, True), + (victim, False), + ] + assert await prisma.db.litellm_usertable.find_unique(where={"user_id": victim}) is None + + +async def test_users_bulk_delete_unknown_query_param_is_400_problem(proxy_client, prisma, scratch, world): + victim = await create_scratch_user(prisma, scratch.prefix, suffix="victim") + + resp = await proxy_client.post( + f"{_URL}?dry_run=1", + headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"user_ids": [victim]}, + ) + assert resp.status_code == 400, resp.text + assert resp.headers["content-type"] == "application/problem+json" + assert resp.json()["type"] == "urn:litellm:error:unknown-query-parameter" + assert "dry_run" in resp.json()["detail"] + assert await prisma.db.litellm_usertable.find_unique(where={"user_id": victim}) is not None + + +async def test_users_bulk_delete_unknown_body_field_is_422_problem(proxy_client, prisma, scratch, world): + victim = await create_scratch_user(prisma, scratch.prefix, suffix="victim") + + resp = await proxy_client.post( + _URL, + headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"user_ids": [victim], "dry_run": True}, + ) + assert resp.status_code == 422, resp.text + assert resp.headers["content-type"] == "application/problem+json" + assert resp.json()["type"] == "urn:litellm:error:invalid-request-body" + assert "dry_run" in resp.json()["detail"] + assert await prisma.db.litellm_usertable.find_unique(where={"user_id": victim}) is not None diff --git a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py index 49aed00177a..077eb90ddce 100644 --- a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py +++ b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py @@ -5,14 +5,14 @@ from contextlib import asynccontextmanager from typing import Final import pytest -from fastapi import HTTPException from pydantic import BaseModel, ConfigDict, ValidationError -from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, MemberDeleteRequest, UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.list_api.common import ManagementProblem from litellm.proxy.management_helpers.bulk_user_deletion import bulk_delete_users, bulk_remove_team_members from litellm.types.proxy.management_endpoints.internal_user_endpoints import BulkDeleteUserRequest -from litellm.types.proxy.management_endpoints.team_endpoints import BulkTeamMemberDeleteRequest +from litellm.types.proxy.management_endpoints.team_endpoints import BulkTeamMemberDeleteRequest, TeamMemberRef ADMIN: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin") INTERNAL: Final = UserAPIKeyAuth(user_id="someone", user_role=LitellmUserRoles.INTERNAL_USER) @@ -237,7 +237,8 @@ async def _remove( cache: UserApiKeyCache | None = None, ): return await bulk_remove_team_members( - data=BulkTeamMemberDeleteRequest(team_id=team_id, members=tuple(MemberDeleteRequest(**m) for m in members)), + team_id=team_id, + data=BulkTeamMemberDeleteRequest(members=tuple(TeamMemberRef(**m) for m in members)), user_api_key_dict=caller, prisma_client=prisma, # pyright: ignore[reportArgumentType] # fake stands in for PrismaClient user_api_key_cache=cache or UserApiKeyCache(), @@ -260,10 +261,10 @@ async def test_bulk_delete_removes_users_from_every_team_and_store(): org_memberships=[{"user_id": "u1", "organization_id": "o1", "user_role": "internal_user"}], ) - response = await _delete(prisma, ["u1", "u2"]) + results = await _delete(prisma, ["u1", "u2"]) - assert (response.total_requested, response.successful_deletions, response.failed_deletions) == (2, 2, 0) - assert [(r.user_id, r.user_email, r.success, r.teams_removed) for r in response.results] == [ + assert len(results) == 2 + assert [(r.user_id, r.user_email, r.success, r.teams_removed) for r in results] == [ ("u1", "u1@example.com", True, ("t1", "t2")), ("u2", "u2@example.com", True, ("t1",)), ] @@ -297,9 +298,9 @@ async def test_bulk_delete_leaves_teammates_who_share_the_deleted_users_email_al ], ) - response = await _delete(prisma, ["u1"]) + results = await _delete(prisma, ["u1"]) - assert [(r.success, r.teams_removed) for r in response.results] == [(True, ("t1",))] + assert [(r.success, r.teams_removed) for r in results] == [(True, ("t1",))] assert _roster(prisma, "t1") == ["twin"] assert set(prisma.db.litellm_usertable.rows) == {"twin"} and prisma.db.litellm_usertable.rows["twin"].teams == [ "t1" @@ -319,9 +320,9 @@ async def test_bulk_delete_removes_the_deleted_users_email_only_roster_entry(): ) prisma = _FakePrisma(users=[_user("u1", "t1"), _user("keep", "t1")], teams=[team]) - response = await _delete(prisma, ["u1"]) + results = await _delete(prisma, ["u1"]) - assert [(r.success, r.teams_removed) for r in response.results] == [(True, ("t1",))] + assert [(r.success, r.teams_removed) for r in results] == [(True, ("t1",))] assert _roster(prisma, "t1") == ["keep"] assert set(prisma.db.litellm_usertable.rows) == {"keep"} @@ -334,9 +335,9 @@ async def test_bulk_delete_finds_teams_through_membership_rows_when_user_teams_a memberships=[("t1", "u1")], ) - response = await _delete(prisma, ["u1"]) + results = await _delete(prisma, ["u1"]) - assert response.results[0].teams_removed == ("t1",) + assert results[0].teams_removed == ("t1",) assert _roster(prisma, "t1") == ["keep"] assert prisma.db.litellm_teammembership.rows == [] @@ -350,9 +351,9 @@ async def test_bulk_delete_reads_roster_under_lock_so_a_concurrent_add_survives( prisma = _FakePrisma(users=[_user("u1", "t1")], teams=[team], on_lock=concurrent_member_add) - response = await _delete(prisma, ["u1"]) + results = await _delete(prisma, ["u1"]) - assert response.results[0].success is True + assert results[0].success is True assert _roster(prisma, "t1") == ["late"] @@ -360,10 +361,10 @@ async def test_bulk_delete_reads_roster_under_lock_so_a_concurrent_add_survives( async def test_bulk_delete_reports_missing_and_duplicate_ids_per_item_and_still_deletes_the_rest(): prisma = _FakePrisma(users=[_user("u1")]) - response = await _delete(prisma, ["u1", "ghost", "u1"]) + results = await _delete(prisma, ["u1", "ghost", "u1"]) - assert (response.successful_deletions, response.failed_deletions) == (1, 2) - assert [(r.user_id, r.success, r.error) for r in response.results] == [ + assert [r.success for r in results].count(True) == 1 + assert [(r.user_id, r.success, r.error) for r in results] == [ ("u1", True, None), ("ghost", False, "User id=ghost not found"), ("u1", False, "Duplicate user_id in request: u1"), @@ -381,9 +382,9 @@ async def test_bulk_delete_rolls_back_every_team_and_user_when_one_team_rewrite_ ) cache = _cache_with("k1") - response = await _delete(prisma, ["u1", "u2"], cache=cache) + results = await _delete(prisma, ["u1", "u2"], cache=cache) - assert [(r.user_id, r.success, r.teams_removed, r.error) for r in response.results] == [ + assert [(r.user_id, r.success, r.teams_removed, r.error) for r in results] == [ ("u1", False, (), "Failed to delete user: lock timeout"), ("u2", False, (), "Failed to delete user: lock timeout"), ] @@ -397,9 +398,9 @@ async def test_bulk_delete_rolls_back_every_team_and_user_when_one_team_rewrite_ async def test_bulk_delete_skips_teams_the_user_still_names_but_which_no_longer_exist(): prisma = _FakePrisma(users=[_user("u1", "gone", "t1")], teams=[_team("t1", "u1", "keep")]) - response = await _delete(prisma, ["u1"]) + results = await _delete(prisma, ["u1"]) - assert [(r.success, r.teams_removed) for r in response.results] == [(True, ("t1",))] + assert [(r.success, r.teams_removed) for r in results] == [(True, ("t1",))] assert prisma.db.litellm_usertable.rows == {} and _roster(prisma, "t1") == ["keep"] assert prisma.locks == ["t1"] @@ -414,9 +415,9 @@ async def test_bulk_delete_rolls_back_every_user_row_and_reports_it_per_row_when ) cache = _cache_with("k1") - response = await _delete(prisma, ["u1", "u2", "ghost"], cache=cache) + results = await _delete(prisma, ["u1", "u2", "ghost"], cache=cache) - assert [(r.user_id, r.success, r.error) for r in response.results] == [ + assert [(r.user_id, r.success, r.error) for r in results] == [ ("u1", False, "Failed to delete user: connection reset"), ("u2", False, "Failed to delete user: connection reset"), ("ghost", False, "User id=ghost not found"), @@ -452,10 +453,10 @@ async def test_bulk_delete_evicts_deleted_keys_and_users_from_the_auth_cache(): async def test_bulk_delete_rejects_non_admin_callers_before_touching_the_db(): prisma = _FakePrisma(users=[_user("u1")]) - with pytest.raises(HTTPException) as exc: + with pytest.raises(ManagementProblem) as exc: await _delete(prisma, ["u1"], caller=INTERNAL) - assert exc.value.status_code == 403 + assert exc.value.problem.status == 403 assert set(prisma.db.litellm_usertable.rows) == {"u1"} @@ -471,10 +472,10 @@ async def test_org_admin_deletes_only_users_fully_inside_their_orgs(): ], ) - response = await _delete(prisma, ["inside", "straddles", "orgless"], caller=ORG_ADMIN) + results = await _delete(prisma, ["inside", "straddles", "orgless"], caller=ORG_ADMIN) - assert [r.success for r in response.results] == [True, False, False] - assert all("not within your admin scope" in (r.error or "") for r in response.results[1:]) + assert [r.success for r in results] == [True, False, False] + assert all("not within your admin scope" in (r.error or "") for r in results[1:]) assert set(prisma.db.litellm_usertable.rows) == {"straddles", "orgless"} assert {(m["user_id"], m["organization_id"]) for m in prisma.db.litellm_organizationmembership.rows} == { ("org-admin", "o1"), @@ -496,10 +497,9 @@ async def test_bulk_member_delete_removes_by_id_and_email_and_keeps_the_rest(): ], ) - response = await _remove(prisma, "t1", [{"user_id": "u1"}, {"user_email": "u2@example.com"}]) + results = await _remove(prisma, "t1", [{"user_id": "u1"}, {"user_email": "u2@example.com"}]) - assert (response.team_id, response.successful_deletions, response.failed_deletions) == ("t1", 2, 0) - assert [(r.user_id, r.user_email, r.success) for r in response.results] == [ + assert [(r.user_id, r.user_email, r.success) for r in results] == [ ("u1", None, True), (None, "u2@example.com", True), ] @@ -516,13 +516,12 @@ async def test_bulk_member_delete_removes_by_id_and_email_and_keeps_the_rest(): async def test_bulk_member_delete_reports_members_not_on_the_team_without_rewriting_the_roster(): prisma = _FakePrisma(users=[_user("u1", "t1"), _user("elsewhere")], teams=[_team("t1", "u1")]) - response = await _remove(prisma, "t1", [{"user_id": "elsewhere"}, {"user_email": "nobody@example.com"}]) + results = await _remove(prisma, "t1", [{"user_id": "elsewhere"}, {"user_email": "nobody@example.com"}]) - assert [(r.success, r.error) for r in response.results] == [ + assert [(r.success, r.error) for r in results] == [ (False, "User not found in team"), (False, "User not found in team"), ] - assert (response.successful_deletions, response.failed_deletions) == (0, 2) assert prisma.db.litellm_teamtable.update_calls == 0 assert _roster(prisma, "t1") == ["u1"] @@ -536,9 +535,9 @@ async def test_bulk_member_delete_leaves_keys_and_memberships_of_unmatched_membe tokens=[{"token": "orphan-key", "user_id": "elsewhere", "team_id": "t1"}], ) - response = await _remove(prisma, "t1", [{"user_id": "elsewhere"}]) + results = await _remove(prisma, "t1", [{"user_id": "elsewhere"}]) - assert response.results[0].success is False + assert results[0].success is False assert prisma.db.litellm_teammembership.rows == [{"team_id": "t1", "user_id": "elsewhere"}] assert [t["token"] for t in prisma.db.litellm_verificationtoken.rows] == ["orphan-key"] @@ -547,17 +546,16 @@ async def test_bulk_member_delete_leaves_keys_and_memberships_of_unmatched_membe async def test_bulk_member_delete_reports_repeated_members_as_duplicates_and_removes_them_once(): prisma = _FakePrisma(users=[_user("u1", "t1"), _user("u2", "t1")], teams=[_team("t1", "u1", "u2", "keep")]) - response = await _remove( + results = await _remove( prisma, "t1", [{"user_id": "u1"}, {"user_id": "u1"}, {"user_email": "u1@example.com"}, {"user_id": "u2"}] ) - assert [(r.success, r.error) for r in response.results] == [ + assert [(r.success, r.error) for r in results] == [ (True, None), (False, "Duplicate member in request"), (True, None), (True, None), ] - assert (response.successful_deletions, response.failed_deletions) == (3, 1) assert _roster(prisma, "t1") == ["keep"] @@ -583,9 +581,9 @@ async def test_bulk_member_delete_evicts_the_removed_team_keys_from_the_auth_cac async def test_bulk_member_delete_cleans_a_user_whose_teams_array_still_names_the_team(): prisma = _FakePrisma(users=[_user("stale", "t1")], teams=[_team("t1", "other")], memberships=[("t1", "stale")]) - response = await _remove(prisma, "t1", [{"user_id": "stale"}]) + results = await _remove(prisma, "t1", [{"user_id": "stale"}]) - assert response.results[0].success is True + assert results[0].success is True assert prisma.db.litellm_usertable.rows["stale"].teams == [] assert prisma.db.litellm_teammembership.rows == [] assert _roster(prisma, "t1") == ["other"] and prisma.db.litellm_teamtable.update_calls == 0 @@ -595,13 +593,13 @@ async def test_bulk_member_delete_cleans_a_user_whose_teams_array_still_names_th async def test_bulk_member_delete_rejects_unknown_team_and_unauthorized_callers(): prisma = _FakePrisma(users=[_user("u1", "t1")], teams=[_team("t1", "u1")]) - with pytest.raises(HTTPException) as missing: + with pytest.raises(ManagementProblem) as missing: await _remove(prisma, "nope", [{"user_id": "u1"}]) - with pytest.raises(HTTPException) as forbidden: + with pytest.raises(ManagementProblem) as forbidden: await _remove(prisma, "t1", [{"user_id": "u1"}], caller=INTERNAL) - assert missing.value.status_code == 400 - assert forbidden.value.status_code == 403 + assert missing.value.problem.status == 404 + assert forbidden.value.problem.status == 403 assert _roster(prisma, "t1") == ["u1"] and prisma.locks == [] @@ -611,9 +609,9 @@ async def test_team_admin_may_bulk_remove_members(): team.members_with_roles[0].role = "admin" prisma = _FakePrisma(users=[_user("lead", "t1"), _user("u1", "t1")], teams=[team]) - response = await _remove(prisma, "t1", [{"user_id": "u1"}], caller=UserAPIKeyAuth(user_id="lead")) + results = await _remove(prisma, "t1", [{"user_id": "u1"}], caller=UserAPIKeyAuth(user_id="lead")) - assert response.results[0].success is True + assert results[0].success is True assert _roster(prisma, "t1") == ["lead"] @@ -623,22 +621,24 @@ def test_request_models_enforce_batch_bounds(): with pytest.raises(ValidationError): BulkDeleteUserRequest(user_ids=tuple(f"u{i}" for i in range(501))) with pytest.raises(ValidationError): - BulkTeamMemberDeleteRequest(team_id="t1", members=()) + BulkTeamMemberDeleteRequest(members=()) with pytest.raises(ValidationError): - BulkTeamMemberDeleteRequest( - team_id="t1", members=tuple(MemberDeleteRequest(user_id=f"u{i}") for i in range(501)) - ) + BulkTeamMemberDeleteRequest(members=tuple(TeamMemberRef(user_id=f"u{i}") for i in range(501))) assert len(BulkDeleteUserRequest(user_ids=tuple(f"u{i}" for i in range(500))).user_ids) == 500 def test_bulk_member_delete_request_requires_exactly_one_identifier_per_member(): with pytest.raises(ValidationError, match="exactly one of user_id or user_email"): - BulkTeamMemberDeleteRequest( - team_id="t1", members=(MemberDeleteRequest(user_id="u1", user_email="other@example.com"),) - ) + BulkTeamMemberDeleteRequest.model_validate({"members": [{"user_id": "u1", "user_email": "other@example.com"}]}) with pytest.raises(ValidationError): - BulkTeamMemberDeleteRequest.model_validate({"team_id": "t1", "members": [{}]}) - assert ( - BulkTeamMemberDeleteRequest(team_id="t1", members=(MemberDeleteRequest(user_id="u1"),)).members[0].user_id - == "u1" - ) + BulkTeamMemberDeleteRequest.model_validate({"members": [{}]}) + assert BulkTeamMemberDeleteRequest(members=(TeamMemberRef(user_id="u1"),)).members[0].user_id == "u1" + + +def test_request_models_reject_unknown_fields(): + with pytest.raises(ValidationError, match="team_id"): + BulkTeamMemberDeleteRequest.model_validate({"team_id": "t1", "members": [{"user_id": "u1"}]}) + with pytest.raises(ValidationError, match="role"): + BulkTeamMemberDeleteRequest.model_validate({"members": [{"user_id": "u1", "role": "admin"}]}) + with pytest.raises(ValidationError, match="dry_run"): + BulkDeleteUserRequest.model_validate({"user_ids": ["u1"], "dry_run": True}) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0c8913d9c6c..b068565e697 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8470,6 +8470,71 @@ export interface paths { patch?: never; trace?: never; }; + "/management/v1/teams/{team_id}/members/bulk_delete": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Bulk Delete Team Members Action + * @description Remove up to 500 members from one team in one call. Same authorization as + * `/team/member_delete`: proxy admins, the team's admins, and admins of the team's + * organization. Each member is named by exactly one of `user_id` or `user_email`; + * unknown body fields are a 422 and an unknown team is a 404. + * + * `data` holds one result per requested member, in request order. A row is + * `success: false` with an `error` when it names nobody on the team or repeats an + * earlier row. The roster is rewritten once, under the team's advisory lock, so a + * concurrent member_add is never overwritten from a stale read. + * + * Example curl: + * ``` + * curl --location 'http://0.0.0.0:4000/management/v1/teams/team-1/members/bulk_delete' --header 'Authorization: Bearer sk-1234' --header 'Content-Type: application/json' --data '{"members": [{"user_id": "user-1"}, {"user_email": "user-2@example.com"}]}' + * ``` + */ + post: operations["bulk_delete_team_members_action_management_v1_teams__team_id__members_bulk_delete_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/management/v1/users/bulk_delete": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Bulk Delete Users Action + * @description Delete up to 500 users in one call, taking each out of every team it belongs to. + * Same authorization as `/user/delete`: proxy admins may delete anyone, org admins + * only users inside organizations they administer. Unknown body fields are a 422. + * + * `data` holds one result per requested `user_id`, in request order. A row is + * `success: false` with an `error` when the id is unknown, repeated in the request, + * or outside the caller's scope. Rows that pass those checks are deleted together, + * in one transaction, so either all of them go or none does. + * + * Example curl: + * ``` + * curl --location 'http://0.0.0.0:4000/management/v1/users/bulk_delete' --header 'Authorization: Bearer sk-1234' --header 'Content-Type: application/json' --data '{"user_ids": ["user-1", "user-2"]}' + * ``` + */ + post: operations["bulk_delete_users_action_management_v1_users_bulk_delete_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/mcp": { parameters: { query?: never; @@ -15056,26 +15121,6 @@ export interface paths { patch?: never; trace?: never; }; - "/team/bulk_member_delete": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** - * Bulk Team Member Delete - * @description Remove up to 500 members from one team; same authorization as `/team/member_delete`. - */ - post: operations["bulk_team_member_delete_team_bulk_member_delete_post"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/team/daily/activity": { parameters: { query?: never; @@ -16498,26 +16543,6 @@ export interface paths { patch?: never; trace?: never; }; - "/user/bulk_delete": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** - * Bulk Delete User - * @description Delete up to 500 users, removing each from every team; same authorization as `/user/delete`. - */ - post: operations["bulk_delete_user_user_bulk_delete_post"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/user/bulk_update": { parameters: { query?: never; @@ -24543,21 +24568,21 @@ export interface components { /** Budgets */ budgets: string[]; }; - /** BulkDeleteUserRequest */ + /** + * BulkDeleteUserRequest + * @description Body of `POST /management/v1/users/bulk_delete`. + */ BulkDeleteUserRequest: { /** User Ids */ user_ids: string[]; }; - /** BulkDeleteUserResponse */ - BulkDeleteUserResponse: { - /** Failed Deletions */ - failed_deletions: number; - /** Results */ - results: components["schemas"]["UserDeleteResult"][]; - /** Successful Deletions */ - successful_deletions: number; - /** Total Requested */ - total_requested: number; + /** + * BulkDeleteUsersResponse + * @description `{data: [...]}` with one `UserDeleteResult` per requested user, in request order. + */ + BulkDeleteUsersResponse: { + /** Data */ + data: components["schemas"]["UserDeleteResult"][]; }; /** * BulkTeamMemberAddRequest @@ -24596,25 +24621,21 @@ export interface components { [key: string]: unknown; } | null; }; - /** BulkTeamMemberDeleteRequest */ + /** + * BulkTeamMemberDeleteRequest + * @description Body of `POST /management/v1/teams/{team_id}/members/bulk_delete`. + */ BulkTeamMemberDeleteRequest: { /** Members */ - members: components["schemas"]["MemberDeleteRequest"][]; - /** Team Id */ - team_id: string; + members: components["schemas"]["TeamMemberRef"][]; }; - /** BulkTeamMemberDeleteResponse */ + /** + * BulkTeamMemberDeleteResponse + * @description `{data: [...]}` with one `TeamMemberDeleteResult` per requested member, in request order. + */ BulkTeamMemberDeleteResponse: { - /** Failed Deletions */ - failed_deletions: number; - /** Results */ - results: components["schemas"]["TeamMemberDeleteResult"][]; - /** Successful Deletions */ - successful_deletions: number; - /** Team Id */ - team_id: string; - /** Total Requested */ - total_requested: number; + /** Data */ + data: components["schemas"]["TeamMemberDeleteResult"][]; }; /** * BulkUpdateKeyRequest @@ -31982,13 +32003,6 @@ export interface components { */ user_id?: string | null; }; - /** MemberDeleteRequest */ - MemberDeleteRequest: { - /** User Email */ - user_email?: string | null; - /** User Id */ - user_id?: string | null; - }; /** MemoryCreateRequest */ MemoryCreateRequest: { /** @@ -37254,7 +37268,7 @@ export interface components { }; /** * TeamMemberDeleteResult - * @description Outcome for one row of `/team/bulk_member_delete`. + * @description Outcome for one requested member, in request order. */ TeamMemberDeleteResult: { /** Error */ @@ -37296,6 +37310,16 @@ export interface components { /** User Id */ user_id: string; }; + /** + * TeamMemberRef + * @description One member to remove, named by exactly one of `user_id` or `user_email`. + */ + TeamMemberRef: { + /** User Email */ + user_email?: string | null; + /** User Id */ + user_id?: string | null; + }; /** TeamMemberUpdateRequest */ TeamMemberUpdateRequest: { /** @@ -39447,7 +39471,7 @@ export interface components { }; /** * UserDeleteResult - * @description Outcome for one row of `/user/bulk_delete`. `teams_removed` lists the teams the user was taken out of. + * @description Outcome for one requested user, in request order. `teams_removed` lists the teams the user left. */ UserDeleteResult: { /** Error */ @@ -51236,6 +51260,77 @@ export interface operations { }; }; }; + bulk_delete_team_members_action_management_v1_teams__team_id__members_bulk_delete_post: { + parameters: { + query?: never; + header?: never; + path: { + team_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BulkTeamMemberDeleteRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BulkTeamMemberDeleteResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + bulk_delete_users_action_management_v1_users_bulk_delete_post: { + parameters: { + query?: never; + header?: { + /** @description Who the caller is acting for; recorded on the audit log entries this call writes. */ + "litellm-changed-by"?: string | null; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BulkDeleteUserRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BulkDeleteUsersResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; aggregate_mcp_route_mcp_get: { parameters: { query?: never; @@ -59006,39 +59101,6 @@ export interface operations { }; }; }; - bulk_team_member_delete_team_bulk_member_delete_post: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["BulkTeamMemberDeleteRequest"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["BulkTeamMemberDeleteResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; get_team_daily_activity_team_daily_activity_get: { parameters: { query?: { @@ -60783,42 +60845,6 @@ export interface operations { }; }; }; - bulk_delete_user_user_bulk_delete_post: { - parameters: { - query?: never; - header?: { - /** @description The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability */ - "litellm-changed-by"?: string | null; - }; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["BulkDeleteUserRequest"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["BulkDeleteUserResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; bulk_user_update_user_bulk_update_post: { parameters: { query?: never; From b0acda28256d2a0b244b2e5518aad4185b338b45 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 14 Sep 2026 23:09:04 +0000 Subject: [PATCH 10/12] fix(proxy): mark validation error TypedDict fields ReadOnly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 88844c04e74..04145eb68ef 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1788,9 +1788,9 @@ class _ExceptionRow(TypedDict, total=False): class _ValidationErrorDetail(TypedDict): - type: str - loc: tuple[int | str, ...] - msg: str + type: ReadOnly[str] + loc: ReadOnly[tuple[int | str, ...]] + msg: ReadOnly[str] def _is_length_error_of_rejected_items(error: _ValidationErrorDetail, errors: Sequence[_ValidationErrorDetail]) -> bool: From 5ace6fa731b0672db095ca64241bc2f4138305de Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 14 Sep 2026 23:51:15 +0000 Subject: [PATCH 11/12] fix(proxy): match an id-only bulk member delete against a legacy email-only roster entry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_helpers/bulk_user_deletion.py | 29 ++++++++++------ .../test_team_bulk_member_delete.py | 33 ++++++++++++++++++- .../test_bulk_user_deletion.py | 21 ++++++++++++ 3 files changed, 72 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py index 53527c42c25..337b3cdeabc 100644 --- a/litellm/proxy/management_helpers/bulk_user_deletion.py +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -169,6 +169,12 @@ def _addresses_member(member: Member, request: MemberDeleteRequest) -> bool: return request.user_id == member.user_id or (member.user_id is None and _same_email(member.user_email, request)) +def _with_row_email(request: MemberDeleteRequest, email_of: Mapping[str, str]) -> MemberDeleteRequest: + if request.user_id is None or request.user_email is not None: + return request + return MemberDeleteRequest(user_id=request.user_id, user_email=email_of.get(request.user_id)) + + def _addresses_user(user: "prisma_models.LiteLLM_UserTable", request: MemberDeleteRequest) -> bool: if request.user_id is None: return _same_email(user.user_email, request) @@ -207,22 +213,25 @@ async def _remove_members_from_team( if roster is None: raise _team_not_found(team_id) - removed_members: Final = tuple(m for m in roster if any(_addresses_member(m, r) for r in members)) - kept_members: Final = tuple(m for m in roster if not any(_addresses_member(m, r) for r in members)) - removed_ids: Final = frozenset(m.user_id for m in removed_members if m.user_id is not None) requested_ids: Final = frozenset(r.user_id for r in members if r.user_id is not None) requested_emails: Final = frozenset(r.user_email for r in members if r.user_id is None and r.user_email) - user_rows: Final = await _user_tx_db(tx).find_many( - where=_any_filter( - _in_filter("user_id", removed_ids | requested_ids), - _in_filter("user_email", requested_emails), - ) + requested_rows: Final = await _user_tx_db(tx).find_many( + where=_any_filter(_in_filter("user_id", requested_ids), _in_filter("user_email", requested_emails)) ) - stale_rows: Final = tuple(u for u in user_rows if team_id in u.teams) + email_of: Final = MappingProxyType({u.user_id: u.user_email for u in requested_rows if u.user_email is not None}) + requests: Final = tuple(_with_row_email(r, email_of) for r in members) + removed_members: Final = tuple(m for m in roster if any(_addresses_member(m, r) for r in requests)) + kept_members: Final = tuple(m for m in roster if not any(_addresses_member(m, r) for r in requests)) + removed_ids: Final = frozenset(m.user_id for m in removed_members if m.user_id is not None) + unfetched_ids: Final = removed_ids - frozenset(u.user_id for u in requested_rows) + removed_rows: Final = ( + await _user_tx_db(tx).find_many(where=_in_filter("user_id", unfetched_ids)) if unfetched_ids else () + ) + stale_rows: Final = tuple(u for u in (*requested_rows, *removed_rows) if team_id in u.teams) cleanup_ids: Final = removed_ids | frozenset(u.user_id for u in stale_rows) matched: Final = frozenset( i - for i, r in enumerate(members) + for i, r in enumerate(requests) if any(_addresses_member(m, r) for m in removed_members) or any(_addresses_user(u, r) for u in stale_rows) ) keys: Final = await _token_tx_db(tx).find_many(where=_team_users_filter(team_id, cleanup_ids)) diff --git a/tests/proxy_behavior/management/test_team_bulk_member_delete.py b/tests/proxy_behavior/management/test_team_bulk_member_delete.py index 5283497c6bf..0818fb25dfe 100644 --- a/tests/proxy_behavior/management/test_team_bulk_member_delete.py +++ b/tests/proxy_behavior/management/test_team_bulk_member_delete.py @@ -1,7 +1,8 @@ import pytest +from prisma import Json from .actors import Actor -from .conftest import create_scratch_team +from .conftest import create_scratch_team, create_scratch_user pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -114,6 +115,36 @@ async def test_team_bulk_member_delete_reports_each_row_in_order(proxy_client, p assert row is not None and _member_ids(row) == [keep] +async def test_team_bulk_member_delete_by_id_removes_a_legacy_email_only_roster_entry( + proxy_client, prisma, scratch, world +): + email = f"{scratch.prefix}@example.com" + victim = await create_scratch_user(prisma, scratch.prefix, suffix="victim", user_email=email) + keep = scratch.tag("keep") + await prisma.db.litellm_teamtable.create( + data={ + "team_id": scratch.prefix, + "team_alias": scratch.prefix, + "organization_id": world.org_a_id, + "members_with_roles": Json([{"user_email": email, "role": "user"}, {"user_id": keep, "role": "user"}]), + } + ) + await prisma.db.litellm_usertable.update(where={"user_id": victim}, data={"teams": [scratch.prefix]}) + + resp = await proxy_client.post( + f"/management/v1/teams/{scratch.prefix}/members/bulk_delete", + headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"members": [{"user_id": victim}]}, + ) + assert resp.status_code == 200, resp.text + assert [(r["user_id"], r["success"]) for r in resp.json()["data"]] == [(victim, True)] + + row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + assert row is not None and [(m["user_id"], m.get("user_email")) for m in row.members_with_roles] == [(keep, None)] + user = await prisma.db.litellm_usertable.find_unique(where={"user_id": victim}) + assert user is not None and user.teams == [] + + async def test_team_bulk_member_delete_row_naming_both_identifiers_is_422(proxy_client, prisma, scratch, world): victim = scratch.tag("victim") await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=[victim]) diff --git a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py index 077eb90ddce..6736f1de08b 100644 --- a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py +++ b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py @@ -589,6 +589,27 @@ async def test_bulk_member_delete_cleans_a_user_whose_teams_array_still_names_th assert _roster(prisma, "t1") == ["other"] and prisma.db.litellm_teamtable.update_calls == 0 +@pytest.mark.asyncio +async def test_bulk_member_delete_by_id_removes_the_members_email_only_roster_entry(): + team = LiteLLM_TeamTable( + team_id="t1", + members_with_roles=[ + Member(user_id=None, user_email="u1@example.com", role="user"), + Member(user_id="twin", user_email="u1@example.com", role="user"), + Member(user_id="keep", user_email="keep@example.com", role="user"), + ], + ) + twin = _UserRow(user_id="twin", user_email="u1@example.com", teams=["t1"]) + prisma = _FakePrisma(users=[_user("u1", "t1"), twin, _user("keep", "t1")], teams=[team]) + + results = await _remove(prisma, "t1", [{"user_id": "u1"}]) + + assert [(r.success, r.error) for r in results] == [(True, None)] + assert _roster(prisma, "t1") == ["twin", "keep"] + users = prisma.db.litellm_usertable.rows + assert users["u1"].teams == [] and users["twin"].teams == ["t1"] + + @pytest.mark.asyncio async def test_bulk_member_delete_rejects_unknown_team_and_unauthorized_callers(): prisma = _FakePrisma(users=[_user("u1", "t1")], teams=[_team("t1", "u1")]) From 98ea14758fd670809d7bc0c61cd636e3db479c7d Mon Sep 17 00:00:00 2001 From: ryan Date: Tue, 15 Sep 2026 00:09:57 +0000 Subject: [PATCH 12/12] fix(proxy): only widen an id-only bulk member delete to its email when the user lists the team Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_helpers/bulk_user_deletion.py | 4 +++- .../test_bulk_user_deletion.py | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py index 337b3cdeabc..1ae83b0004a 100644 --- a/litellm/proxy/management_helpers/bulk_user_deletion.py +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -218,7 +218,9 @@ async def _remove_members_from_team( requested_rows: Final = await _user_tx_db(tx).find_many( where=_any_filter(_in_filter("user_id", requested_ids), _in_filter("user_email", requested_emails)) ) - email_of: Final = MappingProxyType({u.user_id: u.user_email for u in requested_rows if u.user_email is not None}) + email_of: Final = MappingProxyType( + {u.user_id: u.user_email for u in requested_rows if u.user_email is not None and team_id in u.teams} + ) requests: Final = tuple(_with_row_email(r, email_of) for r in members) removed_members: Final = tuple(m for m in roster if any(_addresses_member(m, r) for r in requests)) kept_members: Final = tuple(m for m in roster if not any(_addresses_member(m, r) for r in requests)) diff --git a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py index 6736f1de08b..fc972ccbb75 100644 --- a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py +++ b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py @@ -610,6 +610,26 @@ async def test_bulk_member_delete_by_id_removes_the_members_email_only_roster_en assert users["u1"].teams == [] and users["twin"].teams == ["t1"] +@pytest.mark.asyncio +async def test_bulk_member_delete_by_id_of_a_non_member_leaves_a_same_email_users_roster_entry(): + team = LiteLLM_TeamTable( + team_id="t1", + members_with_roles=[ + Member(user_id=None, user_email="shared@example.com", role="user"), + Member(user_id="keep", user_email="keep@example.com", role="user"), + ], + ) + outsider = _UserRow(user_id="outsider", user_email="shared@example.com", teams=[]) + member = _UserRow(user_id="member", user_email="shared@example.com", teams=["t1"]) + prisma = _FakePrisma(users=[outsider, member, _user("keep", "t1")], teams=[team]) + + results = await _remove(prisma, "t1", [{"user_id": "outsider"}]) + + assert [(r.success, r.error) for r in results] == [(False, "User not found in team")] + assert _roster(prisma, "t1") == [None, "keep"] + assert prisma.db.litellm_usertable.rows["member"].teams == ["t1"] + + @pytest.mark.asyncio async def test_bulk_member_delete_rejects_unknown_team_and_unauthorized_callers(): prisma = _FakePrisma(users=[_user("u1", "t1")], teams=[_team("t1", "u1")])