From 175993910a934073b521d15e85bca6b6d898ee73 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Mon, 16 Nov 2026 08:03:45 +0800 Subject: [PATCH] fix(proxy): batch customer organization cascade deletion (#51969) --- .../management_v1/customers.py | 5 ++ .../management_helpers/customer_hierarchy.py | 26 ++++-- .../management_v1/test_customers.py | 85 ++++++++++++++++++- .../test_customer_hierarchy.py | 20 ++++- 4 files changed, 124 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/management_endpoints/management_v1/customers.py b/litellm/proxy/management_endpoints/management_v1/customers.py index 1df89e60ea1..a999e49ff2e 100644 --- a/litellm/proxy/management_endpoints/management_v1/customers.py +++ b/litellm/proxy/management_endpoints/management_v1/customers.py @@ -58,6 +58,11 @@ _ERROR_DETAILS: Final[Mapping[ErrorCode, tuple[int, str, str]]] = MappingProxyTy "Customer Management busy", "Customer Management capacity is being updated. Retry the request.", ), + ErrorCode.TRANSACTION_TIMEOUT: ( + 504, + "Customer Management operation timed out", + "Customer Management operation timed out. The transaction was rolled back. Retry the request.", + ), ErrorCode.FREE_TIER_LIMIT: ( 403, "Customer Management free tier limit reached", diff --git a/litellm/proxy/management_helpers/customer_hierarchy.py b/litellm/proxy/management_helpers/customer_hierarchy.py index 31ff0d65ac9..7510fd9571b 100644 --- a/litellm/proxy/management_helpers/customer_hierarchy.py +++ b/litellm/proxy/management_helpers/customer_hierarchy.py @@ -92,6 +92,7 @@ class _BudgetSortRow(BaseModel): class ErrorCode(str, Enum): FREE_TIER_LIMIT = "free_tier_limit" CAPACITY_BUSY = "capacity_busy" + TRANSACTION_TIMEOUT = "transaction_timeout" NOT_FOUND = "not_found" PARENT_NOT_FOUND = "parent_not_found" ALREADY_EXISTS = "already_exists" @@ -1269,8 +1270,10 @@ class CustomerHierarchyStore: chunk_size=_DELETE_BATCH_SIZE, ) - async def _delete_team_member_batch(self, team_id: str) -> bool: - rows: Final = await self.members.find_many(where={"parent_team_ids": {"has": team_id}}, take=_DELETE_BATCH_SIZE) + async def _delete_team_member_batch(self, team_ids: tuple[str, ...]) -> bool: + rows: Final = await self.members.find_many( + where={"parent_team_ids": {"has_some": list(team_ids)}}, take=_DELETE_BATCH_SIZE + ) members: Final = tuple(_MemberDeletionRow.model_validate(row) for row in rows) if not members: return False @@ -1282,9 +1285,9 @@ class CustomerHierarchyStore: await self._delete_unused_budgets(budget_ids) return True - async def _delete_team_members(self, team_id: str) -> None: + async def _delete_team_members(self, team_ids: tuple[str, ...]) -> None: while True: - if not await self._delete_team_member_batch(team_id): + if not await self._delete_team_member_batch(team_ids): return async def _delete_organization_team_batch(self, organization_id: str) -> bool: @@ -1292,9 +1295,8 @@ class CustomerHierarchyStore: teams: Final = tuple(_TeamDeletionRow.model_validate(row) for row in rows) if not teams: return False - for team in teams: - await self._delete_team_members(team.customer_team_id) team_ids: Final = tuple(team.customer_team_id for team in teams) + await self._delete_team_members(team_ids) team_budget_ids: Final = chain.from_iterable((team.budget_id, team.budget_override_id) for team in teams) budget_ids: Final = tuple(budget_id for budget_id in team_budget_ids if budget_id is not None) await delete_many_in(self.teams, "customer_team_id", team_ids, atomicity="caller_transaction") @@ -1330,7 +1332,7 @@ class CustomerHierarchyStore: if isinstance(team, Failure): return team if cascade: - await self._delete_team_members(team_id) + await self._delete_team_members((team_id,)) elif await self.members.count(where={"parent_team_ids": {"has": team_id}}): return Failure(ErrorCode.HAS_CHILDREN) await self.teams.delete(where={"customer_team_id": team_id}) @@ -1369,7 +1371,13 @@ class CustomerHierarchy: *, operation_type: TransactionOperation = TransactionOperation.READ_WRITE, ) -> T | Failure: - from prisma.errors import DataError, ForeignKeyViolationError, RecordNotFoundError, UniqueViolationError + from prisma.errors import ( + DataError, + ForeignKeyViolationError, + RecordNotFoundError, + TransactionExpiredError, + UniqueViolationError, + ) try: async with self.db.tx() as tx: @@ -1378,6 +1386,8 @@ class CustomerHierarchy: return Failure(ErrorCode.ALREADY_EXISTS) except RecordNotFoundError: return Failure(ErrorCode.NOT_FOUND) + except TransactionExpiredError: + return Failure(ErrorCode.TRANSACTION_TIMEOUT) except ForeignKeyViolationError: return Failure( ErrorCode.HAS_CHILDREN if operation_type is TransactionOperation.DELETE else ErrorCode.PARENT_NOT_FOUND diff --git a/tests/unit/proxy/management_endpoints/management_v1/test_customers.py b/tests/unit/proxy/management_endpoints/management_v1/test_customers.py index 110b11ef1b3..80a31ed2785 100644 --- a/tests/unit/proxy/management_endpoints/management_v1/test_customers.py +++ b/tests/unit/proxy/management_endpoints/management_v1/test_customers.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Final, cast import pytest from fastapi import FastAPI, Request from fastapi.testclient import TestClient -from prisma.errors import DataError, RawQueryError +from prisma.errors import DataError, RawQueryError, TransactionExpiredError from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.customer_hierarchy_auth import customer_member_cache_key, customer_org_alias_cache_key @@ -1145,6 +1145,89 @@ def test_team_cascade_removes_only_its_members_and_owned_budgets() -> None: assert http.get("/management/v1/customer_members/sibling-member").status_code == 200 +@pytest.mark.parametrize("with_members", (False, True)) +def test_organization_cascade_completes_with_many_teams_within_a_bounded_query_budget(with_members: bool) -> None: + database: Final = Database(member_read_limit=20) + http: Final = client(database=database) + family(http) + created_member(http, "member", "team", override=True) + team: Final = database.litellm_customerteamtable.rows["team"] + member: Final = database.litellm_customermembertable.rows["member"] + team_budget_id: Final = str(team.model_dump()["budget_id"]) + member_budget_id: Final = str(member.model_dump()["budget_override_id"]) + database.litellm_customerteamtable.rows = { + **database.litellm_customerteamtable.rows, + **{ + f"team-{index}": team.model_copy( + update={"customer_team_id": f"team-{index}", "budget_id": f"team-budget-{index}"} + ) + for index in range(204) + }, + } + database.litellm_customermembertable.rows = ( + { + **database.litellm_customermembertable.rows, + **{ + f"member-{index}": member.model_copy( + update={ + "customer_member_id": f"member-{index}", + "parent_team_ids": [f"team-{index}", f"team-{(index + 1) % 204}"], + "budget_override_id": f"member-budget-{index}", + } + ) + for index in range(204) + }, + } + if with_members + else {} + ) + database.litellm_budgettable.rows = { + **database.litellm_budgettable.rows, + **{ + f"team-budget-{index}": database.litellm_budgettable.rows[team_budget_id].model_copy( + update={"budget_id": f"team-budget-{index}"} + ) + for index in range(204) + }, + **{ + f"member-budget-{index}": database.litellm_budgettable.rows[member_budget_id].model_copy( + update={"budget_id": f"member-budget-{index}"} + ) + for index in range(204) + if with_members + }, + } + response: Final = http.delete("/management/v1/customer_organizations/org?cascade=true") + assert response.status_code == 204, response.text + assert not database.litellm_customerorganizationtable.rows + assert not database.litellm_customerteamtable.rows + assert not database.litellm_customermembertable.rows + assert frozenset(database.litellm_budgettable.rows) == (frozenset() if with_members else {member_budget_id}) + + +def test_organization_cascade_expiry_reports_timeout_and_rolls_back_children_and_budgets() -> None: + database: Final = Database(organization_delete_error=TransactionExpiredError("Transaction already closed")) + http: Final = client(database=database) + family(http) + created_member(http, "member", "team", override=True) + tables: Final = ( + database.litellm_customerorganizationtable, + database.litellm_customerteamtable, + database.litellm_customermembertable, + database.litellm_budgettable, + ) + snapshots: Final = tuple(table.rows for table in tables) + response: Final = http.delete("/management/v1/customer_organizations/org?cascade=true") + assert response.status_code == 504, response.text + assert response.json() == { + "type": "urn:litellm:error:customer-hierarchy-transaction-timeout", + "title": "Customer Management operation timed out", + "status": 504, + "detail": "Customer Management operation timed out. The transaction was rolled back. Retry the request.", + } + assert tuple(table.rows for table in tables) == snapshots + + def test_organization_cascade_preserves_the_sibling_customer_and_its_budgets() -> None: database: Final = Database() http: Final = client(database=database) diff --git a/tests/unit/proxy/management_helpers/test_customer_hierarchy.py b/tests/unit/proxy/management_helpers/test_customer_hierarchy.py index ff2c2bfc85d..4c01fdcf58e 100644 --- a/tests/unit/proxy/management_helpers/test_customer_hierarchy.py +++ b/tests/unit/proxy/management_helpers/test_customer_hierarchy.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Final, cast import pytest from prisma import Json -from prisma.errors import DataError, RawQueryError +from prisma.errors import DataError, RawQueryError, TransactionExpiredError from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.models.budget import BudgetLimit, LiteLLM_BudgetTable @@ -96,12 +96,17 @@ class Table: *, create_error: DataError | None = None, hydration_limit: int | None = None, + read_limit: int | None = None, + delete_error: Exception | None = None, ) -> None: self.id_field = id_field self.database = database self.create_error = create_error self.rows: Mapping[str, Record] = {} self.hydration_limit = hydration_limit + self.read_limit = read_limit + self.read_count = 0 + self.delete_error = delete_error async def include(self, row: Record, include: Mapping[str, object] | None) -> Record: if not include: @@ -185,6 +190,9 @@ class Table: skip: int = 0, take: int | None = None, ) -> tuple[Record, ...]: + self.read_count += 1 + if self.read_limit is not None and self.read_count > self.read_limit: + raise TransactionExpiredError("Transaction exhausted its query budget") matches: Final = tuple(row for row in self.rows.values() if self.matches(row, where or {})) ordered: Final = reduce( lambda rows, key: tuple( @@ -206,6 +214,8 @@ class Table: return len(await self.find_many(where=where)) async def delete(self, where: Mapping[str, object]) -> Record | None: + if self.delete_error is not None: + raise self.delete_error row_id: Final = TypeAdapter(str).validate_python(where[self.id_field]) row: Final = self.rows.get(row_id) self.rows = {key: value for key, value in self.rows.items() if key != row_id} @@ -290,9 +300,13 @@ class Database: *, team_create_error: DataError | None = None, hydration_limit: int | None = None, + member_read_limit: int | None = None, + organization_delete_error: Exception | None = None, ) -> None: self.litellm_customerorganizationtable = ( - Table("customer_org_id", self) if source is None else source.litellm_customerorganizationtable + Table("customer_org_id", self, delete_error=organization_delete_error) + if source is None + else source.litellm_customerorganizationtable ) self.litellm_customerteamtable = ( Table("customer_team_id", self, create_error=team_create_error, hydration_limit=hydration_limit) @@ -300,7 +314,7 @@ class Database: else source.litellm_customerteamtable ) self.litellm_customermembertable = ( - Table("customer_member_id", self, hydration_limit=hydration_limit) + Table("customer_member_id", self, hydration_limit=hydration_limit, read_limit=member_read_limit) if source is None else source.litellm_customermembertable )