fix(proxy): batch customer organization cascade deletion (#51969)

This commit is contained in:
moe-berri 2026-11-16 08:03:45 +08:00 • committed by GitHub
parent ceb458c929
commit 175993910a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 124 additions and 12 deletions

View file

@ -58,6 +58,11 @@ _ERROR_DETAILS: Final[Mapping[ErrorCode, tuple[int, str, str]]] = MappingProxyTy
"Customer Management busy",
"Customer Management capacity is being updated. Retry the request.",
),
ErrorCode.TRANSACTION_TIMEOUT: (
504,
"Customer Management operation timed out",
"Customer Management operation timed out. The transaction was rolled back. Retry the request.",
),
ErrorCode.FREE_TIER_LIMIT: (
403,
"Customer Management free tier limit reached",

View file

@ -92,6 +92,7 @@ class _BudgetSortRow(BaseModel):
class ErrorCode(str, Enum):
FREE_TIER_LIMIT = "free_tier_limit"
CAPACITY_BUSY = "capacity_busy"
TRANSACTION_TIMEOUT = "transaction_timeout"
NOT_FOUND = "not_found"
PARENT_NOT_FOUND = "parent_not_found"
ALREADY_EXISTS = "already_exists"
@ -1269,8 +1270,10 @@ class CustomerHierarchyStore:
chunk_size=_DELETE_BATCH_SIZE,
)
async def _delete_team_member_batch(self, team_id: str) -> bool:
rows: Final = await self.members.find_many(where={"parent_team_ids": {"has": team_id}}, take=_DELETE_BATCH_SIZE)
async def _delete_team_member_batch(self, team_ids: tuple[str, ...]) -> bool:
rows: Final = await self.members.find_many(
where={"parent_team_ids": {"has_some": list(team_ids)}}, take=_DELETE_BATCH_SIZE
)
members: Final = tuple(_MemberDeletionRow.model_validate(row) for row in rows)
if not members:
return False
@ -1282,9 +1285,9 @@ class CustomerHierarchyStore:
await self._delete_unused_budgets(budget_ids)
return True
async def _delete_team_members(self, team_id: str) -> None:
async def _delete_team_members(self, team_ids: tuple[str, ...]) -> None:
while True:
if not await self._delete_team_member_batch(team_id):
if not await self._delete_team_member_batch(team_ids):
return
async def _delete_organization_team_batch(self, organization_id: str) -> bool:
@ -1292,9 +1295,8 @@ class CustomerHierarchyStore:
teams: Final = tuple(_TeamDeletionRow.model_validate(row) for row in rows)
if not teams:
return False
for team in teams:
await self._delete_team_members(team.customer_team_id)
team_ids: Final = tuple(team.customer_team_id for team in teams)
await self._delete_team_members(team_ids)
team_budget_ids: Final = chain.from_iterable((team.budget_id, team.budget_override_id) for team in teams)
budget_ids: Final = tuple(budget_id for budget_id in team_budget_ids if budget_id is not None)
await delete_many_in(self.teams, "customer_team_id", team_ids, atomicity="caller_transaction")
@ -1330,7 +1332,7 @@ class CustomerHierarchyStore:
if isinstance(team, Failure):
return team
if cascade:
await self._delete_team_members(team_id)
await self._delete_team_members((team_id,))
elif await self.members.count(where={"parent_team_ids": {"has": team_id}}):
return Failure(ErrorCode.HAS_CHILDREN)
await self.teams.delete(where={"customer_team_id": team_id})
@ -1369,7 +1371,13 @@ class CustomerHierarchy:
*,
operation_type: TransactionOperation = TransactionOperation.READ_WRITE,
) -> T | Failure:
from prisma.errors import DataError, ForeignKeyViolationError, RecordNotFoundError, UniqueViolationError
from prisma.errors import (
DataError,
ForeignKeyViolationError,
RecordNotFoundError,
TransactionExpiredError,
UniqueViolationError,
)
try:
async with self.db.tx() as tx:
@ -1378,6 +1386,8 @@ class CustomerHierarchy:
return Failure(ErrorCode.ALREADY_EXISTS)
except RecordNotFoundError:
return Failure(ErrorCode.NOT_FOUND)
except TransactionExpiredError:
return Failure(ErrorCode.TRANSACTION_TIMEOUT)
except ForeignKeyViolationError:
return Failure(
ErrorCode.HAS_CHILDREN if operation_type is TransactionOperation.DELETE else ErrorCode.PARENT_NOT_FOUND

View file

@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Final, cast
import pytest
from fastapi import FastAPI, Request
from fastapi.testclient import TestClient
from prisma.errors import DataError, RawQueryError
from prisma.errors import DataError, RawQueryError, TransactionExpiredError
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.customer_hierarchy_auth import customer_member_cache_key, customer_org_alias_cache_key
@ -1145,6 +1145,89 @@ def test_team_cascade_removes_only_its_members_and_owned_budgets() -> None:
assert http.get("/management/v1/customer_members/sibling-member").status_code == 200
@pytest.mark.parametrize("with_members", (False, True))
def test_organization_cascade_completes_with_many_teams_within_a_bounded_query_budget(with_members: bool) -> None:
database: Final = Database(member_read_limit=20)
http: Final = client(database=database)
family(http)
created_member(http, "member", "team", override=True)
team: Final = database.litellm_customerteamtable.rows["team"]
member: Final = database.litellm_customermembertable.rows["member"]
team_budget_id: Final = str(team.model_dump()["budget_id"])
member_budget_id: Final = str(member.model_dump()["budget_override_id"])
database.litellm_customerteamtable.rows = {
**database.litellm_customerteamtable.rows,
**{
f"team-{index}": team.model_copy(
update={"customer_team_id": f"team-{index}", "budget_id": f"team-budget-{index}"}
)
for index in range(204)
},
}
database.litellm_customermembertable.rows = (
{
**database.litellm_customermembertable.rows,
**{
f"member-{index}": member.model_copy(
update={
"customer_member_id": f"member-{index}",
"parent_team_ids": [f"team-{index}", f"team-{(index + 1) % 204}"],
"budget_override_id": f"member-budget-{index}",
}
)
for index in range(204)
},
}
if with_members
else {}
)
database.litellm_budgettable.rows = {
**database.litellm_budgettable.rows,
**{
f"team-budget-{index}": database.litellm_budgettable.rows[team_budget_id].model_copy(
update={"budget_id": f"team-budget-{index}"}
)
for index in range(204)
},
**{
f"member-budget-{index}": database.litellm_budgettable.rows[member_budget_id].model_copy(
update={"budget_id": f"member-budget-{index}"}
)
for index in range(204)
if with_members
},
}
response: Final = http.delete("/management/v1/customer_organizations/org?cascade=true")
assert response.status_code == 204, response.text
assert not database.litellm_customerorganizationtable.rows
assert not database.litellm_customerteamtable.rows
assert not database.litellm_customermembertable.rows
assert frozenset(database.litellm_budgettable.rows) == (frozenset() if with_members else {member_budget_id})
def test_organization_cascade_expiry_reports_timeout_and_rolls_back_children_and_budgets() -> None:
database: Final = Database(organization_delete_error=TransactionExpiredError("Transaction already closed"))
http: Final = client(database=database)
family(http)
created_member(http, "member", "team", override=True)
tables: Final = (
database.litellm_customerorganizationtable,
database.litellm_customerteamtable,
database.litellm_customermembertable,
database.litellm_budgettable,
)
snapshots: Final = tuple(table.rows for table in tables)
response: Final = http.delete("/management/v1/customer_organizations/org?cascade=true")
assert response.status_code == 504, response.text
assert response.json() == {
"type": "urn:litellm:error:customer-hierarchy-transaction-timeout",
"title": "Customer Management operation timed out",
"status": 504,
"detail": "Customer Management operation timed out. The transaction was rolled back. Retry the request.",
}
assert tuple(table.rows for table in tables) == snapshots
def test_organization_cascade_preserves_the_sibling_customer_and_its_budgets() -> None:
database: Final = Database()
http: Final = client(database=database)

View file

@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Final, cast
import pytest
from prisma import Json
from prisma.errors import DataError, RawQueryError
from prisma.errors import DataError, RawQueryError, TransactionExpiredError
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm.models.budget import BudgetLimit, LiteLLM_BudgetTable
@ -96,12 +96,17 @@ class Table:
*,
create_error: DataError | None = None,
hydration_limit: int | None = None,
read_limit: int | None = None,
delete_error: Exception | None = None,
) -> None:
self.id_field = id_field
self.database = database
self.create_error = create_error
self.rows: Mapping[str, Record] = {}
self.hydration_limit = hydration_limit
self.read_limit = read_limit
self.read_count = 0
self.delete_error = delete_error
async def include(self, row: Record, include: Mapping[str, object] | None) -> Record:
if not include:
@ -185,6 +190,9 @@ class Table:
skip: int = 0,
take: int | None = None,
) -> tuple[Record, ...]:
self.read_count += 1
if self.read_limit is not None and self.read_count > self.read_limit:
raise TransactionExpiredError("Transaction exhausted its query budget")
matches: Final = tuple(row for row in self.rows.values() if self.matches(row, where or {}))
ordered: Final = reduce(
lambda rows, key: tuple(
@ -206,6 +214,8 @@ class Table:
return len(await self.find_many(where=where))
async def delete(self, where: Mapping[str, object]) -> Record | None:
if self.delete_error is not None:
raise self.delete_error
row_id: Final = TypeAdapter(str).validate_python(where[self.id_field])
row: Final = self.rows.get(row_id)
self.rows = {key: value for key, value in self.rows.items() if key != row_id}
@ -290,9 +300,13 @@ class Database:
*,
team_create_error: DataError | None = None,
hydration_limit: int | None = None,
member_read_limit: int | None = None,
organization_delete_error: Exception | None = None,
) -> None:
self.litellm_customerorganizationtable = (
Table("customer_org_id", self) if source is None else source.litellm_customerorganizationtable
Table("customer_org_id", self, delete_error=organization_delete_error)
if source is None
else source.litellm_customerorganizationtable
)
self.litellm_customerteamtable = (
Table("customer_team_id", self, create_error=team_create_error, hydration_limit=hydration_limit)
@ -300,7 +314,7 @@ class Database:
else source.litellm_customerteamtable
)
self.litellm_customermembertable = (
Table("customer_member_id", self, hydration_limit=hydration_limit)
Table("customer_member_id", self, hydration_limit=hydration_limit, read_limit=member_read_limit)
if source is None
else source.litellm_customermembertable
)