From d06ee225a1071081152715db2388bb19265336bc Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Fri, 23 Oct 2026 16:28:06 -0700 Subject: [PATCH] test(teams): cover the backfill's selection, shared copies and refusals in unit tests The real Postgres suite only runs on main, so pin the Python side of the backfill against an in-memory team as well: who moves, who is skipped for another reset schedule, shared budget copies, the member limit, a missing default, the reported counts and a team deleted while waiting for its lock. --- .../test_upsert_budget_membership.py | 21 ++ .../test_team_endpoints.py | 135 +++++++++ .../test_team_member_default_budget.py | 278 +++++++++++++++++- 3 files changed, 432 insertions(+), 2 deletions(-) diff --git a/tests/unit/proxy/common_utils/test_upsert_budget_membership.py b/tests/unit/proxy/common_utils/test_upsert_budget_membership.py index 9bd3217067a..b0680133325 100644 --- a/tests/unit/proxy/common_utils/test_upsert_budget_membership.py +++ b/tests/unit/proxy/common_utils/test_upsert_budget_membership.py @@ -105,6 +105,27 @@ async def test_clearing_all_limits_links_the_member_to_the_team_default(mock_tx, ) +@pytest.mark.asyncio +async def test_emptying_a_member_on_a_limitless_team_default_keeps_them_on_it(mock_tx, fake_user): + mock_tx.litellm_budgettable.find_unique = AsyncMock(return_value=budget_row(budget_id="team-default")) + + await upsert_budget_and_membership( + mock_tx, + team_id="team-1", + user_id="user-1", + existing_budget_id="team-default", + user_api_key_dict=fake_user, + budget_patch={"max_budget": None}, + team_default_budget_id="team-default", + ) + + mock_tx.litellm_teammembership.update.assert_awaited_once_with( + where={"user_id_team_id": {"user_id": "user-1", "team_id": "team-1"}}, + data={"litellm_budget_table": {"connect": {"budget_id": "team-default"}}}, + ) + mock_tx.litellm_budgettable.create.assert_not_called() + + # TEST: clearing every limit on a member's private budget disconnects it, so the # member falls back to the team default instead of keeping an empty private row. @pytest.mark.asyncio diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index 8b0b5bda334..a420e19e7fe 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -10609,6 +10609,141 @@ async def test_clearing_the_member_default_evicts_every_roster_members_cached_me assert "standalone-team-123_at-old-cap" in evicted +def _backfill_ready_prisma(answer): + from types import SimpleNamespace + + existing: Final = _existing_team_with_model_caps({}) + existing.metadata = {"team_member_budget_id": "member-budget-1"} + existing.model_dump.return_value["metadata"] = existing.metadata + updated: Final = _existing_team_with_model_caps({}) + updated.metadata = existing.metadata + updated.litellm_model_table = None + prisma: Final = MagicMock() + prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=existing) + prisma.jsonify_team_object = lambda db_data: db_data + prisma.db.litellm_teamtable.update = AsyncMock(return_value=updated) + tx: Final = _transaction_over(prisma) + tx.query_raw = AsyncMock(side_effect=answer) + default_row: Final = {"max_budget": 50.0, "budget_duration": "30d"} + prisma.db.litellm_budgettable.find_unique = AsyncMock( + return_value=SimpleNamespace(**default_row, model_dump=lambda: default_row) + ) + return prisma + + +async def _raise_to_default(prisma): + from fastapi import Request + + from litellm.proxy._types import UpdateTeamRequest + from litellm.proxy.management_endpoints.team_endpoints import update_team + + cache: Final = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + cache.async_set_cache = AsyncMock() + cache.async_delete_cache = AsyncMock() + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch("litellm.proxy.proxy_server.user_api_key_cache", cache), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + ): + return await update_team( + data=UpdateTeamRequest( + team_id="standalone-team-123", team_member_budget=50.0, team_member_budget_backfill="raise_to_default" + ), + http_request=MagicMock(spec=Request), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), + ) + + +@pytest.mark.asyncio +async def test_a_backfill_reports_how_many_members_moved(disable_audit_logging_for_mocked_team): + from litellm.proxy._types import TeamMemberBudgetBackfillResult + + async def answer(sql: str, *args: object) -> list[dict[str, object]]: + if "GROUP BY b.budget_duration" in sql: + return [{"duration": "30d", "members": 1}, {"duration": "7d", "members": 2}] + if "b.budget_duration = ANY" in sql: + return [{"user_id": "below", "budget_id": "own-below", "budget_duration": "30d"}] + if "HAVING COUNT(*) > 1" in sql: + return [] + return [{"members_with_roles": [{"user_id": "below", "role": "user"}]}] + + response: Final = await _raise_to_default(_backfill_ready_prisma(answer)) + + assert response["team_member_budget_backfill_result"] == TeamMemberBudgetBackfillResult( + moved_to_default=1, skipped_other_reset_period=2 + ) + + +@pytest.mark.asyncio +async def test_a_team_deleted_while_its_update_waits_for_the_lock_is_not_written(disable_audit_logging_for_mocked_team): + from litellm.proxy._types import ProxyException + + async def answer(sql: str, *args: object) -> list[dict[str, object]]: + return [] + + prisma: Final = _backfill_ready_prisma(answer) + + with pytest.raises(ProxyException) as gone: + await _raise_to_default(prisma) + + assert gone.value.code == "404" + prisma.db.litellm_teamtable.update.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_a_backfill_over_the_member_limit_is_refused_without_writing_the_team( + disable_audit_logging_for_mocked_team, +): + from types import SimpleNamespace + + from fastapi import Request + + from litellm.proxy._types import ProxyException, UpdateTeamRequest + from litellm.proxy.management_endpoints.team_endpoints import update_team + from litellm.proxy.management_helpers.team_member_default_budget import MAX_MEMBER_BUDGETS_MOVED_PER_UPDATE + + existing: Final = _existing_team_with_model_caps({}) + existing.metadata = {"team_member_budget_id": "member-budget-1"} + existing.model_dump.return_value["metadata"] = existing.metadata + prisma: Final = MagicMock() + over_limit: Final = MAX_MEMBER_BUDGETS_MOVED_PER_UPDATE + 1 + + async def answer(sql: str, *args: object) -> list[dict[str, object]]: + if "GROUP BY b.budget_duration" in sql: + return [{"duration": "30d", "members": over_limit}] + return [{"members_with_roles": []}] + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + ): + prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=existing) + prisma.jsonify_team_object = lambda db_data: db_data + prisma.db.litellm_teamtable.update = AsyncMock() + tx: Final = _transaction_over(prisma) + tx.query_raw = AsyncMock(side_effect=answer) + prisma.db.litellm_budgettable.find_unique = AsyncMock( + return_value=SimpleNamespace(max_budget=50.0, budget_duration="30d") + ) + + with pytest.raises(ProxyException) as refused: + await update_team( + data=UpdateTeamRequest( + team_id="standalone-team-123", + team_member_budget=50.0, + team_member_budget_backfill="raise_to_default", + ), + http_request=MagicMock(spec=Request), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), + ) + + assert refused.value.code == "422" + assert str(over_limit) in refused.value.message + prisma.db.litellm_teamtable.update.assert_not_awaited() + + @pytest.mark.asyncio async def test_update_team_persists_member_model_budget_to_default_budget_row( disable_audit_logging_for_mocked_team, diff --git a/tests/unit/proxy/management_helpers/test_team_member_default_budget.py b/tests/unit/proxy/management_helpers/test_team_member_default_budget.py index 3fd741429a5..c675f24e8aa 100644 --- a/tests/unit/proxy/management_helpers/test_team_member_default_budget.py +++ b/tests/unit/proxy/management_helpers/test_team_member_default_budget.py @@ -1,11 +1,19 @@ import json from datetime import datetime, timezone +from types import SimpleNamespace import pytest from pydantic import ValidationError -from litellm.proxy._types import UpdateTeamRequest -from litellm.proxy.management_helpers.team_member_default_budget import plan_team_member_default_budget_write +from litellm.proxy._types import TeamMemberBudgetBackfillResult, UpdateTeamRequest, UserAPIKeyAuth +from litellm.proxy.management_helpers.team_member_default_budget import ( + MAX_MEMBER_BUDGETS_MOVED_PER_UPDATE, + MovedMemberBudgets, + TooManyMemberBudgetsToMove, + move_member_budgets_to_default, + plan_team_member_default_budget_write, + write_team_member_default_budget, +) def _plan(existing_budget_id: str | None = None, team_budget_duration: str | None = None, **fields: object): @@ -88,3 +96,269 @@ def test_per_model_member_budgets_are_stored_as_plain_json(): assert write is not None assert json.loads(json.dumps(write.update["model_max_budget"]))["gpt-4o"]["max_budget"] == 5.0 + + +class _InMemoryTeam: + """One team's memberships and budget rows, changed the way the backfill's statements change Postgres.""" + + def __init__(self, default: dict[str, object], members: dict[str, dict[str, object] | None]) -> None: + self.budgets: dict[str, dict[str, object]] = {"default": {"budget_id": "default", **default}} + self.memberships: dict[str, str | None] = {} + for user_id, budget in members.items(): + if budget is None: + self.memberships[user_id] = "default" + continue + self.budgets[f"own-{user_id}"] = {"budget_id": f"own-{user_id}", **budget} + self.memberships[user_id] = f"own-{user_id}" + self.litellm_budgettable = SimpleNamespace( + find_unique=self._find_budget, upsert=self._upsert_budget, create=self._create_budget + ) + self.litellm_teammembership = SimpleNamespace(upsert=self._link_membership) + + def share(self, budget_id: str, *user_ids: str) -> None: + for user_id in user_ids: + self.memberships[user_id] = budget_id + + async def _create_budget(self, data: dict[str, object], include: dict[str, bool] | None = None): + budget_id = f"copy-{len(self.budgets)}" + self.budgets[budget_id] = {"budget_id": budget_id, **data} + return SimpleNamespace(budget_id=budget_id) + + async def _link_membership(self, where: dict[str, dict[str, str]], data: dict[str, dict[str, object]]) -> None: + self.memberships[where["user_id_team_id"]["user_id"]] = data["update"]["litellm_budget_table"]["connect"][ + "budget_id" + ] + + def budget_of(self, user_id: str) -> dict[str, object]: + return self.budgets[self.memberships[user_id]] + + async def _find_budget(self, where: dict[str, str]): + stored = self.budgets.get(where["budget_id"]) + if stored is None: + return None + row = {**_ROW_DEFAULTS, **stored} + return SimpleNamespace(**row, model_dump=lambda: row) + + async def _upsert_budget(self, where: dict[str, str], data: dict[str, dict[str, object]]) -> None: + current = self.budgets.get(where["budget_id"]) + self.budgets[where["budget_id"]] = {**(current or {}), **(data["update"] if current else data["create"])} + + def _on_chosen_side(self, budget: dict[str, object], direction: str, amount: float) -> bool: + own = budget.get("max_budget") + if budget["budget_id"] == "default" or own is None: + return False + return {"raise_to_default": own < amount, "lower_to_default": own > amount, "all_to_default": own != amount}[ + direction + ] + + async def query_raw(self, sql: str, *args: object) -> list[dict[str, object]]: + if "HAVING COUNT(*) > 1" in sql: + budget_ids = list(self.memberships.values()) + return [{"budget_id": b} for b in set(args[0]) if budget_ids.count(b) > 1] + _team, _default, direction, amount = args[:4] + chosen = [ + (u, self.budgets[b]) + for u, b in self.memberships.items() + if b and self._on_chosen_side(self.budgets[b], direction, amount) + ] + if "GROUP BY" in sql: + durations = [budget.get("budget_duration") for _, budget in chosen] + return [{"duration": d, "members": durations.count(d)} for d in set(durations)] + durations, include_none = args[4], args[5] + return [ + {**_ROW_DEFAULTS, **budget, "user_id": user_id} + for user_id, budget in chosen + if budget.get("budget_duration") in durations or (include_none and budget.get("budget_duration") is None) + ] + + async def execute_raw(self, sql: str, *args: object) -> int: + if "INSERT INTO" in sql: + for user_id in args[2]: + self.memberships.setdefault(user_id, args[1]) + elif "budget_id IS NULL" in sql: + self.memberships = {u: b or args[1] for u, b in self.memberships.items()} + elif "SET budget_reset_at" in sql: + for user_id in args[2]: + if self.memberships[user_id] != args[1]: + self.budget_of(user_id)["budget_reset_at"] = self.budgets[args[1]].get("budget_reset_at") + elif "SET max_budget = NULL" in sql: + for budget_id in args[0]: + self.budgets[budget_id]["max_budget"] = None + elif "SET budget_id = $2" in sql: + for user_id in args[2]: + self.memberships[user_id] = args[1] + return 0 + + +_ROW_DEFAULTS = { + "soft_budget": None, + "max_parallel_requests": None, + "tpm_limit": None, + "rpm_limit": None, + "model_max_budget": None, + "budget_duration": None, + "allowed_models": None, + "temp_budget_increase": None, + "temp_budget_expiry": None, +} +_DEFAULT_RESET = datetime(2030, 1, 1, tzinfo=timezone.utc) +_ADMIN = UserAPIKeyAuth(user_id="admin") + + +async def _backfill(team: _InMemoryTeam, backfill: str): + return await move_member_budgets_to_default( + team, team_id="team-1", default_budget_id="default", backfill=backfill, changed_by=_ADMIN + ) + + +def _team(**members: dict[str, object] | None) -> _InMemoryTeam: + return _InMemoryTeam({"max_budget": 50.0, "budget_duration": "30d", "budget_reset_at": _DEFAULT_RESET}, members) + + +@pytest.mark.asyncio +async def test_raise_moves_members_below_the_default_on_its_schedule_and_counts_the_rest(): + team = _team( + below={"max_budget": 10.0, "budget_duration": "30d"}, + own_rpm={"max_budget": 10.0, "budget_duration": "30d", "rpm_limit": 5, "budget_reset_at": "2030-02-01"}, + weekly={"max_budget": 10.0, "budget_duration": "7d"}, + above={"max_budget": 80.0, "budget_duration": "30d"}, + on_default=None, + ) + + moved = await _backfill(team, "raise_to_default") + + assert isinstance(moved, MovedMemberBudgets) + assert sorted(moved.user_ids) == ["below", "own_rpm"] + assert moved.result() == TeamMemberBudgetBackfillResult(moved_to_default=2, skipped_other_reset_period=1) + assert team.memberships["below"] == "default" + assert team.budget_of("own_rpm") == { + "budget_id": "own-own_rpm", + "max_budget": None, + "budget_duration": "30d", + "rpm_limit": 5, + "budget_reset_at": _DEFAULT_RESET, + } + assert team.budget_of("weekly")["max_budget"] == 10.0 + assert team.budget_of("above")["max_budget"] == 80.0 + + +@pytest.mark.asyncio +async def test_a_model_list_matching_the_default_still_keeps_the_members_own_row(): + team = _team(scoped={"max_budget": 10.0, "budget_duration": "30d", "allowed_models": ["gpt-5.4-mini"]}) + team.budgets["default"]["allowed_models"] = ["gpt-5.4-mini"] + + await _backfill(team, "raise_to_default") + + assert team.memberships["scoped"] == "own-scoped" + assert team.budget_of("scoped")["max_budget"] is None + + +@pytest.mark.asyncio +async def test_more_members_than_one_update_may_move_changes_nobody(): + members = { + f"m{i}": {"max_budget": 10.0, "budget_duration": "30d"} for i in range(MAX_MEMBER_BUDGETS_MOVED_PER_UPDATE + 1) + } + team = _team(**members) + + moved = await _backfill(team, "raise_to_default") + + assert moved == TooManyMemberBudgetsToMove(count=MAX_MEMBER_BUDGETS_MOVED_PER_UPDATE + 1) + assert all(budget_id != "default" for budget_id in team.memberships.values()) + + +@pytest.mark.asyncio +async def test_members_sharing_a_budget_with_their_own_limits_each_get_a_copy_on_the_default(): + team = _team() + team.budgets["shared"] = {"budget_id": "shared", "max_budget": 10.0, "budget_duration": "30d", "rpm_limit": 5} + team.share("shared", "first", "second") + + moved = await _backfill(team, "raise_to_default") + + assert isinstance(moved, MovedMemberBudgets) + assert sorted(moved.user_ids) == ["first", "second"] + copies = {team.memberships["first"], team.memberships["second"]} + assert len(copies) == 2 and "shared" not in copies + assert [(team.budgets[c]["max_budget"], team.budgets[c]["rpm_limit"]) for c in sorted(copies)] == [ + (None, 5), + (None, 5), + ] + assert team.budgets["shared"]["max_budget"] == 10.0 + + +@pytest.mark.asyncio +async def test_only_members_on_another_schedule_are_counted_and_left_alone(): + team = _team(weekly={"max_budget": 10.0, "budget_duration": "7d"}) + + moved = await _backfill(team, "raise_to_default") + + assert moved == MovedMemberBudgets(user_ids=(), skipped_other_reset_period=1) + assert team.memberships["weekly"] == "own-weekly" + + +@pytest.mark.asyncio +async def test_a_missing_default_row_moves_nobody(): + team = _team(below={"max_budget": 10.0, "budget_duration": "30d"}) + del team.budgets["default"] + + assert await _backfill(team, "raise_to_default") == MovedMemberBudgets(user_ids=(), skipped_other_reset_period=0) + + +def test_a_first_default_on_a_team_without_an_alias_gets_a_generic_id(): + write = plan_team_member_default_budget_write( + data=UpdateTeamRequest(team_id="team-1", team_member_budget=5.0), + existing_budget_id=None, + team_alias=None, + team_budget_duration=None, + ) + + assert write is not None + assert write.budget_id.startswith("team-budget-") + + +@pytest.mark.asyncio +async def test_keeping_custom_budgets_moves_nobody(): + team = _team(below={"max_budget": 10.0, "budget_duration": "30d"}) + + assert await _backfill(team, "none") == MovedMemberBudgets(user_ids=(), skipped_other_reset_period=0) + assert team.memberships["below"] == "own-below" + + +@pytest.mark.asyncio +async def test_writing_a_default_links_roster_members_without_a_budget_to_it(): + team = _team(own={"max_budget": 10.0, "budget_duration": "30d"}) + team.memberships["unbudgeted"] = None + write = _plan( + existing_budget_id="default", + team_member_budget=60.0, + team_member_model_max_budget={"gpt-4o": {"max_budget": 1.0}}, + ) + assert write is not None + + await write_team_member_default_budget( + team, write, team_id="team-1", member_user_ids=("own", "unbudgeted", "no-row"), changed_by="admin" + ) + + assert team.budgets["default"]["max_budget"] == 60.0 + assert team.budgets["default"]["model_max_budget"].data == { + "gpt-4o": {"max_budget": 1.0, "budget_duration": None, "tpm_limit": None, "rpm_limit": None} + } + assert (team.memberships["unbudgeted"], team.memberships["no-row"], team.memberships["own"]) == ( + "default", + "default", + "own-own", + ) + + +@pytest.mark.asyncio +async def test_clearing_the_default_does_not_link_members_without_a_budget_to_it(): + team = _team() + team.memberships["unbudgeted"] = None + write = _plan(existing_budget_id="default", team_member_budget=None) + assert write is not None + + await write_team_member_default_budget( + team, write, team_id="team-1", member_user_ids=("unbudgeted", "no-row"), changed_by="admin" + ) + + assert team.budgets["default"]["max_budget"] is None + assert team.memberships == {"unbudgeted": None}