From 4eed565e5a6f798fe0ad473befbf69dcdeec6bd0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:46:05 -0700 Subject: [PATCH] fix(proxy): take the access group lock with pg_try_advisory_xact_lock and retry while another edit holds it --- litellm/proxy/_lazy_openapi_snapshot.json | 2 +- ...model_access_group_management_endpoints.py | 43 ++++++++++------ .../test_access_group_management.py | 51 +++++++++++++++---- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 4 files changed, 69 insertions(+), 29 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 40b46fa1811..9ef858ceb57 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -1648,7 +1648,7 @@ }, "/access_group/{access_group}/update": { "put": { - "description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit keeps one of the group's models locked for 30 s", + "description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit holds the group or one of its models for 30 s", "operationId": "update_access_group_access_group__access_group__update_put", "parameters": [ { diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py index f4f2fb5ac6d..3a2de12d108 100644 --- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py @@ -200,17 +200,17 @@ def _model_table(prisma_client: PrismaClient) -> _ModelTableClient: return ModelRepository(WriterPinnedClient(prisma_client.db)).table -_SERIALIZE_ACCESS_GROUP_WRITES: Final = ( - "SELECT 1 AS locked FROM pg_advisory_xact_lock(hashtext('model_access_group'), hashtext($1))" +_TRY_LOCK_ACCESS_GROUP: Final = ( + "SELECT pg_try_advisory_xact_lock(hashtext('model_access_group'), hashtext($1)) AS locked" ) _LOCK_MODEL_ROWS: Final = ( 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = ANY($1::text[]) ' "ORDER BY model_id FOR UPDATE SKIP LOCKED" ) _MEMBERSHIP_TX_TIMEOUT: Final = timedelta(seconds=30) -_ROW_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds() -_ROW_LOCK_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8) -_ROW_LOCK_LONGEST_RETRY_DELAY_SECONDS: Final = 1.0 +_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds() +_LOCK_RETRY_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8) +_LOCK_RETRY_LONGEST_DELAY_SECONDS: Final = 1.0 class _LockedModelRow(BaseModel): @@ -222,6 +222,15 @@ class _LockedModelRow(BaseModel): _LOCKED_MODEL_ROWS: Final = TypeAdapter(tuple[_LockedModelRow, ...]) +class _AccessGroupLock(BaseModel): + model_config = ConfigDict(frozen=True) + + locked: bool + + +_ACCESS_GROUP_LOCK: Final = TypeAdapter(tuple[_AccessGroupLock]) + + def _model_table_tx(prisma_client: PrismaClient) -> contextlib.AbstractAsyncContextManager[_ModelTableTx]: return writer_wrapper(prisma_client.db).tx(timeout=_MEMBERSHIP_TX_TIMEOUT) @@ -516,11 +525,13 @@ async def _raise_404_unless_the_primary_carries(access_group: str, prisma_client raise _http_error(404, f"Access group '{access_group}' not found") -async def _rewrite_membership_unless_a_row_is_locked( +async def _rewrite_membership_unless_locked( prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest ) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None: async with _model_table_tx(prisma_client) as tx: - await tx.query_raw(_SERIALIZE_ACCESS_GROUP_WRITES, access_group) + (group_lock,) = _ACCESS_GROUP_LOCK.validate_python(await tx.query_raw(_TRY_LOCK_ACCESS_GROUP, access_group)) + if not group_lock.locked: + return None model_ids_to_lock: Final = _model_ids_to_lock( deployments=await tx.litellm_proxymodeltable.find_many(), access_group=access_group, data=data ) @@ -554,22 +565,22 @@ async def _rewrite_membership_unless_a_row_is_locked( return selected_deployments, rewrites -def _row_lock_retry_delays() -> Iterator[float]: - return itertools.chain(_ROW_LOCK_BACKOFF_SECONDS, itertools.repeat(_ROW_LOCK_LONGEST_RETRY_DELAY_SECONDS)) +def _lock_retry_delays() -> Iterator[float]: + return itertools.chain(_LOCK_RETRY_BACKOFF_SECONDS, itertools.repeat(_LOCK_RETRY_LONGEST_DELAY_SECONDS)) -async def _rewrite_membership_once_its_rows_are_free( +async def _rewrite_membership_once_unlocked( prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest, - retry_budget_seconds: float = _ROW_LOCK_RETRY_BUDGET_SECONDS, + retry_budget_seconds: float = _LOCK_RETRY_BUDGET_SECONDS, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, clock: Callable[[], float] = time.monotonic, ) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]: deadline: Final = clock() + retry_budget_seconds - for delay in _row_lock_retry_delays(): + for delay in _lock_retry_delays(): if ( - rewritten := await _rewrite_membership_unless_a_row_is_locked( + rewritten := await _rewrite_membership_unless_locked( prisma_client=prisma_client, access_group=access_group, data=data ) ) is not None: @@ -577,7 +588,7 @@ async def _rewrite_membership_once_its_rows_are_free( if clock() + delay > deadline: break await sleep(delay) - raise _http_error(409, f"Another edit holds a model in access group '{access_group}', try again") + raise _http_error(409, f"Another edit holds access group '{access_group}' or one of its models, try again") def _readable_model_info_or_raise(model_id: str, model_info: object) -> Mapping[str, object]: @@ -1073,7 +1084,7 @@ async def update_access_group( Raises: - HTTPException 400: If any model names don't exist - HTTPException 404: If access group not found - - HTTPException 409: If another edit keeps one of the group's models locked for 30 s + - HTTPException 409: If another edit holds the group or one of its models for 30 s """ from litellm.proxy.proxy_server import llm_router, prisma_client @@ -1116,7 +1127,7 @@ async def update_access_group( ) try: - selected_deployments, rewrites = await _rewrite_membership_once_its_rows_are_free( + selected_deployments, rewrites = await _rewrite_membership_once_unlocked( prisma_client=prisma_client, access_group=access_group, data=data ) if rewrites: diff --git a/tests/unit/proxy/management_endpoints/test_access_group_management.py b/tests/unit/proxy/management_endpoints/test_access_group_management.py index 3d1d5bd6dac..d79af0af519 100644 --- a/tests/unit/proxy/management_endpoints/test_access_group_management.py +++ b/tests/unit/proxy/management_endpoints/test_access_group_management.py @@ -707,6 +707,7 @@ class _FakeModelTable: self.failing_model_ids = failing_model_ids self.updates: list[tuple[dict, dict]] = [] self.rows_locked_by_another_session: frozenset[str] = frozenset() + self.group_locked_by_another_session = False self.edits_the_other_session_commits: dict[str, dict] = {} self.other_session_commits_after_refusals: int | None = None self.lock_refusals = 0 @@ -715,16 +716,27 @@ class _FakeModelTable: model_names = (where or {}).get("model_name", {}).get("in") return [d for d in self.deployments if model_names is None or d.model_name in model_names] + def lock_group(self, wait): + if not self.group_locked_by_another_session: + return True + if wait: + raise TimeoutError("lock_timeout expired waiting for the access group lock") + self._refuse_lock() + return False + def lock_rows(self, model_ids, skip_locked): held = self.rows_locked_by_another_session.intersection(model_ids) if not held: return list(model_ids) if not skip_locked: raise TimeoutError(f"timed out waiting for the row lock on {sorted(held)}") + self._refuse_lock() + return [model_id for model_id in model_ids if model_id not in held] + + def _refuse_lock(self): self.lock_refusals += 1 if self.lock_refusals == self.other_session_commits_after_refusals: self._other_session_commits() - return [model_id for model_id in model_ids if model_id not in held] def _other_session_commits(self): self.deployments = [ @@ -736,6 +748,7 @@ class _FakeModelTable: for d in self.deployments ] self.rows_locked_by_another_session = frozenset() + self.group_locked_by_another_session = False async def find_unique(self, where, include=None): return next((d for d in self.deployments if d.model_id == where["model_id"]), None) @@ -785,9 +798,9 @@ class _FakeModelTableTx: self.litellm_proxymodeltable = SimpleNamespace(find_many=self._find_many) async def query_raw(self, query, *args): - if "pg_advisory_xact_lock" in query: + if "advisory_xact_lock" in query: self.journal.append(f"access_group.lock:{args[0]}") - return [{"locked": 1}] + return [{"locked": self.model_table.lock_group(wait="pg_try_" not in query)}] if "FOR UPDATE" in query: model_ids = sorted(args[0] if args else (d.model_id for d in self.model_table.deployments)) self.journal.append(f"model_table.lock:{','.join(model_ids)}") @@ -1781,12 +1794,25 @@ async def test_an_access_group_emptied_while_the_update_retried_a_locked_row_ans assert prisma.model_table.updates == [] +def _another_session_holds_a_member_row(model_table): + model_table.rows_locked_by_another_session = frozenset({"leaves"}) + + +def _another_edit_holds_the_group(model_table): + model_table.group_locked_by_another_session = True + + @pytest.mark.asyncio -async def test_an_access_group_update_answers_409_and_writes_nothing_while_another_session_keeps_a_member_row_locked(): +@pytest.mark.parametrize( + "hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"] +) +async def test_an_access_group_update_answers_409_and_writes_nothing_while_another_session_keeps_a_lock_it_needs( + hold, +): from fastapi import HTTPException from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( - _rewrite_membership_once_its_rows_are_free, + _rewrite_membership_once_unlocked, ) from litellm.types.proxy.management_endpoints.model_management_endpoints import ( UpdateModelGroupRequest, @@ -1799,11 +1825,11 @@ async def test_an_access_group_update_answers_409_and_writes_nothing_while_anoth _deployment(model_id="leaves", model_name="gpt-leaves"), ], ) - prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"}) + hold(prisma.model_table) clock = _FakeClock() with pytest.raises(HTTPException) as raised: - await _rewrite_membership_once_its_rows_are_free( + await _rewrite_membership_once_unlocked( prisma_client=prisma, access_group="prod-models", data=UpdateModelGroupRequest(model_ids=["stays"]), @@ -1819,9 +1845,12 @@ async def test_an_access_group_update_answers_409_and_writes_nothing_while_anoth @pytest.mark.asyncio -async def test_an_access_group_update_waits_out_a_member_row_held_for_over_ten_seconds_and_writes_the_edit(): +@pytest.mark.parametrize( + "hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"] +) +async def test_an_access_group_update_waits_out_a_lock_held_for_over_ten_seconds_and_writes_the_edit(hold): from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( - _rewrite_membership_once_its_rows_are_free, + _rewrite_membership_once_unlocked, ) from litellm.types.proxy.management_endpoints.model_management_endpoints import ( UpdateModelGroupRequest, @@ -1834,11 +1863,11 @@ async def test_an_access_group_update_waits_out_a_member_row_held_for_over_ten_s _deployment(model_id="leaves", model_name="gpt-leaves"), ], ) - prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"}) + hold(prisma.model_table) prisma.model_table.other_session_commits_after_refusals = 20 clock = _FakeClock() - _, rewrites = await _rewrite_membership_once_its_rows_are_free( + _, rewrites = await _rewrite_membership_once_unlocked( prisma_client=prisma, access_group="prod-models", data=UpdateModelGroupRequest(model_ids=["stays"]), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bedd9cf2e8a..e0dee75a1ee 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -722,7 +722,7 @@ export interface paths { * Raises: * - HTTPException 400: If any model names don't exist * - HTTPException 404: If access group not found - * - HTTPException 409: If another edit keeps one of the group's models locked for 30 s + * - HTTPException 409: If another edit holds the group or one of its models for 30 s */ put: operations["update_access_group_access_group__access_group__update_put"]; post?: never;