fix(proxy): take the access group lock with pg_try_advisory_xact_lock and retry while another edit holds it

This commit is contained in:
mateo-berri 2026-10-29 20:46:05 -07:00
parent 9a6cf12207
commit 4eed565e5a
4 changed files with 69 additions and 29 deletions

View file

@ -1648,7 +1648,7 @@
},
"/access_group/{access_group}/update": {
"put": {
"description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit keeps one of the group's models locked for 30 s",
"description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit holds the group or one of its models for 30 s",
"operationId": "update_access_group_access_group__access_group__update_put",
"parameters": [
{

View file

@ -200,17 +200,17 @@ def _model_table(prisma_client: PrismaClient) -> _ModelTableClient:
return ModelRepository(WriterPinnedClient(prisma_client.db)).table
_SERIALIZE_ACCESS_GROUP_WRITES: Final = (
"SELECT 1 AS locked FROM pg_advisory_xact_lock(hashtext('model_access_group'), hashtext($1))"
_TRY_LOCK_ACCESS_GROUP: Final = (
"SELECT pg_try_advisory_xact_lock(hashtext('model_access_group'), hashtext($1)) AS locked"
)
_LOCK_MODEL_ROWS: Final = (
'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = ANY($1::text[]) '
"ORDER BY model_id FOR UPDATE SKIP LOCKED"
)
_MEMBERSHIP_TX_TIMEOUT: Final = timedelta(seconds=30)
_ROW_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds()
_ROW_LOCK_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8)
_ROW_LOCK_LONGEST_RETRY_DELAY_SECONDS: Final = 1.0
_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds()
_LOCK_RETRY_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8)
_LOCK_RETRY_LONGEST_DELAY_SECONDS: Final = 1.0
class _LockedModelRow(BaseModel):
@ -222,6 +222,15 @@ class _LockedModelRow(BaseModel):
_LOCKED_MODEL_ROWS: Final = TypeAdapter(tuple[_LockedModelRow, ...])
class _AccessGroupLock(BaseModel):
model_config = ConfigDict(frozen=True)
locked: bool
_ACCESS_GROUP_LOCK: Final = TypeAdapter(tuple[_AccessGroupLock])
def _model_table_tx(prisma_client: PrismaClient) -> contextlib.AbstractAsyncContextManager[_ModelTableTx]:
return writer_wrapper(prisma_client.db).tx(timeout=_MEMBERSHIP_TX_TIMEOUT)
@ -516,11 +525,13 @@ async def _raise_404_unless_the_primary_carries(access_group: str, prisma_client
raise _http_error(404, f"Access group '{access_group}' not found")
async def _rewrite_membership_unless_a_row_is_locked(
async def _rewrite_membership_unless_locked(
prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None:
async with _model_table_tx(prisma_client) as tx:
await tx.query_raw(_SERIALIZE_ACCESS_GROUP_WRITES, access_group)
(group_lock,) = _ACCESS_GROUP_LOCK.validate_python(await tx.query_raw(_TRY_LOCK_ACCESS_GROUP, access_group))
if not group_lock.locked:
return None
model_ids_to_lock: Final = _model_ids_to_lock(
deployments=await tx.litellm_proxymodeltable.find_many(), access_group=access_group, data=data
)
@ -554,22 +565,22 @@ async def _rewrite_membership_unless_a_row_is_locked(
return selected_deployments, rewrites
def _row_lock_retry_delays() -> Iterator[float]:
return itertools.chain(_ROW_LOCK_BACKOFF_SECONDS, itertools.repeat(_ROW_LOCK_LONGEST_RETRY_DELAY_SECONDS))
def _lock_retry_delays() -> Iterator[float]:
return itertools.chain(_LOCK_RETRY_BACKOFF_SECONDS, itertools.repeat(_LOCK_RETRY_LONGEST_DELAY_SECONDS))
async def _rewrite_membership_once_its_rows_are_free(
async def _rewrite_membership_once_unlocked(
prisma_client: PrismaClient,
access_group: str,
data: UpdateModelGroupRequest,
retry_budget_seconds: float = _ROW_LOCK_RETRY_BUDGET_SECONDS,
retry_budget_seconds: float = _LOCK_RETRY_BUDGET_SECONDS,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
clock: Callable[[], float] = time.monotonic,
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]:
deadline: Final = clock() + retry_budget_seconds
for delay in _row_lock_retry_delays():
for delay in _lock_retry_delays():
if (
rewritten := await _rewrite_membership_unless_a_row_is_locked(
rewritten := await _rewrite_membership_unless_locked(
prisma_client=prisma_client, access_group=access_group, data=data
)
) is not None:
@ -577,7 +588,7 @@ async def _rewrite_membership_once_its_rows_are_free(
if clock() + delay > deadline:
break
await sleep(delay)
raise _http_error(409, f"Another edit holds a model in access group '{access_group}', try again")
raise _http_error(409, f"Another edit holds access group '{access_group}' or one of its models, try again")
def _readable_model_info_or_raise(model_id: str, model_info: object) -> Mapping[str, object]:
@ -1073,7 +1084,7 @@ async def update_access_group(
Raises:
- HTTPException 400: If any model names don't exist
- HTTPException 404: If access group not found
- HTTPException 409: If another edit keeps one of the group's models locked for 30 s
- HTTPException 409: If another edit holds the group or one of its models for 30 s
"""
from litellm.proxy.proxy_server import llm_router, prisma_client
@ -1116,7 +1127,7 @@ async def update_access_group(
)
try:
selected_deployments, rewrites = await _rewrite_membership_once_its_rows_are_free(
selected_deployments, rewrites = await _rewrite_membership_once_unlocked(
prisma_client=prisma_client, access_group=access_group, data=data
)
if rewrites:

View file

@ -707,6 +707,7 @@ class _FakeModelTable:
self.failing_model_ids = failing_model_ids
self.updates: list[tuple[dict, dict]] = []
self.rows_locked_by_another_session: frozenset[str] = frozenset()
self.group_locked_by_another_session = False
self.edits_the_other_session_commits: dict[str, dict] = {}
self.other_session_commits_after_refusals: int | None = None
self.lock_refusals = 0
@ -715,16 +716,27 @@ class _FakeModelTable:
model_names = (where or {}).get("model_name", {}).get("in")
return [d for d in self.deployments if model_names is None or d.model_name in model_names]
def lock_group(self, wait):
if not self.group_locked_by_another_session:
return True
if wait:
raise TimeoutError("lock_timeout expired waiting for the access group lock")
self._refuse_lock()
return False
def lock_rows(self, model_ids, skip_locked):
held = self.rows_locked_by_another_session.intersection(model_ids)
if not held:
return list(model_ids)
if not skip_locked:
raise TimeoutError(f"timed out waiting for the row lock on {sorted(held)}")
self._refuse_lock()
return [model_id for model_id in model_ids if model_id not in held]
def _refuse_lock(self):
self.lock_refusals += 1
if self.lock_refusals == self.other_session_commits_after_refusals:
self._other_session_commits()
return [model_id for model_id in model_ids if model_id not in held]
def _other_session_commits(self):
self.deployments = [
@ -736,6 +748,7 @@ class _FakeModelTable:
for d in self.deployments
]
self.rows_locked_by_another_session = frozenset()
self.group_locked_by_another_session = False
async def find_unique(self, where, include=None):
return next((d for d in self.deployments if d.model_id == where["model_id"]), None)
@ -785,9 +798,9 @@ class _FakeModelTableTx:
self.litellm_proxymodeltable = SimpleNamespace(find_many=self._find_many)
async def query_raw(self, query, *args):
if "pg_advisory_xact_lock" in query:
if "advisory_xact_lock" in query:
self.journal.append(f"access_group.lock:{args[0]}")
return [{"locked": 1}]
return [{"locked": self.model_table.lock_group(wait="pg_try_" not in query)}]
if "FOR UPDATE" in query:
model_ids = sorted(args[0] if args else (d.model_id for d in self.model_table.deployments))
self.journal.append(f"model_table.lock:{','.join(model_ids)}")
@ -1781,12 +1794,25 @@ async def test_an_access_group_emptied_while_the_update_retried_a_locked_row_ans
assert prisma.model_table.updates == []
def _another_session_holds_a_member_row(model_table):
model_table.rows_locked_by_another_session = frozenset({"leaves"})
def _another_edit_holds_the_group(model_table):
model_table.group_locked_by_another_session = True
@pytest.mark.asyncio
async def test_an_access_group_update_answers_409_and_writes_nothing_while_another_session_keeps_a_member_row_locked():
@pytest.mark.parametrize(
"hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"]
)
async def test_an_access_group_update_answers_409_and_writes_nothing_while_another_session_keeps_a_lock_it_needs(
hold,
):
from fastapi import HTTPException
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
_rewrite_membership_once_its_rows_are_free,
_rewrite_membership_once_unlocked,
)
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
UpdateModelGroupRequest,
@ -1799,11 +1825,11 @@ async def test_an_access_group_update_answers_409_and_writes_nothing_while_anoth
_deployment(model_id="leaves", model_name="gpt-leaves"),
],
)
prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"})
hold(prisma.model_table)
clock = _FakeClock()
with pytest.raises(HTTPException) as raised:
await _rewrite_membership_once_its_rows_are_free(
await _rewrite_membership_once_unlocked(
prisma_client=prisma,
access_group="prod-models",
data=UpdateModelGroupRequest(model_ids=["stays"]),
@ -1819,9 +1845,12 @@ async def test_an_access_group_update_answers_409_and_writes_nothing_while_anoth
@pytest.mark.asyncio
async def test_an_access_group_update_waits_out_a_member_row_held_for_over_ten_seconds_and_writes_the_edit():
@pytest.mark.parametrize(
"hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"]
)
async def test_an_access_group_update_waits_out_a_lock_held_for_over_ten_seconds_and_writes_the_edit(hold):
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
_rewrite_membership_once_its_rows_are_free,
_rewrite_membership_once_unlocked,
)
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
UpdateModelGroupRequest,
@ -1834,11 +1863,11 @@ async def test_an_access_group_update_waits_out_a_member_row_held_for_over_ten_s
_deployment(model_id="leaves", model_name="gpt-leaves"),
],
)
prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"})
hold(prisma.model_table)
prisma.model_table.other_session_commits_after_refusals = 20
clock = _FakeClock()
_, rewrites = await _rewrite_membership_once_its_rows_are_free(
_, rewrites = await _rewrite_membership_once_unlocked(
prisma_client=prisma,
access_group="prod-models",
data=UpdateModelGroupRequest(model_ids=["stays"]),

View file

@ -722,7 +722,7 @@ export interface paths {
* Raises:
* - HTTPException 400: If any model names don't exist
* - HTTPException 404: If access group not found
* - HTTPException 409: If another edit keeps one of the group's models locked for 30 s
* - HTTPException 409: If another edit holds the group or one of its models for 30 s
*/
put: operations["update_access_group_access_group__access_group__update_put"];
post?: never;