fix(proxy): retry access group row locks for 30 s with SKIP LOCKED and lock only the rows model_ids select

This commit is contained in:
mateo-berri 2026-10-29 19:03:54 -07:00
parent 04e51b41d6
commit 9a6cf12207
4 changed files with 99 additions and 54 deletions

View file

@ -1648,7 +1648,7 @@
},
"/access_group/{access_group}/update": {
"put": {
"description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit keeps one of the group's models locked for about 10 s",
"description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit keeps one of the group's models locked for 30 s",
"operationId": "update_access_group_access_group__access_group__update_put",
"parameters": [
{

View file

@ -14,13 +14,16 @@ Endpoints here:
import asyncio
import contextlib
import itertools
import json
from collections.abc import Awaitable, Callable, Mapping, Sequence
import time
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from datetime import datetime, timedelta
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Any, Final, Protocol
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
@ -32,7 +35,6 @@ from litellm.proxy.common_utils.user_api_key_cache import (
model_access_group_cache_key,
model_access_group_registry_cache_key,
)
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient, writer_wrapper
from litellm.proxy.management_endpoints.common_utils import validate_budget_duration
@ -203,10 +205,21 @@ _SERIALIZE_ACCESS_GROUP_WRITES: Final = (
)
_LOCK_MODEL_ROWS: Final = (
'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = ANY($1::text[]) '
"ORDER BY model_id FOR UPDATE NOWAIT"
"ORDER BY model_id FOR UPDATE SKIP LOCKED"
)
_MEMBERSHIP_TX_TIMEOUT: Final = timedelta(seconds=30)
_ROW_LOCK_RETRY_DELAYS_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8, *(1.0,) * 9)
_ROW_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds()
_ROW_LOCK_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8)
_ROW_LOCK_LONGEST_RETRY_DELAY_SECONDS: Final = 1.0
class _LockedModelRow(BaseModel):
model_config = ConfigDict(frozen=True)
model_id: str
_LOCKED_MODEL_ROWS: Final = TypeAdapter(tuple[_LockedModelRow, ...])
def _model_table_tx(prisma_client: PrismaClient) -> contextlib.AbstractAsyncContextManager[_ModelTableTx]:
@ -488,7 +501,7 @@ def _model_ids_to_lock(
deployments: Sequence[_DeploymentRow], access_group: str, data: UpdateModelGroupRequest
) -> frozenset[str]:
selected_model_ids: Final = frozenset(data.model_ids or ())
selected_model_names: Final = frozenset(data.model_names or ())
selected_model_names: Final = frozenset(() if data.model_ids else data.model_names or ())
return frozenset(
deployment.model_id
for deployment in deployments
@ -503,15 +516,19 @@ async def _raise_404_unless_the_primary_carries(access_group: str, prisma_client
raise _http_error(404, f"Access group '{access_group}' not found")
async def _rewrite_membership_under_row_locks(
async def _rewrite_membership_unless_a_row_is_locked(
prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]:
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None:
async with _model_table_tx(prisma_client) as tx:
await tx.query_raw(_SERIALIZE_ACCESS_GROUP_WRITES, access_group)
model_ids_to_lock: Final = _model_ids_to_lock(
deployments=await tx.litellm_proxymodeltable.find_many(), access_group=access_group, data=data
)
await tx.query_raw(_LOCK_MODEL_ROWS, sorted(model_ids_to_lock))
locked_rows: Final = _LOCKED_MODEL_ROWS.validate_python(
await tx.query_raw(_LOCK_MODEL_ROWS, sorted(model_ids_to_lock))
)
if frozenset(row.model_id for row in locked_rows) != model_ids_to_lock:
return None
deployments: Final = tuple(
deployment
for deployment in await tx.litellm_proxymodeltable.find_many()
@ -537,35 +554,29 @@ async def _rewrite_membership_under_row_locks(
return selected_deployments, rewrites
async def _rewrite_membership_unless_a_row_is_locked(
prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None:
try:
return await _rewrite_membership_under_row_locks(
prisma_client=prisma_client, access_group=access_group, data=data
)
except Exception as e:
if PrismaDBExceptionHandler.is_lock_timeout_error(e):
return None
raise
def _row_lock_retry_delays() -> Iterator[float]:
return itertools.chain(_ROW_LOCK_BACKOFF_SECONDS, itertools.repeat(_ROW_LOCK_LONGEST_RETRY_DELAY_SECONDS))
async def _rewrite_membership_once_its_rows_are_free(
prisma_client: PrismaClient,
access_group: str,
data: UpdateModelGroupRequest,
retry_delays_seconds: Sequence[float] = _ROW_LOCK_RETRY_DELAYS_SECONDS,
retry_budget_seconds: float = _ROW_LOCK_RETRY_BUDGET_SECONDS,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
clock: Callable[[], float] = time.monotonic,
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]:
for delay in (*retry_delays_seconds, None):
deadline: Final = clock() + retry_budget_seconds
for delay in _row_lock_retry_delays():
if (
rewritten := await _rewrite_membership_unless_a_row_is_locked(
prisma_client=prisma_client, access_group=access_group, data=data
)
) is not None:
return rewritten
if delay is not None:
await sleep(delay)
if clock() + delay > deadline:
break
await sleep(delay)
raise _http_error(409, f"Another edit holds a model in access group '{access_group}', try again")
@ -1062,7 +1073,7 @@ async def update_access_group(
Raises:
- HTTPException 400: If any model names don't exist
- HTTPException 404: If access group not found
- HTTPException 409: If another edit keeps one of the group's models locked for about 10 s
- HTTPException 409: If another edit keeps one of the group's models locked for 30 s
"""
from litellm.proxy.proxy_server import llm_router, prisma_client

View file

@ -687,17 +687,17 @@ class _FakeAccessGroupBudgetTable:
return self.rows.pop(access_group_name, None)
def _row_lock_not_available(held_model_ids):
import prisma
class _FakeClock:
def __init__(self) -> None:
self.now = 0.0
self.slept: list[float] = []
return prisma.errors.RawQueryError(
{
"user_facing_error": {
"error_code": "P2010",
"meta": {"code": "55P03", "message": f"could not obtain lock on row in relation for {held_model_ids}"},
}
}
)
def __call__(self) -> float:
return self.now
async def sleep(self, seconds: float) -> None:
self.slept.append(seconds)
self.now += seconds
class _FakeModelTable:
@ -715,16 +715,16 @@ class _FakeModelTable:
model_names = (where or {}).get("model_name", {}).get("in")
return [d for d in self.deployments if model_names is None or d.model_name in model_names]
def lock_rows(self, model_ids, nowait):
held = sorted(self.rows_locked_by_another_session.intersection(model_ids))
def lock_rows(self, model_ids, skip_locked):
held = self.rows_locked_by_another_session.intersection(model_ids)
if not held:
return
if not nowait:
raise TimeoutError(f"timed out waiting for the row lock on {held}")
return list(model_ids)
if not skip_locked:
raise TimeoutError(f"timed out waiting for the row lock on {sorted(held)}")
self.lock_refusals += 1
if self.lock_refusals == self.other_session_commits_after_refusals:
self._other_session_commits()
raise _row_lock_not_available(held)
return [model_id for model_id in model_ids if model_id not in held]
def _other_session_commits(self):
self.deployments = [
@ -791,8 +791,8 @@ class _FakeModelTableTx:
if "FOR UPDATE" in query:
model_ids = sorted(args[0] if args else (d.model_id for d in self.model_table.deployments))
self.journal.append(f"model_table.lock:{','.join(model_ids)}")
self.model_table.lock_rows(model_ids, nowait="NOWAIT" in query)
return [{"model_id": model_id} for model_id in model_ids]
locked = self.model_table.lock_rows(model_ids, skip_locked="SKIP LOCKED" in query)
return [{"model_id": model_id} for model_id in locked]
self.journal.append(f"model_table.query:{query}")
return []
@ -1674,7 +1674,10 @@ async def test_an_access_group_update_that_changes_no_membership_writes_and_anno
@pytest.mark.asyncio
async def test_an_access_group_update_goes_through_while_another_session_holds_a_model_row_outside_the_group():
@pytest.mark.parametrize("ignored_model_names", [None, ["gpt-unrelated"]])
async def test_an_access_group_update_goes_through_while_another_session_holds_a_model_row_outside_the_group(
ignored_model_names,
):
import json
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
@ -1698,7 +1701,7 @@ async def test_an_access_group_update_goes_through_while_another_session_holds_a
with _proxy_with_stubbed_reload(prisma):
response = await update_access_group(
access_group="prod-models",
data=UpdateModelGroupRequest(model_ids=["stays", "joins"]),
data=UpdateModelGroupRequest(model_ids=["stays", "joins"], model_names=ignored_model_names),
user_api_key_dict=_admin(),
)
@ -1797,26 +1800,57 @@ async def test_an_access_group_update_answers_409_and_writes_nothing_while_anoth
],
)
prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"})
slept = []
async def record_sleep(seconds):
slept.append(seconds)
clock = _FakeClock()
with pytest.raises(HTTPException) as raised:
await _rewrite_membership_once_its_rows_are_free(
prisma_client=prisma,
access_group="prod-models",
data=UpdateModelGroupRequest(model_ids=["stays"]),
retry_delays_seconds=(0.1, 0.2),
sleep=record_sleep,
retry_budget_seconds=0.5,
sleep=clock.sleep,
clock=clock,
)
assert raised.value.status_code == 409
assert slept == [0.1, 0.2]
assert prisma.model_table.lock_refusals == 3
assert 0 < clock.now <= 0.5
assert prisma.model_table.lock_refusals == len(clock.slept) + 1
assert prisma.model_table.updates == []
@pytest.mark.asyncio
async def test_an_access_group_update_waits_out_a_member_row_held_for_over_ten_seconds_and_writes_the_edit():
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
_rewrite_membership_once_its_rows_are_free,
)
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
UpdateModelGroupRequest,
)
prisma = _FakePrismaClient(
[],
deployments=[
_deployment(model_id="stays", model_name="gpt-stays"),
_deployment(model_id="leaves", model_name="gpt-leaves"),
],
)
prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"})
prisma.model_table.other_session_commits_after_refusals = 20
clock = _FakeClock()
_, rewrites = await _rewrite_membership_once_its_rows_are_free(
prisma_client=prisma,
access_group="prod-models",
data=UpdateModelGroupRequest(model_ids=["stays"]),
sleep=clock.sleep,
clock=clock,
)
assert clock.now > 10
assert [model_id for model_id, _ in rewrites] == ["leaves"]
assert [where["model_id"] for where, _ in prisma.model_table.updates] == ["leaves"]
@pytest.mark.asyncio
async def test_updating_a_missing_access_group_answers_404_before_checking_the_model_names():
from fastapi import HTTPException

View file

@ -722,7 +722,7 @@ export interface paths {
* Raises:
* - HTTPException 400: If any model names don't exist
* - HTTPException 404: If access group not found
* - HTTPException 409: If another edit keeps one of the group's models locked for about 10 s
* - HTTPException 409: If another edit keeps one of the group's models locked for 30 s
*/
put: operations["update_access_group_access_group__access_group__update_put"];
post?: never;