mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): take the access group lock with pg_try_advisory_xact_lock and retry while another edit holds it
This commit is contained in:
parent
9a6cf12207
commit
4eed565e5a
4 changed files with 69 additions and 29 deletions
|
|
@ -1648,7 +1648,7 @@
|
|||
},
|
||||
"/access_group/{access_group}/update": {
|
||||
"put": {
|
||||
"description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit keeps one of the group's models locked for 30 s",
|
||||
"description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit holds the group or one of its models for 30 s",
|
||||
"operationId": "update_access_group_access_group__access_group__update_put",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -200,17 +200,17 @@ def _model_table(prisma_client: PrismaClient) -> _ModelTableClient:
|
|||
return ModelRepository(WriterPinnedClient(prisma_client.db)).table
|
||||
|
||||
|
||||
_SERIALIZE_ACCESS_GROUP_WRITES: Final = (
|
||||
"SELECT 1 AS locked FROM pg_advisory_xact_lock(hashtext('model_access_group'), hashtext($1))"
|
||||
_TRY_LOCK_ACCESS_GROUP: Final = (
|
||||
"SELECT pg_try_advisory_xact_lock(hashtext('model_access_group'), hashtext($1)) AS locked"
|
||||
)
|
||||
_LOCK_MODEL_ROWS: Final = (
|
||||
'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = ANY($1::text[]) '
|
||||
"ORDER BY model_id FOR UPDATE SKIP LOCKED"
|
||||
)
|
||||
_MEMBERSHIP_TX_TIMEOUT: Final = timedelta(seconds=30)
|
||||
_ROW_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds()
|
||||
_ROW_LOCK_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8)
|
||||
_ROW_LOCK_LONGEST_RETRY_DELAY_SECONDS: Final = 1.0
|
||||
_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds()
|
||||
_LOCK_RETRY_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8)
|
||||
_LOCK_RETRY_LONGEST_DELAY_SECONDS: Final = 1.0
|
||||
|
||||
|
||||
class _LockedModelRow(BaseModel):
|
||||
|
|
@ -222,6 +222,15 @@ class _LockedModelRow(BaseModel):
|
|||
_LOCKED_MODEL_ROWS: Final = TypeAdapter(tuple[_LockedModelRow, ...])
|
||||
|
||||
|
||||
class _AccessGroupLock(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
locked: bool
|
||||
|
||||
|
||||
_ACCESS_GROUP_LOCK: Final = TypeAdapter(tuple[_AccessGroupLock])
|
||||
|
||||
|
||||
def _model_table_tx(prisma_client: PrismaClient) -> contextlib.AbstractAsyncContextManager[_ModelTableTx]:
|
||||
return writer_wrapper(prisma_client.db).tx(timeout=_MEMBERSHIP_TX_TIMEOUT)
|
||||
|
||||
|
|
@ -516,11 +525,13 @@ async def _raise_404_unless_the_primary_carries(access_group: str, prisma_client
|
|||
raise _http_error(404, f"Access group '{access_group}' not found")
|
||||
|
||||
|
||||
async def _rewrite_membership_unless_a_row_is_locked(
|
||||
async def _rewrite_membership_unless_locked(
|
||||
prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest
|
||||
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None:
|
||||
async with _model_table_tx(prisma_client) as tx:
|
||||
await tx.query_raw(_SERIALIZE_ACCESS_GROUP_WRITES, access_group)
|
||||
(group_lock,) = _ACCESS_GROUP_LOCK.validate_python(await tx.query_raw(_TRY_LOCK_ACCESS_GROUP, access_group))
|
||||
if not group_lock.locked:
|
||||
return None
|
||||
model_ids_to_lock: Final = _model_ids_to_lock(
|
||||
deployments=await tx.litellm_proxymodeltable.find_many(), access_group=access_group, data=data
|
||||
)
|
||||
|
|
@ -554,22 +565,22 @@ async def _rewrite_membership_unless_a_row_is_locked(
|
|||
return selected_deployments, rewrites
|
||||
|
||||
|
||||
def _row_lock_retry_delays() -> Iterator[float]:
|
||||
return itertools.chain(_ROW_LOCK_BACKOFF_SECONDS, itertools.repeat(_ROW_LOCK_LONGEST_RETRY_DELAY_SECONDS))
|
||||
def _lock_retry_delays() -> Iterator[float]:
|
||||
return itertools.chain(_LOCK_RETRY_BACKOFF_SECONDS, itertools.repeat(_LOCK_RETRY_LONGEST_DELAY_SECONDS))
|
||||
|
||||
|
||||
async def _rewrite_membership_once_its_rows_are_free(
|
||||
async def _rewrite_membership_once_unlocked(
|
||||
prisma_client: PrismaClient,
|
||||
access_group: str,
|
||||
data: UpdateModelGroupRequest,
|
||||
retry_budget_seconds: float = _ROW_LOCK_RETRY_BUDGET_SECONDS,
|
||||
retry_budget_seconds: float = _LOCK_RETRY_BUDGET_SECONDS,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]:
|
||||
deadline: Final = clock() + retry_budget_seconds
|
||||
for delay in _row_lock_retry_delays():
|
||||
for delay in _lock_retry_delays():
|
||||
if (
|
||||
rewritten := await _rewrite_membership_unless_a_row_is_locked(
|
||||
rewritten := await _rewrite_membership_unless_locked(
|
||||
prisma_client=prisma_client, access_group=access_group, data=data
|
||||
)
|
||||
) is not None:
|
||||
|
|
@ -577,7 +588,7 @@ async def _rewrite_membership_once_its_rows_are_free(
|
|||
if clock() + delay > deadline:
|
||||
break
|
||||
await sleep(delay)
|
||||
raise _http_error(409, f"Another edit holds a model in access group '{access_group}', try again")
|
||||
raise _http_error(409, f"Another edit holds access group '{access_group}' or one of its models, try again")
|
||||
|
||||
|
||||
def _readable_model_info_or_raise(model_id: str, model_info: object) -> Mapping[str, object]:
|
||||
|
|
@ -1073,7 +1084,7 @@ async def update_access_group(
|
|||
Raises:
|
||||
- HTTPException 400: If any model names don't exist
|
||||
- HTTPException 404: If access group not found
|
||||
- HTTPException 409: If another edit keeps one of the group's models locked for 30 s
|
||||
- HTTPException 409: If another edit holds the group or one of its models for 30 s
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_router, prisma_client
|
||||
|
||||
|
|
@ -1116,7 +1127,7 @@ async def update_access_group(
|
|||
)
|
||||
|
||||
try:
|
||||
selected_deployments, rewrites = await _rewrite_membership_once_its_rows_are_free(
|
||||
selected_deployments, rewrites = await _rewrite_membership_once_unlocked(
|
||||
prisma_client=prisma_client, access_group=access_group, data=data
|
||||
)
|
||||
if rewrites:
|
||||
|
|
|
|||
|
|
@ -707,6 +707,7 @@ class _FakeModelTable:
|
|||
self.failing_model_ids = failing_model_ids
|
||||
self.updates: list[tuple[dict, dict]] = []
|
||||
self.rows_locked_by_another_session: frozenset[str] = frozenset()
|
||||
self.group_locked_by_another_session = False
|
||||
self.edits_the_other_session_commits: dict[str, dict] = {}
|
||||
self.other_session_commits_after_refusals: int | None = None
|
||||
self.lock_refusals = 0
|
||||
|
|
@ -715,16 +716,27 @@ class _FakeModelTable:
|
|||
model_names = (where or {}).get("model_name", {}).get("in")
|
||||
return [d for d in self.deployments if model_names is None or d.model_name in model_names]
|
||||
|
||||
def lock_group(self, wait):
|
||||
if not self.group_locked_by_another_session:
|
||||
return True
|
||||
if wait:
|
||||
raise TimeoutError("lock_timeout expired waiting for the access group lock")
|
||||
self._refuse_lock()
|
||||
return False
|
||||
|
||||
def lock_rows(self, model_ids, skip_locked):
|
||||
held = self.rows_locked_by_another_session.intersection(model_ids)
|
||||
if not held:
|
||||
return list(model_ids)
|
||||
if not skip_locked:
|
||||
raise TimeoutError(f"timed out waiting for the row lock on {sorted(held)}")
|
||||
self._refuse_lock()
|
||||
return [model_id for model_id in model_ids if model_id not in held]
|
||||
|
||||
def _refuse_lock(self):
|
||||
self.lock_refusals += 1
|
||||
if self.lock_refusals == self.other_session_commits_after_refusals:
|
||||
self._other_session_commits()
|
||||
return [model_id for model_id in model_ids if model_id not in held]
|
||||
|
||||
def _other_session_commits(self):
|
||||
self.deployments = [
|
||||
|
|
@ -736,6 +748,7 @@ class _FakeModelTable:
|
|||
for d in self.deployments
|
||||
]
|
||||
self.rows_locked_by_another_session = frozenset()
|
||||
self.group_locked_by_another_session = False
|
||||
|
||||
async def find_unique(self, where, include=None):
|
||||
return next((d for d in self.deployments if d.model_id == where["model_id"]), None)
|
||||
|
|
@ -785,9 +798,9 @@ class _FakeModelTableTx:
|
|||
self.litellm_proxymodeltable = SimpleNamespace(find_many=self._find_many)
|
||||
|
||||
async def query_raw(self, query, *args):
|
||||
if "pg_advisory_xact_lock" in query:
|
||||
if "advisory_xact_lock" in query:
|
||||
self.journal.append(f"access_group.lock:{args[0]}")
|
||||
return [{"locked": 1}]
|
||||
return [{"locked": self.model_table.lock_group(wait="pg_try_" not in query)}]
|
||||
if "FOR UPDATE" in query:
|
||||
model_ids = sorted(args[0] if args else (d.model_id for d in self.model_table.deployments))
|
||||
self.journal.append(f"model_table.lock:{','.join(model_ids)}")
|
||||
|
|
@ -1781,12 +1794,25 @@ async def test_an_access_group_emptied_while_the_update_retried_a_locked_row_ans
|
|||
assert prisma.model_table.updates == []
|
||||
|
||||
|
||||
def _another_session_holds_a_member_row(model_table):
|
||||
model_table.rows_locked_by_another_session = frozenset({"leaves"})
|
||||
|
||||
|
||||
def _another_edit_holds_the_group(model_table):
|
||||
model_table.group_locked_by_another_session = True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_access_group_update_answers_409_and_writes_nothing_while_another_session_keeps_a_member_row_locked():
|
||||
@pytest.mark.parametrize(
|
||||
"hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"]
|
||||
)
|
||||
async def test_an_access_group_update_answers_409_and_writes_nothing_while_another_session_keeps_a_lock_it_needs(
|
||||
hold,
|
||||
):
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
|
||||
_rewrite_membership_once_its_rows_are_free,
|
||||
_rewrite_membership_once_unlocked,
|
||||
)
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
UpdateModelGroupRequest,
|
||||
|
|
@ -1799,11 +1825,11 @@ async def test_an_access_group_update_answers_409_and_writes_nothing_while_anoth
|
|||
_deployment(model_id="leaves", model_name="gpt-leaves"),
|
||||
],
|
||||
)
|
||||
prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"})
|
||||
hold(prisma.model_table)
|
||||
clock = _FakeClock()
|
||||
|
||||
with pytest.raises(HTTPException) as raised:
|
||||
await _rewrite_membership_once_its_rows_are_free(
|
||||
await _rewrite_membership_once_unlocked(
|
||||
prisma_client=prisma,
|
||||
access_group="prod-models",
|
||||
data=UpdateModelGroupRequest(model_ids=["stays"]),
|
||||
|
|
@ -1819,9 +1845,12 @@ async def test_an_access_group_update_answers_409_and_writes_nothing_while_anoth
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_access_group_update_waits_out_a_member_row_held_for_over_ten_seconds_and_writes_the_edit():
|
||||
@pytest.mark.parametrize(
|
||||
"hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"]
|
||||
)
|
||||
async def test_an_access_group_update_waits_out_a_lock_held_for_over_ten_seconds_and_writes_the_edit(hold):
|
||||
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
|
||||
_rewrite_membership_once_its_rows_are_free,
|
||||
_rewrite_membership_once_unlocked,
|
||||
)
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
UpdateModelGroupRequest,
|
||||
|
|
@ -1834,11 +1863,11 @@ async def test_an_access_group_update_waits_out_a_member_row_held_for_over_ten_s
|
|||
_deployment(model_id="leaves", model_name="gpt-leaves"),
|
||||
],
|
||||
)
|
||||
prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"})
|
||||
hold(prisma.model_table)
|
||||
prisma.model_table.other_session_commits_after_refusals = 20
|
||||
clock = _FakeClock()
|
||||
|
||||
_, rewrites = await _rewrite_membership_once_its_rows_are_free(
|
||||
_, rewrites = await _rewrite_membership_once_unlocked(
|
||||
prisma_client=prisma,
|
||||
access_group="prod-models",
|
||||
data=UpdateModelGroupRequest(model_ids=["stays"]),
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -722,7 +722,7 @@ export interface paths {
|
|||
* Raises:
|
||||
* - HTTPException 400: If any model names don't exist
|
||||
* - HTTPException 404: If access group not found
|
||||
* - HTTPException 409: If another edit keeps one of the group's models locked for 30 s
|
||||
* - HTTPException 409: If another edit holds the group or one of its models for 30 s
|
||||
*/
|
||||
put: operations["update_access_group_access_group__access_group__update_put"];
|
||||
post?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue