mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): retry access group row locks for 30 s with SKIP LOCKED and lock only the rows model_ids select
This commit is contained in:
parent
04e51b41d6
commit
9a6cf12207
4 changed files with 99 additions and 54 deletions
|
|
@ -1648,7 +1648,7 @@
|
|||
},
|
||||
"/access_group/{access_group}/update": {
|
||||
"put": {
|
||||
"description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit keeps one of the group's models locked for about 10 s",
|
||||
"description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit keeps one of the group's models locked for 30 s",
|
||||
"operationId": "update_access_group_access_group__access_group__update_put",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -14,13 +14,16 @@ Endpoints here:
|
|||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import itertools
|
||||
import json
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from datetime import datetime, timedelta
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Any, Final, Protocol
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -32,7 +35,6 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
model_access_group_cache_key,
|
||||
model_access_group_registry_cache_key,
|
||||
)
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient, writer_wrapper
|
||||
from litellm.proxy.management_endpoints.common_utils import validate_budget_duration
|
||||
|
||||
|
|
@ -203,10 +205,21 @@ _SERIALIZE_ACCESS_GROUP_WRITES: Final = (
|
|||
)
|
||||
_LOCK_MODEL_ROWS: Final = (
|
||||
'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = ANY($1::text[]) '
|
||||
"ORDER BY model_id FOR UPDATE NOWAIT"
|
||||
"ORDER BY model_id FOR UPDATE SKIP LOCKED"
|
||||
)
|
||||
_MEMBERSHIP_TX_TIMEOUT: Final = timedelta(seconds=30)
|
||||
_ROW_LOCK_RETRY_DELAYS_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8, *(1.0,) * 9)
|
||||
_ROW_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds()
|
||||
_ROW_LOCK_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8)
|
||||
_ROW_LOCK_LONGEST_RETRY_DELAY_SECONDS: Final = 1.0
|
||||
|
||||
|
||||
class _LockedModelRow(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
model_id: str
|
||||
|
||||
|
||||
_LOCKED_MODEL_ROWS: Final = TypeAdapter(tuple[_LockedModelRow, ...])
|
||||
|
||||
|
||||
def _model_table_tx(prisma_client: PrismaClient) -> contextlib.AbstractAsyncContextManager[_ModelTableTx]:
|
||||
|
|
@ -488,7 +501,7 @@ def _model_ids_to_lock(
|
|||
deployments: Sequence[_DeploymentRow], access_group: str, data: UpdateModelGroupRequest
|
||||
) -> frozenset[str]:
|
||||
selected_model_ids: Final = frozenset(data.model_ids or ())
|
||||
selected_model_names: Final = frozenset(data.model_names or ())
|
||||
selected_model_names: Final = frozenset(() if data.model_ids else data.model_names or ())
|
||||
return frozenset(
|
||||
deployment.model_id
|
||||
for deployment in deployments
|
||||
|
|
@ -503,15 +516,19 @@ async def _raise_404_unless_the_primary_carries(access_group: str, prisma_client
|
|||
raise _http_error(404, f"Access group '{access_group}' not found")
|
||||
|
||||
|
||||
async def _rewrite_membership_under_row_locks(
|
||||
async def _rewrite_membership_unless_a_row_is_locked(
|
||||
prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest
|
||||
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]:
|
||||
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None:
|
||||
async with _model_table_tx(prisma_client) as tx:
|
||||
await tx.query_raw(_SERIALIZE_ACCESS_GROUP_WRITES, access_group)
|
||||
model_ids_to_lock: Final = _model_ids_to_lock(
|
||||
deployments=await tx.litellm_proxymodeltable.find_many(), access_group=access_group, data=data
|
||||
)
|
||||
await tx.query_raw(_LOCK_MODEL_ROWS, sorted(model_ids_to_lock))
|
||||
locked_rows: Final = _LOCKED_MODEL_ROWS.validate_python(
|
||||
await tx.query_raw(_LOCK_MODEL_ROWS, sorted(model_ids_to_lock))
|
||||
)
|
||||
if frozenset(row.model_id for row in locked_rows) != model_ids_to_lock:
|
||||
return None
|
||||
deployments: Final = tuple(
|
||||
deployment
|
||||
for deployment in await tx.litellm_proxymodeltable.find_many()
|
||||
|
|
@ -537,35 +554,29 @@ async def _rewrite_membership_under_row_locks(
|
|||
return selected_deployments, rewrites
|
||||
|
||||
|
||||
async def _rewrite_membership_unless_a_row_is_locked(
|
||||
prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest
|
||||
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None:
|
||||
try:
|
||||
return await _rewrite_membership_under_row_locks(
|
||||
prisma_client=prisma_client, access_group=access_group, data=data
|
||||
)
|
||||
except Exception as e:
|
||||
if PrismaDBExceptionHandler.is_lock_timeout_error(e):
|
||||
return None
|
||||
raise
|
||||
def _row_lock_retry_delays() -> Iterator[float]:
|
||||
return itertools.chain(_ROW_LOCK_BACKOFF_SECONDS, itertools.repeat(_ROW_LOCK_LONGEST_RETRY_DELAY_SECONDS))
|
||||
|
||||
|
||||
async def _rewrite_membership_once_its_rows_are_free(
|
||||
prisma_client: PrismaClient,
|
||||
access_group: str,
|
||||
data: UpdateModelGroupRequest,
|
||||
retry_delays_seconds: Sequence[float] = _ROW_LOCK_RETRY_DELAYS_SECONDS,
|
||||
retry_budget_seconds: float = _ROW_LOCK_RETRY_BUDGET_SECONDS,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]:
|
||||
for delay in (*retry_delays_seconds, None):
|
||||
deadline: Final = clock() + retry_budget_seconds
|
||||
for delay in _row_lock_retry_delays():
|
||||
if (
|
||||
rewritten := await _rewrite_membership_unless_a_row_is_locked(
|
||||
prisma_client=prisma_client, access_group=access_group, data=data
|
||||
)
|
||||
) is not None:
|
||||
return rewritten
|
||||
if delay is not None:
|
||||
await sleep(delay)
|
||||
if clock() + delay > deadline:
|
||||
break
|
||||
await sleep(delay)
|
||||
raise _http_error(409, f"Another edit holds a model in access group '{access_group}', try again")
|
||||
|
||||
|
||||
|
|
@ -1062,7 +1073,7 @@ async def update_access_group(
|
|||
Raises:
|
||||
- HTTPException 400: If any model names don't exist
|
||||
- HTTPException 404: If access group not found
|
||||
- HTTPException 409: If another edit keeps one of the group's models locked for about 10 s
|
||||
- HTTPException 409: If another edit keeps one of the group's models locked for 30 s
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_router, prisma_client
|
||||
|
||||
|
|
|
|||
|
|
@ -687,17 +687,17 @@ class _FakeAccessGroupBudgetTable:
|
|||
return self.rows.pop(access_group_name, None)
|
||||
|
||||
|
||||
def _row_lock_not_available(held_model_ids):
|
||||
import prisma
|
||||
class _FakeClock:
|
||||
def __init__(self) -> None:
|
||||
self.now = 0.0
|
||||
self.slept: list[float] = []
|
||||
|
||||
return prisma.errors.RawQueryError(
|
||||
{
|
||||
"user_facing_error": {
|
||||
"error_code": "P2010",
|
||||
"meta": {"code": "55P03", "message": f"could not obtain lock on row in relation for {held_model_ids}"},
|
||||
}
|
||||
}
|
||||
)
|
||||
def __call__(self) -> float:
|
||||
return self.now
|
||||
|
||||
async def sleep(self, seconds: float) -> None:
|
||||
self.slept.append(seconds)
|
||||
self.now += seconds
|
||||
|
||||
|
||||
class _FakeModelTable:
|
||||
|
|
@ -715,16 +715,16 @@ class _FakeModelTable:
|
|||
model_names = (where or {}).get("model_name", {}).get("in")
|
||||
return [d for d in self.deployments if model_names is None or d.model_name in model_names]
|
||||
|
||||
def lock_rows(self, model_ids, nowait):
|
||||
held = sorted(self.rows_locked_by_another_session.intersection(model_ids))
|
||||
def lock_rows(self, model_ids, skip_locked):
|
||||
held = self.rows_locked_by_another_session.intersection(model_ids)
|
||||
if not held:
|
||||
return
|
||||
if not nowait:
|
||||
raise TimeoutError(f"timed out waiting for the row lock on {held}")
|
||||
return list(model_ids)
|
||||
if not skip_locked:
|
||||
raise TimeoutError(f"timed out waiting for the row lock on {sorted(held)}")
|
||||
self.lock_refusals += 1
|
||||
if self.lock_refusals == self.other_session_commits_after_refusals:
|
||||
self._other_session_commits()
|
||||
raise _row_lock_not_available(held)
|
||||
return [model_id for model_id in model_ids if model_id not in held]
|
||||
|
||||
def _other_session_commits(self):
|
||||
self.deployments = [
|
||||
|
|
@ -791,8 +791,8 @@ class _FakeModelTableTx:
|
|||
if "FOR UPDATE" in query:
|
||||
model_ids = sorted(args[0] if args else (d.model_id for d in self.model_table.deployments))
|
||||
self.journal.append(f"model_table.lock:{','.join(model_ids)}")
|
||||
self.model_table.lock_rows(model_ids, nowait="NOWAIT" in query)
|
||||
return [{"model_id": model_id} for model_id in model_ids]
|
||||
locked = self.model_table.lock_rows(model_ids, skip_locked="SKIP LOCKED" in query)
|
||||
return [{"model_id": model_id} for model_id in locked]
|
||||
self.journal.append(f"model_table.query:{query}")
|
||||
return []
|
||||
|
||||
|
|
@ -1674,7 +1674,10 @@ async def test_an_access_group_update_that_changes_no_membership_writes_and_anno
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_access_group_update_goes_through_while_another_session_holds_a_model_row_outside_the_group():
|
||||
@pytest.mark.parametrize("ignored_model_names", [None, ["gpt-unrelated"]])
|
||||
async def test_an_access_group_update_goes_through_while_another_session_holds_a_model_row_outside_the_group(
|
||||
ignored_model_names,
|
||||
):
|
||||
import json
|
||||
|
||||
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
|
||||
|
|
@ -1698,7 +1701,7 @@ async def test_an_access_group_update_goes_through_while_another_session_holds_a
|
|||
with _proxy_with_stubbed_reload(prisma):
|
||||
response = await update_access_group(
|
||||
access_group="prod-models",
|
||||
data=UpdateModelGroupRequest(model_ids=["stays", "joins"]),
|
||||
data=UpdateModelGroupRequest(model_ids=["stays", "joins"], model_names=ignored_model_names),
|
||||
user_api_key_dict=_admin(),
|
||||
)
|
||||
|
||||
|
|
@ -1797,26 +1800,57 @@ async def test_an_access_group_update_answers_409_and_writes_nothing_while_anoth
|
|||
],
|
||||
)
|
||||
prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"})
|
||||
slept = []
|
||||
|
||||
async def record_sleep(seconds):
|
||||
slept.append(seconds)
|
||||
clock = _FakeClock()
|
||||
|
||||
with pytest.raises(HTTPException) as raised:
|
||||
await _rewrite_membership_once_its_rows_are_free(
|
||||
prisma_client=prisma,
|
||||
access_group="prod-models",
|
||||
data=UpdateModelGroupRequest(model_ids=["stays"]),
|
||||
retry_delays_seconds=(0.1, 0.2),
|
||||
sleep=record_sleep,
|
||||
retry_budget_seconds=0.5,
|
||||
sleep=clock.sleep,
|
||||
clock=clock,
|
||||
)
|
||||
|
||||
assert raised.value.status_code == 409
|
||||
assert slept == [0.1, 0.2]
|
||||
assert prisma.model_table.lock_refusals == 3
|
||||
assert 0 < clock.now <= 0.5
|
||||
assert prisma.model_table.lock_refusals == len(clock.slept) + 1
|
||||
assert prisma.model_table.updates == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_access_group_update_waits_out_a_member_row_held_for_over_ten_seconds_and_writes_the_edit():
|
||||
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
|
||||
_rewrite_membership_once_its_rows_are_free,
|
||||
)
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
UpdateModelGroupRequest,
|
||||
)
|
||||
|
||||
prisma = _FakePrismaClient(
|
||||
[],
|
||||
deployments=[
|
||||
_deployment(model_id="stays", model_name="gpt-stays"),
|
||||
_deployment(model_id="leaves", model_name="gpt-leaves"),
|
||||
],
|
||||
)
|
||||
prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"})
|
||||
prisma.model_table.other_session_commits_after_refusals = 20
|
||||
clock = _FakeClock()
|
||||
|
||||
_, rewrites = await _rewrite_membership_once_its_rows_are_free(
|
||||
prisma_client=prisma,
|
||||
access_group="prod-models",
|
||||
data=UpdateModelGroupRequest(model_ids=["stays"]),
|
||||
sleep=clock.sleep,
|
||||
clock=clock,
|
||||
)
|
||||
|
||||
assert clock.now > 10
|
||||
assert [model_id for model_id, _ in rewrites] == ["leaves"]
|
||||
assert [where["model_id"] for where, _ in prisma.model_table.updates] == ["leaves"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_updating_a_missing_access_group_answers_404_before_checking_the_model_names():
|
||||
from fastapi import HTTPException
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -722,7 +722,7 @@ export interface paths {
|
|||
* Raises:
|
||||
* - HTTPException 400: If any model names don't exist
|
||||
* - HTTPException 404: If access group not found
|
||||
* - HTTPException 409: If another edit keeps one of the group's models locked for about 10 s
|
||||
* - HTTPException 409: If another edit keeps one of the group's models locked for 30 s
|
||||
*/
|
||||
put: operations["update_access_group_access_group__access_group__update_put"];
|
||||
post?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue