mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
fix(proxy): load DB models in a deterministic order
_get_models_from_db ran find_many() with no ORDER BY, so the model list came back in Postgres' heap-scan order. Toggling a model rewrites its row under MVCC and clear_cache() rebuilds the router from a fresh unordered scan, so the toggled model jumps position and the UI list reshuffles on every reload; worse when a filter shows only a slice. Order by created_at then model_id so the order is stable regardless of updates.
This commit is contained in:
parent
360adbe765
commit
74a57e51ce
2 changed files with 60 additions and 1 deletions
|
|
@ -6016,7 +6016,9 @@ class ProxyConfig:
|
|||
as "all models deleted" and must not evict existing router deployments.
|
||||
"""
|
||||
try:
|
||||
new_models = await ModelRepository(prisma_client).table.find_many()
|
||||
new_models = await ModelRepository(prisma_client).table.find_many(
|
||||
order=[{"created_at": "asc"}, {"model_id": "asc"}]
|
||||
)
|
||||
return new_models
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception(
|
||||
|
|
|
|||
|
|
@ -8450,3 +8450,60 @@ def test_config_field_info_returns_raw_secrets_for_full_admin(monkeypatch):
|
|||
)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
class _FakeModelTable:
|
||||
"""Fake prisma model table whose find_many honors the `order` argument.
|
||||
|
||||
Returns rows in scrambled insertion order when no `order` is given, so a
|
||||
test asserting deterministic output fails if the caller stops passing one.
|
||||
"""
|
||||
|
||||
def __init__(self, rows):
|
||||
self._rows = list(rows)
|
||||
|
||||
async def find_many(self, *args, order=None, where=None, **kwargs):
|
||||
if not order:
|
||||
return list(self._rows)
|
||||
|
||||
def sort_key(row):
|
||||
return tuple(getattr(row, list(o)[0]) for o in order)
|
||||
|
||||
directions = [list(o.values())[0] for o in order]
|
||||
if any(d not in ("asc", "desc") for d in directions):
|
||||
raise ValueError(f"unexpected sort direction in {order}")
|
||||
if any(d == "desc" for d in directions):
|
||||
raise AssertionError("model list is expected ascending, not descending")
|
||||
return sorted(self._rows, key=sort_key)
|
||||
|
||||
|
||||
class TestGetModelsFromDbDeterministicOrder:
|
||||
"""Regression: the model list must come back in a stable order so the UI
|
||||
does not reshuffle on every reload (e.g. after toggling a model, which
|
||||
rewrites the row and changes Postgres' unordered scan position)."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_models_ordered_by_created_at_then_model_id(self):
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
def _row(model_id, created_at):
|
||||
r = MagicMock()
|
||||
r.model_id = model_id
|
||||
r.created_at = created_at
|
||||
return r
|
||||
|
||||
early = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
late = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||
|
||||
scrambled = [
|
||||
_row("z-newest", late),
|
||||
_row("b-oldest", early),
|
||||
_row("a-oldest", early),
|
||||
]
|
||||
|
||||
prisma_client = MagicMock()
|
||||
prisma_client.db.litellm_proxymodeltable = _FakeModelTable(scrambled)
|
||||
|
||||
result = await ProxyConfig()._get_models_from_db(prisma_client=prisma_client)
|
||||
|
||||
assert [r.model_id for r in result] == ["a-oldest", "b-oldest", "z-newest"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue