From 63762b8ee006d8f1c2ca31cb079a08d6769a7fed Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:31:58 +0000 Subject: [PATCH] fix(proxy): pin model reconcile read to the writer DB The router reload triggered by /model/new read the model table through the read replica, so a lagging replica made the reload miss the just committed row and fail the request with a 500 even though the write was durable. Fixes #38556 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/db/routing_prisma_wrapper.py | 15 +++++++++ litellm/proxy/proxy_server.py | 11 ++++++- .../proxy/db/test_routing_prisma_wrapper.py | 25 +++++++++++++++ tests/test_litellm/proxy/test_proxy_server.py | 31 +++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/db/routing_prisma_wrapper.py b/litellm/proxy/db/routing_prisma_wrapper.py index 22fc32a898a..ac747db6ee0 100644 --- a/litellm/proxy/db/routing_prisma_wrapper.py +++ b/litellm/proxy/db/routing_prisma_wrapper.py @@ -57,6 +57,21 @@ class _RoutedActions: return getattr(self._writer_actions, name) +class WriterPinnedClient: + """PrismaClient-shaped view whose `.db` always resolves to the writer. + + Read-after-write paths (e.g. the model reconcile a /model/new triggers to + verify its own just-committed row) must not read through a lagging read + replica: the row is not replayed there yet, so the reconcile concludes the + write is missing and fails the request even though it is durable (#38556). + """ + + __slots__ = ("db",) + + def __init__(self, db: "PrismaWrapper | RoutingPrismaWrapper"): + self.db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) else db + + class RoutingPrismaWrapper: """ Routes Prisma operations between a writer and a reader Prisma client. diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 99c3ccd915f..3ca7e94bc26 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6726,9 +6726,17 @@ class ProxyConfig: - list: the rows (may be empty if no models exist) - None: signals a DB fetch *failure* — callers must not treat this as "all models deleted" and must not evict existing router deployments. + + Pinned to the writer DB: this read reconciles the router against the rows a + model write just committed, and reading it through a lagging read replica + makes the write-triggered reload report its own durable write as missing + (#38556). It also keeps a stale replica snapshot from evicting a deployment + another pod just added. """ try: - new_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository(prisma_client).table.find_many() + new_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository( + WriterPinnedClient(prisma_client.db) + ).table.find_many() return new_models except Exception as e: verbose_proxy_logger.exception( @@ -11974,6 +11982,7 @@ async def run_thread( # ) # async def get_available_routes(user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)): from litellm.llms.base_llm.base_utils import BaseTokenCounter +from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.model_repository import ModelRepository from litellm.repositories.table_repositories import ( diff --git a/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py index dcc0036ff04..c0d651895e6 100644 --- a/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py +++ b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py @@ -101,6 +101,31 @@ def test_per_model_reads_route_to_reader_writes_to_writer(): assert actions.delete_many is writer_inner.litellm_usertable.delete_many +def test_writer_pinned_client_bypasses_reader_routing(): + """Regression for #38556: read-after-write reconciles must see the writer's + just-committed rows, so WriterPinnedClient must resolve reads to the writer + even when a read replica is configured.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper, WriterPinnedClient + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.litellm_proxymodeltable = _model_actions_mock("writer_models") + reader_inner.litellm_proxymodeltable = _model_actions_mock("reader_models") + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + pinned = WriterPinnedClient(routing) + + assert pinned.db is writer + assert pinned.db.litellm_proxymodeltable.find_many is writer_inner.litellm_proxymodeltable.find_many + + +def test_writer_pinned_client_passes_through_single_db(): + from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient + + writer, _, _, _ = _make_wrappers() + + assert WriterPinnedClient(writer).db is writer + + @pytest.mark.asyncio async def test_connect_invokes_both_clients(): from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index f51648faf80..3b55a84e605 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -9518,6 +9518,37 @@ class TestDeleteDeploymentSync: assert result is None, f"Expected None on DB failure to signal fetch error, got {result!r}" + @pytest.mark.asyncio + async def test_get_models_from_db_reads_from_writer_not_replica(self): + """ + Regression for #38556: with DATABASE_URL_READ_REPLICA configured, the model + reconcile after /model/new used to read via the replica, so a lagging replica + made the reload miss the just-committed row and fail the request with a 500. + The reconcile read must be pinned to the writer. + """ + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + from litellm.proxy.proxy_server import ProxyConfig + + writer_inner = MagicMock(name="writer_prisma") + reader_inner = MagicMock(name="reader_prisma") + committed_row = MagicMock(name="just_committed_model_row") + writer_inner.litellm_proxymodeltable.find_many = AsyncMock(return_value=[committed_row]) + reader_inner.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + + mock_prisma = MagicMock() + mock_prisma.db = RoutingPrismaWrapper( + writer=PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False), + reader=PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False), + ) + + result = await ProxyConfig()._get_models_from_db(prisma_client=mock_prisma) + + assert result == [committed_row], f"Expected the writer's just-committed row, got {result!r}" + reader_inner.litellm_proxymodeltable.find_many.assert_not_awaited() + def test_get_config_list_includes_cancel_on_disconnect(monkeypatch): """Follow-up to #30223: the flag must be discoverable via /config/list,