From 77e64c5d401510189a396f8e7a9a41ffe0efa461 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 13 Aug 2026 13:42:57 -0700 Subject: [PATCH 1/4] fix(proxy): tolerate a concurrent creator when creating spend views Every replica booting against the same fresh database sees each view as absent and issues the CREATE. Postgres fails all but one with a duplicate-object error, and that exception propagated out of create_missing_views, so every view after the first was never created and /global/spend* 500'd for the life of the deployment. Losing that race reaches the desired end state, so treat it as success. Genuine DDL errors still propagate. --- litellm/proxy/db/create_views.py | 37 ++++++++++--- litellm/proxy/utils.py | 11 ++-- .../proxy/db/test_create_views.py | 52 +++++++++++++++++++ 3 files changed, 89 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/db/create_views.py b/litellm/proxy/db/create_views.py index 141ce92f172..7b7db19b4e3 100644 --- a/litellm/proxy/db/create_views.py +++ b/litellm/proxy/db/create_views.py @@ -10,6 +10,29 @@ _db = Any # 'undefined function' or 'column undefined_col referenced in query'). _VIEW_NOT_FOUND_MARKERS: Final = ("does not exist", "no such table", "undefined table") +# Markers for the inverse condition: another replica created the view between +# our existence probe and our CREATE. +_VIEW_ALREADY_EXISTS_MARKERS: Final = ("already exists", "duplicate object", "duplicate table") + + +async def create_view_tolerating_race(db: _db, view_name: str, ddl: str) -> None: + """ + Create a view, treating "a concurrent creator won" as success. + + Every replica booting against the same fresh database observes the view as + absent and issues the CREATE; Postgres fails all but one with a + duplicate-object error. The desired end state is still reached, so losing + that race is success. Without this, the loser's exception propagates out of + a detached startup task and the remaining views are never created. + """ + try: + await db.execute_raw(ddl) + verbose_logger.debug("%s Created!", view_name) + except Exception as e: + if not any(marker in str(e).lower() for marker in _VIEW_ALREADY_EXISTS_MARKERS): + raise + verbose_logger.debug("%s already created by a concurrent replica", view_name) + async def create_missing_views(db: _db): """ @@ -34,7 +57,10 @@ async def create_missing_views(db: _db): if not any(marker in error_msg for marker in _VIEW_NOT_FOUND_MARKERS): raise # If an error occurs, the view does not exist, so create it - await db.execute_raw(""" + await create_view_tolerating_race( + db, + "LiteLLM_VerificationTokenView", + """ CREATE VIEW "LiteLLM_VerificationTokenView" AS SELECT v.*, @@ -46,9 +72,8 @@ async def create_missing_views(db: _db): FROM "LiteLLM_VerificationToken" v LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id LEFT JOIN "LiteLLM_ProjectTable" p ON v.project_id = p.project_id; - """) - - verbose_logger.debug("LiteLLM_VerificationTokenView Created!") + """, + ) try: await db.query_raw("""SELECT 1 FROM "MonthlyGlobalSpend" LIMIT 1""") @@ -218,9 +243,7 @@ async def create_missing_views(db: _db): ORDER BY total_spend DESC LIMIT 100; """ - await db.execute_raw(query=sql_query) - - verbose_logger.debug("Last30dTopEndUsersSpend Created!") + await create_view_tolerating_race(db, "Last30dTopEndUsersSpend", sql_query) async def should_create_missing_views(db: _db) -> bool: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8a1fae42789..ef0376ade84 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -106,6 +106,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import publish_config_param_c from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.create_views import ( create_missing_views, + create_view_tolerating_race, should_create_missing_views, ) from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter @@ -3273,7 +3274,10 @@ class PrismaClient: ## check if required view exists ## if ret[0]["view_names"] and required_view not in ret[0]["view_names"]: await self.health_check() # make sure we can connect to db - await self.db.execute_raw(""" + await create_view_tolerating_race( + self.db, + "LiteLLM_VerificationTokenView", + """ CREATE VIEW "LiteLLM_VerificationTokenView" AS SELECT v.*, @@ -3283,9 +3287,8 @@ class PrismaClient: t.rpm_limit AS team_rpm_limit FROM "LiteLLM_VerificationToken" v LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id; - """) - - verbose_proxy_logger.info("LiteLLM_VerificationTokenView Created in DB!") + """, + ) else: should_create_views: Final = await should_create_missing_views(db=self.db) if should_create_views: diff --git a/tests/test_litellm/proxy/db/test_create_views.py b/tests/test_litellm/proxy/db/test_create_views.py index c0c09d0137b..011203c4b5a 100644 --- a/tests/test_litellm/proxy/db/test_create_views.py +++ b/tests/test_litellm/proxy/db/test_create_views.py @@ -189,3 +189,55 @@ async def test_create_views_creates_view_on_undefined_table_error(): await create_missing_views(mock_db) mock_db.execute_raw.assert_called_once() + + +@pytest.mark.asyncio +async def test_create_views_tolerates_concurrent_creator_and_continues(): + """A replica that loses the CREATE race must not abort the remaining views. + + Regression: two proxy pods booting on a fresh DB both see every view as + absent and both issue the CREATE. Postgres fails the loser with a + duplicate-object error, which propagated out of create_missing_views and + left every later view (MonthlyGlobalSpend, DailyTagSpend, ...) uncreated, + so /global/spend* 500'd for the life of the deployment. + """ + from litellm.proxy.db.create_views import create_missing_views + + mock_db = MagicMock() + mock_db.query_raw = AsyncMock(side_effect=Exception("relation does not exist")) + mock_db.execute_raw = AsyncMock( + side_effect=[Exception('relation "LiteLLM_VerificationTokenView" already exists')] + + [None] * 20 + ) + + await create_missing_views(mock_db) + + assert mock_db.execute_raw.await_count > 1, ( + "lost the race on the first view and stopped; later views were never created" + ) + + +@pytest.mark.asyncio +async def test_create_views_reraises_genuine_ddl_error(): + """An already-exists guard must not swallow real DDL failures.""" + from litellm.proxy.db.create_views import create_missing_views + + mock_db = MagicMock() + mock_db.query_raw = AsyncMock(side_effect=Exception("relation does not exist")) + mock_db.execute_raw = AsyncMock(side_effect=Exception("syntax error at or near")) + + with pytest.raises(Exception, match="syntax error"): + await create_missing_views(mock_db) + + +@pytest.mark.asyncio +async def test_create_view_tolerating_race_swallows_only_already_exists(): + from litellm.proxy.db.create_views import create_view_tolerating_race + + mock_db = MagicMock() + mock_db.execute_raw = AsyncMock(side_effect=Exception("duplicate object")) + await create_view_tolerating_race(mock_db, "SomeView", "CREATE VIEW ...") + + mock_db.execute_raw = AsyncMock(side_effect=Exception("permission denied")) + with pytest.raises(Exception, match="permission denied"): + await create_view_tolerating_race(mock_db, "SomeView", "CREATE VIEW ...") From 726292720c840abf1e91aaca0ed8c74264549c9b Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 13 Aug 2026 14:41:59 -0700 Subject: [PATCH 2/4] fix(proxy): guard every view creation, not just the first and last Against a real Postgres the previous commit still died on MonthlyGlobalSpend: only 2 of the 8 creation sites went through the tolerant helper, so the losing replica re-raised on the first unguarded one and skipped the rest. The regression test now makes every CREATE lose the race and asserts all 8 are still attempted, which fails on the partial fix. --- litellm/proxy/db/create_views.py | 24 ++++---------- .../proxy/db/test_create_views.py | 31 +++++++++++++------ 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/litellm/proxy/db/create_views.py b/litellm/proxy/db/create_views.py index 7b7db19b4e3..47207adee80 100644 --- a/litellm/proxy/db/create_views.py +++ b/litellm/proxy/db/create_views.py @@ -94,9 +94,7 @@ async def create_missing_views(db: _db): GROUP BY DATE("startTime"); """ - await db.execute_raw(query=sql_query) - - verbose_logger.debug("MonthlyGlobalSpend Created!") + await create_view_tolerating_race(db, "MonthlyGlobalSpend", sql_query) try: await db.query_raw("""SELECT 1 FROM "Last30dKeysBySpend" LIMIT 1""") @@ -125,9 +123,7 @@ async def create_missing_views(db: _db): ORDER BY total_spend DESC; """ - await db.execute_raw(query=sql_query) - - verbose_logger.debug("Last30dKeysBySpend Created!") + await create_view_tolerating_race(db, "Last30dKeysBySpend", sql_query) try: await db.query_raw("""SELECT 1 FROM "Last30dModelsBySpend" LIMIT 1""") @@ -151,9 +147,7 @@ async def create_missing_views(db: _db): ORDER BY total_spend DESC; """ - await db.execute_raw(query=sql_query) - - verbose_logger.debug("Last30dModelsBySpend Created!") + await create_view_tolerating_race(db, "Last30dModelsBySpend", sql_query) try: await db.query_raw("""SELECT 1 FROM "MonthlyGlobalSpendPerKey" LIMIT 1""") verbose_logger.debug("MonthlyGlobalSpendPerKey Exists!") @@ -175,9 +169,7 @@ async def create_missing_views(db: _db): DATE("startTime"), api_key; """ - await db.execute_raw(query=sql_query) - - verbose_logger.debug("MonthlyGlobalSpendPerKey Created!") + await create_view_tolerating_race(db, "MonthlyGlobalSpendPerKey", sql_query) try: await db.query_raw("""SELECT 1 FROM "MonthlyGlobalSpendPerUserPerKey" LIMIT 1""") verbose_logger.debug("MonthlyGlobalSpendPerUserPerKey Exists!") @@ -201,9 +193,7 @@ async def create_missing_views(db: _db): "user", api_key; """ - await db.execute_raw(query=sql_query) - - verbose_logger.debug("MonthlyGlobalSpendPerUserPerKey Created!") + await create_view_tolerating_race(db, "MonthlyGlobalSpendPerUserPerKey", sql_query) try: await db.query_raw("""SELECT 1 FROM "DailyTagSpend" LIMIT 1""") @@ -222,9 +212,7 @@ async def create_missing_views(db: _db): FROM "LiteLLM_SpendLogs" s GROUP BY individual_request_tag, DATE(s."startTime"); """ - await db.execute_raw(query=sql_query) - - verbose_logger.debug("DailyTagSpend Created!") + await create_view_tolerating_race(db, "DailyTagSpend", sql_query) try: await db.query_raw("""SELECT 1 FROM "Last30dTopEndUsersSpend" LIMIT 1""") diff --git a/tests/test_litellm/proxy/db/test_create_views.py b/tests/test_litellm/proxy/db/test_create_views.py index 011203c4b5a..ecc6d70123e 100644 --- a/tests/test_litellm/proxy/db/test_create_views.py +++ b/tests/test_litellm/proxy/db/test_create_views.py @@ -191,29 +191,40 @@ async def test_create_views_creates_view_on_undefined_table_error(): mock_db.execute_raw.assert_called_once() +# Every view create_missing_views is responsible for. Hard-coded rather than +# derived from the module, so adding a view without guarding it fails here. +EXPECTED_VIEW_COUNT = 8 + + @pytest.mark.asyncio -async def test_create_views_tolerates_concurrent_creator_and_continues(): - """A replica that loses the CREATE race must not abort the remaining views. +async def test_create_views_tolerates_a_concurrent_creator_on_every_view(): + """A replica that loses the CREATE race must attempt every view regardless. Regression: two proxy pods booting on a fresh DB both see every view as - absent and both issue the CREATE. Postgres fails the loser with a - duplicate-object error, which propagated out of create_missing_views and - left every later view (MonthlyGlobalSpend, DailyTagSpend, ...) uncreated, - so /global/spend* 500'd for the life of the deployment. + absent and both issue the CREATE, and Postgres fails the loser with a + duplicate-object error on whichever views the winner got to first. Any + creation site still calling execute_raw unguarded re-raises that error and + aborts the rest of the function. + + Every CREATE loses here, which is what pins the guard to all of them: an + earlier version of this fix converted only the first and the last site and + still died on MonthlyGlobalSpend against a real Postgres. Counting the + attempts is the assertion, because a partial fix simply stops early. """ from litellm.proxy.db.create_views import create_missing_views mock_db = MagicMock() mock_db.query_raw = AsyncMock(side_effect=Exception("relation does not exist")) mock_db.execute_raw = AsyncMock( - side_effect=[Exception('relation "LiteLLM_VerificationTokenView" already exists')] - + [None] * 20 + side_effect=Exception('relation "some_view" already exists') ) await create_missing_views(mock_db) - assert mock_db.execute_raw.await_count > 1, ( - "lost the race on the first view and stopped; later views were never created" + assert mock_db.execute_raw.await_count == EXPECTED_VIEW_COUNT, ( + f"every view must still be attempted when the replica loses every race; " + f"got {mock_db.execute_raw.await_count} of {EXPECTED_VIEW_COUNT}, so a " + f"creation site is still unguarded and aborted the rest" ) From 570b34988b128cd758e03b0ad384f5c4d4f8f6e8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 13 Aug 2026 14:53:59 -0700 Subject: [PATCH 3/4] refactor(proxy): type the race helper's db against a Protocol create_view_tolerating_race took the module's _db = Any. It now takes a Protocol naming the single operation it calls, so the contract is checkable at its call sites without retyping the rest of the module. Kept free of Any deliberately: an earlier version typed the Protocol's parameters as Any and pushed create_views.py from 26 basedpyright errors to 30 by adding reportExplicitAny. This version measures identical to the baseline on both create_views.py (26) and utils.py (1355). --- litellm/proxy/db/create_views.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/db/create_views.py b/litellm/proxy/db/create_views.py index 47207adee80..6743f30f400 100644 --- a/litellm/proxy/db/create_views.py +++ b/litellm/proxy/db/create_views.py @@ -1,9 +1,20 @@ -from typing import Any, Final +from typing import Any, Final, Protocol from litellm import verbose_logger _db = Any + +class SupportsExecuteRaw(Protocol): + """The one database operation create_view_tolerating_race needs. + + Narrower than the `_db = Any` the rest of this module still uses, so the + helper's contract is checkable at its call sites without retyping every + function here. + """ + + async def execute_raw(self, query: str, *args: object) -> int: ... + # Markers that indicate a view/relation does not yet exist in the database. # Keeping these in one place avoids repeating the check across all view blocks # and prevents overly broad matches (e.g. bare 'undefined' would also match @@ -15,7 +26,9 @@ _VIEW_NOT_FOUND_MARKERS: Final = ("does not exist", "no such table", "undefined _VIEW_ALREADY_EXISTS_MARKERS: Final = ("already exists", "duplicate object", "duplicate table") -async def create_view_tolerating_race(db: _db, view_name: str, ddl: str) -> None: +async def create_view_tolerating_race( + db: SupportsExecuteRaw, view_name: str, ddl: str +) -> None: """ Create a view, treating "a concurrent creator won" as success. From e448049163301ec11faf5cf0c2928e0fc77e0824 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 13 Aug 2026 15:04:43 -0700 Subject: [PATCH 4/4] style(proxy): satisfy ruff format in create_views --- litellm/proxy/db/create_views.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/db/create_views.py b/litellm/proxy/db/create_views.py index 6743f30f400..5ea9cba8018 100644 --- a/litellm/proxy/db/create_views.py +++ b/litellm/proxy/db/create_views.py @@ -15,6 +15,7 @@ class SupportsExecuteRaw(Protocol): async def execute_raw(self, query: str, *args: object) -> int: ... + # Markers that indicate a view/relation does not yet exist in the database. # Keeping these in one place avoids repeating the check across all view blocks # and prevents overly broad matches (e.g. bare 'undefined' would also match @@ -26,9 +27,7 @@ _VIEW_NOT_FOUND_MARKERS: Final = ("does not exist", "no such table", "undefined _VIEW_ALREADY_EXISTS_MARKERS: Final = ("already exists", "duplicate object", "duplicate table") -async def create_view_tolerating_race( - db: SupportsExecuteRaw, view_name: str, ddl: str -) -> None: +async def create_view_tolerating_race(db: SupportsExecuteRaw, view_name: str, ddl: str) -> None: """ Create a view, treating "a concurrent creator won" as success.