Merge 2ddb7f0553 into 2dccc0dc79

litellm/proxy/proxy_server.py

@@ -7805,12 +7805,19 @@ class ProxyConfig:
                     db_general_settings=db_general_settings.param_value,
                 )
 
-            # initialize vector stores, guardrails, etc. table in db
-            await self._init_non_llm_objects_in_db(prisma_client=prisma_client)
-
         except Exception as e:
             verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - %s", e)
 
+        # Own try: the model reconcile above must not decide whether guardrails, vector
+        # stores, MCP servers and agents get hydrated. They read their own tables.
+        try:
+            # initialize vector stores, guardrails, etc. table in db
+            await self._init_non_llm_objects_in_db(prisma_client=prisma_client)
+        except Exception as e:  # noqa: BLE001 - same swallow the single handler above already had; a hydration failure must not stop the reconcile
+            verbose_proxy_logger.exception(
+                "litellm.proxy.proxy_server.py::ProxyConfig:add_deployment non-LLM objects - %s", e
+            )
+
         # Read while the lock is still held: once it is released the next reconcile can
         # begin, and clear_cache's leading wipe would make this look like a mass drop.
         return ReconcileOutcome(

tests/test_litellm/proxy/test_proxy_server.py

@@ -1066,6 +1066,83 @@ async def test_initialize_scheduled_jobs_hydrates_mcp_when_store_model_in_db_fal
     mock_proxy_config.init_mcp_servers_from_db.assert_awaited_once()
 
 
+@pytest.mark.asyncio
+async def test_add_deployment_hydrates_non_llm_objects_when_model_reconcile_fails():
+    """
+    Regression: _add_deployment_locked ran the model reconcile and the non-LLM object
+    hydration inside one try/except. A failure in the reconcile skipped guardrails,
+    vector stores, MCP servers, agents and pass-through endpoints for that pass, and
+    the swallowed exception left only a log line behind. add_deployment also runs on a
+    timer, so a transient DB error dropped every non-LLM object until a later pass.
+    """
+    from litellm.proxy.proxy_server import ProxyConfig
+    from litellm.proxy.utils import ProxyLogging
+
+    class FailingReconcileProxyConfig(ProxyConfig):
+        """Reconcile fails the way a DB outage makes it fail; hydration records itself."""
+
+        def __init__(self):
+            super().__init__()
+            self.non_llm_hydrations = 0
+
+        async def _get_models_from_db(self, prisma_client):
+            raise RuntimeError("db unavailable")
+
+        async def _init_non_llm_objects_in_db(self, prisma_client):
+            self.non_llm_hydrations += 1
+
+    proxy_config = FailingReconcileProxyConfig()
+
+    outcome = await proxy_config._add_deployment_locked(
+        prisma_client=MagicMock(),
+        proxy_logging_obj=MagicMock(spec=ProxyLogging),
+    )
+
+    assert proxy_config.non_llm_hydrations == 1
+    assert outcome.still_desired is None
+
+
+@pytest.mark.asyncio
+async def test_add_deployment_survives_a_non_llm_hydration_failure():
+    """
+    The new hydration boundary must behave like the reconcile one: log the error and
+    carry on, so a failing guardrail or vector-store table cannot take down the whole
+    reconcile pass or escape into the scheduler job that calls it.
+    """
+    from litellm.proxy.proxy_server import ProxyConfig
+    from litellm.proxy.utils import ProxyLogging
+
+    class FailingHydrationProxyConfig(ProxyConfig):
+        """Reconcile succeeds, hydration raises the way an unavailable table does."""
+
+        def __init__(self):
+            super().__init__()
+            self.reconcile_ran = False
+            self.hydration_attempted = False
+
+        async def _get_models_from_db(self, prisma_client):
+            return []
+
+        async def _update_llm_router(self, new_models, proxy_logging_obj):
+            self.reconcile_ran = True
+            return frozenset()
+
+        async def _init_non_llm_objects_in_db(self, prisma_client):
+            self.hydration_attempted = True
+            raise RuntimeError("guardrail table unavailable")
+
+    proxy_config = FailingHydrationProxyConfig()
+
+    outcome = await proxy_config._add_deployment_locked(
+        prisma_client=MagicMock(),
+        proxy_logging_obj=MagicMock(spec=ProxyLogging),
+    )
+
+    assert proxy_config.reconcile_ran is True
+    assert proxy_config.hydration_attempted is True
+    assert outcome.still_desired == frozenset()
+
+
 @pytest.mark.asyncio
 async def test_init_mcp_servers_from_db_respects_supported_db_objects(monkeypatch):
     """