Merge a7c9f1c4c0 into 523b87ec71

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -4421,6 +4421,8 @@ def _request_app(request: Request) -> FastAPI:
 async def _get_pass_through_endpoints_from_db(
     endpoint_id: str | None = None,
     user_api_key_dict: UserAPIKeyAuth | None = None,
+    *,
+    raise_on_read_error: bool = False,
 ) -> list[PassThroughGenericEndpoint]:
     from litellm.proxy._types import LitellmUserRoles
     from litellm.proxy.proxy_server import get_config_general_settings
@@ -4431,7 +4433,11 @@ async def _get_pass_through_endpoints_from_db(
         response: Final[ConfigFieldInfo] = await get_config_general_settings(
             field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict
         )
+    except HTTPException:
+        return []
     except Exception:
+        if raise_on_read_error:
+            raise
         return []
 
     pass_through_endpoint_data: Final = _config_field_endpoints(response)
@@ -4918,5 +4924,9 @@ async def initialize_pass_through_endpoints_in_db():
     """
     Gets all pass-through endpoints from db and initializes them in the proxy server.
     """
-    pass_through_endpoints: Final = await _get_pass_through_endpoints_from_db()
+    try:
+        pass_through_endpoints: Final = await _get_pass_through_endpoints_from_db(raise_on_read_error=True)
+    except Exception:  # noqa: BLE001  # a failed read keeps the routes from the last good read until the next pass
+        verbose_proxy_logger.exception("Could not read pass-through endpoints from the DB, keeping the current routes")
+        return
     await initialize_pass_through_endpoints(pass_through_endpoints=pass_through_endpoints)

litellm/proxy/proxy_server.py

@@ -8856,12 +8856,20 @@ class ProxyConfig:
                     db_general_settings=db_general_settings.param_value,
                 )
 
-            # initialize vector stores, guardrails, etc. table in db
-            await self._init_non_llm_objects_in_db(prisma_client=prisma_client)
+            if self._should_load_db_object(object_type="pass_through_endpoints"):
+                await self._init_pass_through_endpoints_in_db()
 
         except Exception as e:
             verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - %s", e)
 
+        try:
+            # initialize vector stores, guardrails, etc. table in db
+            await self._init_non_llm_objects_in_db(prisma_client=prisma_client)
+        except Exception as e:  # noqa: BLE001  # log and carry on, like the reconcile handler above
+            verbose_proxy_logger.exception(
+                "litellm.proxy.proxy_server.py::ProxyConfig:add_deployment non-LLM objects - %s", e
+            )
+
         # Read while the lock is still held: once it is released the next reconcile can
         # begin, and clear_cache's leading wipe would make this look like a mass drop.
         return ReconcileOutcome(
@@ -8950,9 +8958,6 @@ class ProxyConfig:
         if self._should_load_db_object(object_type="agents"):
             await self._init_agents_in_db(prisma_client=prisma_client)
 
-        if self._should_load_db_object(object_type="pass_through_endpoints"):
-            await self._init_pass_through_endpoints_in_db()
-
         if self._should_load_db_object(object_type="prompts"):
             await self._init_prompts_in_db(prisma_client=prisma_client)
 

tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py

@@ -13,6 +13,7 @@ from contextlib import ExitStack, contextmanager
 from dataclasses import dataclass
 from datetime import datetime
 from io import BytesIO
+from pathlib import Path
 from types import MappingProxyType, ModuleType, SimpleNamespace
 from typing import Final, Optional, cast
 from unittest.mock import AsyncMock, MagicMock, call, patch
@@ -11715,6 +11716,105 @@ async def test_a_deleted_db_pass_through_stops_serving_on_the_next_db_sync(tmp_p
     assert [str(request.url) for request in config_upstream] == ["http://config-upstream.test/api"]
 
 
+class _UnreadableConfigTable(_InMemoryConfigTable):
+    async def get_generic_data(self, key: str, value: str, table_name: str) -> _StoredConfigRow | None:
+        raise RuntimeError("permission denied for table LiteLLM_Config")
+
+    async def find_first(self, where: Mapping[str, str]) -> _StoredConfigRow | None:
+        raise RuntimeError("permission denied for table LiteLLM_Config")
+
+    async def find_unique(self, where: Mapping[str, str]) -> _StoredConfigRow | None:
+        raise RuntimeError("permission denied for table LiteLLM_Config")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("config_table_after", "expected_status", "expected_upstream"),
+    [
+        (_UnreadableConfigTable({}), 200, ["http://db-upstream.test/api"]),
+        (_InMemoryConfigTable({"general_settings": {"pass_through_endpoints": []}}), 404, []),
+        (_InMemoryConfigTable({"general_settings": {}}), 404, []),
+    ],
+    ids=["read-fails", "emptied-list", "dropped-key"],
+)
+async def test_the_reload_job_keeps_db_pass_throughs_only_when_the_db_read_fails(
+    tmp_path, monkeypatch, config_table_after, expected_status, expected_upstream
+):
+    from litellm.proxy import proxy_server
+    from litellm.proxy.utils import ProxyLogging
+
+    proxy: Final = await _boot_db_backed_proxy(
+        tmp_path,
+        monkeypatch,
+        config_pass_through_endpoints=[],
+        db_pass_through_endpoints=[
+            {"id": "db-kept", "path": "/db-kept", "target": "http://db-upstream.test/api", "auth": False}
+        ],
+        master_key="sk-pass-through-master",
+    )
+    await _run_db_sync_cycle(proxy)
+    served_before, _ = await _send_through_proxy("/db-kept", {})
+
+    monkeypatch.setitem(proxy_server.general_settings, "supported_db_objects", ["pass_through_endpoints"])
+    monkeypatch.setattr(proxy_server, "prisma_client", config_table_after)
+    await proxy.proxy_config.add_deployment(
+        prisma_client=config_table_after, proxy_logging_obj=MagicMock(spec=ProxyLogging)
+    )
+    served_after, db_upstream = await _send_through_proxy("/db-kept", {})
+
+    assert (served_before.status_code, served_after.status_code) == (200, expected_status)
+    assert [str(request.url) for request in db_upstream] == expected_upstream
+
+
+@pytest.mark.asyncio
+async def test_a_failed_model_reconcile_never_serves_a_new_db_pass_through_with_its_old_auth_flag(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from litellm.proxy import proxy_server
+    from litellm.proxy.utils import ProxyLogging, invalidate_config_param
+
+    proxy: Final = await _boot_db_backed_proxy(
+        tmp_path,
+        monkeypatch,
+        config_pass_through_endpoints=[],
+        db_pass_through_endpoints=[
+            {"id": "db-guarded", "path": "/db-guarded", "target": "http://public-upstream.test/api", "auth": False}
+        ],
+        master_key="sk-pass-through-master",
+    )
+    await _run_db_sync_cycle(proxy)
+    proxy.config_table.rows["general_settings"] = {
+        "pass_through_endpoints": [
+            {
+                "id": "db-guarded",
+                "path": "/db-guarded",
+                "target": "http://protected-upstream.test/api",
+                "headers": {"x-upstream-key": "upstream-secret"},
+                "auth": True,
+            }
+        ]
+    }
+    await invalidate_config_param("general_settings")
+
+    async def failing_router_update(new_models: object, proxy_logging_obj: object) -> frozenset[str]:
+        raise RuntimeError("router update failed")
+
+    monkeypatch.setitem(
+        proxy_server.proxy_config.settings, "supported_db_objects", ["models", "pass_through_endpoints"]
+    )
+    monkeypatch.setattr(proxy_server.proxy_config, "_update_llm_router", failing_router_update)
+    assert proxy_server.prisma_client is not None
+    await proxy_server.proxy_config.add_deployment(
+        prisma_client=proxy_server.prisma_client, proxy_logging_obj=MagicMock(spec=ProxyLogging)
+    )
+    anonymous, anonymous_upstream = await _send_through_proxy("/db-guarded", {})
+
+    assert anonymous.status_code == 200
+    assert [(str(request.url), request.headers.get("x-upstream-key")) for request in anonymous_upstream] == [
+        ("http://public-upstream.test/api", None)
+    ]
+
+
 @pytest.mark.asyncio
 async def test_config_pass_through_reads_its_custom_key_header_when_the_db_holds_pass_throughs(tmp_path, monkeypatch):
     proxy: Final = await _boot_db_backed_proxy(

tests/unit/proxy/test_proxy_server_endpoints_and_startup.py

@@ -1169,6 +1169,87 @@ async def test_initialize_scheduled_jobs_hydrates_mcp_when_store_model_in_db_fal
     mock_proxy_config.init_mcp_servers_from_db.assert_awaited_once()
 
 
+@pytest.mark.asyncio
+async def test_add_deployment_hydrates_non_llm_objects_when_model_reconcile_fails():
+    """
+    Regression: add_deployment_locked ran the model reconcile and the non-LLM object
+    hydration inside one try/except. A failure in the reconcile skipped guardrails,
+    vector stores, MCP servers, agents and pass-through endpoints for that pass, and
+    the swallowed exception left only a log line behind. add_deployment also runs on a
+    timer, so while the reconcile kept failing, no pass loaded new or changed non-LLM
+    objects.
+    """
+    from litellm.proxy.proxy_server import ProxyConfig
+    from litellm.proxy.utils import ProxyLogging
+
+    class FailingReconcileProxyConfig(ProxyConfig):
+        """The router update raises inside the shared try, before the hydration; hydration records itself."""
+
+        def __init__(self):
+            super().__init__()
+            self.non_llm_hydrations = 0
+
+        async def get_models_from_db(self, prisma_client):
+            return []
+
+        async def _update_llm_router(self, new_models, proxy_logging_obj):
+            raise RuntimeError("router update failed")
+
+        async def _init_non_llm_objects_in_db(self, prisma_client):
+            self.non_llm_hydrations += 1
+
+    proxy_config = FailingReconcileProxyConfig()
+
+    outcome = await proxy_config.add_deployment_locked(
+        prisma_client=MagicMock(),
+        proxy_logging_obj=MagicMock(spec=ProxyLogging),
+    )
+
+    assert proxy_config.non_llm_hydrations == 1
+    assert outcome.still_desired is None
+
+
+@pytest.mark.asyncio
+async def test_add_deployment_survives_a_non_llm_hydration_failure():
+    """
+    The new hydration boundary must behave like the reconcile one: log the error and
+    carry on, so a failing guardrail or vector-store table cannot take down the whole
+    reconcile pass or escape into the scheduler job that calls it.
+    """
+    from litellm.proxy.proxy_server import ProxyConfig
+    from litellm.proxy.utils import ProxyLogging
+
+    class FailingHydrationProxyConfig(ProxyConfig):
+        """Reconcile succeeds, hydration raises the way an unavailable table does."""
+
+        def __init__(self):
+            super().__init__()
+            self.reconcile_ran = False
+            self.hydration_attempted = False
+
+        async def get_models_from_db(self, prisma_client):
+            return []
+
+        async def _update_llm_router(self, new_models, proxy_logging_obj):
+            self.reconcile_ran = True
+            return frozenset()
+
+        async def _init_non_llm_objects_in_db(self, prisma_client):
+            self.hydration_attempted = True
+            raise RuntimeError("guardrail table unavailable")
+
+    proxy_config = FailingHydrationProxyConfig()
+
+    outcome = await proxy_config.add_deployment_locked(
+        prisma_client=MagicMock(get_generic_data=AsyncMock(return_value=None)),
+        proxy_logging_obj=MagicMock(spec=ProxyLogging),
+    )
+
+    assert proxy_config.reconcile_ran is True
+    assert proxy_config.hydration_attempted is True
+    assert outcome.still_desired == frozenset()
+
+
 @pytest.mark.asyncio
 async def test_init_mcp_servers_from_db_respects_supported_db_objects(monkeypatch):
     """