diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 7f5983d5799..28c83d904c7 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -4421,6 +4421,8 @@ def _request_app(request: Request) -> FastAPI: async def _get_pass_through_endpoints_from_db( endpoint_id: str | None = None, user_api_key_dict: UserAPIKeyAuth | None = None, + *, + raise_on_read_error: bool = False, ) -> list[PassThroughGenericEndpoint]: from litellm.proxy._types import LitellmUserRoles from litellm.proxy.proxy_server import get_config_general_settings @@ -4431,7 +4433,11 @@ async def _get_pass_through_endpoints_from_db( response: Final[ConfigFieldInfo] = await get_config_general_settings( field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict ) + except HTTPException: + return [] except Exception: + if raise_on_read_error: + raise return [] pass_through_endpoint_data: Final = _config_field_endpoints(response) @@ -4918,5 +4924,9 @@ async def initialize_pass_through_endpoints_in_db(): """ Gets all pass-through endpoints from db and initializes them in the proxy server. """ - pass_through_endpoints: Final = await _get_pass_through_endpoints_from_db() + try: + pass_through_endpoints: Final = await _get_pass_through_endpoints_from_db(raise_on_read_error=True) + except Exception: # noqa: BLE001 # a failed read keeps the routes from the last good read until the next pass + verbose_proxy_logger.exception("Could not read pass-through endpoints from the DB, keeping the current routes") + return await initialize_pass_through_endpoints(pass_through_endpoints=pass_through_endpoints) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 22e5bd6b3bd..e120d95d7ed 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8856,12 +8856,20 @@ class ProxyConfig: db_general_settings=db_general_settings.param_value, ) - # initialize vector stores, guardrails, etc. table in db - await self._init_non_llm_objects_in_db(prisma_client=prisma_client) + if self._should_load_db_object(object_type="pass_through_endpoints"): + await self._init_pass_through_endpoints_in_db() except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - %s", e) + try: + # initialize vector stores, guardrails, etc. table in db + await self._init_non_llm_objects_in_db(prisma_client=prisma_client) + except Exception as e: # noqa: BLE001 # log and carry on, like the reconcile handler above + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.py::ProxyConfig:add_deployment non-LLM objects - %s", e + ) + # Read while the lock is still held: once it is released the next reconcile can # begin, and clear_cache's leading wipe would make this look like a mass drop. return ReconcileOutcome( @@ -8950,9 +8958,6 @@ class ProxyConfig: if self._should_load_db_object(object_type="agents"): await self._init_agents_in_db(prisma_client=prisma_client) - if self._should_load_db_object(object_type="pass_through_endpoints"): - await self._init_pass_through_endpoints_in_db() - if self._should_load_db_object(object_type="prompts"): await self._init_prompts_in_db(prisma_client=prisma_client) diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 824f04aee75..cabe7e405e1 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -13,6 +13,7 @@ from contextlib import ExitStack, contextmanager from dataclasses import dataclass from datetime import datetime from io import BytesIO +from pathlib import Path from types import MappingProxyType, ModuleType, SimpleNamespace from typing import Final, Optional, cast from unittest.mock import AsyncMock, MagicMock, call, patch @@ -11715,6 +11716,105 @@ async def test_a_deleted_db_pass_through_stops_serving_on_the_next_db_sync(tmp_p assert [str(request.url) for request in config_upstream] == ["http://config-upstream.test/api"] +class _UnreadableConfigTable(_InMemoryConfigTable): + async def get_generic_data(self, key: str, value: str, table_name: str) -> _StoredConfigRow | None: + raise RuntimeError("permission denied for table LiteLLM_Config") + + async def find_first(self, where: Mapping[str, str]) -> _StoredConfigRow | None: + raise RuntimeError("permission denied for table LiteLLM_Config") + + async def find_unique(self, where: Mapping[str, str]) -> _StoredConfigRow | None: + raise RuntimeError("permission denied for table LiteLLM_Config") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("config_table_after", "expected_status", "expected_upstream"), + [ + (_UnreadableConfigTable({}), 200, ["http://db-upstream.test/api"]), + (_InMemoryConfigTable({"general_settings": {"pass_through_endpoints": []}}), 404, []), + (_InMemoryConfigTable({"general_settings": {}}), 404, []), + ], + ids=["read-fails", "emptied-list", "dropped-key"], +) +async def test_the_reload_job_keeps_db_pass_throughs_only_when_the_db_read_fails( + tmp_path, monkeypatch, config_table_after, expected_status, expected_upstream +): + from litellm.proxy import proxy_server + from litellm.proxy.utils import ProxyLogging + + proxy: Final = await _boot_db_backed_proxy( + tmp_path, + monkeypatch, + config_pass_through_endpoints=[], + db_pass_through_endpoints=[ + {"id": "db-kept", "path": "/db-kept", "target": "http://db-upstream.test/api", "auth": False} + ], + master_key="sk-pass-through-master", + ) + await _run_db_sync_cycle(proxy) + served_before, _ = await _send_through_proxy("/db-kept", {}) + + monkeypatch.setitem(proxy_server.general_settings, "supported_db_objects", ["pass_through_endpoints"]) + monkeypatch.setattr(proxy_server, "prisma_client", config_table_after) + await proxy.proxy_config.add_deployment( + prisma_client=config_table_after, proxy_logging_obj=MagicMock(spec=ProxyLogging) + ) + served_after, db_upstream = await _send_through_proxy("/db-kept", {}) + + assert (served_before.status_code, served_after.status_code) == (200, expected_status) + assert [str(request.url) for request in db_upstream] == expected_upstream + + +@pytest.mark.asyncio +async def test_a_failed_model_reconcile_never_serves_a_new_db_pass_through_with_its_old_auth_flag( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.utils import ProxyLogging, invalidate_config_param + + proxy: Final = await _boot_db_backed_proxy( + tmp_path, + monkeypatch, + config_pass_through_endpoints=[], + db_pass_through_endpoints=[ + {"id": "db-guarded", "path": "/db-guarded", "target": "http://public-upstream.test/api", "auth": False} + ], + master_key="sk-pass-through-master", + ) + await _run_db_sync_cycle(proxy) + proxy.config_table.rows["general_settings"] = { + "pass_through_endpoints": [ + { + "id": "db-guarded", + "path": "/db-guarded", + "target": "http://protected-upstream.test/api", + "headers": {"x-upstream-key": "upstream-secret"}, + "auth": True, + } + ] + } + await invalidate_config_param("general_settings") + + async def failing_router_update(new_models: object, proxy_logging_obj: object) -> frozenset[str]: + raise RuntimeError("router update failed") + + monkeypatch.setitem( + proxy_server.proxy_config.settings, "supported_db_objects", ["models", "pass_through_endpoints"] + ) + monkeypatch.setattr(proxy_server.proxy_config, "_update_llm_router", failing_router_update) + assert proxy_server.prisma_client is not None + await proxy_server.proxy_config.add_deployment( + prisma_client=proxy_server.prisma_client, proxy_logging_obj=MagicMock(spec=ProxyLogging) + ) + anonymous, anonymous_upstream = await _send_through_proxy("/db-guarded", {}) + + assert anonymous.status_code == 200 + assert [(str(request.url), request.headers.get("x-upstream-key")) for request in anonymous_upstream] == [ + ("http://public-upstream.test/api", None) + ] + + @pytest.mark.asyncio async def test_config_pass_through_reads_its_custom_key_header_when_the_db_holds_pass_throughs(tmp_path, monkeypatch): proxy: Final = await _boot_db_backed_proxy( diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index d09008d62bc..e720e3390da 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1169,6 +1169,87 @@ async def test_initialize_scheduled_jobs_hydrates_mcp_when_store_model_in_db_fal mock_proxy_config.init_mcp_servers_from_db.assert_awaited_once() +@pytest.mark.asyncio +async def test_add_deployment_hydrates_non_llm_objects_when_model_reconcile_fails(): + """ + Regression: add_deployment_locked ran the model reconcile and the non-LLM object + hydration inside one try/except. A failure in the reconcile skipped guardrails, + vector stores, MCP servers, agents and pass-through endpoints for that pass, and + the swallowed exception left only a log line behind. add_deployment also runs on a + timer, so while the reconcile kept failing, no pass loaded new or changed non-LLM + objects. + """ + from litellm.proxy.proxy_server import ProxyConfig + from litellm.proxy.utils import ProxyLogging + + class FailingReconcileProxyConfig(ProxyConfig): + """The router update raises inside the shared try, before the hydration; hydration records itself.""" + + def __init__(self): + super().__init__() + self.non_llm_hydrations = 0 + + async def get_models_from_db(self, prisma_client): + return [] + + async def _update_llm_router(self, new_models, proxy_logging_obj): + raise RuntimeError("router update failed") + + async def _init_non_llm_objects_in_db(self, prisma_client): + self.non_llm_hydrations += 1 + + proxy_config = FailingReconcileProxyConfig() + + outcome = await proxy_config.add_deployment_locked( + prisma_client=MagicMock(), + proxy_logging_obj=MagicMock(spec=ProxyLogging), + ) + + assert proxy_config.non_llm_hydrations == 1 + assert outcome.still_desired is None + + +@pytest.mark.asyncio +async def test_add_deployment_survives_a_non_llm_hydration_failure(): + """ + The new hydration boundary must behave like the reconcile one: log the error and + carry on, so a failing guardrail or vector-store table cannot take down the whole + reconcile pass or escape into the scheduler job that calls it. + """ + from litellm.proxy.proxy_server import ProxyConfig + from litellm.proxy.utils import ProxyLogging + + class FailingHydrationProxyConfig(ProxyConfig): + """Reconcile succeeds, hydration raises the way an unavailable table does.""" + + def __init__(self): + super().__init__() + self.reconcile_ran = False + self.hydration_attempted = False + + async def get_models_from_db(self, prisma_client): + return [] + + async def _update_llm_router(self, new_models, proxy_logging_obj): + self.reconcile_ran = True + return frozenset() + + async def _init_non_llm_objects_in_db(self, prisma_client): + self.hydration_attempted = True + raise RuntimeError("guardrail table unavailable") + + proxy_config = FailingHydrationProxyConfig() + + outcome = await proxy_config.add_deployment_locked( + prisma_client=MagicMock(get_generic_data=AsyncMock(return_value=None)), + proxy_logging_obj=MagicMock(spec=ProxyLogging), + ) + + assert proxy_config.reconcile_ran is True + assert proxy_config.hydration_attempted is True + assert outcome.still_desired == frozenset() + + @pytest.mark.asyncio async def test_init_mcp_servers_from_db_respects_supported_db_objects(monkeypatch): """