From 460f336d8a9f2d94c515a978b324bebcbd7296e1 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 18 Sep 2026 02:00:08 -0700 Subject: [PATCH] fix(proxy): rebuild pass-through routes whenever the resolved list changes The reload only re-registered pass-through endpoints when the stored row still carried the key, so deleting the row left the deleted routes serving traffic until the process restarted. It now compares the resolved list before and after the row is applied and rebuilds on any difference, including a deletion that resolves back to the config file's list or to nothing. This matches what _apply_retention_settings already does with the retention values, so the two reload effects no longer disagree about what counts as a change. The tests assert the proxy's registry of live pass-through routes, which is what decides whether a request is routed upstream or falls through to the auth error, rather than that the registration helper was called. --- litellm/proxy/proxy_server.py | 19 ++++-- tests/test_litellm/proxy/test_proxy_server.py | 64 +++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 02695f7f1fb..1b7807241d4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7188,12 +7188,14 @@ class ProxyConfig: self.settings.load_yaml(_as_settings_mapping(general_settings)) cache_size_was_db: Final = self.settings.source("user_api_key_cache_max_size") == "db" previous_retention_values: Final = self._resolved_retention_values() + previous_pass_through_endpoints: Final = self.settings.get("pass_through_endpoints") self.settings.apply_db_row("general_settings", db_general_settings) _bind_general_settings_store(self.settings) await self._apply_general_settings_side_effects( db_general_settings, cache_size_was_db, previous_retention_values, + previous_pass_through_endpoints, ) def _resolved_retention_values(self) -> tuple[SettingsJsonValue | None, ...]: @@ -7211,10 +7213,11 @@ class ProxyConfig: db_values: Mapping[str, SettingsJsonValue], cache_size_was_db: bool, previous_retention_values: tuple[SettingsJsonValue | None, ...], + previous_pass_through_endpoints: SettingsJsonValue | None, ) -> None: effects: Final = ( self._apply_alerting_settings, - self._apply_pass_through_settings, + partial(self._apply_pass_through_settings, previous_endpoints=previous_pass_through_endpoints), self._apply_boolean_settings, partial(self._apply_cache_size_setting, cache_size_was_db=cache_size_was_db), self._apply_store_model_in_db_setting, @@ -7247,10 +7250,18 @@ class ProxyConfig: if "plugins" in db_values and self.settings.source("plugins") == "db": register_plugins_from_config(self.settings) - async def _apply_pass_through_settings(self, db_values: Mapping[str, SettingsJsonValue]) -> None: + async def _apply_pass_through_settings( + self, + db_values: Mapping[str, SettingsJsonValue], + previous_endpoints: SettingsJsonValue | None, + ) -> None: + del db_values resolved_endpoints: Final = self.settings.get("pass_through_endpoints") - if "pass_through_endpoints" in db_values and isinstance(resolved_endpoints, list): - await initialize_pass_through_endpoints(pass_through_endpoints=resolved_endpoints) + if resolved_endpoints == previous_endpoints: + return + await initialize_pass_through_endpoints( + pass_through_endpoints=resolved_endpoints if isinstance(resolved_endpoints, list) else [] + ) async def _apply_boolean_settings(self, db_values: Mapping[str, SettingsJsonValue]) -> None: for key in ( diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 18eb739180d..bc7ae556e7b 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -9,6 +9,7 @@ import socket import subprocess import time import types +import uuid from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Final @@ -7466,6 +7467,69 @@ async def test_update_general_settings_db_pass_through_endpoint_cannot_override_ assert still_open.api_key is None +@pytest.mark.asyncio +async def test_deleting_the_stored_pass_through_row_takes_the_route_out_of_service(): + """A pass-through route the database declared has to stop serving when that row is + deleted. The proxy's own registry of live pass-through routes is what decides whether + a request is routed upstream or falls through to the auth error, so it has to lose the + entry on the reload rather than at the next process restart.""" + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + from litellm.proxy.proxy_server import ProxyConfig + + path: Final = f"/v1/deleted-{uuid.uuid4().hex[:8]}" + db_endpoint: Final = {"id": "db-1", "path": path, "target": "https://example.com/post"} + + def live_routes() -> set[str]: + return {route for route in InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() if path in route} + + settings: Final = patch("litellm.proxy.proxy_server.general_settings", {}) # test-quality-ok: the method reads this module global; no injection seam + yaml_endpoints: Final = patch("litellm.proxy.proxy_server.config_passthrough_endpoints", None) # test-quality-ok: module global holding the YAML endpoints; this case has none + with settings, yaml_endpoints: + pc = ProxyConfig() + await pc._update_general_settings(db_general_settings={"pass_through_endpoints": [db_endpoint]}) + assert live_routes(), "the stored endpoint should be serving before the row is deleted" + + await pc._update_general_settings(db_general_settings={}) + + assert live_routes() == set() + + +@pytest.mark.asyncio +async def test_a_stored_pass_through_row_never_disturbs_the_config_declared_routes(): + """``pass_through_endpoints`` is config-owned once the file declares it, so writing and then + deleting a stored row resolves to the same list both times and the config file's routes keep + serving untouched. The stored entry never gets a route of its own.""" + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + initialize_pass_through_endpoints, + ) + from litellm.proxy.proxy_server import ProxyConfig + + marker: Final = uuid.uuid4().hex[:8] + config_path: Final = f"/v1/kept-{marker}" + db_path: Final = f"/v1/ignored-{marker}" + config_endpoint: Final = {"id": f"cfg-{marker}", "path": config_path, "target": "https://example.com/post"} + db_endpoint: Final = {"id": f"db-{marker}", "path": db_path, "target": "https://example.com/post"} + + def live_paths() -> set[str]: + registered: Final = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() + return {path for path in (config_path, db_path) if any(path in route for route in registered)} + + settings: Final = patch("litellm.proxy.proxy_server.general_settings", {"pass_through_endpoints": [config_endpoint]}) # test-quality-ok: the method reads this module global; no injection seam + yaml_endpoints: Final = patch("litellm.proxy.proxy_server.config_passthrough_endpoints", [config_endpoint]) # test-quality-ok: module global holding the YAML endpoints the reload merges in + with settings, yaml_endpoints: + await initialize_pass_through_endpoints(pass_through_endpoints=[config_endpoint]) + assert live_paths() == {config_path} + + pc = ProxyConfig() + await pc._update_general_settings(db_general_settings={"pass_through_endpoints": [db_endpoint]}) + assert live_paths() == {config_path} + + await pc._update_general_settings(db_general_settings={}) + + assert live_paths() == {config_path} + + def _fill_user_api_key_cache(cache: DualCache, count: int) -> None: for index in range(count): cache.set_cache(key=f"key-{index}", value={"token": f"key-{index}"}, local_only=True)