From 4614faaa25da389a038664504e3d0c22f2d90608 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:47:07 -0700 Subject: [PATCH] fix(proxy): keep eu/jev pass-through routing stable during updates and method mismatches Swap an updated pass-through's routes with no await between removal and re-registration, so the built-in eu/jev route never sees the path as unconfigured mid-update. The built-in eu/jev handler no longer takes a configured route's target, and the built-in route also yields on a method-only match, so GET and PUT on the no-slash form of a slash route redirect again. --- .../pass_through_endpoints.py | 16 +++--- .../test_llm_pass_through_endpoints.py | 51 +++++++++++++++++++ .../test_pass_through_endpoints.py | 50 ++++++++++++++++++ 3 files changed, 110 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 233d7bf6a43..4ba56e0ca52 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2252,11 +2252,15 @@ def create_pass_through_route( detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.", ) - passthrough_params: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route( - route=path, method=request.method + serves_configured_route: Final = custom_llm_provider != "eu_jev" + passthrough_params: Final = ( + InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path, method=request.method) + if serves_configured_route + else None ) if ( - passthrough_params is None + serves_configured_route + and passthrough_params is None and InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path) is not None ): raise HTTPException( @@ -3532,7 +3536,7 @@ class InitPassThroughEndpointHelpers: class YieldsToConfiguredPassThroughRoute(APIRoute): def matches(self, scope: Scope) -> tuple[Match, Scope]: match, child_scope = super().matches(scope) - if match is not Match.FULL: + if match is Match.NONE: return match, child_scope configured: Final = InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash( route=get_request_route(Request(scope)) @@ -3999,9 +4003,6 @@ async def update_pass_through_endpoints( # Update the list pass_through_endpoint_data[endpoint_index] = endpoint_dict - # Remove old routes from registry before they get re-registered - InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id) - ## Update db updated_data: Final = ConfigFieldUpdate( field_name="pass_through_endpoints", @@ -4016,6 +4017,7 @@ async def update_pass_through_endpoints( _custom_headers = await set_env_variables_in_header(custom_headers=_custom_headers) route_app: Final = _request_app(request) + InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id) if updated_endpoint.include_subpath: InitPassThroughEndpointHelpers.add_subpath_route( app=route_app, diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 70e1da1aa7b..a0b81222e3e 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8803,6 +8803,57 @@ class TestEuJevPassthroughRoute: assert (response.status_code, response.json()) == (200, {"operator": True}) assert route.calls.last.request.headers["x-operator"] == "on" + @pytest.mark.parametrize("method", ["GET", "PUT"]) + def test_a_pass_through_configured_with_a_trailing_slash_still_gets_other_methods_without_one( + self, operator_app: Callable[[list[str], str], FastAPI], method: str + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["GET", "POST", "PUT"], "/eu_jev/v1/systemone/")) as client, + ): + route: Final = upstream.route(method=method, url="https://operator.example/decide").respond( + 200, json={"operator": True} + ) + response: Final = client.request(method, "/eu_jev/v1/systemone", json={"state": "x"}) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + + def test_a_request_the_built_in_route_took_is_not_forwarded_with_a_pass_through_added_meanwhile( + self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") + app: Final = operator_app(["POST"], "/eu_jev/v1/systemone") + InitPassThroughEndpointHelpers.remove_endpoint_routes("operator-eu-jev") + + def add_the_pass_through_after_routing() -> UserAPIKeyAuth: + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path="/eu_jev/v1/systemone", + target="https://operator.example/decide", + custom_headers={"x-operator": "on"}, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=["POST"], + ) + return UserAPIKeyAuth(api_key="sk-virtual") + + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, add_the_pass_through_after_routing) + with respx.mock(assert_all_called=False) as upstream, TestClient(app) as client: + built_in: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + operator: Final = upstream.route(host="operator.example").respond(200, json={"operator": True}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "eu-jeff-latest", "state": "x"}) + + assert (response.status_code, response.json()) == (200, {"answers": {}}) + assert built_in.calls.last.request.headers["authorization"] == "Bearer eu-jev-test-key" + assert not operator.called + def test_a_deleted_pass_through_at_the_same_path_stops_forwarding_when_built_in_routes_come_first( self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 2b121b21c7f..709feea18f0 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -2978,6 +2978,56 @@ async def test_update_pass_through_endpoint_preserves_auth_false(): assert RouteChecks.is_auth_enforced_pass_through_route(route="/public-passthrough", method="POST") is False +@pytest.mark.asyncio +async def test_update_pass_through_endpoint_keeps_serving_the_old_route_until_the_new_one_is_registered(): + from litellm.proxy._types import ConfigFieldInfo, PassThroughGenericEndpoint, UserAPIKeyAuth + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + update_pass_through_endpoints, + ) + + def registered_target() -> object: + registered = InitPassThroughEndpointHelpers.get_registered_pass_through_route(route="/operator") or {} + return registered.get("passthrough_params", {}).get("target") + + targets_during_db_write: list[object] = [] + + async def record_registered_target(**_: object) -> None: + targets_during_db_write.append(registered_target()) + + with ( + patch("litellm.proxy.proxy_server.get_config_general_settings") as mock_get_config, + patch("litellm.proxy.proxy_server.update_config_general_settings", side_effect=record_registered_target), + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", {}), + ): + InitPassThroughEndpointHelpers.add_exact_path_route( + app=FastAPI(), + path="/operator", + target="http://old.example/api", + custom_headers=None, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-route", + methods=["POST"], + ) + mock_get_config.return_value = ConfigFieldInfo( + field_name="pass_through_endpoints", + field_value=[ + {"id": "operator-route", "path": "/operator", "target": "http://old.example/api", "methods": ["POST"]} + ], + ) + await update_pass_through_endpoints( + endpoint_id="operator-route", + data=PassThroughGenericEndpoint(path="/operator", target="http://new.example/api", methods=["POST"]), + request=MagicMock(spec=Request), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + ) + + assert (targets_during_db_write, registered_target()) == (["http://old.example/api"], "http://new.example/api") + + @pytest.mark.asyncio async def test_update_pass_through_endpoint_not_found(): """