mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): keep eu/jev pass-through routing stable during updates and method mismatches
Swap an updated pass-through's routes with no await between removal and re-registration, so the built-in eu/jev route never sees the path as unconfigured mid-update. The built-in eu/jev handler no longer takes a configured route's target, and the built-in route also yields on a method-only match, so GET and PUT on the no-slash form of a slash route redirect again.
This commit is contained in:
parent
0f3624fc5a
commit
4614faaa25
3 changed files with 110 additions and 7 deletions
|
|
@ -2252,11 +2252,15 @@ def create_pass_through_route(
|
|||
detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.",
|
||||
)
|
||||
|
||||
passthrough_params: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route(
|
||||
route=path, method=request.method
|
||||
serves_configured_route: Final = custom_llm_provider != "eu_jev"
|
||||
passthrough_params: Final = (
|
||||
InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path, method=request.method)
|
||||
if serves_configured_route
|
||||
else None
|
||||
)
|
||||
if (
|
||||
passthrough_params is None
|
||||
serves_configured_route
|
||||
and passthrough_params is None
|
||||
and InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path) is not None
|
||||
):
|
||||
raise HTTPException(
|
||||
|
|
@ -3532,7 +3536,7 @@ class InitPassThroughEndpointHelpers:
|
|||
class YieldsToConfiguredPassThroughRoute(APIRoute):
|
||||
def matches(self, scope: Scope) -> tuple[Match, Scope]:
|
||||
match, child_scope = super().matches(scope)
|
||||
if match is not Match.FULL:
|
||||
if match is Match.NONE:
|
||||
return match, child_scope
|
||||
configured: Final = InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash(
|
||||
route=get_request_route(Request(scope))
|
||||
|
|
@ -3999,9 +4003,6 @@ async def update_pass_through_endpoints(
|
|||
# Update the list
|
||||
pass_through_endpoint_data[endpoint_index] = endpoint_dict
|
||||
|
||||
# Remove old routes from registry before they get re-registered
|
||||
InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id)
|
||||
|
||||
## Update db
|
||||
updated_data: Final = ConfigFieldUpdate(
|
||||
field_name="pass_through_endpoints",
|
||||
|
|
@ -4016,6 +4017,7 @@ async def update_pass_through_endpoints(
|
|||
_custom_headers = await set_env_variables_in_header(custom_headers=_custom_headers)
|
||||
|
||||
route_app: Final = _request_app(request)
|
||||
InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id)
|
||||
if updated_endpoint.include_subpath:
|
||||
InitPassThroughEndpointHelpers.add_subpath_route(
|
||||
app=route_app,
|
||||
|
|
|
|||
|
|
@ -8803,6 +8803,57 @@ class TestEuJevPassthroughRoute:
|
|||
assert (response.status_code, response.json()) == (200, {"operator": True})
|
||||
assert route.calls.last.request.headers["x-operator"] == "on"
|
||||
|
||||
@pytest.mark.parametrize("method", ["GET", "PUT"])
|
||||
def test_a_pass_through_configured_with_a_trailing_slash_still_gets_other_methods_without_one(
|
||||
self, operator_app: Callable[[list[str], str], FastAPI], method: str
|
||||
) -> None:
|
||||
with (
|
||||
respx.mock(assert_all_called=True) as upstream,
|
||||
TestClient(operator_app(["GET", "POST", "PUT"], "/eu_jev/v1/systemone/")) as client,
|
||||
):
|
||||
route: Final = upstream.route(method=method, url="https://operator.example/decide").respond(
|
||||
200, json={"operator": True}
|
||||
)
|
||||
response: Final = client.request(method, "/eu_jev/v1/systemone", json={"state": "x"})
|
||||
|
||||
assert (response.status_code, response.json()) == (200, {"operator": True})
|
||||
assert route.calls.last.request.headers["x-operator"] == "on"
|
||||
|
||||
def test_a_request_the_built_in_route_took_is_not_forwarded_with_a_pass_through_added_meanwhile(
|
||||
self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers
|
||||
|
||||
monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key")
|
||||
monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base")
|
||||
app: Final = operator_app(["POST"], "/eu_jev/v1/systemone")
|
||||
InitPassThroughEndpointHelpers.remove_endpoint_routes("operator-eu-jev")
|
||||
|
||||
def add_the_pass_through_after_routing() -> UserAPIKeyAuth:
|
||||
InitPassThroughEndpointHelpers.add_exact_path_route(
|
||||
app=app,
|
||||
path="/eu_jev/v1/systemone",
|
||||
target="https://operator.example/decide",
|
||||
custom_headers={"x-operator": "on"},
|
||||
forward_headers=None,
|
||||
merge_query_params=None,
|
||||
dependencies=None,
|
||||
cost_per_request=None,
|
||||
endpoint_id="operator-eu-jev",
|
||||
methods=["POST"],
|
||||
)
|
||||
return UserAPIKeyAuth(api_key="sk-virtual")
|
||||
|
||||
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, add_the_pass_through_after_routing)
|
||||
with respx.mock(assert_all_called=False) as upstream, TestClient(app) as client:
|
||||
built_in: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}})
|
||||
operator: Final = upstream.route(host="operator.example").respond(200, json={"operator": True})
|
||||
response: Final = client.post("/eu_jev/v1/systemone", json={"model": "eu-jeff-latest", "state": "x"})
|
||||
|
||||
assert (response.status_code, response.json()) == (200, {"answers": {}})
|
||||
assert built_in.calls.last.request.headers["authorization"] == "Bearer eu-jev-test-key"
|
||||
assert not operator.called
|
||||
|
||||
def test_a_deleted_pass_through_at_the_same_path_stops_forwarding_when_built_in_routes_come_first(
|
||||
self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -2978,6 +2978,56 @@ async def test_update_pass_through_endpoint_preserves_auth_false():
|
|||
assert RouteChecks.is_auth_enforced_pass_through_route(route="/public-passthrough", method="POST") is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_pass_through_endpoint_keeps_serving_the_old_route_until_the_new_one_is_registered():
|
||||
from litellm.proxy._types import ConfigFieldInfo, PassThroughGenericEndpoint, UserAPIKeyAuth
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
||||
InitPassThroughEndpointHelpers,
|
||||
update_pass_through_endpoints,
|
||||
)
|
||||
|
||||
def registered_target() -> object:
|
||||
registered = InitPassThroughEndpointHelpers.get_registered_pass_through_route(route="/operator") or {}
|
||||
return registered.get("passthrough_params", {}).get("target")
|
||||
|
||||
targets_during_db_write: list[object] = []
|
||||
|
||||
async def record_registered_target(**_: object) -> None:
|
||||
targets_during_db_write.append(registered_target())
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.get_config_general_settings") as mock_get_config,
|
||||
patch("litellm.proxy.proxy_server.update_config_general_settings", side_effect=record_registered_target),
|
||||
patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", {}),
|
||||
):
|
||||
InitPassThroughEndpointHelpers.add_exact_path_route(
|
||||
app=FastAPI(),
|
||||
path="/operator",
|
||||
target="http://old.example/api",
|
||||
custom_headers=None,
|
||||
forward_headers=None,
|
||||
merge_query_params=None,
|
||||
dependencies=None,
|
||||
cost_per_request=None,
|
||||
endpoint_id="operator-route",
|
||||
methods=["POST"],
|
||||
)
|
||||
mock_get_config.return_value = ConfigFieldInfo(
|
||||
field_name="pass_through_endpoints",
|
||||
field_value=[
|
||||
{"id": "operator-route", "path": "/operator", "target": "http://old.example/api", "methods": ["POST"]}
|
||||
],
|
||||
)
|
||||
await update_pass_through_endpoints(
|
||||
endpoint_id="operator-route",
|
||||
data=PassThroughGenericEndpoint(path="/operator", target="http://new.example/api", methods=["POST"]),
|
||||
request=MagicMock(spec=Request),
|
||||
user_api_key_dict=MagicMock(spec=UserAPIKeyAuth),
|
||||
)
|
||||
|
||||
assert (targets_during_db_write, registered_target()) == (["http://old.example/api"], "http://new.example/api")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_pass_through_endpoint_not_found():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue