diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index e0a4184291e..25d92521bf1 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -27,6 +27,7 @@ from fastapi import ( ) from fastapi.responses import StreamingResponse from starlette.datastructures import UploadFile as StarletteUploadFile +from starlette.routing import Match from starlette.websockets import WebSocketState from websockets.asyncio.client import connect from websockets.exceptions import ( @@ -2898,6 +2899,63 @@ class SafeRouteAdder: return True return False + # Every generic native-provider route (files, batches, and any future ones) is + # registered as "/{provider}/v1/...", so this literal path-parameter name is a + # reliable, future-proof marker -- no need to enumerate specific provider routes. + _GENERIC_PROVIDER_PATH_MARKER: Final = "{provider}" + + @staticmethod + def _move_before_generic_provider_routes(app: FastAPI) -> None: + """ + Custom pass-through routes registered from config.yaml are always appended to + app.routes, since they're added during proxy startup, strictly after every + built-in router (including the generic "/{provider}/v1/files" and + "/{provider}/v1/batches" routes) is mounted at module-import time. Starlette + resolves overlapping path templates by registration order, so an appended + custom route can never win against those generic routes -- they always match + first and misinterpret the custom prefix as a provider name (see + https://github.com/BerriAI/litellm/issues/37925). + + Move the just-appended route (the last item in app.routes) to sit immediately + before the first generic route whose template would actually capture this + route's own path, so it is matched first instead. A route is moved only when + that genuine overlap exists: reordering unconditionally on any "{provider}" + sighting would also promote unrelated wildcard pass-throughs (e.g. a + "/key/{subpath:path}" entry) ahead of every route registered afterward, + including unrelated, authenticated built-in routes like "/key/generate" -- + a real route.matches() check on this route's own path rules that out, since + a route path containing its own "{...}" placeholder never structurally + matches a "/{provider}/..." template. If no generic provider route captures + this path (including when none is registered at all, e.g. a minimal + deployment), leave the route appended -- current behavior is preserved as a + safe fallback. + + Builds the reordered list in one expression and reassigns app.router.routes + wholesale, rather than mutating the existing list in place with pop()/insert(). + """ + routes: Final = app.routes + new_route: Final = routes[-1] + scope: Final = { + "type": "http", + "method": "GET", + "path": new_route.path, + "headers": [], + "query_string": b"", + "root_path": "", + } + for index, route in enumerate(routes[:-1]): + route_path = getattr(route, "path", None) + if not (route_path and SafeRouteAdder._GENERIC_PROVIDER_PATH_MARKER in route_path): + continue + if route.matches(scope)[0] == Match.NONE: + continue + app.router.routes = [ # mutable-ok: framework's list # rebind-ok: reordering is the fix + *routes[:index], + new_route, + *routes[index:-1], + ] + return + @staticmethod def add_api_route_if_not_exists( app: FastAPI, @@ -2933,6 +2991,7 @@ class SafeRouteAdder: methods=methods, dependencies=dependencies, ) + SafeRouteAdder._move_before_generic_provider_routes(app=app) verbose_proxy_logger.debug( "Successfully added route: %s with methods %s", path, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 227921d6150..82ebdea4f6a 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -3548,6 +3548,127 @@ def test_native_provider_routes_are_unchanged(method, path, expected_name): assert _resolve_route_name(method, path) == expected_name +def test_custom_pass_through_endpoint_prefix_wins_over_native_provider_routes(): + """ + A pass_through_endpoints entry registered under an arbitrary, non-built-in + prefix (e.g. a self-hosted Anthropic-compatible endpoint reached via a + "/claude-aws" prefix) must win over the native /{provider}/v1/files and + /{provider}/v1/batches routes, which would otherwise misinterpret the + custom prefix as a provider name and 422/500 instead of forwarding + (see https://github.com/BerriAI/litellm/issues/37925). + """ + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + ) + from litellm.proxy.proxy_server import app + + # app is the real, process-wide proxy app -- registering routes on it leaks + # into every other test (e.g. test_component_allowlists.py's full-route-coverage + # check) unless removed again. Track exactly the paths this test adds and + # filter only those back out afterward, rather than restoring a full + # snapshot -- under pytest-xdist, another test on the same worker can + # legitimately register routes between this test's start and its cleanup, + # and a wholesale snapshot restore would silently drop those too. + added_paths = {f"/claude-aws/v1/{suffix}" for suffix in ("files", "batches")} + try: + for suffix in ("files", "batches"): + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path=f"/claude-aws/v1/{suffix}", + target=f"https://example.com/v1/{suffix}", + custom_headers=None, + forward_headers=False, + merge_query_params=False, + dependencies=None, + cost_per_request=None, + endpoint_id=f"test-claude-aws-{suffix}", + ) + + assert _resolve_route_name("POST", "/claude-aws/v1/files") == "endpoint_func" + assert _resolve_route_name("POST", "/claude-aws/v1/batches") == "endpoint_func" + + # registering a custom prefix must not disturb resolution of unrelated, + # already-registered native-provider routes + assert _resolve_route_name("POST", "/openai/v1/files") == "create_file" + assert _resolve_route_name("GET", "/azure/v1/files") == "list_files" + assert _resolve_route_name("POST", "/v1/files") == "create_file" + assert _resolve_route_name("POST", "/v1/batches") == "create_batch" + finally: + app.router.routes = [ + route for route in app.router.routes + if getattr(route, "path", None) not in added_paths + ] + + +def test_wildcard_pass_through_does_not_shadow_unrelated_built_in_routes(): + """ + A pass_through_endpoints entry with a wildcard subpath (e.g. "/key/{subpath:path}") + must not be promoted ahead of unrelated, authenticated built-in routes like + "/key/generate" just because it lands after the first "/{provider}/..." route in + app.routes once appended. Its own path never structurally matches a + "/{provider}/..." template, so it must be left exactly where it was appended. + """ + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + ) + from litellm.proxy.proxy_server import app + + key_generate_route_name_before = _resolve_route_name("POST", "/key/generate") + assert key_generate_route_name_before is not None + + added_paths = {"/key/{subpath:path}"} + try: + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path="/key/{subpath:path}", + target="https://example.com/", + custom_headers=None, + forward_headers=False, + merge_query_params=False, + dependencies=None, + cost_per_request=None, + endpoint_id="test-key-wildcard", + ) + + assert _resolve_route_name("POST", "/key/generate") == key_generate_route_name_before + finally: + app.router.routes = [ + route for route in app.router.routes + if getattr(route, "path", None) not in added_paths + ] + + +def test_move_before_generic_provider_routes_is_a_no_op_without_a_generic_route(): + """ + If no generic "/{provider}/..." route is registered on the app (e.g. a minimal + deployment without the files/batches routers mounted), the newly-appended custom + route is left exactly where it was appended -- a safe no-op fallback. + """ + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + SafeRouteAdder, + ) + + class _FakeRoute: + def __init__(self, path): + self.path = path + + class _FakeRouter: + def __init__(self, routes): + self.routes = routes + + class _FakeApp: + def __init__(self, routes): + self.routes = routes + self.router = _FakeRouter(routes) + + routes = [_FakeRoute("/health"), _FakeRoute("/claude-aws/v1/files")] + app = _FakeApp(routes) + + SafeRouteAdder._move_before_generic_provider_routes(app=app) + + assert app.router.routes == routes + + @pytest.fixture def openai_passthrough_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: from litellm.proxy.proxy_server import app