Merge 76f78727ee into 74cad08997

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -27,6 +27,7 @@ from fastapi import (
 )
 from fastapi.responses import StreamingResponse
 from starlette.datastructures import UploadFile as StarletteUploadFile
+from starlette.routing import Match
 from starlette.websockets import WebSocketState
 from websockets.asyncio.client import connect
 from websockets.exceptions import (
@@ -2898,6 +2899,63 @@ class SafeRouteAdder:
                     return True
         return False
 
+    # Every generic native-provider route (files, batches, and any future ones) is
+    # registered as "/{provider}/v1/...", so this literal path-parameter name is a
+    # reliable, future-proof marker -- no need to enumerate specific provider routes.
+    _GENERIC_PROVIDER_PATH_MARKER: Final = "{provider}"
+
+    @staticmethod
+    def _move_before_generic_provider_routes(app: FastAPI) -> None:
+        """
+        Custom pass-through routes registered from config.yaml are always appended to
+        app.routes, since they're added during proxy startup, strictly after every
+        built-in router (including the generic "/{provider}/v1/files" and
+        "/{provider}/v1/batches" routes) is mounted at module-import time. Starlette
+        resolves overlapping path templates by registration order, so an appended
+        custom route can never win against those generic routes -- they always match
+        first and misinterpret the custom prefix as a provider name (see
+        https://github.com/BerriAI/litellm/issues/37925).
+
+        Move the just-appended route (the last item in app.routes) to sit immediately
+        before the first generic route whose template would actually capture this
+        route's own path, so it is matched first instead. A route is moved only when
+        that genuine overlap exists: reordering unconditionally on any "{provider}"
+        sighting would also promote unrelated wildcard pass-throughs (e.g. a
+        "/key/{subpath:path}" entry) ahead of every route registered afterward,
+        including unrelated, authenticated built-in routes like "/key/generate" --
+        a real route.matches() check on this route's own path rules that out, since
+        a route path containing its own "{...}" placeholder never structurally
+        matches a "/{provider}/..." template. If no generic provider route captures
+        this path (including when none is registered at all, e.g. a minimal
+        deployment), leave the route appended -- current behavior is preserved as a
+        safe fallback.
+
+        Builds the reordered list in one expression and reassigns app.router.routes
+        wholesale, rather than mutating the existing list in place with pop()/insert().
+        """
+        routes: Final = app.routes
+        new_route: Final = routes[-1]
+        scope: Final = {
+            "type": "http",
+            "method": "GET",
+            "path": new_route.path,
+            "headers": [],
+            "query_string": b"",
+            "root_path": "",
+        }
+        for index, route in enumerate(routes[:-1]):
+            route_path = getattr(route, "path", None)
+            if not (route_path and SafeRouteAdder._GENERIC_PROVIDER_PATH_MARKER in route_path):
+                continue
+            if route.matches(scope)[0] == Match.NONE:
+                continue
+            app.router.routes = [  # mutable-ok: framework's list  # rebind-ok: reordering is the fix
+                *routes[:index],
+                new_route,
+                *routes[index:-1],
+            ]
+            return
+
     @staticmethod
     def add_api_route_if_not_exists(
         app: FastAPI,
@@ -2933,6 +2991,7 @@ class SafeRouteAdder:
             methods=methods,
             dependencies=dependencies,
         )
+        SafeRouteAdder._move_before_generic_provider_routes(app=app)
         verbose_proxy_logger.debug(
             "Successfully added route: %s with methods %s",
             path,

tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py

@@ -3548,6 +3548,127 @@ def test_native_provider_routes_are_unchanged(method, path, expected_name):
     assert _resolve_route_name(method, path) == expected_name
 
 
+def test_custom_pass_through_endpoint_prefix_wins_over_native_provider_routes():
+    """
+    A pass_through_endpoints entry registered under an arbitrary, non-built-in
+    prefix (e.g. a self-hosted Anthropic-compatible endpoint reached via a
+    "/claude-aws" prefix) must win over the native /{provider}/v1/files and
+    /{provider}/v1/batches routes, which would otherwise misinterpret the
+    custom prefix as a provider name and 422/500 instead of forwarding
+    (see https://github.com/BerriAI/litellm/issues/37925).
+    """
+    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+        InitPassThroughEndpointHelpers,
+    )
+    from litellm.proxy.proxy_server import app
+
+    # app is the real, process-wide proxy app -- registering routes on it leaks
+    # into every other test (e.g. test_component_allowlists.py's full-route-coverage
+    # check) unless removed again. Track exactly the paths this test adds and
+    # filter only those back out afterward, rather than restoring a full
+    # snapshot -- under pytest-xdist, another test on the same worker can
+    # legitimately register routes between this test's start and its cleanup,
+    # and a wholesale snapshot restore would silently drop those too.
+    added_paths = {f"/claude-aws/v1/{suffix}" for suffix in ("files", "batches")}
+    try:
+        for suffix in ("files", "batches"):
+            InitPassThroughEndpointHelpers.add_exact_path_route(
+                app=app,
+                path=f"/claude-aws/v1/{suffix}",
+                target=f"https://example.com/v1/{suffix}",
+                custom_headers=None,
+                forward_headers=False,
+                merge_query_params=False,
+                dependencies=None,
+                cost_per_request=None,
+                endpoint_id=f"test-claude-aws-{suffix}",
+            )
+
+        assert _resolve_route_name("POST", "/claude-aws/v1/files") == "endpoint_func"
+        assert _resolve_route_name("POST", "/claude-aws/v1/batches") == "endpoint_func"
+
+        # registering a custom prefix must not disturb resolution of unrelated,
+        # already-registered native-provider routes
+        assert _resolve_route_name("POST", "/openai/v1/files") == "create_file"
+        assert _resolve_route_name("GET", "/azure/v1/files") == "list_files"
+        assert _resolve_route_name("POST", "/v1/files") == "create_file"
+        assert _resolve_route_name("POST", "/v1/batches") == "create_batch"
+    finally:
+        app.router.routes = [
+            route for route in app.router.routes
+            if getattr(route, "path", None) not in added_paths
+        ]
+
+
+def test_wildcard_pass_through_does_not_shadow_unrelated_built_in_routes():
+    """
+    A pass_through_endpoints entry with a wildcard subpath (e.g. "/key/{subpath:path}")
+    must not be promoted ahead of unrelated, authenticated built-in routes like
+    "/key/generate" just because it lands after the first "/{provider}/..." route in
+    app.routes once appended. Its own path never structurally matches a
+    "/{provider}/..." template, so it must be left exactly where it was appended.
+    """
+    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+        InitPassThroughEndpointHelpers,
+    )
+    from litellm.proxy.proxy_server import app
+
+    key_generate_route_name_before = _resolve_route_name("POST", "/key/generate")
+    assert key_generate_route_name_before is not None
+
+    added_paths = {"/key/{subpath:path}"}
+    try:
+        InitPassThroughEndpointHelpers.add_exact_path_route(
+            app=app,
+            path="/key/{subpath:path}",
+            target="https://example.com/",
+            custom_headers=None,
+            forward_headers=False,
+            merge_query_params=False,
+            dependencies=None,
+            cost_per_request=None,
+            endpoint_id="test-key-wildcard",
+        )
+
+        assert _resolve_route_name("POST", "/key/generate") == key_generate_route_name_before
+    finally:
+        app.router.routes = [
+            route for route in app.router.routes
+            if getattr(route, "path", None) not in added_paths
+        ]
+
+
+def test_move_before_generic_provider_routes_is_a_no_op_without_a_generic_route():
+    """
+    If no generic "/{provider}/..." route is registered on the app (e.g. a minimal
+    deployment without the files/batches routers mounted), the newly-appended custom
+    route is left exactly where it was appended -- a safe no-op fallback.
+    """
+    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+        SafeRouteAdder,
+    )
+
+    class _FakeRoute:
+        def __init__(self, path):
+            self.path = path
+
+    class _FakeRouter:
+        def __init__(self, routes):
+            self.routes = routes
+
+    class _FakeApp:
+        def __init__(self, routes):
+            self.routes = routes
+            self.router = _FakeRouter(routes)
+
+    routes = [_FakeRoute("/health"), _FakeRoute("/claude-aws/v1/files")]
+    app = _FakeApp(routes)
+
+    SafeRouteAdder._move_before_generic_provider_routes(app=app)
+
+    assert app.router.routes == routes
+
+
 @pytest.fixture
 def openai_passthrough_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
     from litellm.proxy.proxy_server import app