Merge 6e82516d84 into 0070ce23b3

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -23,7 +23,8 @@ from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast
 
 import httpx
 import openai
-from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket
+from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket, WebSocketException
+from fastapi.requests import HTTPConnection
 from fastapi.responses import StreamingResponse
 from pydantic import ConfigDict, TypeAdapter, ValidationError
 from starlette.websockets import WebSocketState
@@ -163,7 +164,43 @@ else:
     ProxyConfig = Any  # rebind-ok: runtime fallback
 
 vertex_llm_base: Final = VertexBase()
-router: Final = APIRouter()
+
+
+_ALLOWED_LLM_PASSTHROUGH_ROUTES_SETTING: Final = "allowed_llm_passthrough_routes"
+_ALLOWED_LLM_PASSTHROUGH_ROUTES: Final[TypeAdapter[frozenset[str] | None]] = TypeAdapter(frozenset[str] | None)
+
+
+def check_llm_passthrough_route_enabled(connection: HTTPConnection) -> None:
+    """Answer 404 for a provider pass-through route the operator has not enabled.
+
+    `general_settings.allowed_llm_passthrough_routes` lists the route prefixes
+    the proxy serves, such as `["/vertex_ai"]`; an empty list turns every
+    provider pass-through route off. Unset, every route stays on. A route that
+    is off answers like a path the proxy does not have, so an unconfigured
+    provider never fails with a 500 or forwards the request without a
+    credential.
+    """
+    try:
+        allowed_routes: Final = _ALLOWED_LLM_PASSTHROUGH_ROUTES.validate_python(
+            _proxy_general_settings().get(_ALLOWED_LLM_PASSTHROUGH_ROUTES_SETTING)
+        )
+    except ValidationError as error:
+        raise HTTPException(
+            status_code=500,
+            detail=f"general_settings.{_ALLOWED_LLM_PASSTHROUGH_ROUTES_SETTING} must be a list of strings",
+        ) from error
+    if allowed_routes is None:
+        return
+    route_path: Final[str] = getattr(connection.scope.get("route"), "path", "")
+    route_prefix: Final = "/" + route_path.lstrip("/").split("/", 1)[0]
+    if route_prefix in frozenset("/" + allowed.strip("/") for allowed in allowed_routes):
+        return
+    if connection.scope["type"] == "websocket":
+        raise WebSocketException(code=1008, reason="Not Found")  # policy violation
+    raise HTTPException(status_code=404, detail="Not Found")
+
+
+router: Final = APIRouter(dependencies=[Depends(check_llm_passthrough_route_enabled)])
 _DECODED_JSON: Final = TypeAdapter(object)
 default_vertex_config: Final = None
 passthrough_endpoint_router: Final = PassthroughEndpointRouter()

litellm/proxy/pass_through_endpoints/openai_passthrough_endpoints.py

@@ -5,11 +5,19 @@ registered at startup and defers to the lazily loaded handler per call."""
 from typing import Final
 
 from fastapi import APIRouter, Depends, Request, Response
+from fastapi.requests import HTTPConnection
 
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 
-router: Final = APIRouter()
+
+def _llm_passthrough_route_enabled(connection: HTTPConnection) -> None:
+    from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import check_llm_passthrough_route_enabled
+
+    check_llm_passthrough_route_enabled(connection)
+
+
+router: Final = APIRouter(dependencies=[Depends(_llm_passthrough_route_enabled)])
 
 
 @router.api_route(

tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py

@@ -555,3 +555,77 @@ def test_vertex_pass_through_forwards_a_request_without_betas_despite_a_malforme
 
     assert (response.status_code, response.json()) == (200, _VERTEX_CLAUDE_REPLY)
     assert "anthropic-beta" not in upstream.calls.last.request.headers
+
+
+_VERTEX_PASS_THROUGH_PATH: Final = (
+    "/vertex_ai/v1/projects/p/locations/global/publishers/google/models/gemini-2.5-flash:generateContent"
+)
+
+
+@pytest.mark.parametrize(
+    "path",
+    [
+        "/gemini/v1beta/models/gemini-2.5-flash:generateContent",
+        "/openai_passthrough/v1/responses",
+        "/cohere/v1/chat",
+        _VERTEX_PASS_THROUGH_PATH,
+    ],
+)
+def test_a_passthrough_route_left_off_the_allowlist_answers_404_before_auth(
+    monkeypatch: pytest.MonkeyPatch, path: str
+) -> None:
+    from litellm.proxy import proxy_server
+
+    monkeypatch.setitem(proxy_server.general_settings, "allowed_llm_passthrough_routes", [])
+
+    response: Final = TestClient(proxy_server.app).post(path, json={})
+
+    assert response.status_code == 404
+    assert response.json() == {"detail": "Not Found"}
+
+
+@pytest.mark.parametrize("allowed_routes", [None, ["/vertex_ai/"]], ids=["unset", "allowlisted"])
+def test_a_passthrough_route_the_operator_serves_reaches_its_handler(
+    monkeypatch: pytest.MonkeyPatch, allowed_routes: list[str] | None
+) -> None:
+    from litellm.proxy import proxy_server
+
+    if allowed_routes is None:
+        monkeypatch.delitem(proxy_server.general_settings, "allowed_llm_passthrough_routes", raising=False)
+    else:
+        monkeypatch.setitem(proxy_server.general_settings, "allowed_llm_passthrough_routes", allowed_routes)
+    client: Final = TestClient(proxy_server.app)
+
+    response: Final = client.post(_VERTEX_PASS_THROUGH_PATH, json={})
+
+    assert response.status_code == 401
+    assert "No Vertex AI credential is configured" in response.json()["detail"]
+    if allowed_routes is not None:
+        assert client.post("/gemini/v1beta/models/gemini-2.5-flash:generateContent", json={}).status_code == 404
+
+
+def test_a_passthrough_websocket_left_off_the_allowlist_closes_as_not_found(monkeypatch: pytest.MonkeyPatch) -> None:
+    from starlette.websockets import WebSocketDisconnect
+
+    from litellm.proxy import proxy_server
+
+    monkeypatch.setitem(proxy_server.general_settings, "allowed_llm_passthrough_routes", [])
+
+    with pytest.raises(WebSocketDisconnect) as closed:
+        with TestClient(proxy_server.app).websocket_connect("/deepgram/listen"):
+            pass
+
+    assert (closed.value.code, closed.value.reason) == (1008, "Not Found")
+
+
+def test_a_malformed_passthrough_allowlist_is_refused_rather_than_read_as_characters(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from litellm.proxy import proxy_server
+
+    monkeypatch.setitem(proxy_server.general_settings, "allowed_llm_passthrough_routes", "/vertex_ai")
+
+    response: Final = TestClient(proxy_server.app).post(_VERTEX_PASS_THROUGH_PATH, json={})
+
+    assert response.status_code == 500
+    assert "allowed_llm_passthrough_routes must be a list of strings" in response.text