From 6e82516d8420ca25690127e9ea5ef4a6aa39413b Mon Sep 17 00:00:00 2001 From: Andrew Shaw Care Date: Tue, 27 Oct 2026 10:24:53 -0400 Subject: [PATCH] feat(proxy): let operators turn provider pass-through routes off general_settings.allowed_llm_passthrough_routes lists the provider pass-through route prefixes the proxy serves. A route left off answers 404 before auth, like a path the proxy does not have, so a provider without a credential no longer fails with a 500 or forwards the caller's request without one. Unset keeps every route on --- .../llm_passthrough_endpoints.py | 41 +++++++++- .../openai_passthrough_endpoints.py | 10 ++- .../test_llm_passthrough_endpoints.py | 74 +++++++++++++++++++ 3 files changed, 122 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 9eef4f0ef9d..de12b99175a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -23,7 +23,8 @@ from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast import httpx import openai -from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket +from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket, WebSocketException +from fastapi.requests import HTTPConnection from fastapi.responses import StreamingResponse from pydantic import ConfigDict, TypeAdapter, ValidationError from starlette.websockets import WebSocketState @@ -151,7 +152,43 @@ else: ProxyConfig = Any # rebind-ok: runtime fallback vertex_llm_base: Final = VertexBase() -router: Final = APIRouter() + + +_ALLOWED_LLM_PASSTHROUGH_ROUTES_SETTING: Final = "allowed_llm_passthrough_routes" +_ALLOWED_LLM_PASSTHROUGH_ROUTES: Final[TypeAdapter[frozenset[str] | None]] = TypeAdapter(frozenset[str] | None) + + +def check_llm_passthrough_route_enabled(connection: HTTPConnection) -> None: + """Answer 404 for a provider pass-through route the operator has not enabled. + + `general_settings.allowed_llm_passthrough_routes` lists the route prefixes + the proxy serves, such as `["/vertex_ai"]`; an empty list turns every + provider pass-through route off. Unset, every route stays on. A route that + is off answers like a path the proxy does not have, so an unconfigured + provider never fails with a 500 or forwards the request without a + credential. + """ + try: + allowed_routes: Final = _ALLOWED_LLM_PASSTHROUGH_ROUTES.validate_python( + _proxy_general_settings().get(_ALLOWED_LLM_PASSTHROUGH_ROUTES_SETTING) + ) + except ValidationError as error: + raise HTTPException( + status_code=500, + detail=f"general_settings.{_ALLOWED_LLM_PASSTHROUGH_ROUTES_SETTING} must be a list of strings", + ) from error + if allowed_routes is None: + return + route_path: Final[str] = getattr(connection.scope.get("route"), "path", "") + route_prefix: Final = "/" + route_path.lstrip("/").split("/", 1)[0] + if route_prefix in frozenset("/" + allowed.strip("/") for allowed in allowed_routes): + return + if connection.scope["type"] == "websocket": + raise WebSocketException(code=1008, reason="Not Found") # policy violation + raise HTTPException(status_code=404, detail="Not Found") + + +router: Final = APIRouter(dependencies=[Depends(check_llm_passthrough_route_enabled)]) _DECODED_JSON: Final = TypeAdapter(object) default_vertex_config: Final = None passthrough_endpoint_router: Final = PassthroughEndpointRouter() diff --git a/litellm/proxy/pass_through_endpoints/openai_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/openai_passthrough_endpoints.py index f56a59dd560..9f643ddd086 100644 --- a/litellm/proxy/pass_through_endpoints/openai_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/openai_passthrough_endpoints.py @@ -5,11 +5,19 @@ registered at startup and defers to the lazily loaded handler per call.""" from typing import Final from fastapi import APIRouter, Depends, Request, Response +from fastapi.requests import HTTPConnection from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -router: Final = APIRouter() + +def _llm_passthrough_route_enabled(connection: HTTPConnection) -> None: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import check_llm_passthrough_route_enabled + + check_llm_passthrough_route_enabled(connection) + + +router: Final = APIRouter(dependencies=[Depends(_llm_passthrough_route_enabled)]) @router.api_route( diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py index e654c6d444b..dd295e8761a 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py @@ -555,3 +555,77 @@ def test_vertex_pass_through_forwards_a_request_without_betas_despite_a_malforme assert (response.status_code, response.json()) == (200, _VERTEX_CLAUDE_REPLY) assert "anthropic-beta" not in upstream.calls.last.request.headers + + +_VERTEX_PASS_THROUGH_PATH: Final = ( + "/vertex_ai/v1/projects/p/locations/global/publishers/google/models/gemini-2.5-flash:generateContent" +) + + +@pytest.mark.parametrize( + "path", + [ + "/gemini/v1beta/models/gemini-2.5-flash:generateContent", + "/openai_passthrough/v1/responses", + "/cohere/v1/chat", + _VERTEX_PASS_THROUGH_PATH, + ], +) +def test_a_passthrough_route_left_off_the_allowlist_answers_404_before_auth( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "allowed_llm_passthrough_routes", []) + + response: Final = TestClient(proxy_server.app).post(path, json={}) + + assert response.status_code == 404 + assert response.json() == {"detail": "Not Found"} + + +@pytest.mark.parametrize("allowed_routes", [None, ["/vertex_ai/"]], ids=["unset", "allowlisted"]) +def test_a_passthrough_route_the_operator_serves_reaches_its_handler( + monkeypatch: pytest.MonkeyPatch, allowed_routes: list[str] | None +) -> None: + from litellm.proxy import proxy_server + + if allowed_routes is None: + monkeypatch.delitem(proxy_server.general_settings, "allowed_llm_passthrough_routes", raising=False) + else: + monkeypatch.setitem(proxy_server.general_settings, "allowed_llm_passthrough_routes", allowed_routes) + client: Final = TestClient(proxy_server.app) + + response: Final = client.post(_VERTEX_PASS_THROUGH_PATH, json={}) + + assert response.status_code == 401 + assert "No Vertex AI credential is configured" in response.json()["detail"] + if allowed_routes is not None: + assert client.post("/gemini/v1beta/models/gemini-2.5-flash:generateContent", json={}).status_code == 404 + + +def test_a_passthrough_websocket_left_off_the_allowlist_closes_as_not_found(monkeypatch: pytest.MonkeyPatch) -> None: + from starlette.websockets import WebSocketDisconnect + + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "allowed_llm_passthrough_routes", []) + + with pytest.raises(WebSocketDisconnect) as closed: + with TestClient(proxy_server.app).websocket_connect("/deepgram/listen"): + pass + + assert (closed.value.code, closed.value.reason) == (1008, "Not Found") + + +def test_a_malformed_passthrough_allowlist_is_refused_rather_than_read_as_characters( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "allowed_llm_passthrough_routes", "/vertex_ai") + + response: Final = TestClient(proxy_server.app).post(_VERTEX_PASS_THROUGH_PATH, json={}) + + assert response.status_code == 500 + assert "allowed_llm_passthrough_routes must be a list of strings" in response.text