From b6b0e2f1c55be1db1f473ef01843dfdb3043ed14 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:13:09 -0700 Subject: [PATCH 1/4] test(proxy): cover model access on the Vertex AI Live websocket passthrough (internal copy of #40078) Co-authored-by: Marty Sullivan --- .../test_pass_through_endpoints.py | 483 +++++++++++++++++- 1 file changed, 481 insertions(+), 2 deletions(-) diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index f0baceecdaa..953aefe7f7c 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -35,7 +35,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._lazy_features import LazyFeature, attach_lazy_features -from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import router as llm_passthrough_router from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( @@ -7175,6 +7175,39 @@ class RecordingUpstreamWebSocket: raise StopAsyncIteration +class ClosableUpstreamWebSocket: + """An upstream whose ``recv`` ends when ``close`` does, and whose close keeps yielding after that. + + A real close handshake outlives the ``recv`` it unblocks, so the reader task finishes while the closing + task is still suspended. That is the interleaving that leaves a refusal stranded in a cancelled task + """ + + CLOSE_HANDSHAKE_YIELDS = 5 + + def __init__(self): + self._closing = asyncio.Event() + self.send = AsyncMock() + self.close = AsyncMock(side_effect=self._close) + + async def _close(self): + self._closing.set() + for _ in range(self.CLOSE_HANDSHAKE_YIELDS): + await asyncio.sleep(0) + + async def recv(self, decode: bool = True): + from websockets.exceptions import ConnectionClosedOK + from websockets.frames import Close + + await self._closing.wait() + raise ConnectionClosedOK(rcvd=Close(1000, ""), sent=Close(1000, ""), rcvd_then_sent=True) + + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + def _client_websocket(receive): from starlette.websockets import WebSocketState @@ -7210,7 +7243,7 @@ def _patched_websocket_passthrough_environment(upstream_ws): mock_worker.ensure_initialized_and_enqueue = MagicMock( side_effect=lambda async_coroutine: async_coroutine.close() ) - yield + yield SimpleNamespace(proxy_logging=mock_proxy_logging, logging_worker=mock_worker) async def _pending_receive(): @@ -7336,6 +7369,452 @@ async def test_websocket_passthrough_rewrites_gateway_alias_setup_model(): assert sent_setup["model"] == "projects/proj-db/locations/global/publishers/google/models/gemini-live-2.5-flash" +VERTEX_LIVE_TARGET = "wss://aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent" + + +VERTEX_LIVE_GROUP = "gemini-live-2.5-flash-native-audio" + +VERTEX_LIVE_ALIAS_GROUP = "gemini-live-native-audio" + +VERTEX_LIVE_CLIENT_ADDRESSINGS = ( + "{model}", + "models/{model}", + "vertex_ai/{model}", + "publishers/google/models/{model}", + "projects/proj-db/locations/global/publishers/google/models/{model}", +) + + +def _vertex_live_router(): + return litellm.Router( + model_list=[ + { + "model_name": "gemini-live-2.5-flash", + "litellm_params": {"model": "vertex_ai/gemini-live-2.5-flash"}, + }, + { + "model_name": VERTEX_LIVE_GROUP, + "litellm_params": {"model": f"vertex_ai/{VERTEX_LIVE_GROUP}"}, + }, + { + "model_name": VERTEX_LIVE_ALIAS_GROUP, + "litellm_params": {"model": f"vertex_ai/{VERTEX_LIVE_GROUP}"}, + }, + { + "model_name": "gemini-3-pro-live", + "litellm_params": {"model": "vertex_ai/gemini-3-pro-live"}, + }, + ] + ) + + +async def _run_vertex_live_gated_passthrough(client_frames, valid_token): + """Drive /vertex_ai/live with the real frame gate, returning the client and upstream sockets""" + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + _build_vertex_live_client_frame_gate, + _build_vertex_live_setup_model_rewriter, + ) + + llm_router = _vertex_live_router() + upstream_ws = RecordingUpstreamWebSocket() + websocket = _client_websocket( + AsyncMock(side_effect=[*client_frames, {"type": "websocket.disconnect"}]), + ) + + with _patched_websocket_passthrough_environment(upstream_ws): + await websocket_passthrough_request( + websocket=websocket, + target=VERTEX_LIVE_TARGET, + custom_headers={"Authorization": "Bearer token"}, + user_api_key_dict=valid_token, + forward_headers=False, + endpoint="/vertex_ai/live", + accept_websocket=False, + setup_model_rewriter=_build_vertex_live_setup_model_rewriter( + vertex_project="proj-db", + vertex_location="global", + llm_router=llm_router, + ), + client_frame_gate=_build_vertex_live_client_frame_gate( + llm_model_list=llm_router.get_model_list(), + llm_router=llm_router, + ), + ) + + return websocket, upstream_ws + + +def _setup_frame(model: str) -> dict: + return { + "type": "websocket.receive", + "text": json.dumps({"setup": {"model": model, "generationConfig": {"responseModalities": ["TEXT"]}}}), + } + + +def _policy_close_reason(websocket) -> str | None: + return next( + (call.kwargs.get("reason") for call in websocket.close.await_args_list if call.kwargs.get("code") == 1008), + None, + ) + + +@pytest.mark.asyncio +async def test_vertex_live_setup_frame_naming_a_model_the_key_cannot_call_is_refused(): + """ + ?model= is optional on this route, so a key that names an unlisted model only in the setup frame used to + reach Vertex with the gateway's own service-account credentials + """ + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame("totally-unlisted-model-abc")], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + ) + + upstream_ws.send.assert_not_awaited() + reason = _policy_close_reason(websocket) + assert reason is not None and "totally-unlisted-model-abc" in reason + upstream_ws.close.assert_awaited() + + +@pytest.mark.asyncio +async def test_vertex_live_setup_frame_naming_a_permitted_model_still_reaches_vertex(): + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame("gemini-live-2.5-flash")], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + ) + + upstream_ws.send.assert_awaited_once() + sent_setup = json.loads(upstream_ws.send.await_args.args[0])["setup"] + assert sent_setup["model"] == "projects/proj-db/locations/global/publishers/google/models/gemini-live-2.5-flash" + assert _policy_close_reason(websocket) is None + + +@pytest.mark.asyncio +async def test_vertex_live_setup_frame_from_an_unrestricted_key_still_reaches_vertex(): + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame("gemini-3-pro-live")], + UserAPIKeyAuth(token="hashed", models=[]), + ) + + upstream_ws.send.assert_awaited_once() + assert _policy_close_reason(websocket) is None + + +@pytest.mark.asyncio +async def test_vertex_live_setup_frame_is_checked_against_the_team_for_an_all_team_models_key(): + """ + ``all-team-models`` skips the key allowlist, so the team's own restriction is the only thing left to apply + """ + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame("gemini-3-pro-live")], + UserAPIKeyAuth( + token="hashed", + models=["all-team-models"], + team_id="team-1", + team_models=["gemini-live-2.5-flash"], + ), + ) + + upstream_ws.send.assert_not_awaited() + reason = _policy_close_reason(websocket) + assert reason is not None and "gemini-3-pro-live" in reason + + +@pytest.mark.asyncio +async def test_vertex_live_setup_frame_is_gated_when_it_arrives_as_binary(): + """A client that sends its setup frame as bytes must not skip the check the text path applies""" + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [ + { + "type": "websocket.receive", + "bytes": json.dumps({"setup": {"model": "totally-unlisted-model-abc"}}).encode(), + } + ], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + ) + + upstream_ws.send.assert_not_awaited() + assert _policy_close_reason(websocket) is not None + + +@pytest.mark.asyncio +async def test_vertex_live_setup_frame_with_an_escaped_key_is_still_gated(): + """Large audio frames skip the parse on a substring test, so a JSON-escaped key must not slip past it""" + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [{"type": "websocket.receive", "text": '{"\\u0073etup": {"model": "totally-unlisted-model-abc"}}'}], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + ) + + upstream_ws.send.assert_not_awaited() + assert _policy_close_reason(websocket) is not None + + +@pytest.mark.asyncio +async def test_vertex_live_non_setup_frames_are_forwarded_without_a_model_check(): + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [ + _setup_frame("gemini-live-2.5-flash"), + {"type": "websocket.receive", "text": json.dumps({"realtimeInput": {"audio": {"data": "AAAA"}}})}, + ], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + ) + + assert upstream_ws.send.await_count == 2 + assert _policy_close_reason(websocket) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS) +@pytest.mark.parametrize("group", [VERTEX_LIVE_GROUP, VERTEX_LIVE_ALIAS_GROUP]) +async def test_vertex_live_setup_frame_authorizes_a_permitted_group_in_every_addressing(addressing, group): + """ + The Live SDK wraps the caller's model as ``models/`` and the docs tell callers to send the full + resource path, so a key holding the group has to work whichever addressing its client actually sends + """ + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame(addressing.format(model=group))], + UserAPIKeyAuth(token="hashed", models=[group]), + ) + + upstream_ws.send.assert_awaited_once() + sent_setup = json.loads(upstream_ws.send.await_args.args[0])["setup"] + assert sent_setup["model"].startswith("projects/") + assert _policy_close_reason(websocket) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS) +async def test_vertex_live_setup_frame_refuses_an_unpermitted_group_in_every_addressing(addressing): + """Normalizing the addressing must not become a way to reach a group the key does not hold""" + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame(addressing.format(model="gemini-3-pro-live"))], + UserAPIKeyAuth(token="hashed", models=[VERTEX_LIVE_GROUP]), + ) + + upstream_ws.send.assert_not_awaited() + reason = _policy_close_reason(websocket) + assert reason is not None and "gemini-3-pro-live" in reason + + +@pytest.mark.asyncio +@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS) +async def test_vertex_live_setup_frame_naming_no_group_at_all_is_refused_in_every_addressing(addressing): + """A name that resolves to no group is authorized as it arrived, so an unknown model cannot ride a prefix in""" + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame(addressing.format(model="totally-unlisted-model-abc"))], + UserAPIKeyAuth(token="hashed", models=[VERTEX_LIVE_GROUP]), + ) + + upstream_ws.send.assert_not_awaited() + assert _policy_close_reason(websocket) is not None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS[1:]) +async def test_vertex_live_setup_frame_does_not_strip_a_prefix_off_a_model_no_group_serves(addressing): + """ + Stripping the addressing off a name the router does not serve would let a wildcard key reach an arbitrary + publisher model, and a full path would carry a project of the caller's choosing with it + """ + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame(addressing.format(model="gemini-4-pro-live-unserved"))], + UserAPIKeyAuth(token="hashed", models=["gemini-*"]), + ) + + upstream_ws.send.assert_not_awaited() + assert _policy_close_reason(websocket) is not None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS) +async def test_vertex_live_setup_frame_is_checked_against_the_team_in_every_addressing(addressing): + """The team allowlist is keyed by group too, so the addressing must not decide whether the team is checked""" + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame(addressing.format(model="gemini-3-pro-live"))], + UserAPIKeyAuth( + token="hashed", + models=["all-team-models"], + team_id="team-1", + team_models=[VERTEX_LIVE_GROUP], + ), + ) + + upstream_ws.send.assert_not_awaited() + reason = _policy_close_reason(websocket) + assert reason is not None and "gemini-3-pro-live" in reason + + +@pytest.mark.asyncio +@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS) +async def test_vertex_live_setup_frame_resolves_an_access_group_grant_in_every_addressing(addressing): + """ + ``get_model_access_groups`` is keyed by group as well, and returns nothing for a prefixed name, so a key + entitled through an access group rather than a model name needs the same normalization + """ + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + _build_vertex_live_client_frame_gate, + ) + + llm_router = litellm.Router( + model_list=[ + { + "model_name": VERTEX_LIVE_GROUP, + "litellm_params": {"model": f"vertex_ai/{VERTEX_LIVE_GROUP}"}, + "model_info": {"access_groups": ["live-team"]}, + }, + ] + ) + gate = _build_vertex_live_client_frame_gate( + llm_model_list=llm_router.get_model_list(), + llm_router=llm_router, + ) + + frame = json.dumps({"setup": {"model": addressing.format(model=VERTEX_LIVE_GROUP)}}) + denial = await gate(frame, UserAPIKeyAuth(token="hashed", models=["live-team"])) + + assert denial is None + + +async def _run_vertex_live_route(client_frames, valid_token): + """Drive the /vertex_ai/live route, so the gate under test is whichever one the route itself wires""" + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + vertex_ai_live_websocket_passthrough, + vertex_llm_base, + ) + + upstream_ws = RecordingUpstreamWebSocket() + websocket = _client_websocket( + AsyncMock(side_effect=[*client_frames, {"type": "websocket.disconnect"}]), + ) + + with ( + _patched_websocket_passthrough_environment(upstream_ws) as environment, + patch.object( # test-quality-ok: an OAuth exchange with Google, so this is the boundary being faked + vertex_llm_base, + "_ensure_access_token_async", + AsyncMock(return_value=("token-abc", "proj-db")), + ), + ): + await vertex_ai_live_websocket_passthrough( + websocket=websocket, + model="gemini-live-2.5-flash", + vertex_project="proj-db", + vertex_location="global", + user_api_key_dict=valid_token, + ) + + return websocket, upstream_ws, environment + + +@pytest.mark.asyncio +async def test_vertex_live_route_gates_the_setup_frame_even_when_the_query_model_is_permitted(): + """ + ?model= is all connect-time auth sees, so a key that passes it and then names a different model in its setup + frame reaches Vertex on the gateway's own credentials unless the route gates the frames as well + """ + websocket, upstream_ws, environment = await _run_vertex_live_route( + [_setup_frame("gemini-3-pro-live")], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + ) + + upstream_ws.send.assert_not_awaited() + reason = _policy_close_reason(websocket) + assert reason is not None and "gemini-3-pro-live" in reason + refusal = environment.proxy_logging.post_call_failure_hook.await_args.kwargs["original_exception"] + assert refusal.code == "403" + assert refusal.type == ProxyErrorTypes.key_model_access_denied + + +@pytest.mark.asyncio +async def test_vertex_live_route_keeps_the_key_allowlist_out_of_the_close_reason_and_in_the_proxy_log(caplog): + """The client learns which model was refused, while only the proxy log names the models the key may reach""" + with caplog.at_level("WARNING", logger="LiteLLM Proxy"): + websocket, _, _ = await _run_vertex_live_route( + [_setup_frame("gemini-3-pro-live")], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + ) + + reason = _policy_close_reason(websocket) + assert reason is not None and "gemini-live-2.5-flash" not in reason + assert any( + "gemini-3-pro-live" in record.getMessage() and "gemini-live-2.5-flash" in record.getMessage() + for record in caplog.records + ) + + +@pytest.mark.asyncio +async def test_vertex_live_route_forwards_a_setup_frame_the_key_is_allowed_to_run(): + websocket, upstream_ws, _ = await _run_vertex_live_route( + [_setup_frame("gemini-live-2.5-flash")], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + ) + + upstream_ws.send.assert_awaited_once() + assert _policy_close_reason(websocket) is None + + +async def _run_passthrough_with_frame_gate(gate): + upstream_ws = ClosableUpstreamWebSocket() + websocket = _client_websocket( + AsyncMock( + side_effect=[ + {"type": "websocket.receive", "text": json.dumps({"setup": {"model": "gemini-live-2.5-flash"}})}, + {"type": "websocket.disconnect"}, + ] + ), + ) + + with _patched_websocket_passthrough_environment(upstream_ws) as environment: + await websocket_passthrough_request( + websocket=websocket, + target=VERTEX_LIVE_TARGET, + custom_headers={"Authorization": "Bearer token"}, + user_api_key_dict=UserAPIKeyAuth(token="hashed"), + forward_headers=False, + endpoint="/vertex_ai/live", + accept_websocket=False, + client_frame_gate=gate, + ) + + return websocket, upstream_ws, environment + + +@pytest.mark.asyncio +async def test_websocket_passthrough_reports_a_refused_client_frame_as_a_failure(): + """A denied session logged through the success hooks tells spend tracking and audit that it succeeded""" + denial = ProxyException( + message="gemini-live-2.5-flash: key not allowed to access model", + type=ProxyErrorTypes.key_model_access_denied, + param="model", + code=403, + ) + + async def refusing_gate(frame_data, valid_token): + return denial + + websocket, upstream_ws, environment = await _run_passthrough_with_frame_gate(refusing_gate) + + assert _policy_close_reason(websocket) == "gemini-live-2.5-flash: key not allowed to access model" + upstream_ws.close.assert_awaited() + environment.proxy_logging.post_call_success_hook.assert_not_awaited() + environment.logging_worker.ensure_initialized_and_enqueue.assert_not_called() + environment.proxy_logging.post_call_failure_hook.assert_awaited_once() + failure_kwargs = environment.proxy_logging.post_call_failure_hook.await_args.kwargs + assert failure_kwargs["original_exception"] is denial + assert failure_kwargs["request_data"]["litellm_logging_obj"].litellm_call_id + + +@pytest.mark.asyncio +async def test_websocket_passthrough_still_reports_an_allowed_session_as_a_success(): + async def permitting_gate(frame_data, valid_token): + return None + + _, _, environment = await _run_passthrough_with_frame_gate(permitting_gate) + + environment.proxy_logging.post_call_failure_hook.assert_not_awaited() + environment.proxy_logging.post_call_success_hook.assert_awaited_once() + environment.logging_worker.ensure_initialized_and_enqueue.assert_called_once() + + @pytest.mark.parametrize( "setup_model", ["gemini-live-2.5-flash", "models/gemini-live-2.5-flash", "publishers/google/models/gemini-live-2.5-flash"], From 408db64293a5425a2530bf34ceb3583ec1b5bb2a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:01:04 -0700 Subject: [PATCH 2/4] fix(proxy): enforce model access on the Vertex AI Live websocket passthrough The Live route takes its model from the client's setup frame after connect-time auth, so a key limited to one model could open a session on any Vertex Live model. Each setup frame's model is now checked against the key, team and project allowlists, and a refused frame closes the socket with 1008 before anything reaches Vertex. Vertex also reads setup frames that strict JSON cannot parse (single quotes, trailing commas, bare keys), so such a frame is refused instead of being forwarded unchecked. Co-authored-by: Marty Sullivan --- .../llm_passthrough_endpoints.py | 139 ++++++++++++++++-- .../pass_through_endpoints.py | 123 ++++++++++------ .../test_pass_through_endpoints.py | 36 ++++- 3 files changed, 237 insertions(+), 61 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 16fbd1de376..6afd8c53974 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -72,7 +72,8 @@ from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, valid from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.passthrough.main import AsyncPassthroughStreamingResponse, is_provider_error from litellm.proxy._types import * -from litellm.proxy.auth.auth_checks import enforced_model_allowlists +from litellm.proxy.auth.auth_checks import can_key_call_resolved_model, enforced_model_allowlists +from litellm.proxy.auth.auth_utils import log_model_access_denial from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401 # legacy module exports @@ -110,6 +111,7 @@ from litellm.proxy.pass_through_endpoints.injected_stream_usage import ( ) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( HttpPassThroughEndpointHelpers, + WebsocketClientFrameGate, create_pass_through_route, create_websocket_passthrough_route, websocket_passthrough_request, @@ -145,6 +147,7 @@ from .passthrough_endpoint_router import PassthroughEndpointRouter if TYPE_CHECKING: from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig from litellm.router import Router + from litellm.types.router import DeploymentTypedDict ProxyConfig = _ProxyConfig # rebind-ok: conditional type alias else: @@ -3803,6 +3806,12 @@ VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON: Final = ( "Vertex AI auth failed: set a use_in_pass_through vertex model, default_vertex_config, or DEFAULT_VERTEXAI_* env" ) +VERTEX_LIVE_UNVERIFIED_MODEL_CLOSE_REASON: Final = ( + "Could not check your key's access to the model this setup frame names; ask a proxy admin to read the logs" +) + +VERTEX_LIVE_UNREADABLE_SETUP_CLOSE_REASON: Final = "Send setup frames as strict JSON so the proxy can check their model" + VERTEX_PUBLISHER_MODEL_PREFIX: Final = "publishers/google/models/" VERTEX_PUBLISHERS_SEGMENT: Final = "publishers/" @@ -3827,6 +3836,120 @@ def _get_llm_router() -> Router | None: return llm_router +@dataclass(frozen=True, slots=True) +class _UnreadableVertexLiveFrame: + pass + + +class _VertexLiveSetup(TypedDict, total=False): + model: ReadOnly[str] + + +class _VertexLiveClientFrame(TypedDict, total=False): + setup: ReadOnly[_VertexLiveSetup] + + +_VERTEX_LIVE_CLIENT_FRAME: Final = TypeAdapter(_VertexLiveClientFrame) + + +def _may_carry_vertex_live_setup(frame_data: str | bytes) -> bool: + """ + Whether a frame is worth parsing: audio and video frames stream continuously and are large, and one that + cannot spell ``setup`` cannot carry one. Vertex also reads single-quoted and bare keys, so the bare word is + the test, and a backslash means the key may be escaped, which only a parse settles + """ + if isinstance(frame_data, bytes): + return b"setup" in frame_data or b"\\" in frame_data + return "setup" in frame_data or "\\" in frame_data + + +def _vertex_live_setup_model(frame_data: str | bytes) -> str | _UnreadableVertexLiveFrame | None: + """ + The model a Vertex AI Live ``setup`` frame names, which is the only place BidiGenerateContent carries one. + + Vertex parses trailing commas, single quotes and bare keys that ``json.loads`` refuses, so a frame that may + carry a setup but fails a strict parse is unreadable rather than setup-free + """ + if not _may_carry_vertex_live_setup(frame_data): + return None + try: + frame: Final = _VERTEX_LIVE_CLIENT_FRAME.validate_python(json.loads(frame_data)) + except (json.JSONDecodeError, UnicodeDecodeError): + return _UnreadableVertexLiveFrame() + except ValidationError: + return None + setup: Final = frame.get("setup") + return None if setup is None else setup.get("model") or None + + +def _vertex_live_deployment(setup_model: str, llm_router: Router | None) -> DeploymentTypedDict | None: + """ + The router deployment whose group a ``setup`` frame's model names, whichever addressing the client used. + + The Live SDK wraps whatever the caller typed as ``models/``, and clients also send LiteLLM ids and + full Vertex resource paths, so the group is read from the last segment as well as from the whole string + """ + if llm_router is None: + return None + candidates: Final = (setup_model, setup_model.rsplit("/", 1)[-1]) + return next( + ( + deployment + for deployment in (llm_router.get_model_list() or ()) + if deployment.get("model_name") in candidates + ), + None, + ) + + +def _build_vertex_live_client_frame_gate(llm_router: Router | None) -> WebsocketClientFrameGate: + """ + Apply the key's, team's and project's model access to the model a client's ``setup`` frame names. + + ``?model=`` is optional on this route and the Live protocol carries the real model in the first client + frame, so connect-time auth has no model to check and every frame that names one has to be authorized + here instead. + + The allowlists and the access groups behind them are keyed by model group, and every addressing a client + sends carries that group inside a prefix, so the group is what gets authorized. A frame naming no group + is still checked as it arrived, which is the strictest reading available for a name nothing recognises + """ + + async def gate(frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None: + model: Final = _vertex_live_setup_model(frame_data) + if model is None: + return None + if isinstance(model, _UnreadableVertexLiveFrame): + return ProxyException( + message=VERTEX_LIVE_UNREADABLE_SETUP_CLOSE_REASON, + type=ProxyErrorTypes.bad_request_error, + param="setup", + code=400, + ) + deployment: Final = _vertex_live_deployment(model, llm_router) + try: + await can_key_call_resolved_model( + model=model if deployment is None else deployment["model_name"], + llm_model_list=None if llm_router is None else llm_router.get_model_list(), + valid_token=valid_token, + llm_router=llm_router, + ) + except ProxyException as denial: + log_model_access_denial(denial) + return denial + except Exception: # noqa: BLE001 # fail closed: a gate that cannot answer refuses, and says so in band + verbose_proxy_logger.exception("Vertex AI Live passthrough: model access check failed for %s", model) + return ProxyException( + message=VERTEX_LIVE_UNVERIFIED_MODEL_CLOSE_REASON, + type=ProxyErrorTypes.internal_server_error, + param="model", + code=500, + ) + return None + + return gate + + def _resolve_vertex_live_credentials( vertex_project: str | None, vertex_location: str | None, @@ -3885,17 +4008,8 @@ def _resolve_alias_to_upstream_model(setup_model: str, llm_router: Router | None """ The Live SDK wraps whatever the caller typed as ``models/``, so a gateway alias arrives prefixed """ - if llm_router is None: - return setup_model - candidates: Final = (setup_model, setup_model.rsplit("/", 1)[-1]) - upstream: Final = next( - ( - deployment["litellm_params"].get("model") - for deployment in (llm_router.get_model_list() or ()) - if deployment.get("model_name") in candidates - ), - None, - ) + deployment: Final = _vertex_live_deployment(setup_model, llm_router) + upstream: Final = None if deployment is None else deployment["litellm_params"].get("model") if upstream is None: return setup_model try: @@ -4009,6 +4123,7 @@ async def vertex_ai_live_websocket_passthrough( vertex_location=resolved_location, llm_router=_get_llm_router(), ), + client_frame_gate=_build_vertex_live_client_frame_gate(llm_router=_get_llm_router()), ) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index fe7ac30e434..6eda40e369f 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from datetime import datetime from itertools import chain, count, groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, TypedDict, cast +from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, cast from urllib.parse import urlencode, urlparse import httpx @@ -30,7 +30,7 @@ from pydantic import TypeAdapter from starlette.datastructures import UploadFile as StarletteUploadFile from starlette.routing import BaseRoute, Route from starlette.websockets import WebSocketState -from websockets.asyncio.client import connect +from websockets.asyncio.client import ClientConnection, connect from websockets.exceptions import ( ConnectionClosedError, ConnectionClosedOK, @@ -2454,6 +2454,39 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict return inject_trace_context(headers, parent_span=parent_span) +class WebsocketClientFrameGate(Protocol): + """Authorize one client frame, returning the denial to report or ``None`` to let it through. + + A websocket passthrough authenticates before any frame arrives, so a protocol that names its model + inside a frame has no model to authorize at connect time. The frame, not the extracted model, is the + argument because where a model sits is part of each provider's wire protocol + """ + + async def __call__(self, frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None: ... + + +async def _relay_client_frame( + websocket: WebSocket, + upstream_ws: ClientConnection, + checked_frame: str | bytes, + forwarded_frame: str | bytes, + gate: WebsocketClientFrameGate | None, + valid_token: UserAPIKeyAuth, +) -> ProxyException | None: + """Forward one client frame upstream, or refuse it and hand back the denial that ended the session. + + The denial travels out so the caller reports a failed session instead of leaving the success hooks to + record a refusal as a success, and tearing the upstream down is the caller's job: awaiting that close + here lets the upstream reader finish first, which cancels this task mid-close and loses the denial + """ + refusal: Final = None if gate is None else await gate(checked_frame, valid_token) + if refusal is None: + await upstream_ws.send(forwarded_frame) + return None + await websocket.close(code=CloseCode.POLICY_VIOLATION, reason=_truncated_close_reason(refusal.message)) + return refusal + + async def websocket_passthrough_request( websocket: WebSocket, target: str, @@ -2464,6 +2497,7 @@ async def websocket_passthrough_request( cost_per_request: float | None = None, accept_websocket: bool = True, setup_model_rewriter: Callable[[str], str] | None = None, + client_frame_gate: WebsocketClientFrameGate | None = None, ): """ WebSocket passthrough request handler. @@ -2477,6 +2511,7 @@ async def websocket_passthrough_request( endpoint: The endpoint path (for logging purposes) cost_per_request: Optional field - cost per request to the target endpoint setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream + client_frame_gate: Optional per-frame authorization, applied before a frame reaches the upstream """ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj @@ -2572,6 +2607,14 @@ async def websocket_passthrough_request( ) logging_obj.model_call_details["litellm_call_id"] = litellm_call_id + async def log_failure(exception: Exception, traceback_str: str | None = None) -> None: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=exception, + request_data={**kwargs, "litellm_logging_obj": logging_obj}, + traceback_str=traceback_str, + ) + # Pre-call logging logging_obj.pre_call( input=[{"role": "user", "content": "WebSocket connection"}], @@ -2603,8 +2646,8 @@ async def websocket_passthrough_request( "WebSocket passthrough (%s): Upstream connection established successfully", endpoint ) - async def forward_client_to_upstream() -> None: - """Forward messages from client to upstream WebSocket""" + async def forward_client_to_upstream() -> ProxyException | None: + """Forward messages from client to upstream WebSocket, returning any denial that ended it""" try: while True: message = await websocket.receive() @@ -2673,9 +2716,27 @@ async def websocket_passthrough_request( ) # Not a JSON message or doesn't contain setup data - await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter)) + refusal = await _relay_client_frame( + websocket, + upstream_ws, + text_data, + _rewrite_vertex_live_setup_model(text_data, setup_model_rewriter), + client_frame_gate, + user_api_key_dict, + ) + if refusal is not None: + return refusal elif bytes_data is not None: - await upstream_ws.send(bytes_data) + refusal = await _relay_client_frame( + websocket, + upstream_ws, + bytes_data, + bytes_data, + client_frame_gate, + user_api_key_dict, + ) + if refusal is not None: + return refusal except asyncio.CancelledError: raise except Exception: @@ -2748,13 +2809,25 @@ async def websocket_passthrough_request( except asyncio.CancelledError: pass + task_results: Final = tuple(task.result() for task in done if task.exception() is None) + + # A refusal outranks any exception it caused: closing the client socket is what makes the + # upstream reader's next send fail, so raising that instead would report the wrong cause + client_frame_refusal: Final = next( + (result for result in task_results if isinstance(result, ProxyException)), None + ) + if client_frame_refusal is not None: + await upstream_ws.close() + await log_failure(client_frame_refusal) + return + # Check for exceptions in completed tasks for task in done: exception = task.exception() if exception is not None: raise exception - upstream_close: Final = _upstream_close_to_relay(task.result() for task in done) + upstream_close: Final = _upstream_close_to_relay(task_results) if upstream_close is not None and _client_socket_is_open(websocket): await websocket.close( code=upstream_close.code, @@ -2829,23 +2902,7 @@ async def websocket_passthrough_request( except InvalidStatus as exc: verbose_proxy_logger.exception("WebSocket passthrough (%s): upstream rejected WebSocket connection", endpoint) - # Prepare request payload for logging - request_payload = {} - if kwargs: - for key, value in kwargs.items(): - request_payload[key] = value - if logging_obj is not None: - request_payload["litellm_logging_obj"] = logging_obj - - # Log the connection failure using the same pattern as HTTP - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, - original_exception=exc, - request_data=request_payload, - traceback_str=traceback.format_exc( - limit=MAXIMUM_TRACEBACK_LINES_TO_LOG, - ), - ) + await log_failure(exc, traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG)) if _client_socket_is_open(websocket): await websocket.close( @@ -2857,23 +2914,7 @@ async def websocket_passthrough_request( "WebSocket passthrough (%s): unexpected error while proxying WebSocket", endpoint ) - # Prepare request payload for logging - request_payload = {} - if kwargs: - for key, value in kwargs.items(): - request_payload[key] = value - if logging_obj is not None: - request_payload["litellm_logging_obj"] = logging_obj - - # Log the unexpected error using the same pattern as HTTP - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, - original_exception=e, - request_data=request_payload, - traceback_str=traceback.format_exc( - limit=MAXIMUM_TRACEBACK_LINES_TO_LOG, - ), - ) + await log_failure(e, traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG)) if _client_socket_is_open(websocket): await websocket.close(code=1011, reason="WebSocket passthrough error") diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 953aefe7f7c..1fac949fe46 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -7435,10 +7435,7 @@ async def _run_vertex_live_gated_passthrough(client_frames, valid_token): vertex_location="global", llm_router=llm_router, ), - client_frame_gate=_build_vertex_live_client_frame_gate( - llm_model_list=llm_router.get_model_list(), - llm_router=llm_router, - ), + client_frame_gate=_build_vertex_live_client_frame_gate(llm_router=llm_router), ) return websocket, upstream_ws @@ -7548,6 +7545,32 @@ async def test_vertex_live_setup_frame_with_an_escaped_key_is_still_gated(): assert _policy_close_reason(websocket) is not None +@pytest.mark.asyncio +@pytest.mark.parametrize("frame_kind", ["text", "bytes"]) +@pytest.mark.parametrize( + "lenient_setup", + [ + '{"setup": {"model": "totally-unlisted-model-abc",},}', + "{'setup': {'model': 'totally-unlisted-model-abc'}}", + '{setup: {model: "totally-unlisted-model-abc"}}', + ], +) +async def test_vertex_live_setup_frame_only_vertex_can_parse_is_refused(lenient_setup, frame_kind): + """Vertex reads trailing commas, single quotes and bare keys, which strict JSON refuses to parse""" + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [ + { + "type": "websocket.receive", + frame_kind: lenient_setup if frame_kind == "text" else lenient_setup.encode(), + } + ], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + ) + + upstream_ws.send.assert_not_awaited() + assert _policy_close_reason(websocket) is not None + + @pytest.mark.asyncio async def test_vertex_live_non_setup_frames_are_forwarded_without_a_model_check(): websocket, upstream_ws = await _run_vertex_live_gated_passthrough( @@ -7663,10 +7686,7 @@ async def test_vertex_live_setup_frame_resolves_an_access_group_grant_in_every_a }, ] ) - gate = _build_vertex_live_client_frame_gate( - llm_model_list=llm_router.get_model_list(), - llm_router=llm_router, - ) + gate = _build_vertex_live_client_frame_gate(llm_router=llm_router) frame = json.dumps({"setup": {"model": addressing.format(model=VERTEX_LIVE_GROUP)}}) denial = await gate(frame, UserAPIKeyAuth(token="hashed", models=["live-team"])) From 42de0e6c893587f19c24cbe231e1a119549308d5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:06:39 -0700 Subject: [PATCH 3/4] fix(proxy): bill turns before a refused Vertex Live frame and authorize publisher ids Turns served before a refused setup frame are logged and billed as a success at the model they ran on, and the frame is relayed or refused before its model is recorded. A name no model group serves is checked by its publisher id, bare or vertex_ai/ prefixed, and an exact group match wins over the last path segment. Frames the proxy cannot read (lenient JSON, a repeated key, or the wrong shape) are refused only for callers with a model limit. The gate now also runs the user and agent model checks, and the close reason is cut at the last whole sentence that fits. --- litellm/proxy/auth/auth_checks.py | 39 +++ .../llm_passthrough_endpoints.py | 143 ++++++++--- .../pass_through_endpoints.py | 33 +-- .../test_pass_through_endpoints.py | 228 ++++++++++++++++-- 4 files changed, 375 insertions(+), 68 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 8887d86570a..adfb4bc14f7 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -5677,6 +5677,45 @@ async def can_key_call_resolved_model( ) +async def can_caller_call_resolved_model( + model: str, + valid_token: UserAPIKeyAuth, + llm_router: litellm.Router | None, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, +) -> None: + """``can_key_call_resolved_model`` plus the agent grant and a personal key owner's own models, which + ``common_checks`` applies to a model named at auth time and a model named after auth needs as well""" + await can_key_call_resolved_model( + model=model, + llm_model_list=None if llm_router is None else llm_router.get_model_list(), + valid_token=valid_token, + llm_router=llm_router, + ) + await can_agent_call_model( + model=model, + valid_token=valid_token, + llm_router=llm_router, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if valid_token.team_id is not None or prisma_client is None: + return + await can_user_call_model( + model=model, + llm_router=llm_router, + user_object=await _user_object_or_none( + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ), + key_model_aliases=key_model_aliases_for_auth_check(valid_token), + ) + + def can_org_access_model( model: str, org_object: LiteLLM_OrganizationTable | None, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 6afd8c53974..5acceb54ce3 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -72,7 +72,7 @@ from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, valid from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.passthrough.main import AsyncPassthroughStreamingResponse, is_provider_error from litellm.proxy._types import * -from litellm.proxy.auth.auth_checks import can_key_call_resolved_model, enforced_model_allowlists +from litellm.proxy.auth.auth_checks import can_caller_call_resolved_model, enforced_model_allowlists from litellm.proxy.auth.auth_utils import log_model_access_denial from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.route_checks import RouteChecks @@ -3812,6 +3812,8 @@ VERTEX_LIVE_UNVERIFIED_MODEL_CLOSE_REASON: Final = ( VERTEX_LIVE_UNREADABLE_SETUP_CLOSE_REASON: Final = "Send setup frames as strict JSON so the proxy can check their model" +VERTEX_LIVE_NAME_NO_ALLOWLIST_HOLDS: Final = "litellm-probe/a-setup-frame-model-the-proxy-could-not-read" + VERTEX_PUBLISHER_MODEL_PREFIX: Final = "publishers/google/models/" VERTEX_PUBLISHERS_SEGMENT: Final = "publishers/" @@ -3863,80 +3865,146 @@ def _may_carry_vertex_live_setup(frame_data: str | bytes) -> bool: return "setup" in frame_data or "\\" in frame_data +def _object_with_unique_keys(pairs: Sequence[tuple[str, object]]) -> Mapping[str, object]: + unique: Final = MappingProxyType(dict(pairs)) + if len(unique) < len(pairs): + raise ValueError("a JSON object repeats a key") + return unique + + def _vertex_live_setup_model(frame_data: str | bytes) -> str | _UnreadableVertexLiveFrame | None: """ The model a Vertex AI Live ``setup`` frame names, which is the only place BidiGenerateContent carries one. - Vertex parses trailing commas, single quotes and bare keys that ``json.loads`` refuses, so a frame that may - carry a setup but fails a strict parse is unreadable rather than setup-free + Vertex parses trailing commas, single quotes and bare keys that ``json.loads`` refuses, and keeps the first of + two keys when the second is null, so a frame that may carry a setup but fails a strict parse, repeats a key, + or holds a setup of the wrong shape is unreadable rather than setup-free """ if not _may_carry_vertex_live_setup(frame_data): return None try: - frame: Final = _VERTEX_LIVE_CLIENT_FRAME.validate_python(json.loads(frame_data)) - except (json.JSONDecodeError, UnicodeDecodeError): + frame: Final = _VERTEX_LIVE_CLIENT_FRAME.validate_python( + json.loads(frame_data, object_pairs_hook=_object_with_unique_keys) + ) + except ValueError: return _UnreadableVertexLiveFrame() - except ValidationError: - return None setup: Final = frame.get("setup") return None if setup is None else setup.get("model") or None +def _deployment_named(model_name: str, deployments: Sequence[DeploymentTypedDict]) -> DeploymentTypedDict | None: + return next((deployment for deployment in deployments if deployment.get("model_name") == model_name), None) + + def _vertex_live_deployment(setup_model: str, llm_router: Router | None) -> DeploymentTypedDict | None: """ The router deployment whose group a ``setup`` frame's model names, whichever addressing the client used. The Live SDK wraps whatever the caller typed as ``models/``, and clients also send LiteLLM ids and - full Vertex resource paths, so the group is read from the last segment as well as from the whole string + full Vertex resource paths, so the group is read from the last segment when the whole string names none """ if llm_router is None: return None - candidates: Final = (setup_model, setup_model.rsplit("/", 1)[-1]) - return next( - ( - deployment - for deployment in (llm_router.get_model_list() or ()) - if deployment.get("model_name") in candidates - ), - None, + deployments: Final = tuple(llm_router.get_model_list() or ()) + exact: Final = _deployment_named(setup_model, deployments) + return exact if exact is not None else _deployment_named(setup_model.rsplit("/", 1)[-1], deployments) + + +def _vertex_live_model_names(setup_model: str, llm_router: Router | None) -> tuple[str, ...]: + """ + The names a key can hold for the model a ``setup`` frame runs: the router group the setup rewriter sends it + through, else the publisher model id Vertex runs, bare or under LiteLLM's ``vertex_ai/`` prefix. + + A full resource path skips the rewriter and reaches Vertex as sent, so only its publisher model id counts, and + a wildcard is matched against that id alone so addressing in front of it cannot satisfy the pattern + """ + deployment: Final = ( + None if setup_model.startswith("projects/") else _vertex_live_deployment(setup_model, llm_router) ) + if deployment is not None: + return (deployment["model_name"],) + publisher_model: Final = setup_model.rsplit("/", 1)[-1] + return (publisher_model, f"vertex_ai/{publisher_model}") + + +async def _model_access_denial( + model: str, valid_token: UserAPIKeyAuth, llm_router: Router | None +) -> ProxyException | None: + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + try: + await can_caller_call_resolved_model( + model=model, + valid_token=valid_token, + llm_router=llm_router, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + except ProxyException as denial: + return denial + except HTTPException as refusal: + return ProxyException( + message=str(refusal.detail), + type=ProxyErrorTypes.auth_error, + param="model", + code=refusal.status_code, + ) + return None + + +async def _any_name_allowed_denial( + model_names: tuple[str, ...], valid_token: UserAPIKeyAuth, llm_router: Router | None +) -> ProxyException | None: + denials: Final = tuple([await _model_access_denial(name, valid_token, llm_router) for name in model_names]) + return None if None in denials else denials[0] + + +async def _vertex_live_setup_denial( + setup_model: str | _UnreadableVertexLiveFrame, valid_token: UserAPIKeyAuth, llm_router: Router | None +) -> ProxyException | None: + match setup_model: + case str(): + denial: Final = await _any_name_allowed_denial( + _vertex_live_model_names(setup_model, llm_router), valid_token, llm_router + ) + if denial is not None: + log_model_access_denial(denial) + return denial + case _UnreadableVertexLiveFrame(): + unlimited: Final = await _any_name_allowed_denial( + (VERTEX_LIVE_NAME_NO_ALLOWLIST_HOLDS,), valid_token, llm_router + ) + return ( + None + if unlimited is None + else ProxyException( + message=VERTEX_LIVE_UNREADABLE_SETUP_CLOSE_REASON, + type=ProxyErrorTypes.bad_request_error, + param="setup", + code=400, + ) + ) def _build_vertex_live_client_frame_gate(llm_router: Router | None) -> WebsocketClientFrameGate: """ - Apply the key's, team's and project's model access to the model a client's ``setup`` frame names. + Apply the checks a model named at auth time gets to the model a client's ``setup`` frame names. ``?model=`` is optional on this route and the Live protocol carries the real model in the first client frame, so connect-time auth has no model to check and every frame that names one has to be authorized here instead. - The allowlists and the access groups behind them are keyed by model group, and every addressing a client - sends carries that group inside a prefix, so the group is what gets authorized. A frame naming no group - is still checked as it arrived, which is the strictest reading available for a name nothing recognises + A frame the proxy cannot read the way Vertex does only reaches Vertex for a caller who may call any model, + which is a caller whose checks accept a name no allowlist holds """ async def gate(frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None: model: Final = _vertex_live_setup_model(frame_data) if model is None: return None - if isinstance(model, _UnreadableVertexLiveFrame): - return ProxyException( - message=VERTEX_LIVE_UNREADABLE_SETUP_CLOSE_REASON, - type=ProxyErrorTypes.bad_request_error, - param="setup", - code=400, - ) - deployment: Final = _vertex_live_deployment(model, llm_router) try: - await can_key_call_resolved_model( - model=model if deployment is None else deployment["model_name"], - llm_model_list=None if llm_router is None else llm_router.get_model_list(), - valid_token=valid_token, - llm_router=llm_router, - ) - except ProxyException as denial: - log_model_access_denial(denial) - return denial + return await _vertex_live_setup_denial(model, valid_token, llm_router) except Exception: # noqa: BLE001 # fail closed: a gate that cannot answer refuses, and says so in band verbose_proxy_logger.exception("Vertex AI Live passthrough: model access check failed for %s", model) return ProxyException( @@ -3945,7 +4013,6 @@ def _build_vertex_live_client_frame_gate(llm_router: Router | None) -> Websocket param="model", code=500, ) - return None return gate diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 6eda40e369f..9fdb0b74455 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2403,12 +2403,15 @@ def _json_object_frame(frame: str | bytes) -> dict[str, object] | None: def _truncated_close_reason(reason: str) -> str: """ - Fit a close reason inside the byte budget a WebSocket close frame allows, without splitting a character + Fit a close reason inside the byte budget a WebSocket close frame allows, ending on the last whole sentence + that fits when there is one and never splitting a character """ encoded: Final = reason.encode("utf-8") if len(encoded) <= WEBSOCKET_CLOSE_REASON_MAX_BYTES: return reason - return encoded[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode("utf-8", errors="ignore") + clipped: Final = encoded[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode("utf-8", errors="ignore") + sentence_end: Final = clipped.rfind(". ") + return clipped if sentence_end == -1 else clipped[: sentence_end + 1] SENDABLE_CLOSE_CODES: Final = frozenset(CloseCode) - frozenset( @@ -2660,6 +2663,16 @@ async def websocket_passthrough_request( bytes_data: bytes | None = message.get("bytes") if text_data is not None: + refusal = await _relay_client_frame( + websocket, + upstream_ws, + text_data, + _rewrite_vertex_live_setup_model(text_data, setup_model_rewriter), + client_frame_gate, + user_api_key_dict, + ) + if refusal is not None: + return refusal # Try to extract model from client setup message for Vertex AI Live if endpoint and "/vertex_ai/live" in endpoint: verbose_proxy_logger.debug( @@ -2716,16 +2729,6 @@ async def websocket_passthrough_request( ) # Not a JSON message or doesn't contain setup data - refusal = await _relay_client_frame( - websocket, - upstream_ws, - text_data, - _rewrite_vertex_live_setup_model(text_data, setup_model_rewriter), - client_frame_gate, - user_api_key_dict, - ) - if refusal is not None: - return refusal elif bytes_data is not None: refusal = await _relay_client_frame( websocket, @@ -2812,19 +2815,21 @@ async def websocket_passthrough_request( task_results: Final = tuple(task.result() for task in done if task.exception() is None) # A refusal outranks any exception it caused: closing the client socket is what makes the - # upstream reader's next send fail, so raising that instead would report the wrong cause + # upstream reader's next send fail, so raising that instead would report the wrong cause. + # Turns that completed before a refusal were served, so they are logged and billed as a success client_frame_refusal: Final = next( (result for result in task_results if isinstance(result, ProxyException)), None ) if client_frame_refusal is not None: await upstream_ws.close() + if client_frame_refusal is not None and not websocket_messages: await log_failure(client_frame_refusal) return # Check for exceptions in completed tasks for task in done: exception = task.exception() - if exception is not None: + if exception is not None and client_frame_refusal is None: raise exception upstream_close: Final = _upstream_close_to_relay(task_results) diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 1fac949fe46..c7163b22666 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -7240,10 +7240,16 @@ def _patched_websocket_passthrough_environment(upstream_ws): mock_proxy_logging.pre_call_hook = AsyncMock(return_value={}) mock_proxy_logging.post_call_success_hook = AsyncMock() mock_proxy_logging.post_call_failure_hook = AsyncMock() + enqueued_handler = MagicMock() mock_worker.ensure_initialized_and_enqueue = MagicMock( - side_effect=lambda async_coroutine: async_coroutine.close() + side_effect=lambda async_coroutine: ( + enqueued_handler(**async_coroutine.cr_frame.f_locals), + async_coroutine.close(), + ) + ) + yield SimpleNamespace( + proxy_logging=mock_proxy_logging, logging_worker=mock_worker, enqueued_handler=enqueued_handler ) - yield SimpleNamespace(proxy_logging=mock_proxy_logging, logging_worker=mock_worker) async def _pending_receive(): @@ -7408,20 +7414,27 @@ def _vertex_live_router(): ) -async def _run_vertex_live_gated_passthrough(client_frames, valid_token): +async def _run_vertex_live_gated_passthrough(client_frames, valid_token, llm_router=None): """Drive /vertex_ai/live with the real frame gate, returning the client and upstream sockets""" + upstream_ws = RecordingUpstreamWebSocket() + websocket, _ = await _run_vertex_live_gated_session( + AsyncMock(side_effect=[*client_frames, {"type": "websocket.disconnect"}]), + upstream_ws, + valid_token, + llm_router or _vertex_live_router(), + ) + return websocket, upstream_ws + + +async def _run_vertex_live_gated_session(receive, upstream_ws, valid_token, llm_router): from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _build_vertex_live_client_frame_gate, _build_vertex_live_setup_model_rewriter, ) - llm_router = _vertex_live_router() - upstream_ws = RecordingUpstreamWebSocket() - websocket = _client_websocket( - AsyncMock(side_effect=[*client_frames, {"type": "websocket.disconnect"}]), - ) + websocket = _client_websocket(receive) - with _patched_websocket_passthrough_environment(upstream_ws): + with _patched_websocket_passthrough_environment(upstream_ws) as environment: await websocket_passthrough_request( websocket=websocket, target=VERTEX_LIVE_TARGET, @@ -7438,7 +7451,7 @@ async def _run_vertex_live_gated_passthrough(client_frames, valid_token): client_frame_gate=_build_vertex_live_client_frame_gate(llm_router=llm_router), ) - return websocket, upstream_ws + return websocket, environment def _setup_frame(model: str) -> dict: @@ -7469,6 +7482,7 @@ async def test_vertex_live_setup_frame_naming_a_model_the_key_cannot_call_is_ref upstream_ws.send.assert_not_awaited() reason = _policy_close_reason(websocket) assert reason is not None and "totally-unlisted-model-abc" in reason + assert reason.endswith(".") upstream_ws.close.assert_awaited() @@ -7571,6 +7585,69 @@ async def test_vertex_live_setup_frame_only_vertex_can_parse_is_refused(lenient_ assert _policy_close_reason(websocket) is not None +@pytest.mark.asyncio +@pytest.mark.parametrize( + "lenient_setup", + [ + '{"setup": {"model": "gemini-3-pro-live",},}', + "{'setup': {'model': 'gemini-3-pro-live'}}", + '{setup: {model: "gemini-3-pro-live"}}', + ], +) +async def test_vertex_live_setup_frame_only_vertex_can_parse_still_reaches_vertex_for_an_unrestricted_key( + lenient_setup, +): + """A key that may call any model loses nothing the proxy cannot read, so its lenient frames still go through""" + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [{"type": "websocket.receive", "text": lenient_setup}], + UserAPIKeyAuth(token="hashed", models=[]), + ) + + upstream_ws.send.assert_awaited_once() + assert _policy_close_reason(websocket) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "repeated_key_setup", + [ + '{"setup": {"model": "gemini-3-pro-live", "model": null}}', + '{"setup": {"model": "gemini-3-pro-live"}, "setup": null}', + ], +) +async def test_vertex_live_setup_frame_repeating_a_key_as_null_is_refused(repeated_key_setup): + """ + Vertex keeps the first of two keys when the second is null, while ``json.loads`` keeps the last, so the + proxy would read no model where Vertex runs the first one + """ + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [{"type": "websocket.receive", "text": repeated_key_setup}], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + ) + + upstream_ws.send.assert_not_awaited() + assert _policy_close_reason(websocket) is not None + + +@pytest.mark.asyncio +async def test_vertex_live_client_content_that_mentions_setup_is_forwarded(): + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [ + _setup_frame("gemini-live-2.5-flash"), + { + "type": "websocket.receive", + "text": json.dumps( + {"clientContent": {"turns": [{"role": "user", "parts": [{"text": "help me setup a meeting"}]}]}} + ), + }, + ], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + ) + + assert upstream_ws.send.await_count == 2 + assert _policy_close_reason(websocket) is None + + @pytest.mark.asyncio async def test_vertex_live_non_setup_frames_are_forwarded_without_a_model_check(): websocket, upstream_ws = await _run_vertex_live_gated_passthrough( @@ -7632,19 +7709,79 @@ async def test_vertex_live_setup_frame_naming_no_group_at_all_is_refused_in_ever @pytest.mark.asyncio -@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS[1:]) -async def test_vertex_live_setup_frame_does_not_strip_a_prefix_off_a_model_no_group_serves(addressing): +@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS) +@pytest.mark.parametrize("key_model", ["gemini-4-pro-live-unserved", "vertex_ai/gemini-4-pro-live-unserved"]) +async def test_vertex_live_setup_frame_authorizes_a_model_no_group_serves_by_its_publisher_id(addressing, key_model): """ - Stripping the addressing off a name the router does not serve would let a wildcard key reach an arbitrary - publisher model, and a full path would carry a project of the caller's choosing with it + With no group to resolve, the setup rewriter sends Vertex the last path segment, so a key holding that + publisher model, bare or as a LiteLLM ``vertex_ai/`` id, can open it whichever addressing its client sends """ websocket, upstream_ws = await _run_vertex_live_gated_passthrough( [_setup_frame(addressing.format(model="gemini-4-pro-live-unserved"))], - UserAPIKeyAuth(token="hashed", models=["gemini-*"]), + UserAPIKeyAuth(token="hashed", models=[key_model]), + ) + + upstream_ws.send.assert_awaited_once() + sent_setup = json.loads(upstream_ws.send.await_args.args[0])["setup"] + assert sent_setup["model"].endswith("/models/gemini-4-pro-live-unserved") + assert _policy_close_reason(websocket) is None + + +@pytest.mark.asyncio +async def test_vertex_live_setup_frame_addressing_cannot_satisfy_a_wildcard_for_another_publisher_model(): + """Vertex runs only the last segment, so the segments ahead of it must not be what matches the key's wildcard""" + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame("vertex_ai/gemini-anything/forbidden-model")], + UserAPIKeyAuth(token="hashed", models=["vertex_ai/gemini-*"]), ) upstream_ws.send.assert_not_awaited() - assert _policy_close_reason(websocket) is not None + reason = _policy_close_reason(websocket) + assert reason is not None and "forbidden-model" in reason + + +@pytest.mark.asyncio +async def test_vertex_live_setup_frame_prefers_the_group_its_whole_name_matches(): + """A group named ``vertex_ai/`` is the one the rewriter sends, so it is the one the key must hold""" + llm_router = litellm.Router( + model_list=[ + {"model_name": "gemini-3-pro-live", "litellm_params": {"model": "vertex_ai/gemini-3-pro-live"}}, + {"model_name": "vertex_ai/gemini-3-pro-live", "litellm_params": {"model": "vertex_ai/gemini-3-pro-live"}}, + ] + ) + + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame("vertex_ai/gemini-3-pro-live")], + UserAPIKeyAuth(token="hashed", models=["vertex_ai/gemini-3-pro-live"]), + llm_router=llm_router, + ) + + upstream_ws.send.assert_awaited_once() + assert _policy_close_reason(websocket) is None + + +@pytest.mark.asyncio +async def test_vertex_live_setup_frame_is_checked_against_the_owner_of_a_personal_key(): + """A personal key's owner can be limited to fewer models than the key lists, and auth enforces that limit too""" + from litellm.proxy.proxy_server import user_api_key_cache + + user_id = f"live-user-{uuid.uuid4().hex}" + await user_api_key_cache.async_set_cache( + key=user_id, + value={"user_id": user_id, "user_email": "live@example.com", "models": ["gemini-live-2.5-flash"]}, + ) + + with patch( # test-quality-ok: read at call time, and the user row comes from the cache seeded above + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ): + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame("gemini-3-pro-live")], + UserAPIKeyAuth(token="hashed", user_id=user_id, models=[]), + ) + + upstream_ws.send.assert_not_awaited() + reason = _policy_close_reason(websocket) + assert reason is not None and "gemini-3-pro-live" in reason @pytest.mark.asyncio @@ -7772,6 +7909,65 @@ async def test_vertex_live_route_forwards_a_setup_frame_the_key_is_allowed_to_ru assert _policy_close_reason(websocket) is None +class ScriptedUpstreamWebSocket: + """An upstream that sends the given frames in order, then stays open with nothing more to say""" + + def __init__(self, frames): + self._frames = iter(frames) + self.sent_every_frame = asyncio.Event() + self.send = AsyncMock() + self.close = AsyncMock() + + async def recv(self, decode: bool = True): + frame = next(self._frames, None) + if frame is not None: + return frame + self.sent_every_frame.set() + await asyncio.Event().wait() + + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + +@pytest.mark.asyncio +async def test_vertex_live_turns_served_before_a_refused_frame_are_still_billed(): + """ + A refusal used to log the whole session as a failure, which records no spend, so a key could run allowed + turns and then send a refused frame to have them go unbilled. The refused frame's model must not become + the model those turns are priced at either + """ + upstream_ws = ScriptedUpstreamWebSocket( + ( + json.dumps({"setupComplete": {}}), + json.dumps({"serverContent": {"turnComplete": True}, "usageMetadata": {"totalTokenCount": 60}}), + ) + ) + + async def client_frames(): + yield _setup_frame("gemini-live-2.5-flash") + await upstream_ws.sent_every_frame.wait() + yield _setup_frame("gemini-3-pro-live") + + websocket, environment = await _run_vertex_live_gated_session( + client_frames().__anext__, + upstream_ws, + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + _vertex_live_router(), + ) + + reason = _policy_close_reason(websocket) + assert reason is not None and "gemini-3-pro-live" in reason + upstream_ws.send.assert_awaited_once() + upstream_ws.close.assert_awaited() + environment.logging_worker.ensure_initialized_and_enqueue.assert_called_once() + assert environment.enqueued_handler.call_args.kwargs["logging_obj"].model == "gemini-live-2.5-flash" + environment.proxy_logging.post_call_success_hook.assert_awaited_once() + environment.proxy_logging.post_call_failure_hook.assert_not_awaited() + + async def _run_passthrough_with_frame_gate(gate): upstream_ws = ClosableUpstreamWebSocket() websocket = _client_websocket( From 3ee80f6acabe3c1bc75a58c5486de449a43a4ee0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:33:40 -0700 Subject: [PATCH 4/4] fix(proxy): check the Vertex Live model the setup frame forwards, not an alias of it A key, team, router or global alias name no longer grants a Live setup frame, since the frame reaches Vertex under the name sent. A full projects/ path is allowed for any group serving that publisher model. Frames the proxy cannot read are refused for any caller with a model limit, and upstream close reasons keep the byte cut. --- .../llm_passthrough_endpoints.py | 115 +++++++--- .../pass_through_endpoints.py | 58 +++-- .../test_pass_through_endpoints.py | 217 +++++++++++++++--- 3 files changed, 311 insertions(+), 79 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 5acceb54ce3..f57d1a1af63 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -72,9 +72,16 @@ from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, valid from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.passthrough.main import AsyncPassthroughStreamingResponse, is_provider_error from litellm.proxy._types import * -from litellm.proxy.auth.auth_checks import can_caller_call_resolved_model, enforced_model_allowlists +from litellm.proxy.auth.auth_checks import ( + can_caller_call_resolved_model, + enforced_model_allowlists, + key_model_aliases_for_auth_check, + personal_key_team_model_access_applies, + team_model_aliases_for_auth_check, +) from litellm.proxy.auth.auth_utils import log_model_access_denial from litellm.proxy.auth.handle_jwt import JWTHandler +from litellm.proxy.auth.model_access_denied import model_access_denied_client_message from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401 # legacy module exports _get_bearer_token, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -3812,8 +3819,6 @@ VERTEX_LIVE_UNVERIFIED_MODEL_CLOSE_REASON: Final = ( VERTEX_LIVE_UNREADABLE_SETUP_CLOSE_REASON: Final = "Send setup frames as strict JSON so the proxy can check their model" -VERTEX_LIVE_NAME_NO_ALLOWLIST_HOLDS: Final = "litellm-probe/a-setup-frame-model-the-proxy-could-not-read" - VERTEX_PUBLISHER_MODEL_PREFIX: Final = "publishers/google/models/" VERTEX_PUBLISHERS_SEGMENT: Final = "publishers/" @@ -3910,12 +3915,24 @@ def _vertex_live_deployment(setup_model: str, llm_router: Router | None) -> Depl return exact if exact is not None else _deployment_named(setup_model.rsplit("/", 1)[-1], deployments) +def _deployment_upstream_model(deployment: DeploymentTypedDict) -> str | None: + upstream: Final = deployment["litellm_params"].get("model") + if upstream is None: + return None + try: + _, provider, _, _ = litellm.get_llm_provider(model=upstream) + except litellm.exceptions.BadRequestError: + return upstream + return upstream.removeprefix(f"{provider}/") + + def _vertex_live_model_names(setup_model: str, llm_router: Router | None) -> tuple[str, ...]: """ The names a key can hold for the model a ``setup`` frame runs: the router group the setup rewriter sends it - through, else the publisher model id Vertex runs, bare or under LiteLLM's ``vertex_ai/`` prefix. + through, else every group serving the publisher model Vertex runs, and that model's id bare or under LiteLLM's + ``vertex_ai/`` prefix unless a group holds that name for another model. - A full resource path skips the rewriter and reaches Vertex as sent, so only its publisher model id counts, and + A full resource path skips the rewriter and reaches Vertex as sent, so only its publisher model counts, and a wildcard is matched against that id alone so addressing in front of it cannot satisfy the pattern """ deployment: Final = ( @@ -3923,8 +3940,30 @@ def _vertex_live_model_names(setup_model: str, llm_router: Router | None) -> tup ) if deployment is not None: return (deployment["model_name"],) - publisher_model: Final = setup_model.rsplit("/", 1)[-1] - return (publisher_model, f"vertex_ai/{publisher_model}") + deployments: Final = () if llm_router is None else tuple(llm_router.get_model_list() or ()) + publisher_model: Final = _vertex_publisher_model_suffix(setup_model) + serving_groups: Final = tuple( + dict.fromkeys( + candidate["model_name"] + for candidate in deployments + if _vertex_publisher_model_suffix(_deployment_upstream_model(candidate) or "") == publisher_model + ) + ) + group_names: Final = frozenset(candidate["model_name"] for candidate in deployments) + publisher_model_id: Final = setup_model.rsplit("/", 1)[-1] + return ( + *serving_groups, + *(name for name in (publisher_model_id, f"vertex_ai/{publisher_model_id}") if name not in group_names), + ) + + +def _is_model_alias(name: str, valid_token: UserAPIKeyAuth, llm_router: Router | None) -> bool: + return ( + name in (key_model_aliases_for_auth_check(valid_token) or ()) + or name in (team_model_aliases_for_auth_check(valid_token) or ()) + or name in litellm.model_alias_map + or (llm_router is not None and name in llm_router.model_group_alias) + ) async def _model_access_denial( @@ -3953,11 +3992,42 @@ async def _model_access_denial( return None -async def _any_name_allowed_denial( - model_names: tuple[str, ...], valid_token: UserAPIKeyAuth, llm_router: Router | None +async def _vertex_live_model_denial( + setup_model: str, valid_token: UserAPIKeyAuth, llm_router: Router | None ) -> ProxyException | None: + """ + The setup rewriter forwards a name as sent, so a name an alias maps to another model grants nothing here + """ + model_names: Final = tuple( + name + for name in _vertex_live_model_names(setup_model, llm_router) + if not _is_model_alias(name, valid_token, llm_router) + ) denials: Final = tuple([await _model_access_denial(name, valid_token, llm_router) for name in model_names]) - return None if None in denials else denials[0] + if None in denials: + return None + return ( + denials[0] + if denials + else ModelAccessDeniedProxyException( + message=model_access_denied_client_message(model=setup_model), + internal_message=f"Vertex AI Live setup model {setup_model} is only reachable through a model alias", + type=ProxyErrorTypes.key_model_access_denied, + param="model", + code=403, + ) + ) + + +async def _caller_has_model_limits(valid_token: UserAPIKeyAuth) -> bool: + return ( + valid_token.agent_id is not None + or valid_token.managed_agent_policy is not None + or valid_token.agent_caller is not None + or valid_token.end_user_id is not None + or personal_key_team_model_access_applies(valid_token) + or _has_model_restrictions(await _proxy_model_allowlists()(valid_token)) + ) async def _vertex_live_setup_denial( @@ -3965,25 +4035,20 @@ async def _vertex_live_setup_denial( ) -> ProxyException | None: match setup_model: case str(): - denial: Final = await _any_name_allowed_denial( - _vertex_live_model_names(setup_model, llm_router), valid_token, llm_router - ) + denial: Final = await _vertex_live_model_denial(setup_model, valid_token, llm_router) if denial is not None: log_model_access_denial(denial) return denial case _UnreadableVertexLiveFrame(): - unlimited: Final = await _any_name_allowed_denial( - (VERTEX_LIVE_NAME_NO_ALLOWLIST_HOLDS,), valid_token, llm_router - ) return ( - None - if unlimited is None - else ProxyException( + ProxyException( message=VERTEX_LIVE_UNREADABLE_SETUP_CLOSE_REASON, type=ProxyErrorTypes.bad_request_error, param="setup", code=400, ) + if await _caller_has_model_limits(valid_token) + else None ) @@ -3996,7 +4061,7 @@ def _build_vertex_live_client_frame_gate(llm_router: Router | None) -> Websocket here instead. A frame the proxy cannot read the way Vertex does only reaches Vertex for a caller who may call any model, - which is a caller whose checks accept a name no allowlist holds + which is a caller with no model allowlist and no agent, end user or team grant standing in for one """ async def gate(frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None: @@ -4076,14 +4141,8 @@ def _resolve_alias_to_upstream_model(setup_model: str, llm_router: Router | None The Live SDK wraps whatever the caller typed as ``models/``, so a gateway alias arrives prefixed """ deployment: Final = _vertex_live_deployment(setup_model, llm_router) - upstream: Final = None if deployment is None else deployment["litellm_params"].get("model") - if upstream is None: - return setup_model - try: - _, provider, _, _ = litellm.get_llm_provider(model=upstream) - except litellm.exceptions.BadRequestError: - return upstream - return upstream.removeprefix(f"{provider}/") + upstream: Final = None if deployment is None else _deployment_upstream_model(deployment) + return setup_model if upstream is None else upstream async def vertex_ai_live_websocket_passthrough( diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 9fdb0b74455..c8949dc3a01 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2403,15 +2403,21 @@ def _json_object_frame(frame: str | bytes) -> dict[str, object] | None: def _truncated_close_reason(reason: str) -> str: """ - Fit a close reason inside the byte budget a WebSocket close frame allows, ending on the last whole sentence - that fits when there is one and never splitting a character + Fit a close reason inside the byte budget a WebSocket close frame allows, without splitting a character """ encoded: Final = reason.encode("utf-8") if len(encoded) <= WEBSOCKET_CLOSE_REASON_MAX_BYTES: return reason - clipped: Final = encoded[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode("utf-8", errors="ignore") + return encoded[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode("utf-8", errors="ignore") + + +def _refusal_close_reason(message: str) -> str: + """ + The proxy's own refusal message as a close reason, ending on the last whole sentence that fits + """ + clipped: Final = _truncated_close_reason(message) sentence_end: Final = clipped.rfind(". ") - return clipped if sentence_end == -1 else clipped[: sentence_end + 1] + return clipped if clipped == message or sentence_end == -1 else clipped[: sentence_end + 1] SENDABLE_CLOSE_CODES: Final = frozenset(CloseCode) - frozenset( @@ -2486,7 +2492,7 @@ async def _relay_client_frame( if refusal is None: await upstream_ws.send(forwarded_frame) return None - await websocket.close(code=CloseCode.POLICY_VIOLATION, reason=_truncated_close_reason(refusal.message)) + await websocket.close(code=CloseCode.POLICY_VIOLATION, reason=_refusal_close_reason(refusal.message)) return refusal @@ -2663,16 +2669,17 @@ async def websocket_passthrough_request( bytes_data: bytes | None = message.get("bytes") if text_data is not None: - refusal = await _relay_client_frame( - websocket, - upstream_ws, - text_data, - _rewrite_vertex_live_setup_model(text_data, setup_model_rewriter), - client_frame_gate, - user_api_key_dict, - ) - if refusal is not None: - return refusal + if ( + text_refusal := await _relay_client_frame( + websocket, + upstream_ws, + text_data, + _rewrite_vertex_live_setup_model(text_data, setup_model_rewriter), + client_frame_gate, + user_api_key_dict, + ) + ) is not None: + return text_refusal # Try to extract model from client setup message for Vertex AI Live if endpoint and "/vertex_ai/live" in endpoint: verbose_proxy_logger.debug( @@ -2730,16 +2737,17 @@ async def websocket_passthrough_request( # Not a JSON message or doesn't contain setup data elif bytes_data is not None: - refusal = await _relay_client_frame( - websocket, - upstream_ws, - bytes_data, - bytes_data, - client_frame_gate, - user_api_key_dict, - ) - if refusal is not None: - return refusal + if ( + bytes_refusal := await _relay_client_frame( + websocket, + upstream_ws, + bytes_data, + bytes_data, + client_frame_gate, + user_api_key_dict, + ) + ) is not None: + return bytes_refusal except asyncio.CancelledError: raise except Exception: diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index c7163b22666..e95a132eae2 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -7,7 +7,7 @@ import os import sys import uuid import zlib -from collections.abc import AsyncIterator, Callable, Mapping +from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping, Sequence from contextlib import ExitStack, contextmanager from dataclasses import dataclass from datetime import datetime @@ -30,7 +30,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, WEBSOCKET_CLOSE_REASON_MAX_BYTES from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -53,6 +53,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( initialize_pass_through_endpoints, pass_through_request, resolve_llm_passthrough_timeout, + WebsocketClientFrameGate, resolve_pass_through_request_timeout, set_env_variables_in_header, websocket_passthrough_request, @@ -7184,27 +7185,27 @@ class ClosableUpstreamWebSocket: CLOSE_HANDSHAKE_YIELDS = 5 - def __init__(self): + def __init__(self) -> None: self._closing = asyncio.Event() self.send = AsyncMock() self.close = AsyncMock(side_effect=self._close) - async def _close(self): + async def _close(self) -> None: self._closing.set() for _ in range(self.CLOSE_HANDSHAKE_YIELDS): await asyncio.sleep(0) - async def recv(self, decode: bool = True): + async def recv(self, decode: bool = True) -> str: from websockets.exceptions import ConnectionClosedOK from websockets.frames import Close await self._closing.wait() raise ConnectionClosedOK(rcvd=Close(1000, ""), sent=Close(1000, ""), rcvd_then_sent=True) - def __aiter__(self): + def __aiter__(self) -> "ClosableUpstreamWebSocket": return self - async def __anext__(self): + async def __anext__(self) -> str: raise StopAsyncIteration @@ -7285,6 +7286,36 @@ async def test_websocket_passthrough_relays_upstream_policy_close_to_client(): websocket.close.assert_awaited_once_with(code=1008, reason=upstream_reason) +@pytest.mark.asyncio +async def test_websocket_passthrough_relays_as_much_of_a_long_upstream_close_reason_as_fits(): + from websockets.exceptions import ConnectionClosedError + from websockets.frames import Close + + upstream_reason = ( + "Quota exceeded for concurrent_sessions. Request more quota for aiplatform.googleapis.com " + "BidiGenerateContent sessions in us-central1 from the Google Cloud console quotas page" + ) + upstream_ws = ClosingUpstreamWebSocket( + ConnectionClosedError(rcvd=Close(1011, upstream_reason), sent=Close(1011, ""), rcvd_then_sent=True) + ) + websocket = _client_websocket(_pending_receive) + + with _patched_websocket_passthrough_environment(upstream_ws): + await websocket_passthrough_request( + websocket=websocket, + target="wss://aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent", + custom_headers={"Authorization": "Bearer token"}, + user_api_key_dict=UserAPIKeyAuth(), + forward_headers=False, + endpoint="/vertex_ai/live", + accept_websocket=False, + ) + + relayed_reason = websocket.close.await_args.kwargs["reason"] + assert upstream_reason.startswith(relayed_reason) + assert len(relayed_reason.encode("utf-8")) == WEBSOCKET_CLOSE_REASON_MAX_BYTES + + @pytest.mark.asyncio async def test_websocket_passthrough_keeps_normal_upstream_close_normal(): from websockets.exceptions import ConnectionClosedOK @@ -7382,16 +7413,19 @@ VERTEX_LIVE_GROUP = "gemini-live-2.5-flash-native-audio" VERTEX_LIVE_ALIAS_GROUP = "gemini-live-native-audio" -VERTEX_LIVE_CLIENT_ADDRESSINGS = ( +VERTEX_LIVE_GROUP_ADDRESSINGS = ( "{model}", "models/{model}", "vertex_ai/{model}", "publishers/google/models/{model}", - "projects/proj-db/locations/global/publishers/google/models/{model}", ) +VERTEX_LIVE_FULL_PATH = "projects/proj-db/locations/global/publishers/google/models/{model}" -def _vertex_live_router(): +VERTEX_LIVE_CLIENT_ADDRESSINGS = (*VERTEX_LIVE_GROUP_ADDRESSINGS, VERTEX_LIVE_FULL_PATH) + + +def _vertex_live_router() -> litellm.Router: return litellm.Router( model_list=[ { @@ -7414,7 +7448,11 @@ def _vertex_live_router(): ) -async def _run_vertex_live_gated_passthrough(client_frames, valid_token, llm_router=None): +async def _run_vertex_live_gated_passthrough( + client_frames: Sequence[Mapping[str, str | bytes]], + valid_token: UserAPIKeyAuth, + llm_router: litellm.Router | None = None, +) -> tuple[MagicMock, RecordingUpstreamWebSocket]: """Drive /vertex_ai/live with the real frame gate, returning the client and upstream sockets""" upstream_ws = RecordingUpstreamWebSocket() websocket, _ = await _run_vertex_live_gated_session( @@ -7426,7 +7464,12 @@ async def _run_vertex_live_gated_passthrough(client_frames, valid_token, llm_rou return websocket, upstream_ws -async def _run_vertex_live_gated_session(receive, upstream_ws, valid_token, llm_router): +async def _run_vertex_live_gated_session( + receive: Callable[[], Awaitable[Mapping[str, str | bytes]]], + upstream_ws: "RecordingUpstreamWebSocket | ScriptedUpstreamWebSocket", + valid_token: UserAPIKeyAuth, + llm_router: litellm.Router, +) -> tuple[MagicMock, SimpleNamespace]: from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _build_vertex_live_client_frame_gate, _build_vertex_live_setup_model_rewriter, @@ -7454,14 +7497,14 @@ async def _run_vertex_live_gated_session(receive, upstream_ws, valid_token, llm_ return websocket, environment -def _setup_frame(model: str) -> dict: +def _setup_frame(model: str) -> dict[str, str]: return { "type": "websocket.receive", "text": json.dumps({"setup": {"model": model, "generationConfig": {"responseModalities": ["TEXT"]}}}), } -def _policy_close_reason(websocket) -> str | None: +def _policy_close_reason(websocket: MagicMock) -> str | None: return next( (call.kwargs.get("reason") for call in websocket.close.await_args_list if call.kwargs.get("code") == 1008), None, @@ -7629,6 +7672,37 @@ async def test_vertex_live_setup_frame_repeating_a_key_as_null_is_refused(repeat assert _policy_close_reason(websocket) is not None +@pytest.mark.asyncio +@pytest.mark.parametrize( + "limited_token", + [ + UserAPIKeyAuth(token="hashed", models=["litellm-*"]), + UserAPIKeyAuth(token="hashed", models=["vertex_ai/gemini-*"]), + UserAPIKeyAuth(token="hashed", models=[], team_id="team-1", team_models=["gemini-live-2.5-flash"]), + UserAPIKeyAuth(token="hashed", models=[], agent_id="agent-1"), + UserAPIKeyAuth(token="hashed", models=[], end_user_id="end-user-1"), + ], + ids=["litellm-wildcard", "vertex-wildcard", "team", "agent", "end-user"], +) +async def test_vertex_live_setup_frame_only_vertex_can_parse_is_refused_for_any_limited_caller(limited_token): + """A wildcard can match any name the proxy might test the caller with, so only a caller with no model limit + anywhere may send a frame the proxy cannot read""" + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [ + { + "type": "websocket.receive", + "text": "{'setup': {'model': '" + + VERTEX_LIVE_FULL_PATH.format(model="gemini-3-pro-live") + + "',},}", + } + ], + limited_token, + ) + + upstream_ws.send.assert_not_awaited() + assert _policy_close_reason(websocket) is not None + + @pytest.mark.asyncio async def test_vertex_live_client_content_that_mentions_setup_is_forwarded(): websocket, upstream_ws = await _run_vertex_live_gated_passthrough( @@ -7663,12 +7737,12 @@ async def test_vertex_live_non_setup_frames_are_forwarded_without_a_model_check( @pytest.mark.asyncio -@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS) +@pytest.mark.parametrize("addressing", VERTEX_LIVE_GROUP_ADDRESSINGS) @pytest.mark.parametrize("group", [VERTEX_LIVE_GROUP, VERTEX_LIVE_ALIAS_GROUP]) async def test_vertex_live_setup_frame_authorizes_a_permitted_group_in_every_addressing(addressing, group): """ - The Live SDK wraps the caller's model as ``models/`` and the docs tell callers to send the full - resource path, so a key holding the group has to work whichever addressing its client actually sends + The Live SDK wraps the caller's model as ``models/``, so a key holding the group has to work + whichever addressing its client actually sends """ websocket, upstream_ws = await _run_vertex_live_gated_passthrough( [_setup_frame(addressing.format(model=group))], @@ -7760,6 +7834,93 @@ async def test_vertex_live_setup_frame_prefers_the_group_its_whole_name_matches( assert _policy_close_reason(websocket) is None +@pytest.mark.asyncio +@pytest.mark.parametrize("group", [VERTEX_LIVE_GROUP, VERTEX_LIVE_ALIAS_GROUP]) +async def test_vertex_live_full_path_is_authorized_by_any_group_serving_its_publisher_model(group): + """The docs tell callers to send the full resource path, which Vertex runs as sent, so every group serving + that publisher model grants it, whatever the group is called""" + full_path = VERTEX_LIVE_FULL_PATH.format(model=VERTEX_LIVE_GROUP) + + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame(full_path)], + UserAPIKeyAuth(token="hashed", models=[group]), + ) + + upstream_ws.send.assert_awaited_once() + assert json.loads(upstream_ws.send.await_args.args[0])["setup"]["model"] == full_path + assert _policy_close_reason(websocket) is None + + +@pytest.mark.asyncio +async def test_vertex_live_full_path_is_not_granted_by_a_group_named_after_it_that_serves_another_model(): + llm_router = litellm.Router( + model_list=[ + {"model_name": "gemini-3-pro-live", "litellm_params": {"model": "vertex_ai/gemini-live-2.5-flash"}}, + ] + ) + + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame(VERTEX_LIVE_FULL_PATH.format(model="gemini-3-pro-live"))], + UserAPIKeyAuth(token="hashed", models=["gemini-3-pro-live"]), + llm_router=llm_router, + ) + + upstream_ws.send.assert_not_awaited() + reason = _policy_close_reason(websocket) + assert reason is not None and "gemini-3-pro-live" in reason + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "valid_token", + [ + UserAPIKeyAuth( + token="hashed", + models=["gemini-live-2.5-flash"], + aliases={"gemini-3-pro-live": "gemini-live-2.5-flash"}, + ), + UserAPIKeyAuth( + token="hashed", + models=["all-team-models"], + team_id="team-1", + team_models=["gemini-live-2.5-flash"], + team_model_aliases={"gemini-3-pro-live": "gemini-live-2.5-flash"}, + ), + ], + ids=["key-alias", "team-alias"], +) +async def test_vertex_live_setup_frame_naming_an_alias_cannot_open_the_model_the_alias_shadows(valid_token): + """ + An alias passes the access check by its target, while the setup rewriter sends the name as written, so a + key that could set an alias would open the model the alias name belongs to + """ + websocket, upstream_ws = await _run_vertex_live_gated_passthrough([_setup_frame("gemini-3-pro-live")], valid_token) + + upstream_ws.send.assert_not_awaited() + reason = _policy_close_reason(websocket) + assert reason is not None and "gemini-3-pro-live" in reason + + +@pytest.mark.asyncio +async def test_vertex_live_setup_frame_naming_a_router_alias_cannot_open_the_model_the_alias_shadows(): + llm_router = litellm.Router( + model_list=[ + {"model_name": "gemini-live-2.5-flash", "litellm_params": {"model": "vertex_ai/gemini-live-2.5-flash"}}, + ], + model_group_alias={"gemini-3-pro-live": "gemini-live-2.5-flash"}, + ) + + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame("gemini-3-pro-live")], + UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]), + llm_router=llm_router, + ) + + upstream_ws.send.assert_not_awaited() + reason = _policy_close_reason(websocket) + assert reason is not None and "gemini-3-pro-live" in reason + + @pytest.mark.asyncio async def test_vertex_live_setup_frame_is_checked_against_the_owner_of_a_personal_key(): """A personal key's owner can be limited to fewer models than the key lists, and auth enforces that limit too""" @@ -7831,7 +7992,9 @@ async def test_vertex_live_setup_frame_resolves_an_access_group_grant_in_every_a assert denial is None -async def _run_vertex_live_route(client_frames, valid_token): +async def _run_vertex_live_route( + client_frames: Sequence[Mapping[str, str | bytes]], valid_token: UserAPIKeyAuth +) -> tuple[MagicMock, RecordingUpstreamWebSocket, SimpleNamespace]: """Drive the /vertex_ai/live route, so the gate under test is whichever one the route itself wires""" from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( vertex_ai_live_websocket_passthrough, @@ -7912,23 +8075,23 @@ async def test_vertex_live_route_forwards_a_setup_frame_the_key_is_allowed_to_ru class ScriptedUpstreamWebSocket: """An upstream that sends the given frames in order, then stays open with nothing more to say""" - def __init__(self, frames): + def __init__(self, frames: Iterable[str]) -> None: self._frames = iter(frames) self.sent_every_frame = asyncio.Event() self.send = AsyncMock() self.close = AsyncMock() - async def recv(self, decode: bool = True): + async def recv(self, decode: bool = True) -> str | None: frame = next(self._frames, None) if frame is not None: return frame self.sent_every_frame.set() await asyncio.Event().wait() - def __aiter__(self): + def __aiter__(self) -> "ScriptedUpstreamWebSocket": return self - async def __anext__(self): + async def __anext__(self) -> str: raise StopAsyncIteration @@ -7946,7 +8109,7 @@ async def test_vertex_live_turns_served_before_a_refused_frame_are_still_billed( ) ) - async def client_frames(): + async def client_frames() -> AsyncIterator[dict[str, str]]: yield _setup_frame("gemini-live-2.5-flash") await upstream_ws.sent_every_frame.wait() yield _setup_frame("gemini-3-pro-live") @@ -7968,7 +8131,9 @@ async def test_vertex_live_turns_served_before_a_refused_frame_are_still_billed( environment.proxy_logging.post_call_failure_hook.assert_not_awaited() -async def _run_passthrough_with_frame_gate(gate): +async def _run_passthrough_with_frame_gate( + gate: WebsocketClientFrameGate, +) -> tuple[MagicMock, ClosableUpstreamWebSocket, SimpleNamespace]: upstream_ws = ClosableUpstreamWebSocket() websocket = _client_websocket( AsyncMock( @@ -8004,7 +8169,7 @@ async def test_websocket_passthrough_reports_a_refused_client_frame_as_a_failure code=403, ) - async def refusing_gate(frame_data, valid_token): + async def refusing_gate(frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None: return denial websocket, upstream_ws, environment = await _run_passthrough_with_frame_gate(refusing_gate) @@ -8021,7 +8186,7 @@ async def test_websocket_passthrough_reports_a_refused_client_frame_as_a_failure @pytest.mark.asyncio async def test_websocket_passthrough_still_reports_an_allowed_session_as_a_success(): - async def permitting_gate(frame_data, valid_token): + async def permitting_gate(frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None: return None _, _, environment = await _run_passthrough_with_frame_gate(permitting_gate)