This commit is contained in:
Mateo Wang 2026-10-27 02:24:56 +00:00 • committed by GitHub
commit 377eb91042
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 1260 additions and 66 deletions

View file

@ -5691,6 +5691,45 @@ async def can_key_call_resolved_model(
)
async def can_caller_call_resolved_model(
model: str,
valid_token: UserAPIKeyAuth,
llm_router: litellm.Router | None,
prisma_client: PrismaClient | None,
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,
) -> None:
"""``can_key_call_resolved_model`` plus the agent grant and a personal key owner's own models, which
``common_checks`` applies to a model named at auth time and a model named after auth needs as well"""
await can_key_call_resolved_model(
model=model,
llm_model_list=None if llm_router is None else llm_router.get_model_list(),
valid_token=valid_token,
llm_router=llm_router,
)
await can_agent_call_model(
model=model,
valid_token=valid_token,
llm_router=llm_router,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
if valid_token.team_id is not None or prisma_client is None:
return
await can_user_call_model(
model=model,
llm_router=llm_router,
user_object=await _user_object_or_none(
valid_token=valid_token,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
),
key_model_aliases=key_model_aliases_for_auth_check(valid_token),
)
def can_org_access_model(
model: str,
org_object: LiteLLM_OrganizationTable | None,

View file

@ -72,8 +72,16 @@ from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, valid
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.passthrough.main import AsyncPassthroughStreamingResponse, is_provider_error
from litellm.proxy._types import *
from litellm.proxy.auth.auth_checks import enforced_model_allowlists
from litellm.proxy.auth.auth_checks import (
can_caller_call_resolved_model,
enforced_model_allowlists,
key_model_aliases_for_auth_check,
personal_key_team_model_access_applies,
team_model_aliases_for_auth_check,
)
from litellm.proxy.auth.auth_utils import log_model_access_denial
from litellm.proxy.auth.handle_jwt import JWTHandler
from litellm.proxy.auth.model_access_denied import model_access_denied_client_message
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401 # legacy module exports
_get_bearer_token, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
@ -110,6 +118,7 @@ from litellm.proxy.pass_through_endpoints.injected_stream_usage import (
)
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
HttpPassThroughEndpointHelpers,
WebsocketClientFrameGate,
create_pass_through_route,
create_websocket_passthrough_route,
websocket_passthrough_request,
@ -145,6 +154,7 @@ from .passthrough_endpoint_router import PassthroughEndpointRouter
if TYPE_CHECKING:
from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig
from litellm.router import Router
from litellm.types.router import DeploymentTypedDict
ProxyConfig = _ProxyConfig # rebind-ok: conditional type alias
else:
@ -3803,6 +3813,12 @@ VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON: Final = (
"Vertex AI auth failed: set a use_in_pass_through vertex model, default_vertex_config, or DEFAULT_VERTEXAI_* env"
)
VERTEX_LIVE_UNVERIFIED_MODEL_CLOSE_REASON: Final = (
"Could not check your key's access to the model this setup frame names; ask a proxy admin to read the logs"
)
VERTEX_LIVE_UNREADABLE_SETUP_CLOSE_REASON: Final = "Send setup frames as strict JSON so the proxy can check their model"
VERTEX_PUBLISHER_MODEL_PREFIX: Final = "publishers/google/models/"
VERTEX_PUBLISHERS_SEGMENT: Final = "publishers/"
@ -3827,6 +3843,245 @@ def _get_llm_router() -> Router | None:
return llm_router
@dataclass(frozen=True, slots=True)
class _UnreadableVertexLiveFrame:
pass
class _VertexLiveSetup(TypedDict, total=False):
model: ReadOnly[str]
class _VertexLiveClientFrame(TypedDict, total=False):
setup: ReadOnly[_VertexLiveSetup]
_VERTEX_LIVE_CLIENT_FRAME: Final = TypeAdapter(_VertexLiveClientFrame)
def _may_carry_vertex_live_setup(frame_data: str | bytes) -> bool:
"""
Whether a frame is worth parsing: audio and video frames stream continuously and are large, and one that
cannot spell ``setup`` cannot carry one. Vertex also reads single-quoted and bare keys, so the bare word is
the test, and a backslash means the key may be escaped, which only a parse settles
"""
if isinstance(frame_data, bytes):
return b"setup" in frame_data or b"\\" in frame_data
return "setup" in frame_data or "\\" in frame_data
def _object_with_unique_keys(pairs: Sequence[tuple[str, object]]) -> Mapping[str, object]:
unique: Final = MappingProxyType(dict(pairs))
if len(unique) < len(pairs):
raise ValueError("a JSON object repeats a key")
return unique
def _vertex_live_setup_model(frame_data: str | bytes) -> str | _UnreadableVertexLiveFrame | None:
"""
The model a Vertex AI Live ``setup`` frame names, which is the only place BidiGenerateContent carries one.
Vertex parses trailing commas, single quotes and bare keys that ``json.loads`` refuses, and keeps the first of
two keys when the second is null, so a frame that may carry a setup but fails a strict parse, repeats a key,
or holds a setup of the wrong shape is unreadable rather than setup-free
"""
if not _may_carry_vertex_live_setup(frame_data):
return None
try:
frame: Final = _VERTEX_LIVE_CLIENT_FRAME.validate_python(
json.loads(frame_data, object_pairs_hook=_object_with_unique_keys)
)
except ValueError:
return _UnreadableVertexLiveFrame()
setup: Final = frame.get("setup")
return None if setup is None else setup.get("model") or None
def _deployment_named(model_name: str, deployments: Sequence[DeploymentTypedDict]) -> DeploymentTypedDict | None:
return next((deployment for deployment in deployments if deployment.get("model_name") == model_name), None)
def _vertex_live_deployment(setup_model: str, llm_router: Router | None) -> DeploymentTypedDict | None:
"""
The router deployment whose group a ``setup`` frame's model names, whichever addressing the client used.
The Live SDK wraps whatever the caller typed as ``models/<name>``, and clients also send LiteLLM ids and
full Vertex resource paths, so the group is read from the last segment when the whole string names none
"""
if llm_router is None:
return None
deployments: Final = tuple(llm_router.get_model_list() or ())
exact: Final = _deployment_named(setup_model, deployments)
return exact if exact is not None else _deployment_named(setup_model.rsplit("/", 1)[-1], deployments)
def _deployment_upstream_model(deployment: DeploymentTypedDict) -> str | None:
upstream: Final = deployment["litellm_params"].get("model")
if upstream is None:
return None
try:
_, provider, _, _ = litellm.get_llm_provider(model=upstream)
except litellm.exceptions.BadRequestError:
return upstream
return upstream.removeprefix(f"{provider}/")
def _vertex_live_model_names(setup_model: str, llm_router: Router | None) -> tuple[str, ...]:
"""
The names a key can hold for the model a ``setup`` frame runs: the router group the setup rewriter sends it
through, else every group serving the publisher model Vertex runs, and that model's id bare or under LiteLLM's
``vertex_ai/`` prefix unless a group holds that name for another model.
A full resource path skips the rewriter and reaches Vertex as sent, so only its publisher model counts, and
a wildcard is matched against that id alone so addressing in front of it cannot satisfy the pattern
"""
deployment: Final = (
None if setup_model.startswith("projects/") else _vertex_live_deployment(setup_model, llm_router)
)
if deployment is not None:
return (deployment["model_name"],)
deployments: Final = () if llm_router is None else tuple(llm_router.get_model_list() or ())
publisher_model: Final = _vertex_publisher_model_suffix(setup_model)
serving_groups: Final = tuple(
dict.fromkeys(
candidate["model_name"]
for candidate in deployments
if _vertex_publisher_model_suffix(_deployment_upstream_model(candidate) or "") == publisher_model
)
)
group_names: Final = frozenset(candidate["model_name"] for candidate in deployments)
publisher_model_id: Final = setup_model.rsplit("/", 1)[-1]
return (
*serving_groups,
*(name for name in (publisher_model_id, f"vertex_ai/{publisher_model_id}") if name not in group_names),
)
def _is_model_alias(name: str, valid_token: UserAPIKeyAuth, llm_router: Router | None) -> bool:
return (
name in (key_model_aliases_for_auth_check(valid_token) or ())
or name in (team_model_aliases_for_auth_check(valid_token) or ())
or name in litellm.model_alias_map
or (llm_router is not None and name in llm_router.model_group_alias)
)
async def _model_access_denial(
model: str, valid_token: UserAPIKeyAuth, llm_router: Router | None
) -> ProxyException | None:
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
try:
await can_caller_call_resolved_model(
model=model,
valid_token=valid_token,
llm_router=llm_router,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
except ProxyException as denial:
return denial
except HTTPException as refusal:
return ProxyException(
message=str(refusal.detail),
type=ProxyErrorTypes.auth_error,
param="model",
code=refusal.status_code,
)
return None
async def _vertex_live_model_denial(
setup_model: str, valid_token: UserAPIKeyAuth, llm_router: Router | None
) -> ProxyException | None:
"""
The setup rewriter forwards a name as sent, so a name an alias maps to another model grants nothing here
"""
model_names: Final = tuple(
name
for name in _vertex_live_model_names(setup_model, llm_router)
if not _is_model_alias(name, valid_token, llm_router)
)
denials: Final = tuple([await _model_access_denial(name, valid_token, llm_router) for name in model_names])
if None in denials:
return None
return (
denials[0]
if denials
else ModelAccessDeniedProxyException(
message=model_access_denied_client_message(model=setup_model),
internal_message=f"Vertex AI Live setup model {setup_model} is only reachable through a model alias",
type=ProxyErrorTypes.key_model_access_denied,
param="model",
code=403,
)
)
async def _caller_has_model_limits(valid_token: UserAPIKeyAuth) -> bool:
return (
valid_token.agent_id is not None
or valid_token.managed_agent_policy is not None
or valid_token.agent_caller is not None
or valid_token.end_user_id is not None
or personal_key_team_model_access_applies(valid_token)
or _has_model_restrictions(await _proxy_model_allowlists()(valid_token))
)
async def _vertex_live_setup_denial(
setup_model: str | _UnreadableVertexLiveFrame, valid_token: UserAPIKeyAuth, llm_router: Router | None
) -> ProxyException | None:
match setup_model:
case str():
denial: Final = await _vertex_live_model_denial(setup_model, valid_token, llm_router)
if denial is not None:
log_model_access_denial(denial)
return denial
case _UnreadableVertexLiveFrame():
return (
ProxyException(
message=VERTEX_LIVE_UNREADABLE_SETUP_CLOSE_REASON,
type=ProxyErrorTypes.bad_request_error,
param="setup",
code=400,
)
if await _caller_has_model_limits(valid_token)
else None
)
def _build_vertex_live_client_frame_gate(llm_router: Router | None) -> WebsocketClientFrameGate:
"""
Apply the checks a model named at auth time gets to the model a client's ``setup`` frame names.
``?model=`` is optional on this route and the Live protocol carries the real model in the first client
frame, so connect-time auth has no model to check and every frame that names one has to be authorized
here instead.
A frame the proxy cannot read the way Vertex does only reaches Vertex for a caller who may call any model,
which is a caller with no model allowlist and no agent, end user or team grant standing in for one
"""
async def gate(frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None:
model: Final = _vertex_live_setup_model(frame_data)
if model is None:
return None
try:
return await _vertex_live_setup_denial(model, valid_token, llm_router)
except Exception: # noqa: BLE001 # fail closed: a gate that cannot answer refuses, and says so in band
verbose_proxy_logger.exception("Vertex AI Live passthrough: model access check failed for %s", model)
return ProxyException(
message=VERTEX_LIVE_UNVERIFIED_MODEL_CLOSE_REASON,
type=ProxyErrorTypes.internal_server_error,
param="model",
code=500,
)
return gate
def _resolve_vertex_live_credentials(
vertex_project: str | None,
vertex_location: str | None,
@ -3885,24 +4140,9 @@ def _resolve_alias_to_upstream_model(setup_model: str, llm_router: Router | None
"""
The Live SDK wraps whatever the caller typed as ``models/<name>``, so a gateway alias arrives prefixed
"""
if llm_router is None:
return setup_model
candidates: Final = (setup_model, setup_model.rsplit("/", 1)[-1])
upstream: Final = next(
(
deployment["litellm_params"].get("model")
for deployment in (llm_router.get_model_list() or ())
if deployment.get("model_name") in candidates
),
None,
)
if upstream is None:
return setup_model
try:
_, provider, _, _ = litellm.get_llm_provider(model=upstream)
except litellm.exceptions.BadRequestError:
return upstream
return upstream.removeprefix(f"{provider}/")
deployment: Final = _vertex_live_deployment(setup_model, llm_router)
upstream: Final = None if deployment is None else _deployment_upstream_model(deployment)
return setup_model if upstream is None else upstream
async def vertex_ai_live_websocket_passthrough(
@ -4009,6 +4249,7 @@ async def vertex_ai_live_websocket_passthrough(
vertex_location=resolved_location,
llm_router=_get_llm_router(),
),
client_frame_gate=_build_vertex_live_client_frame_gate(llm_router=_get_llm_router()),
)

View file

@ -10,7 +10,7 @@ from dataclasses import dataclass
from datetime import datetime
from itertools import chain, count, groupby
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, cast
from urllib.parse import urlencode, urlparse
import httpx
@ -30,7 +30,7 @@ from pydantic import TypeAdapter
from starlette.datastructures import UploadFile as StarletteUploadFile
from starlette.routing import BaseRoute, Route
from starlette.websockets import WebSocketState
from websockets.asyncio.client import connect
from websockets.asyncio.client import ClientConnection, connect
from websockets.exceptions import (
ConnectionClosedError,
ConnectionClosedOK,
@ -2414,6 +2414,15 @@ def _truncated_close_reason(reason: str) -> str:
return encoded[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode("utf-8", errors="ignore")
def _refusal_close_reason(message: str) -> str:
"""
The proxy's own refusal message as a close reason, ending on the last whole sentence that fits
"""
clipped: Final = _truncated_close_reason(message)
sentence_end: Final = clipped.rfind(". ")
return clipped if clipped == message or sentence_end == -1 else clipped[: sentence_end + 1]
SENDABLE_CLOSE_CODES: Final = frozenset(CloseCode) - frozenset(
{CloseCode.NO_STATUS_RCVD, CloseCode.ABNORMAL_CLOSURE, CloseCode.TLS_HANDSHAKE}
)
@ -2457,6 +2466,39 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict
return inject_trace_context(headers, parent_span=parent_span)
class WebsocketClientFrameGate(Protocol):
"""Authorize one client frame, returning the denial to report or ``None`` to let it through.
A websocket passthrough authenticates before any frame arrives, so a protocol that names its model
inside a frame has no model to authorize at connect time. The frame, not the extracted model, is the
argument because where a model sits is part of each provider's wire protocol
"""
async def __call__(self, frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None: ...
async def _relay_client_frame(
websocket: WebSocket,
upstream_ws: ClientConnection,
checked_frame: str | bytes,
forwarded_frame: str | bytes,
gate: WebsocketClientFrameGate | None,
valid_token: UserAPIKeyAuth,
) -> ProxyException | None:
"""Forward one client frame upstream, or refuse it and hand back the denial that ended the session.
The denial travels out so the caller reports a failed session instead of leaving the success hooks to
record a refusal as a success, and tearing the upstream down is the caller's job: awaiting that close
here lets the upstream reader finish first, which cancels this task mid-close and loses the denial
"""
refusal: Final = None if gate is None else await gate(checked_frame, valid_token)
if refusal is None:
await upstream_ws.send(forwarded_frame)
return None
await websocket.close(code=CloseCode.POLICY_VIOLATION, reason=_refusal_close_reason(refusal.message))
return refusal
async def websocket_passthrough_request(
websocket: WebSocket,
target: str,
@ -2467,6 +2509,7 @@ async def websocket_passthrough_request(
cost_per_request: float | None = None,
accept_websocket: bool = True,
setup_model_rewriter: Callable[[str], str] | None = None,
client_frame_gate: WebsocketClientFrameGate | None = None,
):
"""
WebSocket passthrough request handler.
@ -2480,6 +2523,7 @@ async def websocket_passthrough_request(
endpoint: The endpoint path (for logging purposes)
cost_per_request: Optional field - cost per request to the target endpoint
setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream
client_frame_gate: Optional per-frame authorization, applied before a frame reaches the upstream
"""
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj
@ -2575,6 +2619,14 @@ async def websocket_passthrough_request(
)
logging_obj.model_call_details["litellm_call_id"] = litellm_call_id
async def log_failure(exception: Exception, traceback_str: str | None = None) -> None:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=exception,
request_data={**kwargs, "litellm_logging_obj": logging_obj},
traceback_str=traceback_str,
)
# Pre-call logging
logging_obj.pre_call(
input=[{"role": "user", "content": "WebSocket connection"}],
@ -2606,8 +2658,8 @@ async def websocket_passthrough_request(
"WebSocket passthrough (%s): Upstream connection established successfully", endpoint
)
async def forward_client_to_upstream() -> None:
"""Forward messages from client to upstream WebSocket"""
async def forward_client_to_upstream() -> ProxyException | None:
"""Forward messages from client to upstream WebSocket, returning any denial that ended it"""
try:
while True:
message = await websocket.receive()
@ -2620,6 +2672,17 @@ async def websocket_passthrough_request(
bytes_data: bytes | None = message.get("bytes")
if text_data is not None:
if (
text_refusal := await _relay_client_frame(
websocket,
upstream_ws,
text_data,
_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter),
client_frame_gate,
user_api_key_dict,
)
) is not None:
return text_refusal
# Try to extract model from client setup message for Vertex AI Live
if endpoint and "/vertex_ai/live" in endpoint:
verbose_proxy_logger.debug(
@ -2676,9 +2739,18 @@ async def websocket_passthrough_request(
)
# Not a JSON message or doesn't contain setup data
await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter))
elif bytes_data is not None:
await upstream_ws.send(bytes_data)
if (
bytes_refusal := await _relay_client_frame(
websocket,
upstream_ws,
bytes_data,
bytes_data,
client_frame_gate,
user_api_key_dict,
)
) is not None:
return bytes_refusal
except asyncio.CancelledError:
raise
except Exception:
@ -2751,13 +2823,27 @@ async def websocket_passthrough_request(
except asyncio.CancelledError:
pass
task_results: Final = tuple(task.result() for task in done if task.exception() is None)
# A refusal outranks any exception it caused: closing the client socket is what makes the
# upstream reader's next send fail, so raising that instead would report the wrong cause.
# Turns that completed before a refusal were served, so they are logged and billed as a success
client_frame_refusal: Final = next(
(result for result in task_results if isinstance(result, ProxyException)), None
)
if client_frame_refusal is not None:
await upstream_ws.close()
if client_frame_refusal is not None and not websocket_messages:
await log_failure(client_frame_refusal)
return
# Check for exceptions in completed tasks
for task in done:
exception = task.exception()
if exception is not None:
if exception is not None and client_frame_refusal is None:
raise exception
upstream_close: Final = _upstream_close_to_relay(task.result() for task in done)
upstream_close: Final = _upstream_close_to_relay(task_results)
if upstream_close is not None and _client_socket_is_open(websocket):
await websocket.close(
code=upstream_close.code,
@ -2832,23 +2918,7 @@ async def websocket_passthrough_request(
except InvalidStatus as exc:
verbose_proxy_logger.exception("WebSocket passthrough (%s): upstream rejected WebSocket connection", endpoint)
# Prepare request payload for logging
request_payload = {}
if kwargs:
for key, value in kwargs.items():
request_payload[key] = value
if logging_obj is not None:
request_payload["litellm_logging_obj"] = logging_obj
# Log the connection failure using the same pattern as HTTP
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=exc,
request_data=request_payload,
traceback_str=traceback.format_exc(
limit=MAXIMUM_TRACEBACK_LINES_TO_LOG,
),
)
await log_failure(exc, traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG))
if _client_socket_is_open(websocket):
await websocket.close(
@ -2860,23 +2930,7 @@ async def websocket_passthrough_request(
"WebSocket passthrough (%s): unexpected error while proxying WebSocket", endpoint
)
# Prepare request payload for logging
request_payload = {}
if kwargs:
for key, value in kwargs.items():
request_payload[key] = value
if logging_obj is not None:
request_payload["litellm_logging_obj"] = logging_obj
# Log the unexpected error using the same pattern as HTTP
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=request_payload,
traceback_str=traceback.format_exc(
limit=MAXIMUM_TRACEBACK_LINES_TO_LOG,
),
)
await log_failure(e, traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG))
if _client_socket_is_open(websocket):
await websocket.close(code=1011, reason="WebSocket passthrough error")

View file

@ -7,7 +7,7 @@ import os
import sys
import uuid
import zlib
from collections.abc import AsyncIterator, Callable, Mapping
from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping, Sequence
from contextlib import ExitStack, contextmanager
from dataclasses import dataclass
from datetime import datetime
@ -30,12 +30,12 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, WEBSOCKET_CLOSE_REASON_MAX_BYTES
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._lazy_features import LazyFeature, attach_lazy_features
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import router as llm_passthrough_router
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
@ -53,6 +53,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
initialize_pass_through_endpoints,
pass_through_request,
resolve_llm_passthrough_timeout,
WebsocketClientFrameGate,
resolve_pass_through_request_timeout,
set_env_variables_in_header,
websocket_passthrough_request,
@ -7175,6 +7176,39 @@ class RecordingUpstreamWebSocket:
raise StopAsyncIteration
class ClosableUpstreamWebSocket:
"""An upstream whose ``recv`` ends when ``close`` does, and whose close keeps yielding after that.
A real close handshake outlives the ``recv`` it unblocks, so the reader task finishes while the closing
task is still suspended. That is the interleaving that leaves a refusal stranded in a cancelled task
"""
CLOSE_HANDSHAKE_YIELDS = 5
def __init__(self) -> None:
self._closing = asyncio.Event()
self.send = AsyncMock()
self.close = AsyncMock(side_effect=self._close)
async def _close(self) -> None:
self._closing.set()
for _ in range(self.CLOSE_HANDSHAKE_YIELDS):
await asyncio.sleep(0)
async def recv(self, decode: bool = True) -> str:
from websockets.exceptions import ConnectionClosedOK
from websockets.frames import Close
await self._closing.wait()
raise ConnectionClosedOK(rcvd=Close(1000, ""), sent=Close(1000, ""), rcvd_then_sent=True)
def __aiter__(self) -> "ClosableUpstreamWebSocket":
return self
async def __anext__(self) -> str:
raise StopAsyncIteration
def _client_websocket(receive):
from starlette.websockets import WebSocketState
@ -7207,10 +7241,16 @@ def _patched_websocket_passthrough_environment(upstream_ws):
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
mock_proxy_logging.post_call_success_hook = AsyncMock()
mock_proxy_logging.post_call_failure_hook = AsyncMock()
enqueued_handler = MagicMock()
mock_worker.ensure_initialized_and_enqueue = MagicMock(
side_effect=lambda async_coroutine: async_coroutine.close()
side_effect=lambda async_coroutine: (
enqueued_handler(**async_coroutine.cr_frame.f_locals),
async_coroutine.close(),
)
)
yield SimpleNamespace(
proxy_logging=mock_proxy_logging, logging_worker=mock_worker, enqueued_handler=enqueued_handler
)
yield
async def _pending_receive():
@ -7246,6 +7286,36 @@ async def test_websocket_passthrough_relays_upstream_policy_close_to_client():
websocket.close.assert_awaited_once_with(code=1008, reason=upstream_reason)
@pytest.mark.asyncio
async def test_websocket_passthrough_relays_as_much_of_a_long_upstream_close_reason_as_fits():
from websockets.exceptions import ConnectionClosedError
from websockets.frames import Close
upstream_reason = (
"Quota exceeded for concurrent_sessions. Request more quota for aiplatform.googleapis.com "
"BidiGenerateContent sessions in us-central1 from the Google Cloud console quotas page"
)
upstream_ws = ClosingUpstreamWebSocket(
ConnectionClosedError(rcvd=Close(1011, upstream_reason), sent=Close(1011, ""), rcvd_then_sent=True)
)
websocket = _client_websocket(_pending_receive)
with _patched_websocket_passthrough_environment(upstream_ws):
await websocket_passthrough_request(
websocket=websocket,
target="wss://aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent",
custom_headers={"Authorization": "Bearer token"},
user_api_key_dict=UserAPIKeyAuth(),
forward_headers=False,
endpoint="/vertex_ai/live",
accept_websocket=False,
)
relayed_reason = websocket.close.await_args.kwargs["reason"]
assert upstream_reason.startswith(relayed_reason)
assert len(relayed_reason.encode("utf-8")) == WEBSOCKET_CLOSE_REASON_MAX_BYTES
@pytest.mark.asyncio
async def test_websocket_passthrough_keeps_normal_upstream_close_normal():
from websockets.exceptions import ConnectionClosedOK
@ -7336,6 +7406,796 @@ async def test_websocket_passthrough_rewrites_gateway_alias_setup_model():
assert sent_setup["model"] == "projects/proj-db/locations/global/publishers/google/models/gemini-live-2.5-flash"
VERTEX_LIVE_TARGET = "wss://aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent"
VERTEX_LIVE_GROUP = "gemini-live-2.5-flash-native-audio"
VERTEX_LIVE_ALIAS_GROUP = "gemini-live-native-audio"
VERTEX_LIVE_GROUP_ADDRESSINGS = (
"{model}",
"models/{model}",
"vertex_ai/{model}",
"publishers/google/models/{model}",
)
VERTEX_LIVE_FULL_PATH = "projects/proj-db/locations/global/publishers/google/models/{model}"
VERTEX_LIVE_CLIENT_ADDRESSINGS = (*VERTEX_LIVE_GROUP_ADDRESSINGS, VERTEX_LIVE_FULL_PATH)
def _vertex_live_router() -> litellm.Router:
return litellm.Router(
model_list=[
{
"model_name": "gemini-live-2.5-flash",
"litellm_params": {"model": "vertex_ai/gemini-live-2.5-flash"},
},
{
"model_name": VERTEX_LIVE_GROUP,
"litellm_params": {"model": f"vertex_ai/{VERTEX_LIVE_GROUP}"},
},
{
"model_name": VERTEX_LIVE_ALIAS_GROUP,
"litellm_params": {"model": f"vertex_ai/{VERTEX_LIVE_GROUP}"},
},
{
"model_name": "gemini-3-pro-live",
"litellm_params": {"model": "vertex_ai/gemini-3-pro-live"},
},
]
)
async def _run_vertex_live_gated_passthrough(
client_frames: Sequence[Mapping[str, str | bytes]],
valid_token: UserAPIKeyAuth,
llm_router: litellm.Router | None = None,
) -> tuple[MagicMock, RecordingUpstreamWebSocket]:
"""Drive /vertex_ai/live with the real frame gate, returning the client and upstream sockets"""
upstream_ws = RecordingUpstreamWebSocket()
websocket, _ = await _run_vertex_live_gated_session(
AsyncMock(side_effect=[*client_frames, {"type": "websocket.disconnect"}]),
upstream_ws,
valid_token,
llm_router or _vertex_live_router(),
)
return websocket, upstream_ws
async def _run_vertex_live_gated_session(
receive: Callable[[], Awaitable[Mapping[str, str | bytes]]],
upstream_ws: "RecordingUpstreamWebSocket | ScriptedUpstreamWebSocket",
valid_token: UserAPIKeyAuth,
llm_router: litellm.Router,
) -> tuple[MagicMock, SimpleNamespace]:
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
_build_vertex_live_client_frame_gate,
_build_vertex_live_setup_model_rewriter,
)
websocket = _client_websocket(receive)
with _patched_websocket_passthrough_environment(upstream_ws) as environment:
await websocket_passthrough_request(
websocket=websocket,
target=VERTEX_LIVE_TARGET,
custom_headers={"Authorization": "Bearer token"},
user_api_key_dict=valid_token,
forward_headers=False,
endpoint="/vertex_ai/live",
accept_websocket=False,
setup_model_rewriter=_build_vertex_live_setup_model_rewriter(
vertex_project="proj-db",
vertex_location="global",
llm_router=llm_router,
),
client_frame_gate=_build_vertex_live_client_frame_gate(llm_router=llm_router),
)
return websocket, environment
def _setup_frame(model: str) -> dict[str, str]:
return {
"type": "websocket.receive",
"text": json.dumps({"setup": {"model": model, "generationConfig": {"responseModalities": ["TEXT"]}}}),
}
def _policy_close_reason(websocket: MagicMock) -> str | None:
return next(
(call.kwargs.get("reason") for call in websocket.close.await_args_list if call.kwargs.get("code") == 1008),
None,
)
@pytest.mark.asyncio
async def test_vertex_live_setup_frame_naming_a_model_the_key_cannot_call_is_refused():
"""
?model= is optional on this route, so a key that names an unlisted model only in the setup frame used to
reach Vertex with the gateway's own service-account credentials
"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame("totally-unlisted-model-abc")],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
)
upstream_ws.send.assert_not_awaited()
reason = _policy_close_reason(websocket)
assert reason is not None and "totally-unlisted-model-abc" in reason
assert reason.endswith(".")
upstream_ws.close.assert_awaited()
@pytest.mark.asyncio
async def test_vertex_live_setup_frame_naming_a_permitted_model_still_reaches_vertex():
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame("gemini-live-2.5-flash")],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
)
upstream_ws.send.assert_awaited_once()
sent_setup = json.loads(upstream_ws.send.await_args.args[0])["setup"]
assert sent_setup["model"] == "projects/proj-db/locations/global/publishers/google/models/gemini-live-2.5-flash"
assert _policy_close_reason(websocket) is None
@pytest.mark.asyncio
async def test_vertex_live_setup_frame_from_an_unrestricted_key_still_reaches_vertex():
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame("gemini-3-pro-live")],
UserAPIKeyAuth(token="hashed", models=[]),
)
upstream_ws.send.assert_awaited_once()
assert _policy_close_reason(websocket) is None
@pytest.mark.asyncio
async def test_vertex_live_setup_frame_is_checked_against_the_team_for_an_all_team_models_key():
"""
``all-team-models`` skips the key allowlist, so the team's own restriction is the only thing left to apply
"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame("gemini-3-pro-live")],
UserAPIKeyAuth(
token="hashed",
models=["all-team-models"],
team_id="team-1",
team_models=["gemini-live-2.5-flash"],
),
)
upstream_ws.send.assert_not_awaited()
reason = _policy_close_reason(websocket)
assert reason is not None and "gemini-3-pro-live" in reason
@pytest.mark.asyncio
async def test_vertex_live_setup_frame_is_gated_when_it_arrives_as_binary():
"""A client that sends its setup frame as bytes must not skip the check the text path applies"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[
{
"type": "websocket.receive",
"bytes": json.dumps({"setup": {"model": "totally-unlisted-model-abc"}}).encode(),
}
],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
)
upstream_ws.send.assert_not_awaited()
assert _policy_close_reason(websocket) is not None
@pytest.mark.asyncio
async def test_vertex_live_setup_frame_with_an_escaped_key_is_still_gated():
"""Large audio frames skip the parse on a substring test, so a JSON-escaped key must not slip past it"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[{"type": "websocket.receive", "text": '{"\\u0073etup": {"model": "totally-unlisted-model-abc"}}'}],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
)
upstream_ws.send.assert_not_awaited()
assert _policy_close_reason(websocket) is not None
@pytest.mark.asyncio
@pytest.mark.parametrize("frame_kind", ["text", "bytes"])
@pytest.mark.parametrize(
"lenient_setup",
[
'{"setup": {"model": "totally-unlisted-model-abc",},}',
"{'setup': {'model': 'totally-unlisted-model-abc'}}",
'{setup: {model: "totally-unlisted-model-abc"}}',
],
)
async def test_vertex_live_setup_frame_only_vertex_can_parse_is_refused(lenient_setup, frame_kind):
"""Vertex reads trailing commas, single quotes and bare keys, which strict JSON refuses to parse"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[
{
"type": "websocket.receive",
frame_kind: lenient_setup if frame_kind == "text" else lenient_setup.encode(),
}
],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
)
upstream_ws.send.assert_not_awaited()
assert _policy_close_reason(websocket) is not None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"lenient_setup",
[
'{"setup": {"model": "gemini-3-pro-live",},}',
"{'setup': {'model': 'gemini-3-pro-live'}}",
'{setup: {model: "gemini-3-pro-live"}}',
],
)
async def test_vertex_live_setup_frame_only_vertex_can_parse_still_reaches_vertex_for_an_unrestricted_key(
lenient_setup,
):
"""A key that may call any model loses nothing the proxy cannot read, so its lenient frames still go through"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[{"type": "websocket.receive", "text": lenient_setup}],
UserAPIKeyAuth(token="hashed", models=[]),
)
upstream_ws.send.assert_awaited_once()
assert _policy_close_reason(websocket) is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"repeated_key_setup",
[
'{"setup": {"model": "gemini-3-pro-live", "model": null}}',
'{"setup": {"model": "gemini-3-pro-live"}, "setup": null}',
],
)
async def test_vertex_live_setup_frame_repeating_a_key_as_null_is_refused(repeated_key_setup):
"""
Vertex keeps the first of two keys when the second is null, while ``json.loads`` keeps the last, so the
proxy would read no model where Vertex runs the first one
"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[{"type": "websocket.receive", "text": repeated_key_setup}],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
)
upstream_ws.send.assert_not_awaited()
assert _policy_close_reason(websocket) is not None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"limited_token",
[
UserAPIKeyAuth(token="hashed", models=["litellm-*"]),
UserAPIKeyAuth(token="hashed", models=["vertex_ai/gemini-*"]),
UserAPIKeyAuth(token="hashed", models=[], team_id="team-1", team_models=["gemini-live-2.5-flash"]),
UserAPIKeyAuth(token="hashed", models=[], agent_id="agent-1"),
UserAPIKeyAuth(token="hashed", models=[], end_user_id="end-user-1"),
],
ids=["litellm-wildcard", "vertex-wildcard", "team", "agent", "end-user"],
)
async def test_vertex_live_setup_frame_only_vertex_can_parse_is_refused_for_any_limited_caller(limited_token):
"""A wildcard can match any name the proxy might test the caller with, so only a caller with no model limit
anywhere may send a frame the proxy cannot read"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[
{
"type": "websocket.receive",
"text": "{'setup': {'model': '"
+ VERTEX_LIVE_FULL_PATH.format(model="gemini-3-pro-live")
+ "',},}",
}
],
limited_token,
)
upstream_ws.send.assert_not_awaited()
assert _policy_close_reason(websocket) is not None
@pytest.mark.asyncio
async def test_vertex_live_client_content_that_mentions_setup_is_forwarded():
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[
_setup_frame("gemini-live-2.5-flash"),
{
"type": "websocket.receive",
"text": json.dumps(
{"clientContent": {"turns": [{"role": "user", "parts": [{"text": "help me setup a meeting"}]}]}}
),
},
],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
)
assert upstream_ws.send.await_count == 2
assert _policy_close_reason(websocket) is None
@pytest.mark.asyncio
async def test_vertex_live_non_setup_frames_are_forwarded_without_a_model_check():
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[
_setup_frame("gemini-live-2.5-flash"),
{"type": "websocket.receive", "text": json.dumps({"realtimeInput": {"audio": {"data": "AAAA"}}})},
],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
)
assert upstream_ws.send.await_count == 2
assert _policy_close_reason(websocket) is None
@pytest.mark.asyncio
@pytest.mark.parametrize("addressing", VERTEX_LIVE_GROUP_ADDRESSINGS)
@pytest.mark.parametrize("group", [VERTEX_LIVE_GROUP, VERTEX_LIVE_ALIAS_GROUP])
async def test_vertex_live_setup_frame_authorizes_a_permitted_group_in_every_addressing(addressing, group):
"""
The Live SDK wraps the caller's model as ``models/<name>``, so a key holding the group has to work
whichever addressing its client actually sends
"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame(addressing.format(model=group))],
UserAPIKeyAuth(token="hashed", models=[group]),
)
upstream_ws.send.assert_awaited_once()
sent_setup = json.loads(upstream_ws.send.await_args.args[0])["setup"]
assert sent_setup["model"].startswith("projects/")
assert _policy_close_reason(websocket) is None
@pytest.mark.asyncio
@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS)
async def test_vertex_live_setup_frame_refuses_an_unpermitted_group_in_every_addressing(addressing):
"""Normalizing the addressing must not become a way to reach a group the key does not hold"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame(addressing.format(model="gemini-3-pro-live"))],
UserAPIKeyAuth(token="hashed", models=[VERTEX_LIVE_GROUP]),
)
upstream_ws.send.assert_not_awaited()
reason = _policy_close_reason(websocket)
assert reason is not None and "gemini-3-pro-live" in reason
@pytest.mark.asyncio
@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS)
async def test_vertex_live_setup_frame_naming_no_group_at_all_is_refused_in_every_addressing(addressing):
"""A name that resolves to no group is authorized as it arrived, so an unknown model cannot ride a prefix in"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame(addressing.format(model="totally-unlisted-model-abc"))],
UserAPIKeyAuth(token="hashed", models=[VERTEX_LIVE_GROUP]),
)
upstream_ws.send.assert_not_awaited()
assert _policy_close_reason(websocket) is not None
@pytest.mark.asyncio
@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS)
@pytest.mark.parametrize("key_model", ["gemini-4-pro-live-unserved", "vertex_ai/gemini-4-pro-live-unserved"])
async def test_vertex_live_setup_frame_authorizes_a_model_no_group_serves_by_its_publisher_id(addressing, key_model):
"""
With no group to resolve, the setup rewriter sends Vertex the last path segment, so a key holding that
publisher model, bare or as a LiteLLM ``vertex_ai/`` id, can open it whichever addressing its client sends
"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame(addressing.format(model="gemini-4-pro-live-unserved"))],
UserAPIKeyAuth(token="hashed", models=[key_model]),
)
upstream_ws.send.assert_awaited_once()
sent_setup = json.loads(upstream_ws.send.await_args.args[0])["setup"]
assert sent_setup["model"].endswith("/models/gemini-4-pro-live-unserved")
assert _policy_close_reason(websocket) is None
@pytest.mark.asyncio
async def test_vertex_live_setup_frame_addressing_cannot_satisfy_a_wildcard_for_another_publisher_model():
"""Vertex runs only the last segment, so the segments ahead of it must not be what matches the key's wildcard"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame("vertex_ai/gemini-anything/forbidden-model")],
UserAPIKeyAuth(token="hashed", models=["vertex_ai/gemini-*"]),
)
upstream_ws.send.assert_not_awaited()
reason = _policy_close_reason(websocket)
assert reason is not None and "forbidden-model" in reason
@pytest.mark.asyncio
async def test_vertex_live_setup_frame_prefers_the_group_its_whole_name_matches():
"""A group named ``vertex_ai/<id>`` is the one the rewriter sends, so it is the one the key must hold"""
llm_router = litellm.Router(
model_list=[
{"model_name": "gemini-3-pro-live", "litellm_params": {"model": "vertex_ai/gemini-3-pro-live"}},
{"model_name": "vertex_ai/gemini-3-pro-live", "litellm_params": {"model": "vertex_ai/gemini-3-pro-live"}},
]
)
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame("vertex_ai/gemini-3-pro-live")],
UserAPIKeyAuth(token="hashed", models=["vertex_ai/gemini-3-pro-live"]),
llm_router=llm_router,
)
upstream_ws.send.assert_awaited_once()
assert _policy_close_reason(websocket) is None
@pytest.mark.asyncio
@pytest.mark.parametrize("group", [VERTEX_LIVE_GROUP, VERTEX_LIVE_ALIAS_GROUP])
async def test_vertex_live_full_path_is_authorized_by_any_group_serving_its_publisher_model(group):
"""The docs tell callers to send the full resource path, which Vertex runs as sent, so every group serving
that publisher model grants it, whatever the group is called"""
full_path = VERTEX_LIVE_FULL_PATH.format(model=VERTEX_LIVE_GROUP)
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame(full_path)],
UserAPIKeyAuth(token="hashed", models=[group]),
)
upstream_ws.send.assert_awaited_once()
assert json.loads(upstream_ws.send.await_args.args[0])["setup"]["model"] == full_path
assert _policy_close_reason(websocket) is None
@pytest.mark.asyncio
async def test_vertex_live_full_path_is_not_granted_by_a_group_named_after_it_that_serves_another_model():
llm_router = litellm.Router(
model_list=[
{"model_name": "gemini-3-pro-live", "litellm_params": {"model": "vertex_ai/gemini-live-2.5-flash"}},
]
)
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame(VERTEX_LIVE_FULL_PATH.format(model="gemini-3-pro-live"))],
UserAPIKeyAuth(token="hashed", models=["gemini-3-pro-live"]),
llm_router=llm_router,
)
upstream_ws.send.assert_not_awaited()
reason = _policy_close_reason(websocket)
assert reason is not None and "gemini-3-pro-live" in reason
@pytest.mark.asyncio
@pytest.mark.parametrize(
"valid_token",
[
UserAPIKeyAuth(
token="hashed",
models=["gemini-live-2.5-flash"],
aliases={"gemini-3-pro-live": "gemini-live-2.5-flash"},
),
UserAPIKeyAuth(
token="hashed",
models=["all-team-models"],
team_id="team-1",
team_models=["gemini-live-2.5-flash"],
team_model_aliases={"gemini-3-pro-live": "gemini-live-2.5-flash"},
),
],
ids=["key-alias", "team-alias"],
)
async def test_vertex_live_setup_frame_naming_an_alias_cannot_open_the_model_the_alias_shadows(valid_token):
"""
An alias passes the access check by its target, while the setup rewriter sends the name as written, so a
key that could set an alias would open the model the alias name belongs to
"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough([_setup_frame("gemini-3-pro-live")], valid_token)
upstream_ws.send.assert_not_awaited()
reason = _policy_close_reason(websocket)
assert reason is not None and "gemini-3-pro-live" in reason
@pytest.mark.asyncio
async def test_vertex_live_setup_frame_naming_a_router_alias_cannot_open_the_model_the_alias_shadows():
llm_router = litellm.Router(
model_list=[
{"model_name": "gemini-live-2.5-flash", "litellm_params": {"model": "vertex_ai/gemini-live-2.5-flash"}},
],
model_group_alias={"gemini-3-pro-live": "gemini-live-2.5-flash"},
)
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame("gemini-3-pro-live")],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
llm_router=llm_router,
)
upstream_ws.send.assert_not_awaited()
reason = _policy_close_reason(websocket)
assert reason is not None and "gemini-3-pro-live" in reason
@pytest.mark.asyncio
async def test_vertex_live_setup_frame_is_checked_against_the_owner_of_a_personal_key():
"""A personal key's owner can be limited to fewer models than the key lists, and auth enforces that limit too"""
from litellm.proxy.proxy_server import user_api_key_cache
user_id = f"live-user-{uuid.uuid4().hex}"
await user_api_key_cache.async_set_cache(
key=user_id,
value={"user_id": user_id, "user_email": "live@example.com", "models": ["gemini-live-2.5-flash"]},
)
with patch( # test-quality-ok: read at call time, and the user row comes from the cache seeded above
"litellm.proxy.proxy_server.prisma_client", MagicMock()
):
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame("gemini-3-pro-live")],
UserAPIKeyAuth(token="hashed", user_id=user_id, models=[]),
)
upstream_ws.send.assert_not_awaited()
reason = _policy_close_reason(websocket)
assert reason is not None and "gemini-3-pro-live" in reason
@pytest.mark.asyncio
@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS)
async def test_vertex_live_setup_frame_is_checked_against_the_team_in_every_addressing(addressing):
"""The team allowlist is keyed by group too, so the addressing must not decide whether the team is checked"""
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
[_setup_frame(addressing.format(model="gemini-3-pro-live"))],
UserAPIKeyAuth(
token="hashed",
models=["all-team-models"],
team_id="team-1",
team_models=[VERTEX_LIVE_GROUP],
),
)
upstream_ws.send.assert_not_awaited()
reason = _policy_close_reason(websocket)
assert reason is not None and "gemini-3-pro-live" in reason
@pytest.mark.asyncio
@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS)
async def test_vertex_live_setup_frame_resolves_an_access_group_grant_in_every_addressing(addressing):
"""
``get_model_access_groups`` is keyed by group as well, and returns nothing for a prefixed name, so a key
entitled through an access group rather than a model name needs the same normalization
"""
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
_build_vertex_live_client_frame_gate,
)
llm_router = litellm.Router(
model_list=[
{
"model_name": VERTEX_LIVE_GROUP,
"litellm_params": {"model": f"vertex_ai/{VERTEX_LIVE_GROUP}"},
"model_info": {"access_groups": ["live-team"]},
},
]
)
gate = _build_vertex_live_client_frame_gate(llm_router=llm_router)
frame = json.dumps({"setup": {"model": addressing.format(model=VERTEX_LIVE_GROUP)}})
denial = await gate(frame, UserAPIKeyAuth(token="hashed", models=["live-team"]))
assert denial is None
async def _run_vertex_live_route(
client_frames: Sequence[Mapping[str, str | bytes]], valid_token: UserAPIKeyAuth
) -> tuple[MagicMock, RecordingUpstreamWebSocket, SimpleNamespace]:
"""Drive the /vertex_ai/live route, so the gate under test is whichever one the route itself wires"""
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
vertex_ai_live_websocket_passthrough,
vertex_llm_base,
)
upstream_ws = RecordingUpstreamWebSocket()
websocket = _client_websocket(
AsyncMock(side_effect=[*client_frames, {"type": "websocket.disconnect"}]),
)
with (
_patched_websocket_passthrough_environment(upstream_ws) as environment,
patch.object( # test-quality-ok: an OAuth exchange with Google, so this is the boundary being faked
vertex_llm_base,
"_ensure_access_token_async",
AsyncMock(return_value=("token-abc", "proj-db")),
),
):
await vertex_ai_live_websocket_passthrough(
websocket=websocket,
model="gemini-live-2.5-flash",
vertex_project="proj-db",
vertex_location="global",
user_api_key_dict=valid_token,
)
return websocket, upstream_ws, environment
@pytest.mark.asyncio
async def test_vertex_live_route_gates_the_setup_frame_even_when_the_query_model_is_permitted():
"""
?model= is all connect-time auth sees, so a key that passes it and then names a different model in its setup
frame reaches Vertex on the gateway's own credentials unless the route gates the frames as well
"""
websocket, upstream_ws, environment = await _run_vertex_live_route(
[_setup_frame("gemini-3-pro-live")],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
)
upstream_ws.send.assert_not_awaited()
reason = _policy_close_reason(websocket)
assert reason is not None and "gemini-3-pro-live" in reason
refusal = environment.proxy_logging.post_call_failure_hook.await_args.kwargs["original_exception"]
assert refusal.code == "403"
assert refusal.type == ProxyErrorTypes.key_model_access_denied
@pytest.mark.asyncio
async def test_vertex_live_route_keeps_the_key_allowlist_out_of_the_close_reason_and_in_the_proxy_log(caplog):
"""The client learns which model was refused, while only the proxy log names the models the key may reach"""
with caplog.at_level("WARNING", logger="LiteLLM Proxy"):
websocket, _, _ = await _run_vertex_live_route(
[_setup_frame("gemini-3-pro-live")],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
)
reason = _policy_close_reason(websocket)
assert reason is not None and "gemini-live-2.5-flash" not in reason
assert any(
"gemini-3-pro-live" in record.getMessage() and "gemini-live-2.5-flash" in record.getMessage()
for record in caplog.records
)
@pytest.mark.asyncio
async def test_vertex_live_route_forwards_a_setup_frame_the_key_is_allowed_to_run():
websocket, upstream_ws, _ = await _run_vertex_live_route(
[_setup_frame("gemini-live-2.5-flash")],
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
)
upstream_ws.send.assert_awaited_once()
assert _policy_close_reason(websocket) is None
class ScriptedUpstreamWebSocket:
"""An upstream that sends the given frames in order, then stays open with nothing more to say"""
def __init__(self, frames: Iterable[str]) -> None:
self._frames = iter(frames)
self.sent_every_frame = asyncio.Event()
self.send = AsyncMock()
self.close = AsyncMock()
async def recv(self, decode: bool = True) -> str | None:
frame = next(self._frames, None)
if frame is not None:
return frame
self.sent_every_frame.set()
await asyncio.Event().wait()
def __aiter__(self) -> "ScriptedUpstreamWebSocket":
return self
async def __anext__(self) -> str:
raise StopAsyncIteration
@pytest.mark.asyncio
async def test_vertex_live_turns_served_before_a_refused_frame_are_still_billed():
"""
A refusal used to log the whole session as a failure, which records no spend, so a key could run allowed
turns and then send a refused frame to have them go unbilled. The refused frame's model must not become
the model those turns are priced at either
"""
upstream_ws = ScriptedUpstreamWebSocket(
(
json.dumps({"setupComplete": {}}),
json.dumps({"serverContent": {"turnComplete": True}, "usageMetadata": {"totalTokenCount": 60}}),
)
)
async def client_frames() -> AsyncIterator[dict[str, str]]:
yield _setup_frame("gemini-live-2.5-flash")
await upstream_ws.sent_every_frame.wait()
yield _setup_frame("gemini-3-pro-live")
websocket, environment = await _run_vertex_live_gated_session(
client_frames().__anext__,
upstream_ws,
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
_vertex_live_router(),
)
reason = _policy_close_reason(websocket)
assert reason is not None and "gemini-3-pro-live" in reason
upstream_ws.send.assert_awaited_once()
upstream_ws.close.assert_awaited()
environment.logging_worker.ensure_initialized_and_enqueue.assert_called_once()
assert environment.enqueued_handler.call_args.kwargs["logging_obj"].model == "gemini-live-2.5-flash"
environment.proxy_logging.post_call_success_hook.assert_awaited_once()
environment.proxy_logging.post_call_failure_hook.assert_not_awaited()
async def _run_passthrough_with_frame_gate(
gate: WebsocketClientFrameGate,
) -> tuple[MagicMock, ClosableUpstreamWebSocket, SimpleNamespace]:
upstream_ws = ClosableUpstreamWebSocket()
websocket = _client_websocket(
AsyncMock(
side_effect=[
{"type": "websocket.receive", "text": json.dumps({"setup": {"model": "gemini-live-2.5-flash"}})},
{"type": "websocket.disconnect"},
]
),
)
with _patched_websocket_passthrough_environment(upstream_ws) as environment:
await websocket_passthrough_request(
websocket=websocket,
target=VERTEX_LIVE_TARGET,
custom_headers={"Authorization": "Bearer token"},
user_api_key_dict=UserAPIKeyAuth(token="hashed"),
forward_headers=False,
endpoint="/vertex_ai/live",
accept_websocket=False,
client_frame_gate=gate,
)
return websocket, upstream_ws, environment
@pytest.mark.asyncio
async def test_websocket_passthrough_reports_a_refused_client_frame_as_a_failure():
"""A denied session logged through the success hooks tells spend tracking and audit that it succeeded"""
denial = ProxyException(
message="gemini-live-2.5-flash: key not allowed to access model",
type=ProxyErrorTypes.key_model_access_denied,
param="model",
code=403,
)
async def refusing_gate(frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None:
return denial
websocket, upstream_ws, environment = await _run_passthrough_with_frame_gate(refusing_gate)
assert _policy_close_reason(websocket) == "gemini-live-2.5-flash: key not allowed to access model"
upstream_ws.close.assert_awaited()
environment.proxy_logging.post_call_success_hook.assert_not_awaited()
environment.logging_worker.ensure_initialized_and_enqueue.assert_not_called()
environment.proxy_logging.post_call_failure_hook.assert_awaited_once()
failure_kwargs = environment.proxy_logging.post_call_failure_hook.await_args.kwargs
assert failure_kwargs["original_exception"] is denial
assert failure_kwargs["request_data"]["litellm_logging_obj"].litellm_call_id
@pytest.mark.asyncio
async def test_websocket_passthrough_still_reports_an_allowed_session_as_a_success():
async def permitting_gate(frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None:
return None
_, _, environment = await _run_passthrough_with_frame_gate(permitting_gate)
environment.proxy_logging.post_call_failure_hook.assert_not_awaited()
environment.proxy_logging.post_call_success_hook.assert_awaited_once()
environment.logging_worker.ensure_initialized_and_enqueue.assert_called_once()
@pytest.mark.parametrize(
"setup_model",
["gemini-live-2.5-flash", "models/gemini-live-2.5-flash", "publishers/google/models/gemini-live-2.5-flash"],