mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 3ee80f6aca into e16d79d635
This commit is contained in:
commit
377eb91042
4 changed files with 1260 additions and 66 deletions
|
|
@ -5691,6 +5691,45 @@ async def can_key_call_resolved_model(
|
|||
)
|
||||
|
||||
|
||||
async def can_caller_call_resolved_model(
|
||||
model: str,
|
||||
valid_token: UserAPIKeyAuth,
|
||||
llm_router: litellm.Router | None,
|
||||
prisma_client: PrismaClient | None,
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
) -> None:
|
||||
"""``can_key_call_resolved_model`` plus the agent grant and a personal key owner's own models, which
|
||||
``common_checks`` applies to a model named at auth time and a model named after auth needs as well"""
|
||||
await can_key_call_resolved_model(
|
||||
model=model,
|
||||
llm_model_list=None if llm_router is None else llm_router.get_model_list(),
|
||||
valid_token=valid_token,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
await can_agent_call_model(
|
||||
model=model,
|
||||
valid_token=valid_token,
|
||||
llm_router=llm_router,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
if valid_token.team_id is not None or prisma_client is None:
|
||||
return
|
||||
await can_user_call_model(
|
||||
model=model,
|
||||
llm_router=llm_router,
|
||||
user_object=await _user_object_or_none(
|
||||
valid_token=valid_token,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
),
|
||||
key_model_aliases=key_model_aliases_for_auth_check(valid_token),
|
||||
)
|
||||
|
||||
|
||||
def can_org_access_model(
|
||||
model: str,
|
||||
org_object: LiteLLM_OrganizationTable | None,
|
||||
|
|
|
|||
|
|
@ -72,8 +72,16 @@ from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, valid
|
|||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
from litellm.passthrough.main import AsyncPassthroughStreamingResponse, is_provider_error
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.auth_checks import enforced_model_allowlists
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
can_caller_call_resolved_model,
|
||||
enforced_model_allowlists,
|
||||
key_model_aliases_for_auth_check,
|
||||
personal_key_team_model_access_applies,
|
||||
team_model_aliases_for_auth_check,
|
||||
)
|
||||
from litellm.proxy.auth.auth_utils import log_model_access_denial
|
||||
from litellm.proxy.auth.handle_jwt import JWTHandler
|
||||
from litellm.proxy.auth.model_access_denied import model_access_denied_client_message
|
||||
from litellm.proxy.auth.route_checks import RouteChecks
|
||||
from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401 # legacy module exports
|
||||
_get_bearer_token, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
|
|
@ -110,6 +118,7 @@ from litellm.proxy.pass_through_endpoints.injected_stream_usage import (
|
|||
)
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
||||
HttpPassThroughEndpointHelpers,
|
||||
WebsocketClientFrameGate,
|
||||
create_pass_through_route,
|
||||
create_websocket_passthrough_route,
|
||||
websocket_passthrough_request,
|
||||
|
|
@ -145,6 +154,7 @@ from .passthrough_endpoint_router import PassthroughEndpointRouter
|
|||
if TYPE_CHECKING:
|
||||
from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig
|
||||
from litellm.router import Router
|
||||
from litellm.types.router import DeploymentTypedDict
|
||||
|
||||
ProxyConfig = _ProxyConfig # rebind-ok: conditional type alias
|
||||
else:
|
||||
|
|
@ -3803,6 +3813,12 @@ VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON: Final = (
|
|||
"Vertex AI auth failed: set a use_in_pass_through vertex model, default_vertex_config, or DEFAULT_VERTEXAI_* env"
|
||||
)
|
||||
|
||||
VERTEX_LIVE_UNVERIFIED_MODEL_CLOSE_REASON: Final = (
|
||||
"Could not check your key's access to the model this setup frame names; ask a proxy admin to read the logs"
|
||||
)
|
||||
|
||||
VERTEX_LIVE_UNREADABLE_SETUP_CLOSE_REASON: Final = "Send setup frames as strict JSON so the proxy can check their model"
|
||||
|
||||
VERTEX_PUBLISHER_MODEL_PREFIX: Final = "publishers/google/models/"
|
||||
|
||||
VERTEX_PUBLISHERS_SEGMENT: Final = "publishers/"
|
||||
|
|
@ -3827,6 +3843,245 @@ def _get_llm_router() -> Router | None:
|
|||
return llm_router
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _UnreadableVertexLiveFrame:
|
||||
pass
|
||||
|
||||
|
||||
class _VertexLiveSetup(TypedDict, total=False):
|
||||
model: ReadOnly[str]
|
||||
|
||||
|
||||
class _VertexLiveClientFrame(TypedDict, total=False):
|
||||
setup: ReadOnly[_VertexLiveSetup]
|
||||
|
||||
|
||||
_VERTEX_LIVE_CLIENT_FRAME: Final = TypeAdapter(_VertexLiveClientFrame)
|
||||
|
||||
|
||||
def _may_carry_vertex_live_setup(frame_data: str | bytes) -> bool:
|
||||
"""
|
||||
Whether a frame is worth parsing: audio and video frames stream continuously and are large, and one that
|
||||
cannot spell ``setup`` cannot carry one. Vertex also reads single-quoted and bare keys, so the bare word is
|
||||
the test, and a backslash means the key may be escaped, which only a parse settles
|
||||
"""
|
||||
if isinstance(frame_data, bytes):
|
||||
return b"setup" in frame_data or b"\\" in frame_data
|
||||
return "setup" in frame_data or "\\" in frame_data
|
||||
|
||||
|
||||
def _object_with_unique_keys(pairs: Sequence[tuple[str, object]]) -> Mapping[str, object]:
|
||||
unique: Final = MappingProxyType(dict(pairs))
|
||||
if len(unique) < len(pairs):
|
||||
raise ValueError("a JSON object repeats a key")
|
||||
return unique
|
||||
|
||||
|
||||
def _vertex_live_setup_model(frame_data: str | bytes) -> str | _UnreadableVertexLiveFrame | None:
|
||||
"""
|
||||
The model a Vertex AI Live ``setup`` frame names, which is the only place BidiGenerateContent carries one.
|
||||
|
||||
Vertex parses trailing commas, single quotes and bare keys that ``json.loads`` refuses, and keeps the first of
|
||||
two keys when the second is null, so a frame that may carry a setup but fails a strict parse, repeats a key,
|
||||
or holds a setup of the wrong shape is unreadable rather than setup-free
|
||||
"""
|
||||
if not _may_carry_vertex_live_setup(frame_data):
|
||||
return None
|
||||
try:
|
||||
frame: Final = _VERTEX_LIVE_CLIENT_FRAME.validate_python(
|
||||
json.loads(frame_data, object_pairs_hook=_object_with_unique_keys)
|
||||
)
|
||||
except ValueError:
|
||||
return _UnreadableVertexLiveFrame()
|
||||
setup: Final = frame.get("setup")
|
||||
return None if setup is None else setup.get("model") or None
|
||||
|
||||
|
||||
def _deployment_named(model_name: str, deployments: Sequence[DeploymentTypedDict]) -> DeploymentTypedDict | None:
|
||||
return next((deployment for deployment in deployments if deployment.get("model_name") == model_name), None)
|
||||
|
||||
|
||||
def _vertex_live_deployment(setup_model: str, llm_router: Router | None) -> DeploymentTypedDict | None:
|
||||
"""
|
||||
The router deployment whose group a ``setup`` frame's model names, whichever addressing the client used.
|
||||
|
||||
The Live SDK wraps whatever the caller typed as ``models/<name>``, and clients also send LiteLLM ids and
|
||||
full Vertex resource paths, so the group is read from the last segment when the whole string names none
|
||||
"""
|
||||
if llm_router is None:
|
||||
return None
|
||||
deployments: Final = tuple(llm_router.get_model_list() or ())
|
||||
exact: Final = _deployment_named(setup_model, deployments)
|
||||
return exact if exact is not None else _deployment_named(setup_model.rsplit("/", 1)[-1], deployments)
|
||||
|
||||
|
||||
def _deployment_upstream_model(deployment: DeploymentTypedDict) -> str | None:
|
||||
upstream: Final = deployment["litellm_params"].get("model")
|
||||
if upstream is None:
|
||||
return None
|
||||
try:
|
||||
_, provider, _, _ = litellm.get_llm_provider(model=upstream)
|
||||
except litellm.exceptions.BadRequestError:
|
||||
return upstream
|
||||
return upstream.removeprefix(f"{provider}/")
|
||||
|
||||
|
||||
def _vertex_live_model_names(setup_model: str, llm_router: Router | None) -> tuple[str, ...]:
|
||||
"""
|
||||
The names a key can hold for the model a ``setup`` frame runs: the router group the setup rewriter sends it
|
||||
through, else every group serving the publisher model Vertex runs, and that model's id bare or under LiteLLM's
|
||||
``vertex_ai/`` prefix unless a group holds that name for another model.
|
||||
|
||||
A full resource path skips the rewriter and reaches Vertex as sent, so only its publisher model counts, and
|
||||
a wildcard is matched against that id alone so addressing in front of it cannot satisfy the pattern
|
||||
"""
|
||||
deployment: Final = (
|
||||
None if setup_model.startswith("projects/") else _vertex_live_deployment(setup_model, llm_router)
|
||||
)
|
||||
if deployment is not None:
|
||||
return (deployment["model_name"],)
|
||||
deployments: Final = () if llm_router is None else tuple(llm_router.get_model_list() or ())
|
||||
publisher_model: Final = _vertex_publisher_model_suffix(setup_model)
|
||||
serving_groups: Final = tuple(
|
||||
dict.fromkeys(
|
||||
candidate["model_name"]
|
||||
for candidate in deployments
|
||||
if _vertex_publisher_model_suffix(_deployment_upstream_model(candidate) or "") == publisher_model
|
||||
)
|
||||
)
|
||||
group_names: Final = frozenset(candidate["model_name"] for candidate in deployments)
|
||||
publisher_model_id: Final = setup_model.rsplit("/", 1)[-1]
|
||||
return (
|
||||
*serving_groups,
|
||||
*(name for name in (publisher_model_id, f"vertex_ai/{publisher_model_id}") if name not in group_names),
|
||||
)
|
||||
|
||||
|
||||
def _is_model_alias(name: str, valid_token: UserAPIKeyAuth, llm_router: Router | None) -> bool:
|
||||
return (
|
||||
name in (key_model_aliases_for_auth_check(valid_token) or ())
|
||||
or name in (team_model_aliases_for_auth_check(valid_token) or ())
|
||||
or name in litellm.model_alias_map
|
||||
or (llm_router is not None and name in llm_router.model_group_alias)
|
||||
)
|
||||
|
||||
|
||||
async def _model_access_denial(
|
||||
model: str, valid_token: UserAPIKeyAuth, llm_router: Router | None
|
||||
) -> ProxyException | None:
|
||||
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
|
||||
|
||||
try:
|
||||
await can_caller_call_resolved_model(
|
||||
model=model,
|
||||
valid_token=valid_token,
|
||||
llm_router=llm_router,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
except ProxyException as denial:
|
||||
return denial
|
||||
except HTTPException as refusal:
|
||||
return ProxyException(
|
||||
message=str(refusal.detail),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="model",
|
||||
code=refusal.status_code,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def _vertex_live_model_denial(
|
||||
setup_model: str, valid_token: UserAPIKeyAuth, llm_router: Router | None
|
||||
) -> ProxyException | None:
|
||||
"""
|
||||
The setup rewriter forwards a name as sent, so a name an alias maps to another model grants nothing here
|
||||
"""
|
||||
model_names: Final = tuple(
|
||||
name
|
||||
for name in _vertex_live_model_names(setup_model, llm_router)
|
||||
if not _is_model_alias(name, valid_token, llm_router)
|
||||
)
|
||||
denials: Final = tuple([await _model_access_denial(name, valid_token, llm_router) for name in model_names])
|
||||
if None in denials:
|
||||
return None
|
||||
return (
|
||||
denials[0]
|
||||
if denials
|
||||
else ModelAccessDeniedProxyException(
|
||||
message=model_access_denied_client_message(model=setup_model),
|
||||
internal_message=f"Vertex AI Live setup model {setup_model} is only reachable through a model alias",
|
||||
type=ProxyErrorTypes.key_model_access_denied,
|
||||
param="model",
|
||||
code=403,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _caller_has_model_limits(valid_token: UserAPIKeyAuth) -> bool:
|
||||
return (
|
||||
valid_token.agent_id is not None
|
||||
or valid_token.managed_agent_policy is not None
|
||||
or valid_token.agent_caller is not None
|
||||
or valid_token.end_user_id is not None
|
||||
or personal_key_team_model_access_applies(valid_token)
|
||||
or _has_model_restrictions(await _proxy_model_allowlists()(valid_token))
|
||||
)
|
||||
|
||||
|
||||
async def _vertex_live_setup_denial(
|
||||
setup_model: str | _UnreadableVertexLiveFrame, valid_token: UserAPIKeyAuth, llm_router: Router | None
|
||||
) -> ProxyException | None:
|
||||
match setup_model:
|
||||
case str():
|
||||
denial: Final = await _vertex_live_model_denial(setup_model, valid_token, llm_router)
|
||||
if denial is not None:
|
||||
log_model_access_denial(denial)
|
||||
return denial
|
||||
case _UnreadableVertexLiveFrame():
|
||||
return (
|
||||
ProxyException(
|
||||
message=VERTEX_LIVE_UNREADABLE_SETUP_CLOSE_REASON,
|
||||
type=ProxyErrorTypes.bad_request_error,
|
||||
param="setup",
|
||||
code=400,
|
||||
)
|
||||
if await _caller_has_model_limits(valid_token)
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def _build_vertex_live_client_frame_gate(llm_router: Router | None) -> WebsocketClientFrameGate:
|
||||
"""
|
||||
Apply the checks a model named at auth time gets to the model a client's ``setup`` frame names.
|
||||
|
||||
``?model=`` is optional on this route and the Live protocol carries the real model in the first client
|
||||
frame, so connect-time auth has no model to check and every frame that names one has to be authorized
|
||||
here instead.
|
||||
|
||||
A frame the proxy cannot read the way Vertex does only reaches Vertex for a caller who may call any model,
|
||||
which is a caller with no model allowlist and no agent, end user or team grant standing in for one
|
||||
"""
|
||||
|
||||
async def gate(frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None:
|
||||
model: Final = _vertex_live_setup_model(frame_data)
|
||||
if model is None:
|
||||
return None
|
||||
try:
|
||||
return await _vertex_live_setup_denial(model, valid_token, llm_router)
|
||||
except Exception: # noqa: BLE001 # fail closed: a gate that cannot answer refuses, and says so in band
|
||||
verbose_proxy_logger.exception("Vertex AI Live passthrough: model access check failed for %s", model)
|
||||
return ProxyException(
|
||||
message=VERTEX_LIVE_UNVERIFIED_MODEL_CLOSE_REASON,
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
param="model",
|
||||
code=500,
|
||||
)
|
||||
|
||||
return gate
|
||||
|
||||
|
||||
def _resolve_vertex_live_credentials(
|
||||
vertex_project: str | None,
|
||||
vertex_location: str | None,
|
||||
|
|
@ -3885,24 +4140,9 @@ def _resolve_alias_to_upstream_model(setup_model: str, llm_router: Router | None
|
|||
"""
|
||||
The Live SDK wraps whatever the caller typed as ``models/<name>``, so a gateway alias arrives prefixed
|
||||
"""
|
||||
if llm_router is None:
|
||||
return setup_model
|
||||
candidates: Final = (setup_model, setup_model.rsplit("/", 1)[-1])
|
||||
upstream: Final = next(
|
||||
(
|
||||
deployment["litellm_params"].get("model")
|
||||
for deployment in (llm_router.get_model_list() or ())
|
||||
if deployment.get("model_name") in candidates
|
||||
),
|
||||
None,
|
||||
)
|
||||
if upstream is None:
|
||||
return setup_model
|
||||
try:
|
||||
_, provider, _, _ = litellm.get_llm_provider(model=upstream)
|
||||
except litellm.exceptions.BadRequestError:
|
||||
return upstream
|
||||
return upstream.removeprefix(f"{provider}/")
|
||||
deployment: Final = _vertex_live_deployment(setup_model, llm_router)
|
||||
upstream: Final = None if deployment is None else _deployment_upstream_model(deployment)
|
||||
return setup_model if upstream is None else upstream
|
||||
|
||||
|
||||
async def vertex_ai_live_websocket_passthrough(
|
||||
|
|
@ -4009,6 +4249,7 @@ async def vertex_ai_live_websocket_passthrough(
|
|||
vertex_location=resolved_location,
|
||||
llm_router=_get_llm_router(),
|
||||
),
|
||||
client_frame_gate=_build_vertex_live_client_frame_gate(llm_router=_get_llm_router()),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from dataclasses import dataclass
|
|||
from datetime import datetime
|
||||
from itertools import chain, count, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, cast
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
import httpx
|
||||
|
|
@ -30,7 +30,7 @@ from pydantic import TypeAdapter
|
|||
from starlette.datastructures import UploadFile as StarletteUploadFile
|
||||
from starlette.routing import BaseRoute, Route
|
||||
from starlette.websockets import WebSocketState
|
||||
from websockets.asyncio.client import connect
|
||||
from websockets.asyncio.client import ClientConnection, connect
|
||||
from websockets.exceptions import (
|
||||
ConnectionClosedError,
|
||||
ConnectionClosedOK,
|
||||
|
|
@ -2414,6 +2414,15 @@ def _truncated_close_reason(reason: str) -> str:
|
|||
return encoded[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode("utf-8", errors="ignore")
|
||||
|
||||
|
||||
def _refusal_close_reason(message: str) -> str:
|
||||
"""
|
||||
The proxy's own refusal message as a close reason, ending on the last whole sentence that fits
|
||||
"""
|
||||
clipped: Final = _truncated_close_reason(message)
|
||||
sentence_end: Final = clipped.rfind(". ")
|
||||
return clipped if clipped == message or sentence_end == -1 else clipped[: sentence_end + 1]
|
||||
|
||||
|
||||
SENDABLE_CLOSE_CODES: Final = frozenset(CloseCode) - frozenset(
|
||||
{CloseCode.NO_STATUS_RCVD, CloseCode.ABNORMAL_CLOSURE, CloseCode.TLS_HANDSHAKE}
|
||||
)
|
||||
|
|
@ -2457,6 +2466,39 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict
|
|||
return inject_trace_context(headers, parent_span=parent_span)
|
||||
|
||||
|
||||
class WebsocketClientFrameGate(Protocol):
|
||||
"""Authorize one client frame, returning the denial to report or ``None`` to let it through.
|
||||
|
||||
A websocket passthrough authenticates before any frame arrives, so a protocol that names its model
|
||||
inside a frame has no model to authorize at connect time. The frame, not the extracted model, is the
|
||||
argument because where a model sits is part of each provider's wire protocol
|
||||
"""
|
||||
|
||||
async def __call__(self, frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None: ...
|
||||
|
||||
|
||||
async def _relay_client_frame(
|
||||
websocket: WebSocket,
|
||||
upstream_ws: ClientConnection,
|
||||
checked_frame: str | bytes,
|
||||
forwarded_frame: str | bytes,
|
||||
gate: WebsocketClientFrameGate | None,
|
||||
valid_token: UserAPIKeyAuth,
|
||||
) -> ProxyException | None:
|
||||
"""Forward one client frame upstream, or refuse it and hand back the denial that ended the session.
|
||||
|
||||
The denial travels out so the caller reports a failed session instead of leaving the success hooks to
|
||||
record a refusal as a success, and tearing the upstream down is the caller's job: awaiting that close
|
||||
here lets the upstream reader finish first, which cancels this task mid-close and loses the denial
|
||||
"""
|
||||
refusal: Final = None if gate is None else await gate(checked_frame, valid_token)
|
||||
if refusal is None:
|
||||
await upstream_ws.send(forwarded_frame)
|
||||
return None
|
||||
await websocket.close(code=CloseCode.POLICY_VIOLATION, reason=_refusal_close_reason(refusal.message))
|
||||
return refusal
|
||||
|
||||
|
||||
async def websocket_passthrough_request(
|
||||
websocket: WebSocket,
|
||||
target: str,
|
||||
|
|
@ -2467,6 +2509,7 @@ async def websocket_passthrough_request(
|
|||
cost_per_request: float | None = None,
|
||||
accept_websocket: bool = True,
|
||||
setup_model_rewriter: Callable[[str], str] | None = None,
|
||||
client_frame_gate: WebsocketClientFrameGate | None = None,
|
||||
):
|
||||
"""
|
||||
WebSocket passthrough request handler.
|
||||
|
|
@ -2480,6 +2523,7 @@ async def websocket_passthrough_request(
|
|||
endpoint: The endpoint path (for logging purposes)
|
||||
cost_per_request: Optional field - cost per request to the target endpoint
|
||||
setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream
|
||||
client_frame_gate: Optional per-frame authorization, applied before a frame reaches the upstream
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj
|
||||
|
|
@ -2575,6 +2619,14 @@ async def websocket_passthrough_request(
|
|||
)
|
||||
logging_obj.model_call_details["litellm_call_id"] = litellm_call_id
|
||||
|
||||
async def log_failure(exception: Exception, traceback_str: str | None = None) -> None:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=exception,
|
||||
request_data={**kwargs, "litellm_logging_obj": logging_obj},
|
||||
traceback_str=traceback_str,
|
||||
)
|
||||
|
||||
# Pre-call logging
|
||||
logging_obj.pre_call(
|
||||
input=[{"role": "user", "content": "WebSocket connection"}],
|
||||
|
|
@ -2606,8 +2658,8 @@ async def websocket_passthrough_request(
|
|||
"WebSocket passthrough (%s): Upstream connection established successfully", endpoint
|
||||
)
|
||||
|
||||
async def forward_client_to_upstream() -> None:
|
||||
"""Forward messages from client to upstream WebSocket"""
|
||||
async def forward_client_to_upstream() -> ProxyException | None:
|
||||
"""Forward messages from client to upstream WebSocket, returning any denial that ended it"""
|
||||
try:
|
||||
while True:
|
||||
message = await websocket.receive()
|
||||
|
|
@ -2620,6 +2672,17 @@ async def websocket_passthrough_request(
|
|||
bytes_data: bytes | None = message.get("bytes")
|
||||
|
||||
if text_data is not None:
|
||||
if (
|
||||
text_refusal := await _relay_client_frame(
|
||||
websocket,
|
||||
upstream_ws,
|
||||
text_data,
|
||||
_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter),
|
||||
client_frame_gate,
|
||||
user_api_key_dict,
|
||||
)
|
||||
) is not None:
|
||||
return text_refusal
|
||||
# Try to extract model from client setup message for Vertex AI Live
|
||||
if endpoint and "/vertex_ai/live" in endpoint:
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
@ -2676,9 +2739,18 @@ async def websocket_passthrough_request(
|
|||
)
|
||||
# Not a JSON message or doesn't contain setup data
|
||||
|
||||
await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter))
|
||||
elif bytes_data is not None:
|
||||
await upstream_ws.send(bytes_data)
|
||||
if (
|
||||
bytes_refusal := await _relay_client_frame(
|
||||
websocket,
|
||||
upstream_ws,
|
||||
bytes_data,
|
||||
bytes_data,
|
||||
client_frame_gate,
|
||||
user_api_key_dict,
|
||||
)
|
||||
) is not None:
|
||||
return bytes_refusal
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
|
|
@ -2751,13 +2823,27 @@ async def websocket_passthrough_request(
|
|||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
task_results: Final = tuple(task.result() for task in done if task.exception() is None)
|
||||
|
||||
# A refusal outranks any exception it caused: closing the client socket is what makes the
|
||||
# upstream reader's next send fail, so raising that instead would report the wrong cause.
|
||||
# Turns that completed before a refusal were served, so they are logged and billed as a success
|
||||
client_frame_refusal: Final = next(
|
||||
(result for result in task_results if isinstance(result, ProxyException)), None
|
||||
)
|
||||
if client_frame_refusal is not None:
|
||||
await upstream_ws.close()
|
||||
if client_frame_refusal is not None and not websocket_messages:
|
||||
await log_failure(client_frame_refusal)
|
||||
return
|
||||
|
||||
# Check for exceptions in completed tasks
|
||||
for task in done:
|
||||
exception = task.exception()
|
||||
if exception is not None:
|
||||
if exception is not None and client_frame_refusal is None:
|
||||
raise exception
|
||||
|
||||
upstream_close: Final = _upstream_close_to_relay(task.result() for task in done)
|
||||
upstream_close: Final = _upstream_close_to_relay(task_results)
|
||||
if upstream_close is not None and _client_socket_is_open(websocket):
|
||||
await websocket.close(
|
||||
code=upstream_close.code,
|
||||
|
|
@ -2832,23 +2918,7 @@ async def websocket_passthrough_request(
|
|||
except InvalidStatus as exc:
|
||||
verbose_proxy_logger.exception("WebSocket passthrough (%s): upstream rejected WebSocket connection", endpoint)
|
||||
|
||||
# Prepare request payload for logging
|
||||
request_payload = {}
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
request_payload[key] = value
|
||||
if logging_obj is not None:
|
||||
request_payload["litellm_logging_obj"] = logging_obj
|
||||
|
||||
# Log the connection failure using the same pattern as HTTP
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=exc,
|
||||
request_data=request_payload,
|
||||
traceback_str=traceback.format_exc(
|
||||
limit=MAXIMUM_TRACEBACK_LINES_TO_LOG,
|
||||
),
|
||||
)
|
||||
await log_failure(exc, traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG))
|
||||
|
||||
if _client_socket_is_open(websocket):
|
||||
await websocket.close(
|
||||
|
|
@ -2860,23 +2930,7 @@ async def websocket_passthrough_request(
|
|||
"WebSocket passthrough (%s): unexpected error while proxying WebSocket", endpoint
|
||||
)
|
||||
|
||||
# Prepare request payload for logging
|
||||
request_payload = {}
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
request_payload[key] = value
|
||||
if logging_obj is not None:
|
||||
request_payload["litellm_logging_obj"] = logging_obj
|
||||
|
||||
# Log the unexpected error using the same pattern as HTTP
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
request_data=request_payload,
|
||||
traceback_str=traceback.format_exc(
|
||||
limit=MAXIMUM_TRACEBACK_LINES_TO_LOG,
|
||||
),
|
||||
)
|
||||
await log_failure(e, traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG))
|
||||
|
||||
if _client_socket_is_open(websocket):
|
||||
await websocket.close(code=1011, reason="WebSocket passthrough error")
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import os
|
|||
import sys
|
||||
import uuid
|
||||
import zlib
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
|
@ -30,12 +30,12 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
|
||||
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, WEBSOCKET_CLOSE_REASON_MAX_BYTES
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy._lazy_features import LazyFeature, attach_lazy_features
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import router as llm_passthrough_router
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
||||
|
|
@ -53,6 +53,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
|||
initialize_pass_through_endpoints,
|
||||
pass_through_request,
|
||||
resolve_llm_passthrough_timeout,
|
||||
WebsocketClientFrameGate,
|
||||
resolve_pass_through_request_timeout,
|
||||
set_env_variables_in_header,
|
||||
websocket_passthrough_request,
|
||||
|
|
@ -7175,6 +7176,39 @@ class RecordingUpstreamWebSocket:
|
|||
raise StopAsyncIteration
|
||||
|
||||
|
||||
class ClosableUpstreamWebSocket:
|
||||
"""An upstream whose ``recv`` ends when ``close`` does, and whose close keeps yielding after that.
|
||||
|
||||
A real close handshake outlives the ``recv`` it unblocks, so the reader task finishes while the closing
|
||||
task is still suspended. That is the interleaving that leaves a refusal stranded in a cancelled task
|
||||
"""
|
||||
|
||||
CLOSE_HANDSHAKE_YIELDS = 5
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._closing = asyncio.Event()
|
||||
self.send = AsyncMock()
|
||||
self.close = AsyncMock(side_effect=self._close)
|
||||
|
||||
async def _close(self) -> None:
|
||||
self._closing.set()
|
||||
for _ in range(self.CLOSE_HANDSHAKE_YIELDS):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
async def recv(self, decode: bool = True) -> str:
|
||||
from websockets.exceptions import ConnectionClosedOK
|
||||
from websockets.frames import Close
|
||||
|
||||
await self._closing.wait()
|
||||
raise ConnectionClosedOK(rcvd=Close(1000, ""), sent=Close(1000, ""), rcvd_then_sent=True)
|
||||
|
||||
def __aiter__(self) -> "ClosableUpstreamWebSocket":
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> str:
|
||||
raise StopAsyncIteration
|
||||
|
||||
|
||||
def _client_websocket(receive):
|
||||
from starlette.websockets import WebSocketState
|
||||
|
||||
|
|
@ -7207,10 +7241,16 @@ def _patched_websocket_passthrough_environment(upstream_ws):
|
|||
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
|
||||
mock_proxy_logging.post_call_success_hook = AsyncMock()
|
||||
mock_proxy_logging.post_call_failure_hook = AsyncMock()
|
||||
enqueued_handler = MagicMock()
|
||||
mock_worker.ensure_initialized_and_enqueue = MagicMock(
|
||||
side_effect=lambda async_coroutine: async_coroutine.close()
|
||||
side_effect=lambda async_coroutine: (
|
||||
enqueued_handler(**async_coroutine.cr_frame.f_locals),
|
||||
async_coroutine.close(),
|
||||
)
|
||||
)
|
||||
yield SimpleNamespace(
|
||||
proxy_logging=mock_proxy_logging, logging_worker=mock_worker, enqueued_handler=enqueued_handler
|
||||
)
|
||||
yield
|
||||
|
||||
|
||||
async def _pending_receive():
|
||||
|
|
@ -7246,6 +7286,36 @@ async def test_websocket_passthrough_relays_upstream_policy_close_to_client():
|
|||
websocket.close.assert_awaited_once_with(code=1008, reason=upstream_reason)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_relays_as_much_of_a_long_upstream_close_reason_as_fits():
|
||||
from websockets.exceptions import ConnectionClosedError
|
||||
from websockets.frames import Close
|
||||
|
||||
upstream_reason = (
|
||||
"Quota exceeded for concurrent_sessions. Request more quota for aiplatform.googleapis.com "
|
||||
"BidiGenerateContent sessions in us-central1 from the Google Cloud console quotas page"
|
||||
)
|
||||
upstream_ws = ClosingUpstreamWebSocket(
|
||||
ConnectionClosedError(rcvd=Close(1011, upstream_reason), sent=Close(1011, ""), rcvd_then_sent=True)
|
||||
)
|
||||
websocket = _client_websocket(_pending_receive)
|
||||
|
||||
with _patched_websocket_passthrough_environment(upstream_ws):
|
||||
await websocket_passthrough_request(
|
||||
websocket=websocket,
|
||||
target="wss://aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent",
|
||||
custom_headers={"Authorization": "Bearer token"},
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
forward_headers=False,
|
||||
endpoint="/vertex_ai/live",
|
||||
accept_websocket=False,
|
||||
)
|
||||
|
||||
relayed_reason = websocket.close.await_args.kwargs["reason"]
|
||||
assert upstream_reason.startswith(relayed_reason)
|
||||
assert len(relayed_reason.encode("utf-8")) == WEBSOCKET_CLOSE_REASON_MAX_BYTES
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_keeps_normal_upstream_close_normal():
|
||||
from websockets.exceptions import ConnectionClosedOK
|
||||
|
|
@ -7336,6 +7406,796 @@ async def test_websocket_passthrough_rewrites_gateway_alias_setup_model():
|
|||
assert sent_setup["model"] == "projects/proj-db/locations/global/publishers/google/models/gemini-live-2.5-flash"
|
||||
|
||||
|
||||
VERTEX_LIVE_TARGET = "wss://aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent"
|
||||
|
||||
|
||||
VERTEX_LIVE_GROUP = "gemini-live-2.5-flash-native-audio"
|
||||
|
||||
VERTEX_LIVE_ALIAS_GROUP = "gemini-live-native-audio"
|
||||
|
||||
VERTEX_LIVE_GROUP_ADDRESSINGS = (
|
||||
"{model}",
|
||||
"models/{model}",
|
||||
"vertex_ai/{model}",
|
||||
"publishers/google/models/{model}",
|
||||
)
|
||||
|
||||
VERTEX_LIVE_FULL_PATH = "projects/proj-db/locations/global/publishers/google/models/{model}"
|
||||
|
||||
VERTEX_LIVE_CLIENT_ADDRESSINGS = (*VERTEX_LIVE_GROUP_ADDRESSINGS, VERTEX_LIVE_FULL_PATH)
|
||||
|
||||
|
||||
def _vertex_live_router() -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gemini-live-2.5-flash",
|
||||
"litellm_params": {"model": "vertex_ai/gemini-live-2.5-flash"},
|
||||
},
|
||||
{
|
||||
"model_name": VERTEX_LIVE_GROUP,
|
||||
"litellm_params": {"model": f"vertex_ai/{VERTEX_LIVE_GROUP}"},
|
||||
},
|
||||
{
|
||||
"model_name": VERTEX_LIVE_ALIAS_GROUP,
|
||||
"litellm_params": {"model": f"vertex_ai/{VERTEX_LIVE_GROUP}"},
|
||||
},
|
||||
{
|
||||
"model_name": "gemini-3-pro-live",
|
||||
"litellm_params": {"model": "vertex_ai/gemini-3-pro-live"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def _run_vertex_live_gated_passthrough(
|
||||
client_frames: Sequence[Mapping[str, str | bytes]],
|
||||
valid_token: UserAPIKeyAuth,
|
||||
llm_router: litellm.Router | None = None,
|
||||
) -> tuple[MagicMock, RecordingUpstreamWebSocket]:
|
||||
"""Drive /vertex_ai/live with the real frame gate, returning the client and upstream sockets"""
|
||||
upstream_ws = RecordingUpstreamWebSocket()
|
||||
websocket, _ = await _run_vertex_live_gated_session(
|
||||
AsyncMock(side_effect=[*client_frames, {"type": "websocket.disconnect"}]),
|
||||
upstream_ws,
|
||||
valid_token,
|
||||
llm_router or _vertex_live_router(),
|
||||
)
|
||||
return websocket, upstream_ws
|
||||
|
||||
|
||||
async def _run_vertex_live_gated_session(
|
||||
receive: Callable[[], Awaitable[Mapping[str, str | bytes]]],
|
||||
upstream_ws: "RecordingUpstreamWebSocket | ScriptedUpstreamWebSocket",
|
||||
valid_token: UserAPIKeyAuth,
|
||||
llm_router: litellm.Router,
|
||||
) -> tuple[MagicMock, SimpleNamespace]:
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
_build_vertex_live_client_frame_gate,
|
||||
_build_vertex_live_setup_model_rewriter,
|
||||
)
|
||||
|
||||
websocket = _client_websocket(receive)
|
||||
|
||||
with _patched_websocket_passthrough_environment(upstream_ws) as environment:
|
||||
await websocket_passthrough_request(
|
||||
websocket=websocket,
|
||||
target=VERTEX_LIVE_TARGET,
|
||||
custom_headers={"Authorization": "Bearer token"},
|
||||
user_api_key_dict=valid_token,
|
||||
forward_headers=False,
|
||||
endpoint="/vertex_ai/live",
|
||||
accept_websocket=False,
|
||||
setup_model_rewriter=_build_vertex_live_setup_model_rewriter(
|
||||
vertex_project="proj-db",
|
||||
vertex_location="global",
|
||||
llm_router=llm_router,
|
||||
),
|
||||
client_frame_gate=_build_vertex_live_client_frame_gate(llm_router=llm_router),
|
||||
)
|
||||
|
||||
return websocket, environment
|
||||
|
||||
|
||||
def _setup_frame(model: str) -> dict[str, str]:
|
||||
return {
|
||||
"type": "websocket.receive",
|
||||
"text": json.dumps({"setup": {"model": model, "generationConfig": {"responseModalities": ["TEXT"]}}}),
|
||||
}
|
||||
|
||||
|
||||
def _policy_close_reason(websocket: MagicMock) -> str | None:
|
||||
return next(
|
||||
(call.kwargs.get("reason") for call in websocket.close.await_args_list if call.kwargs.get("code") == 1008),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_setup_frame_naming_a_model_the_key_cannot_call_is_refused():
|
||||
"""
|
||||
?model= is optional on this route, so a key that names an unlisted model only in the setup frame used to
|
||||
reach Vertex with the gateway's own service-account credentials
|
||||
"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame("totally-unlisted-model-abc")],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "totally-unlisted-model-abc" in reason
|
||||
assert reason.endswith(".")
|
||||
upstream_ws.close.assert_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_setup_frame_naming_a_permitted_model_still_reaches_vertex():
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame("gemini-live-2.5-flash")],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_awaited_once()
|
||||
sent_setup = json.loads(upstream_ws.send.await_args.args[0])["setup"]
|
||||
assert sent_setup["model"] == "projects/proj-db/locations/global/publishers/google/models/gemini-live-2.5-flash"
|
||||
assert _policy_close_reason(websocket) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_setup_frame_from_an_unrestricted_key_still_reaches_vertex():
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame("gemini-3-pro-live")],
|
||||
UserAPIKeyAuth(token="hashed", models=[]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_awaited_once()
|
||||
assert _policy_close_reason(websocket) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_setup_frame_is_checked_against_the_team_for_an_all_team_models_key():
|
||||
"""
|
||||
``all-team-models`` skips the key allowlist, so the team's own restriction is the only thing left to apply
|
||||
"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame("gemini-3-pro-live")],
|
||||
UserAPIKeyAuth(
|
||||
token="hashed",
|
||||
models=["all-team-models"],
|
||||
team_id="team-1",
|
||||
team_models=["gemini-live-2.5-flash"],
|
||||
),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "gemini-3-pro-live" in reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_setup_frame_is_gated_when_it_arrives_as_binary():
|
||||
"""A client that sends its setup frame as bytes must not skip the check the text path applies"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[
|
||||
{
|
||||
"type": "websocket.receive",
|
||||
"bytes": json.dumps({"setup": {"model": "totally-unlisted-model-abc"}}).encode(),
|
||||
}
|
||||
],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
assert _policy_close_reason(websocket) is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_setup_frame_with_an_escaped_key_is_still_gated():
|
||||
"""Large audio frames skip the parse on a substring test, so a JSON-escaped key must not slip past it"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[{"type": "websocket.receive", "text": '{"\\u0073etup": {"model": "totally-unlisted-model-abc"}}'}],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
assert _policy_close_reason(websocket) is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("frame_kind", ["text", "bytes"])
|
||||
@pytest.mark.parametrize(
|
||||
"lenient_setup",
|
||||
[
|
||||
'{"setup": {"model": "totally-unlisted-model-abc",},}',
|
||||
"{'setup': {'model': 'totally-unlisted-model-abc'}}",
|
||||
'{setup: {model: "totally-unlisted-model-abc"}}',
|
||||
],
|
||||
)
|
||||
async def test_vertex_live_setup_frame_only_vertex_can_parse_is_refused(lenient_setup, frame_kind):
|
||||
"""Vertex reads trailing commas, single quotes and bare keys, which strict JSON refuses to parse"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[
|
||||
{
|
||||
"type": "websocket.receive",
|
||||
frame_kind: lenient_setup if frame_kind == "text" else lenient_setup.encode(),
|
||||
}
|
||||
],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
assert _policy_close_reason(websocket) is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"lenient_setup",
|
||||
[
|
||||
'{"setup": {"model": "gemini-3-pro-live",},}',
|
||||
"{'setup': {'model': 'gemini-3-pro-live'}}",
|
||||
'{setup: {model: "gemini-3-pro-live"}}',
|
||||
],
|
||||
)
|
||||
async def test_vertex_live_setup_frame_only_vertex_can_parse_still_reaches_vertex_for_an_unrestricted_key(
|
||||
lenient_setup,
|
||||
):
|
||||
"""A key that may call any model loses nothing the proxy cannot read, so its lenient frames still go through"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[{"type": "websocket.receive", "text": lenient_setup}],
|
||||
UserAPIKeyAuth(token="hashed", models=[]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_awaited_once()
|
||||
assert _policy_close_reason(websocket) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"repeated_key_setup",
|
||||
[
|
||||
'{"setup": {"model": "gemini-3-pro-live", "model": null}}',
|
||||
'{"setup": {"model": "gemini-3-pro-live"}, "setup": null}',
|
||||
],
|
||||
)
|
||||
async def test_vertex_live_setup_frame_repeating_a_key_as_null_is_refused(repeated_key_setup):
|
||||
"""
|
||||
Vertex keeps the first of two keys when the second is null, while ``json.loads`` keeps the last, so the
|
||||
proxy would read no model where Vertex runs the first one
|
||||
"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[{"type": "websocket.receive", "text": repeated_key_setup}],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
assert _policy_close_reason(websocket) is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"limited_token",
|
||||
[
|
||||
UserAPIKeyAuth(token="hashed", models=["litellm-*"]),
|
||||
UserAPIKeyAuth(token="hashed", models=["vertex_ai/gemini-*"]),
|
||||
UserAPIKeyAuth(token="hashed", models=[], team_id="team-1", team_models=["gemini-live-2.5-flash"]),
|
||||
UserAPIKeyAuth(token="hashed", models=[], agent_id="agent-1"),
|
||||
UserAPIKeyAuth(token="hashed", models=[], end_user_id="end-user-1"),
|
||||
],
|
||||
ids=["litellm-wildcard", "vertex-wildcard", "team", "agent", "end-user"],
|
||||
)
|
||||
async def test_vertex_live_setup_frame_only_vertex_can_parse_is_refused_for_any_limited_caller(limited_token):
|
||||
"""A wildcard can match any name the proxy might test the caller with, so only a caller with no model limit
|
||||
anywhere may send a frame the proxy cannot read"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[
|
||||
{
|
||||
"type": "websocket.receive",
|
||||
"text": "{'setup': {'model': '"
|
||||
+ VERTEX_LIVE_FULL_PATH.format(model="gemini-3-pro-live")
|
||||
+ "',},}",
|
||||
}
|
||||
],
|
||||
limited_token,
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
assert _policy_close_reason(websocket) is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_client_content_that_mentions_setup_is_forwarded():
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[
|
||||
_setup_frame("gemini-live-2.5-flash"),
|
||||
{
|
||||
"type": "websocket.receive",
|
||||
"text": json.dumps(
|
||||
{"clientContent": {"turns": [{"role": "user", "parts": [{"text": "help me setup a meeting"}]}]}}
|
||||
),
|
||||
},
|
||||
],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
)
|
||||
|
||||
assert upstream_ws.send.await_count == 2
|
||||
assert _policy_close_reason(websocket) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_non_setup_frames_are_forwarded_without_a_model_check():
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[
|
||||
_setup_frame("gemini-live-2.5-flash"),
|
||||
{"type": "websocket.receive", "text": json.dumps({"realtimeInput": {"audio": {"data": "AAAA"}}})},
|
||||
],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
)
|
||||
|
||||
assert upstream_ws.send.await_count == 2
|
||||
assert _policy_close_reason(websocket) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("addressing", VERTEX_LIVE_GROUP_ADDRESSINGS)
|
||||
@pytest.mark.parametrize("group", [VERTEX_LIVE_GROUP, VERTEX_LIVE_ALIAS_GROUP])
|
||||
async def test_vertex_live_setup_frame_authorizes_a_permitted_group_in_every_addressing(addressing, group):
|
||||
"""
|
||||
The Live SDK wraps the caller's model as ``models/<name>``, so a key holding the group has to work
|
||||
whichever addressing its client actually sends
|
||||
"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame(addressing.format(model=group))],
|
||||
UserAPIKeyAuth(token="hashed", models=[group]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_awaited_once()
|
||||
sent_setup = json.loads(upstream_ws.send.await_args.args[0])["setup"]
|
||||
assert sent_setup["model"].startswith("projects/")
|
||||
assert _policy_close_reason(websocket) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS)
|
||||
async def test_vertex_live_setup_frame_refuses_an_unpermitted_group_in_every_addressing(addressing):
|
||||
"""Normalizing the addressing must not become a way to reach a group the key does not hold"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame(addressing.format(model="gemini-3-pro-live"))],
|
||||
UserAPIKeyAuth(token="hashed", models=[VERTEX_LIVE_GROUP]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "gemini-3-pro-live" in reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS)
|
||||
async def test_vertex_live_setup_frame_naming_no_group_at_all_is_refused_in_every_addressing(addressing):
|
||||
"""A name that resolves to no group is authorized as it arrived, so an unknown model cannot ride a prefix in"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame(addressing.format(model="totally-unlisted-model-abc"))],
|
||||
UserAPIKeyAuth(token="hashed", models=[VERTEX_LIVE_GROUP]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
assert _policy_close_reason(websocket) is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS)
|
||||
@pytest.mark.parametrize("key_model", ["gemini-4-pro-live-unserved", "vertex_ai/gemini-4-pro-live-unserved"])
|
||||
async def test_vertex_live_setup_frame_authorizes_a_model_no_group_serves_by_its_publisher_id(addressing, key_model):
|
||||
"""
|
||||
With no group to resolve, the setup rewriter sends Vertex the last path segment, so a key holding that
|
||||
publisher model, bare or as a LiteLLM ``vertex_ai/`` id, can open it whichever addressing its client sends
|
||||
"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame(addressing.format(model="gemini-4-pro-live-unserved"))],
|
||||
UserAPIKeyAuth(token="hashed", models=[key_model]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_awaited_once()
|
||||
sent_setup = json.loads(upstream_ws.send.await_args.args[0])["setup"]
|
||||
assert sent_setup["model"].endswith("/models/gemini-4-pro-live-unserved")
|
||||
assert _policy_close_reason(websocket) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_setup_frame_addressing_cannot_satisfy_a_wildcard_for_another_publisher_model():
|
||||
"""Vertex runs only the last segment, so the segments ahead of it must not be what matches the key's wildcard"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame("vertex_ai/gemini-anything/forbidden-model")],
|
||||
UserAPIKeyAuth(token="hashed", models=["vertex_ai/gemini-*"]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "forbidden-model" in reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_setup_frame_prefers_the_group_its_whole_name_matches():
|
||||
"""A group named ``vertex_ai/<id>`` is the one the rewriter sends, so it is the one the key must hold"""
|
||||
llm_router = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "gemini-3-pro-live", "litellm_params": {"model": "vertex_ai/gemini-3-pro-live"}},
|
||||
{"model_name": "vertex_ai/gemini-3-pro-live", "litellm_params": {"model": "vertex_ai/gemini-3-pro-live"}},
|
||||
]
|
||||
)
|
||||
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame("vertex_ai/gemini-3-pro-live")],
|
||||
UserAPIKeyAuth(token="hashed", models=["vertex_ai/gemini-3-pro-live"]),
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_awaited_once()
|
||||
assert _policy_close_reason(websocket) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("group", [VERTEX_LIVE_GROUP, VERTEX_LIVE_ALIAS_GROUP])
|
||||
async def test_vertex_live_full_path_is_authorized_by_any_group_serving_its_publisher_model(group):
|
||||
"""The docs tell callers to send the full resource path, which Vertex runs as sent, so every group serving
|
||||
that publisher model grants it, whatever the group is called"""
|
||||
full_path = VERTEX_LIVE_FULL_PATH.format(model=VERTEX_LIVE_GROUP)
|
||||
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame(full_path)],
|
||||
UserAPIKeyAuth(token="hashed", models=[group]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_awaited_once()
|
||||
assert json.loads(upstream_ws.send.await_args.args[0])["setup"]["model"] == full_path
|
||||
assert _policy_close_reason(websocket) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_full_path_is_not_granted_by_a_group_named_after_it_that_serves_another_model():
|
||||
llm_router = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "gemini-3-pro-live", "litellm_params": {"model": "vertex_ai/gemini-live-2.5-flash"}},
|
||||
]
|
||||
)
|
||||
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame(VERTEX_LIVE_FULL_PATH.format(model="gemini-3-pro-live"))],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-3-pro-live"]),
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "gemini-3-pro-live" in reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"valid_token",
|
||||
[
|
||||
UserAPIKeyAuth(
|
||||
token="hashed",
|
||||
models=["gemini-live-2.5-flash"],
|
||||
aliases={"gemini-3-pro-live": "gemini-live-2.5-flash"},
|
||||
),
|
||||
UserAPIKeyAuth(
|
||||
token="hashed",
|
||||
models=["all-team-models"],
|
||||
team_id="team-1",
|
||||
team_models=["gemini-live-2.5-flash"],
|
||||
team_model_aliases={"gemini-3-pro-live": "gemini-live-2.5-flash"},
|
||||
),
|
||||
],
|
||||
ids=["key-alias", "team-alias"],
|
||||
)
|
||||
async def test_vertex_live_setup_frame_naming_an_alias_cannot_open_the_model_the_alias_shadows(valid_token):
|
||||
"""
|
||||
An alias passes the access check by its target, while the setup rewriter sends the name as written, so a
|
||||
key that could set an alias would open the model the alias name belongs to
|
||||
"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough([_setup_frame("gemini-3-pro-live")], valid_token)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "gemini-3-pro-live" in reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_setup_frame_naming_a_router_alias_cannot_open_the_model_the_alias_shadows():
|
||||
llm_router = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "gemini-live-2.5-flash", "litellm_params": {"model": "vertex_ai/gemini-live-2.5-flash"}},
|
||||
],
|
||||
model_group_alias={"gemini-3-pro-live": "gemini-live-2.5-flash"},
|
||||
)
|
||||
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame("gemini-3-pro-live")],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "gemini-3-pro-live" in reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_setup_frame_is_checked_against_the_owner_of_a_personal_key():
|
||||
"""A personal key's owner can be limited to fewer models than the key lists, and auth enforces that limit too"""
|
||||
from litellm.proxy.proxy_server import user_api_key_cache
|
||||
|
||||
user_id = f"live-user-{uuid.uuid4().hex}"
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=user_id,
|
||||
value={"user_id": user_id, "user_email": "live@example.com", "models": ["gemini-live-2.5-flash"]},
|
||||
)
|
||||
|
||||
with patch( # test-quality-ok: read at call time, and the user row comes from the cache seeded above
|
||||
"litellm.proxy.proxy_server.prisma_client", MagicMock()
|
||||
):
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame("gemini-3-pro-live")],
|
||||
UserAPIKeyAuth(token="hashed", user_id=user_id, models=[]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "gemini-3-pro-live" in reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS)
|
||||
async def test_vertex_live_setup_frame_is_checked_against_the_team_in_every_addressing(addressing):
|
||||
"""The team allowlist is keyed by group too, so the addressing must not decide whether the team is checked"""
|
||||
websocket, upstream_ws = await _run_vertex_live_gated_passthrough(
|
||||
[_setup_frame(addressing.format(model="gemini-3-pro-live"))],
|
||||
UserAPIKeyAuth(
|
||||
token="hashed",
|
||||
models=["all-team-models"],
|
||||
team_id="team-1",
|
||||
team_models=[VERTEX_LIVE_GROUP],
|
||||
),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "gemini-3-pro-live" in reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("addressing", VERTEX_LIVE_CLIENT_ADDRESSINGS)
|
||||
async def test_vertex_live_setup_frame_resolves_an_access_group_grant_in_every_addressing(addressing):
|
||||
"""
|
||||
``get_model_access_groups`` is keyed by group as well, and returns nothing for a prefixed name, so a key
|
||||
entitled through an access group rather than a model name needs the same normalization
|
||||
"""
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
_build_vertex_live_client_frame_gate,
|
||||
)
|
||||
|
||||
llm_router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": VERTEX_LIVE_GROUP,
|
||||
"litellm_params": {"model": f"vertex_ai/{VERTEX_LIVE_GROUP}"},
|
||||
"model_info": {"access_groups": ["live-team"]},
|
||||
},
|
||||
]
|
||||
)
|
||||
gate = _build_vertex_live_client_frame_gate(llm_router=llm_router)
|
||||
|
||||
frame = json.dumps({"setup": {"model": addressing.format(model=VERTEX_LIVE_GROUP)}})
|
||||
denial = await gate(frame, UserAPIKeyAuth(token="hashed", models=["live-team"]))
|
||||
|
||||
assert denial is None
|
||||
|
||||
|
||||
async def _run_vertex_live_route(
|
||||
client_frames: Sequence[Mapping[str, str | bytes]], valid_token: UserAPIKeyAuth
|
||||
) -> tuple[MagicMock, RecordingUpstreamWebSocket, SimpleNamespace]:
|
||||
"""Drive the /vertex_ai/live route, so the gate under test is whichever one the route itself wires"""
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
vertex_ai_live_websocket_passthrough,
|
||||
vertex_llm_base,
|
||||
)
|
||||
|
||||
upstream_ws = RecordingUpstreamWebSocket()
|
||||
websocket = _client_websocket(
|
||||
AsyncMock(side_effect=[*client_frames, {"type": "websocket.disconnect"}]),
|
||||
)
|
||||
|
||||
with (
|
||||
_patched_websocket_passthrough_environment(upstream_ws) as environment,
|
||||
patch.object( # test-quality-ok: an OAuth exchange with Google, so this is the boundary being faked
|
||||
vertex_llm_base,
|
||||
"_ensure_access_token_async",
|
||||
AsyncMock(return_value=("token-abc", "proj-db")),
|
||||
),
|
||||
):
|
||||
await vertex_ai_live_websocket_passthrough(
|
||||
websocket=websocket,
|
||||
model="gemini-live-2.5-flash",
|
||||
vertex_project="proj-db",
|
||||
vertex_location="global",
|
||||
user_api_key_dict=valid_token,
|
||||
)
|
||||
|
||||
return websocket, upstream_ws, environment
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_route_gates_the_setup_frame_even_when_the_query_model_is_permitted():
|
||||
"""
|
||||
?model= is all connect-time auth sees, so a key that passes it and then names a different model in its setup
|
||||
frame reaches Vertex on the gateway's own credentials unless the route gates the frames as well
|
||||
"""
|
||||
websocket, upstream_ws, environment = await _run_vertex_live_route(
|
||||
[_setup_frame("gemini-3-pro-live")],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_not_awaited()
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "gemini-3-pro-live" in reason
|
||||
refusal = environment.proxy_logging.post_call_failure_hook.await_args.kwargs["original_exception"]
|
||||
assert refusal.code == "403"
|
||||
assert refusal.type == ProxyErrorTypes.key_model_access_denied
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_route_keeps_the_key_allowlist_out_of_the_close_reason_and_in_the_proxy_log(caplog):
|
||||
"""The client learns which model was refused, while only the proxy log names the models the key may reach"""
|
||||
with caplog.at_level("WARNING", logger="LiteLLM Proxy"):
|
||||
websocket, _, _ = await _run_vertex_live_route(
|
||||
[_setup_frame("gemini-3-pro-live")],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
)
|
||||
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "gemini-live-2.5-flash" not in reason
|
||||
assert any(
|
||||
"gemini-3-pro-live" in record.getMessage() and "gemini-live-2.5-flash" in record.getMessage()
|
||||
for record in caplog.records
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_route_forwards_a_setup_frame_the_key_is_allowed_to_run():
|
||||
websocket, upstream_ws, _ = await _run_vertex_live_route(
|
||||
[_setup_frame("gemini-live-2.5-flash")],
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
)
|
||||
|
||||
upstream_ws.send.assert_awaited_once()
|
||||
assert _policy_close_reason(websocket) is None
|
||||
|
||||
|
||||
class ScriptedUpstreamWebSocket:
|
||||
"""An upstream that sends the given frames in order, then stays open with nothing more to say"""
|
||||
|
||||
def __init__(self, frames: Iterable[str]) -> None:
|
||||
self._frames = iter(frames)
|
||||
self.sent_every_frame = asyncio.Event()
|
||||
self.send = AsyncMock()
|
||||
self.close = AsyncMock()
|
||||
|
||||
async def recv(self, decode: bool = True) -> str | None:
|
||||
frame = next(self._frames, None)
|
||||
if frame is not None:
|
||||
return frame
|
||||
self.sent_every_frame.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
def __aiter__(self) -> "ScriptedUpstreamWebSocket":
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> str:
|
||||
raise StopAsyncIteration
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_live_turns_served_before_a_refused_frame_are_still_billed():
|
||||
"""
|
||||
A refusal used to log the whole session as a failure, which records no spend, so a key could run allowed
|
||||
turns and then send a refused frame to have them go unbilled. The refused frame's model must not become
|
||||
the model those turns are priced at either
|
||||
"""
|
||||
upstream_ws = ScriptedUpstreamWebSocket(
|
||||
(
|
||||
json.dumps({"setupComplete": {}}),
|
||||
json.dumps({"serverContent": {"turnComplete": True}, "usageMetadata": {"totalTokenCount": 60}}),
|
||||
)
|
||||
)
|
||||
|
||||
async def client_frames() -> AsyncIterator[dict[str, str]]:
|
||||
yield _setup_frame("gemini-live-2.5-flash")
|
||||
await upstream_ws.sent_every_frame.wait()
|
||||
yield _setup_frame("gemini-3-pro-live")
|
||||
|
||||
websocket, environment = await _run_vertex_live_gated_session(
|
||||
client_frames().__anext__,
|
||||
upstream_ws,
|
||||
UserAPIKeyAuth(token="hashed", models=["gemini-live-2.5-flash"]),
|
||||
_vertex_live_router(),
|
||||
)
|
||||
|
||||
reason = _policy_close_reason(websocket)
|
||||
assert reason is not None and "gemini-3-pro-live" in reason
|
||||
upstream_ws.send.assert_awaited_once()
|
||||
upstream_ws.close.assert_awaited()
|
||||
environment.logging_worker.ensure_initialized_and_enqueue.assert_called_once()
|
||||
assert environment.enqueued_handler.call_args.kwargs["logging_obj"].model == "gemini-live-2.5-flash"
|
||||
environment.proxy_logging.post_call_success_hook.assert_awaited_once()
|
||||
environment.proxy_logging.post_call_failure_hook.assert_not_awaited()
|
||||
|
||||
|
||||
async def _run_passthrough_with_frame_gate(
|
||||
gate: WebsocketClientFrameGate,
|
||||
) -> tuple[MagicMock, ClosableUpstreamWebSocket, SimpleNamespace]:
|
||||
upstream_ws = ClosableUpstreamWebSocket()
|
||||
websocket = _client_websocket(
|
||||
AsyncMock(
|
||||
side_effect=[
|
||||
{"type": "websocket.receive", "text": json.dumps({"setup": {"model": "gemini-live-2.5-flash"}})},
|
||||
{"type": "websocket.disconnect"},
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
with _patched_websocket_passthrough_environment(upstream_ws) as environment:
|
||||
await websocket_passthrough_request(
|
||||
websocket=websocket,
|
||||
target=VERTEX_LIVE_TARGET,
|
||||
custom_headers={"Authorization": "Bearer token"},
|
||||
user_api_key_dict=UserAPIKeyAuth(token="hashed"),
|
||||
forward_headers=False,
|
||||
endpoint="/vertex_ai/live",
|
||||
accept_websocket=False,
|
||||
client_frame_gate=gate,
|
||||
)
|
||||
|
||||
return websocket, upstream_ws, environment
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_reports_a_refused_client_frame_as_a_failure():
|
||||
"""A denied session logged through the success hooks tells spend tracking and audit that it succeeded"""
|
||||
denial = ProxyException(
|
||||
message="gemini-live-2.5-flash: key not allowed to access model",
|
||||
type=ProxyErrorTypes.key_model_access_denied,
|
||||
param="model",
|
||||
code=403,
|
||||
)
|
||||
|
||||
async def refusing_gate(frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None:
|
||||
return denial
|
||||
|
||||
websocket, upstream_ws, environment = await _run_passthrough_with_frame_gate(refusing_gate)
|
||||
|
||||
assert _policy_close_reason(websocket) == "gemini-live-2.5-flash: key not allowed to access model"
|
||||
upstream_ws.close.assert_awaited()
|
||||
environment.proxy_logging.post_call_success_hook.assert_not_awaited()
|
||||
environment.logging_worker.ensure_initialized_and_enqueue.assert_not_called()
|
||||
environment.proxy_logging.post_call_failure_hook.assert_awaited_once()
|
||||
failure_kwargs = environment.proxy_logging.post_call_failure_hook.await_args.kwargs
|
||||
assert failure_kwargs["original_exception"] is denial
|
||||
assert failure_kwargs["request_data"]["litellm_logging_obj"].litellm_call_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_still_reports_an_allowed_session_as_a_success():
|
||||
async def permitting_gate(frame_data: str | bytes, valid_token: UserAPIKeyAuth, /) -> ProxyException | None:
|
||||
return None
|
||||
|
||||
_, _, environment = await _run_passthrough_with_frame_gate(permitting_gate)
|
||||
|
||||
environment.proxy_logging.post_call_failure_hook.assert_not_awaited()
|
||||
environment.proxy_logging.post_call_success_hook.assert_awaited_once()
|
||||
environment.logging_worker.ensure_initialized_and_enqueue.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"setup_model",
|
||||
["gemini-live-2.5-flash", "models/gemini-live-2.5-flash", "publishers/google/models/gemini-live-2.5-flash"],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue