From 601d6ff2c8ad63c28a68bb8047fe44586adfb9b8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:26:18 -0700 Subject: [PATCH] [e2e] Pin the OpenAI websocket passthrough prefixes The websocket routes under /openai_passthrough and /openai had no e2e coverage, so nothing catches the regression from issue #36088, where both prefixes carried HTTP routes only and refused every upgrade with a 403 before a socket ever existed. Two tests cover it. The realtime one opens /openai_passthrough/v1/realtime and asserts OpenAI's own session.created frame comes back, which proves the route is registered and relayed upstream. The responses one asserts /openai/v1/responses accepts the upgrade, since a responses.connect socket waits for the client to speak first and has no opening frame to check. A refused upgrade is an HTTP response rather than a close frame, so both assert on the handshake. ws_base_url moves into e2e_config now that a second suite needs it --- .../coverage_registry/llm_conversational.yaml | 1 + .../llm_nonconversational.yaml | 1 + tests/e2e/e2e_config.py | 9 +++ .../e2e/llm_translation/passthrough_client.py | 58 +++++++++++++++++++ .../realtime/realtime_client.py | 9 +-- .../test_realtime_pipecat_audio_e2e.py | 2 +- .../realtime/test_realtime_pipecat_e2e.py | 2 +- .../llm_translation/test_passthrough_e2e.py | 52 +++++++++++++++++ 8 files changed, 124 insertions(+), 10 deletions(-) diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 44ed5765e38..1d4e1e028ca 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -64,6 +64,7 @@ - {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"} - {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"} - {id: llm.responses.openai.passthrough.stream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "A streamed POST /openai_passthrough/v1/responses is costed and keyed by the provider response id; it used to log a zero-cost row under a random id (GitHub issue #36523)"} +- {id: llm.responses.openai.passthrough_websocket.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], fail_before_fix: proven, source: "test_passthrough_e2e.py", rationale: "A websocket upgrade on /openai/v1/responses is accepted, so a responses.connect client reaches OpenAI through the same prefix its HTTP traffic uses; the prefix carried no websocket route and refused the upgrade with a 403 (GitHub issue #36088)"} - {id: llm.responses.openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Tool calls via Responses API"} - {id: llm.responses.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vision via Responses API"} - {id: llm.responses.anthropic.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Responses w/ Anthropic translation (smoke)"} diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 8ae7dd01b5a..f0a990a7f1f 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -44,6 +44,7 @@ - {id: llm.google_native.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: google_native, route: gemini, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "LIT-4076 / proxy/google_endpoints/endpoints.py", fail_before_fix: proven, rationale: "google-native generateContent must stamp x-litellm-response-cost so SDK traffic reconciles against spend"} - {id: llm.google_native.gemini.basic.stream.works, module: llm, tier: P0, subject_endpoint: google_native, route: gemini, capability: basic, streaming: stream, assertions: [works], source: "PR #28213 / proxy/proxy_server.py async_data_generator", fail_before_fix: proven, rationale: "streamGenerateContent must relay single-prefixed SSE frames with no [DONE] sentinel; doubled data: prefixes and the OpenAI terminator both break the Vertex Java SDK"} - {id: llm.realtime.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: realtime, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.19 / LIT-4778", rationale: "HTTP /v1/realtime/client_secrets returns an ephemeral credential"} +- {id: llm.realtime.openai.passthrough.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: basic, streaming: stream, assertions: [works], fail_before_fix: proven, source: "test_passthrough_e2e.py", rationale: "A websocket upgrade on /openai_passthrough/v1/realtime is accepted and relayed to OpenAI; only HTTP routes were registered under the prefix, so realtime clients were refused with a 403 before a socket existed (GitHub issue #36088)"} - {id: llm.vector_stores.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store create/list/retrieve/delete lifecycle"} - {id: llm.vector_stores.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store search and invalid id errors"} - {id: llm.bedrock_native.bedrock_converse.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native converse happy path"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 8bf39f6021f..0266c75e1a7 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -150,6 +150,15 @@ ANOMALY_SPEND_SETTLE_SECONDS = float( ) +def ws_base_url() -> str: + """PROXY_BASE_URL with its scheme swapped for the websocket one, so a suite + opening a socket points at the same proxy every HTTP suite uses.""" + for scheme, ws_scheme in (("https://", "wss://"), ("http://", "ws://")): + if PROXY_BASE_URL.startswith(scheme): + return ws_scheme + PROXY_BASE_URL[len(scheme) :] + return PROXY_BASE_URL + + def datadog_mcp_url(*, toolsets: str = "core") -> str: """Regional Datadog remote MCP endpoint for this process's DD_SITE. diff --git a/tests/e2e/llm_translation/passthrough_client.py b/tests/e2e/llm_translation/passthrough_client.py index e0dfae679a9..20a8592db20 100644 --- a/tests/e2e/llm_translation/passthrough_client.py +++ b/tests/e2e/llm_translation/passthrough_client.py @@ -11,9 +11,13 @@ native request models are co-located here because only this suite uses them. from __future__ import annotations from dataclasses import dataclass +from urllib.parse import urlencode from pydantic import BaseModel, Field +from websockets.exceptions import InvalidStatus +from websockets.sync.client import connect +from e2e_config import ws_base_url from proxy_client import ProxyClient from e2e_http import FileUploadForm, Headers, NoBody, Result, StreamingResponse from models import ChatMessage @@ -175,6 +179,26 @@ class OpenAIEmbeddingBody(BaseModel): input: str +class WebsocketEnvelope(BaseModel): + """The one field every provider event carries, so the first frame off a + passthrough socket identifies itself without the suite parsing raw dicts.""" + + type: str + + +class WebsocketHandshake(BaseModel): + """What the proxy did with a websocket upgrade on a passthrough prefix. + + `rejected_status` is the HTTP status of a refused upgrade: a prefix carrying no + websocket route answers 403, before any socket exists. `first_event_type` is the + type of the first frame an accepted socket delivered, which is None when the + provider waits for the client to speak first. + """ + + rejected_status: int | None = None + first_event_type: str | None = None + + class PassthroughBatchList(BaseModel): """OpenAI's own batch page, relayed verbatim. `object` is required so a body that is not an OpenAI list fails validation instead of passing vacuously.""" @@ -339,5 +363,39 @@ class PassthroughClient: ), ) + # ---- OpenAI websocket passthrough ---------------------------------- + # + # The same prefixes over an upgrade instead of a POST, for the provider APIs + # that only speak websocket (realtime, responses.connect). + + def openai_passthrough_websocket( + self, + key: str, + path: str, + *, + model: str | None = None, + open_timeout: float = 30.0, + first_event_timeout: float = 30.0, + ) -> WebsocketHandshake: + query = f"?{urlencode({'model': model})}" if model is not None else "" + try: + connection = connect( + f"{ws_base_url()}{path}{query}", + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=open_timeout, + ) + except InvalidStatus as rejected: + return WebsocketHandshake(rejected_status=rejected.response.status_code) + with connection: + try: + frame = connection.recv(timeout=first_event_timeout) + except TimeoutError: + return WebsocketHandshake() + text = frame.decode("utf-8") if isinstance(frame, bytes) else frame + return WebsocketHandshake( + first_event_type=WebsocketEnvelope.model_validate_json(text).type + ) + + def build_client(proxy: ProxyClient) -> PassthroughClient: return PassthroughClient(proxy=proxy) diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index e6c5c19cbd1..632a9cf7e57 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -21,20 +21,13 @@ from pydantic import BaseModel, ConfigDict from websockets.sync.client import connect from websockets.sync.connection import Connection -from e2e_config import PROXY_BASE_URL, unique_marker +from e2e_config import unique_marker, ws_base_url from proxy_client import ProxyClient from models import LiteLLMParamsBody _M = TypeVar("_M", bound=BaseModel) -def ws_base_url() -> str: - for scheme, ws_scheme in (("https://", "wss://"), ("http://", "ws://")): - if PROXY_BASE_URL.startswith(scheme): - return ws_scheme + PROXY_BASE_URL[len(scheme) :] - return PROXY_BASE_URL - - def realtime_ws_url(model: str) -> str: return f"{ws_base_url()}/v1/realtime?{urlencode({'model': model})}" diff --git a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py index 78955974cd5..2e9cfcfe648 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py @@ -27,10 +27,10 @@ from pathlib import Path import pytest +from e2e_config import ws_base_url from realtime_client import ( PROVIDERS, RealtimeProvider, - ws_base_url, realtime_model, ) diff --git a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py index 16628fd257a..f84ce197f88 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py @@ -25,10 +25,10 @@ import asyncio import pytest +from e2e_config import ws_base_url from realtime_client import ( PROVIDERS, RealtimeProvider, - ws_base_url, realtime_model, ) diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index 17b0dbe1ae5..78b68de8389 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -28,6 +28,9 @@ from passthrough_client import ( ) EMBEDDING_MODEL = "text-embedding-3-small" +# Relayed to OpenAI untranslated, so this is OpenAI's own realtime model name +# rather than a gateway deployment alias. +REALTIME_MODEL = "gpt-realtime-2" pytestmark = pytest.mark.e2e @@ -339,3 +342,52 @@ class TestOpenAIPassthroughSpend: f"the embeddings row logged no prompt tokens, so whatever cost it carries " f"was not computed from the real usage: {row}" ) + + +class TestOpenAIPassthroughWebsocket: + """The OpenAI passthrough prefixes must answer a websocket upgrade, not only a POST. + + The customer points realtime and responses.connect clients at the same prefixes + their HTTP traffic already uses. Only HTTP routes were registered under those + prefixes, so every upgrade was refused before a socket existed and those clients + could not reach the gateway at all. A refused upgrade is an HTTP response, not a + close frame, which is why these assert on the handshake rather than a close code. + """ + + @pytest.mark.covers("llm.realtime.openai.passthrough.stream.works") + def test_realtime_upgrade_reaches_openai_through_the_passthrough_prefix( + self, client: PassthroughClient, scoped_key: str + ) -> None: + """Pins GitHub issue #36088: /openai_passthrough/v1/realtime accepts the + upgrade and relays OpenAI's own session, instead of rejecting it with a 403.""" + handshake = client.openai_passthrough_websocket( + scoped_key, "/openai_passthrough/v1/realtime", model=REALTIME_MODEL + ) + + assert handshake.rejected_status is None, ( + f"/openai_passthrough/v1/realtime refused the websocket upgrade with HTTP " + f"{handshake.rejected_status}, so a realtime client cannot connect through " + "the gateway at all" + ) + assert handshake.first_event_type == "session.created", ( + "the accepted socket never carried OpenAI's opening session event, so the " + f"upgrade was not relayed upstream; the first frame was " + f"{handshake.first_event_type}" + ) + + @pytest.mark.covers("llm.responses.openai.passthrough_websocket.stream.works") + def test_responses_upgrade_is_accepted_on_the_openai_prefix( + self, client: PassthroughClient, scoped_key: str + ) -> None: + """Pins GitHub issue #36088 on the second prefix: /openai/v1/responses upgrades + as well. A responses.connect socket waits for the client to speak first, so the + accepted handshake is the whole signal here.""" + handshake = client.openai_passthrough_websocket( + scoped_key, "/openai/v1/responses", first_event_timeout=2.0 + ) + + assert handshake.rejected_status is None, ( + f"/openai/v1/responses refused the websocket upgrade with HTTP " + f"{handshake.rejected_status}; the prefix relays this route over HTTP but " + "drops a responses.connect client before the socket opens" + )