fix(proxy): refuse unparseable and unroutable websocket frames and hold legacy slots per socket

A client frame too deep or too long for the JSON parser closed the provider
socket mid-turn, so the turn in flight was never billed. The gate now refuses
that frame with an error event and keeps draining the open turn.

A stream_id OpenAI rejects comes back as an error that names no lane, which
dropped another turn's authorization. The gate now refuses those frames before
they reach OpenAI.

With the legacy parallel request limiter, each per-turn spend row freed the
key's slot while the socket stayed open. Websocket passthrough rows now free
nothing, and the socket frees its slot once when it closes.
This commit is contained in:
mateo-berri 2026-11-02 06:45:37 -08:00
parent b7f3e03115
commit 4ec2d7a0d4
3 changed files with 181 additions and 9 deletions

View file

@ -13,7 +13,7 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.exceptions import RateLimitType
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_parent_otel_span_from_kwargs
from litellm.proxy._types import CommonProxyErrors, CurrentItemRateLimit, UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import (
get_key_model_rpm_limit,
@ -53,9 +53,16 @@ def _response_total_tokens(kwargs: Mapping[str, object], response_obj: object) -
_CONNECTION_SCOPED_CALL_TYPES: Final = frozenset({CallTypes.aresponses_websocket.value})
def _is_websocket_passthrough_row(kwargs: Mapping[str, object]) -> bool:
passthrough_payload: Final = as_str_mapping(kwargs.get("passthrough_logging_payload"))
return passthrough_payload is not None and passthrough_payload.get("request_method") == "WEBSOCKET"
def _slots_released(kwargs: Mapping[str, object]) -> int:
"""A websocket turn callback releases nothing; the socket's slot is released once at disconnect."""
return 0 if kwargs.get("call_type") in _CONNECTION_SCOPED_CALL_TYPES else 1
if kwargs.get("call_type") in _CONNECTION_SCOPED_CALL_TYPES or _is_websocket_passthrough_row(kwargs):
return 0
return 1
class CacheObject(TypedDict):

View file

@ -2705,6 +2705,12 @@ _NO_CLIENT_FRAME_REQUESTS: Final = _ClientFrameRequests(
starts_turn=False, lane=None, turn_model=None, models=(), unverifiable_model_refusal=None
)
_FORWARD_WITHOUT_TURN: Final = _ForwardClientFrame(starts_turn=False, turn=None, lane=None)
_UNREADABLE_CLIENT_FRAME: Final = _RefuseClientFrame(
refusal=WebsocketFrameModelRefused(
message="The gateway cannot parse this frame, so it cannot check which model it asks for.",
error_type="invalid_request_error",
)
)
_TURN_STARTING_FRAME: Final = "response.create"
_FRAME_TEXT_WORTH_PARSING: Final = (f'"{_TURN_STARTING_FRAME}"', '"model"', '"prompt"', '"tools"', "\\")
_TURN_MODEL_PLACEMENTS: Final = (("model",), ("response", "model"), ("session", "model"))
@ -2725,6 +2731,16 @@ _UNNAMED_TOOL_MODEL_REFUSAL: Final = (
"The gateway cannot check which model an unnamed one runs."
)
_FIRST_SENTENCE: Final = re.compile(r".*?[.!?](?=\s|$)")
_LANE_NAME: Final = re.compile(r"[A-Za-z0-9_.-]{1,256}")
_UNROUTABLE_LANE_REFUSAL: Final = _RefuseClientFrame(
refusal=WebsocketFrameModelRefused(
message=(
"This stream_id is not a lane name the provider accepts. "
"The provider's error for it names no lane, so the gateway cannot tell which turn it refuses."
),
error_type="invalid_request_error",
)
)
def _may_name_a_model(text_data: str) -> bool:
@ -2736,6 +2752,11 @@ def _frame_lane(frame: Mapping[str, object]) -> str | None:
return stream_id if isinstance(stream_id, str) else None
def _names_an_unroutable_lane(frame: Mapping[str, object]) -> bool:
stream_id: Final = frame.get("stream_id")
return stream_id is not None and not (isinstance(stream_id, str) and _LANE_NAME.fullmatch(stream_id) is not None)
def _frames_of_lane(lane: str | None) -> Callable[[Mapping[str, object]], bool]:
def in_lane(frame: Mapping[str, object]) -> bool:
return _frame_lane(frame) == lane
@ -2795,12 +2816,17 @@ def _unverifiable_model_refusal(frame: Mapping[str, object], tools: Sequence[Map
return None
def _client_frame_requests(frame_data: str | bytes) -> _ClientFrameRequests:
def _client_frame_requests(frame_data: str | bytes) -> _ClientFrameRequests | _RefuseClientFrame:
if isinstance(frame_data, str) and not _may_name_a_model(frame_data):
return _NO_CLIENT_FRAME_REQUESTS
frame: Final = _json_object_frame(frame_data)
try:
frame: Final = _json_object_frame(frame_data)
except (RecursionError, ValueError):
return _UNREADABLE_CLIENT_FRAME
if frame is None:
return _NO_CLIENT_FRAME_REQUESTS
if _names_an_unroutable_lane(frame):
return _UNROUTABLE_LANE_REFUSAL
turn_models: Final = _placed_models(frame, _TURN_MODEL_PLACEMENTS)
tools: Final = _placed_tools(frame)
tool_models: Final = (model for model in (tool.get("model") for tool in tools) if isinstance(model, str))
@ -2827,6 +2853,8 @@ async def _client_frame_verdict(
if gate is None:
return _FORWARD_WITHOUT_TURN
requests: Final = _client_frame_requests(frame_data)
if isinstance(requests, _RefuseClientFrame):
return requests
verdicts: Final = tuple([await gate(websocket, model) for model in requests.models])
refusal: Final = next((verdict for verdict in verdicts if isinstance(verdict, WebsocketFrameModelRefused)), None)
if refusal is not None:
@ -3535,6 +3563,7 @@ async def websocket_passthrough_request(
if _client_socket_is_open(websocket):
await websocket.close(code=1011, reason="WebSocket passthrough error")
finally:
await proxy_logging_obj.arelease_responses_websocket_slot(user_api_key_dict)
if _client_socket_is_open(websocket):
await websocket.close()

View file

@ -41,6 +41,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.proxy._lazy_features import LazyFeature, attach_lazy_features
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.parallel_request_limiter import PROXY_MaxParallelRequestsHandler
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import router as llm_passthrough_router
@ -7395,6 +7396,7 @@ async def test_websocket_passthrough_forwards_non_ascii_first_frame():
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
mock_proxy_logging.post_call_success_hook = AsyncMock()
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock()
mock_worker.ensure_initialized_and_enqueue = MagicMock(
side_effect=lambda async_coroutine: async_coroutine.close()
)
@ -7447,6 +7449,7 @@ async def test_websocket_passthrough_propagates_active_trace_context(
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
mock_proxy_logging.post_call_success_hook = AsyncMock()
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock()
mock_worker = MagicMock()
mock_worker.ensure_initialized_and_enqueue = MagicMock(side_effect=lambda async_coroutine: async_coroutine.close())
monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging)
@ -7513,6 +7516,7 @@ async def test_websocket_passthrough_never_forwards_caller_credentials_upstream(
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
mock_proxy_logging.post_call_success_hook = AsyncMock()
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock()
mock_worker = MagicMock()
mock_worker.ensure_initialized_and_enqueue = MagicMock(side_effect=lambda async_coroutine: async_coroutine.close())
monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging)
@ -7606,6 +7610,7 @@ def _patched_websocket_passthrough_environment(upstream_ws):
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
mock_proxy_logging.post_call_success_hook = AsyncMock()
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock()
mock_worker.ensure_initialized_and_enqueue = MagicMock(
side_effect=lambda async_coroutine: async_coroutine.close()
)
@ -8558,6 +8563,105 @@ async def test_websocket_passthrough_holds_the_keys_parallel_slot_until_the_sess
assert (while_open, after_close) == ([False], True)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"models, upstream_refuses, while_open",
[
((), False, [False]),
(("gpt-5.4-mini",), False, [False]),
(("gpt-5.4-mini", "gpt-5.4"), False, [False]),
((), True, []),
],
ids=["no-turn", "one-turn", "two-turns", "upstream-refused"],
)
async def test_websocket_passthrough_frees_one_legacy_parallel_slot_only_when_the_session_ends(
models: tuple[str, ...], upstream_refuses: bool, while_open: list[bool]
):
limiter: Final = PROXY_MaxParallelRequestsHandler(
internal_usage_cache=InternalUsageCache(DualCache()), clock=lambda: datetime(2026, 1, 31, 23, 59, 30)
)
proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache())
proxy_logging_obj.proxy_hook_mapping["parallel_request_limiter"] = limiter
key: Final = UserAPIKeyAuth(api_key=hash_token("sk-legacy-socket"), max_parallel_requests=2)
logged_rows: Final[list[asyncio.Task[None]]] = []
seen_while_open: Final[list[bool]] = []
async def admit(user_api_key_dict: UserAPIKeyAuth, data: Mapping[str, object], call_type: str) -> None:
await limiter.async_pre_call_hook(
user_api_key_dict=user_api_key_dict, cache=DualCache(), data=data, call_type=call_type
)
async def log_row(**row: object) -> None:
await limiter.async_log_success_event(
kwargs=row, response_obj=None, start_time=row["start_time"], end_time=row["end_time"]
)
async def opens_another_session() -> bool:
await asyncio.gather(*logged_rows)
try:
await admit(key, {}, "pass_through_endpoint")
except ProxyRateLimitError:
return False
return True
client: Final = _turn_taking_client(
[json.dumps({"type": "response.create", "model": model}) for model in models],
answered_by='"response.completed"',
)
take_frame: Final = client.receive
async def receive() -> Mapping[str, str | bytes]:
message: Final = await take_frame()
if message["type"] == "websocket.disconnect":
seen_while_open.append(await opens_another_session())
return message
client.receive = receive
await admit(key, {}, "pass_through_endpoint")
with ExitStack() as stack:
proxy_logging: Final = stack.enter_context(
_patched_websocket_passthrough_environment(
ScriptedUpstreamWebSocket(replies=[_responses_turn(model) for model in models])
)
)
worker: Final = stack.enter_context(
patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.GLOBAL_LOGGING_WORKER")
)
stack.enter_context(patch(SUCCESS_HANDLER, new_callable=AsyncMock, side_effect=log_row))
if upstream_refuses:
stack.enter_context(
patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.connect",
side_effect=OSError("upstream refused the socket"),
)
)
proxy_logging.pre_call_hook = AsyncMock(side_effect=admit)
proxy_logging.arelease_responses_websocket_slot = proxy_logging_obj.arelease_responses_websocket_slot
worker.ensure_initialized_and_enqueue = MagicMock(
side_effect=lambda async_coroutine: logged_rows.append(asyncio.create_task(async_coroutine))
)
await asyncio.wait_for(
websocket_passthrough_request(
websocket=client,
target="wss://api.openai.com/v1/responses",
custom_headers={"Authorization": "Bearer sk-provider"},
user_api_key_dict=key,
forward_headers=False,
endpoint="/openai_passthrough/v1/responses",
accept_websocket=False,
client_frame_model_gate=RecordingFrameModelGate(),
upstream_frame_kind=openai_websocket_frame_kind,
walkout_turn_drain_seconds=0.05,
),
timeout=10,
)
after_close: Final = (await opens_another_session(), await opens_another_session())
assert len(logged_rows) == len(models)
assert (seen_while_open, after_close) == (while_open, (True, False))
@pytest.mark.asyncio
async def test_websocket_passthrough_charges_pipelined_turns_to_the_frame_that_asked_for_each():
"""
@ -8687,6 +8791,28 @@ async def test_websocket_passthrough_charges_turns_on_parallel_lanes_to_the_fram
]
@pytest.mark.asyncio
@pytest.mark.parametrize("stream_id", ["", 7])
async def test_websocket_passthrough_refuses_a_frame_on_a_lane_the_provider_error_cannot_name(stream_id):
"""
The provider refuses a frame with an invalid stream_id through an error that
names no lane (OpenAI WebSocket mode guide, "Errors to handle", read
2026-11-02, which also says an empty stream_id is invalid). Relayed, that
error would refuse a turn waiting on the default lane instead, and that turn
would reach spend without the groups its own frame asked for.
"""
gate: Final = RecordingFrameModelGate()
allowed: Final = json.dumps({"type": "response.create", "model": "gpt-5.4-mini"})
relayed: Final = await _relay_client_frames(
[allowed, json.dumps({"type": "response.create", "stream_id": stream_id, "model": "gpt-5.4-mini"})], gate
)
assert [sent.args[0] for sent in relayed.upstream.send.await_args_list] == [allowed]
assert json.loads(relayed.client.send_text.await_args.args[0])["type"] == "error"
assert relayed.client.close.await_args.kwargs["code"] == 1008
@pytest.mark.asyncio
async def test_websocket_passthrough_charges_realtime_turns_to_the_session_the_key_opened():
"""
@ -9224,7 +9350,20 @@ async def test_websocket_passthrough_keeps_the_session_update_a_clip_ran_under_u
@pytest.mark.asyncio
async def test_websocket_passthrough_still_bills_the_answer_in_flight_when_a_later_frame_is_refused():
@pytest.mark.parametrize(
"later_frame",
[
json.dumps({"type": "response.create", "model": "gpt-5.4"}),
'{"type": "response.create", "model": "gpt-5.4-mini", "metadata": ' + "[" * 100_000 + "]" * 100_000 + "}",
(
'{"type": "response.create", "model": "gpt-5.4-mini", "seed": '
+ "9" * (sys.get_int_max_str_digits() + 1)
+ "}"
),
],
ids=["names_a_denied_model", "nests_too_deep_to_parse", "holds_a_number_too_long_to_parse"],
)
async def test_websocket_passthrough_still_bills_the_answer_in_flight_when_a_later_frame_is_refused(later_frame: str):
"""
A refused frame closes the client while the provider may still be sending the
answer to an earlier, allowed frame. That answer cannot go to a closed client,
@ -9233,10 +9372,7 @@ async def test_websocket_passthrough_still_bills_the_answer_in_flight_when_a_lat
"""
gate: Final = RecordingFrameModelGate(MappingProxyType({"gpt-5.4": KEY_MODEL_REFUSAL}))
client: Final = _turn_taking_client(
[
json.dumps({"type": "response.create", "model": "gpt-5.4-mini"}),
json.dumps({"type": "response.create", "model": "gpt-5.4"}),
],
[json.dumps({"type": "response.create", "model": "gpt-5.4-mini"}), later_frame],
answered_by='"response.created"',
)
refused: Final = asyncio.Event()