mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(proxy): refuse unparseable and unroutable websocket frames and hold legacy slots per socket
A client frame too deep or too long for the JSON parser closed the provider socket mid-turn, so the turn in flight was never billed. The gate now refuses that frame with an error event and keeps draining the open turn. A stream_id OpenAI rejects comes back as an error that names no lane, which dropped another turn's authorization. The gate now refuses those frames before they reach OpenAI. With the legacy parallel request limiter, each per-turn spend row freed the key's slot while the socket stayed open. Websocket passthrough rows now free nothing, and the socket frees its slot once when it closes.
This commit is contained in:
parent
b7f3e03115
commit
4ec2d7a0d4
3 changed files with 181 additions and 9 deletions
|
|
@ -13,7 +13,7 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.exceptions import RateLimitType
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_parent_otel_span_from_kwargs
|
||||
from litellm.proxy._types import CommonProxyErrors, CurrentItemRateLimit, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
get_key_model_rpm_limit,
|
||||
|
|
@ -53,9 +53,16 @@ def _response_total_tokens(kwargs: Mapping[str, object], response_obj: object) -
|
|||
_CONNECTION_SCOPED_CALL_TYPES: Final = frozenset({CallTypes.aresponses_websocket.value})
|
||||
|
||||
|
||||
def _is_websocket_passthrough_row(kwargs: Mapping[str, object]) -> bool:
|
||||
passthrough_payload: Final = as_str_mapping(kwargs.get("passthrough_logging_payload"))
|
||||
return passthrough_payload is not None and passthrough_payload.get("request_method") == "WEBSOCKET"
|
||||
|
||||
|
||||
def _slots_released(kwargs: Mapping[str, object]) -> int:
|
||||
"""A websocket turn callback releases nothing; the socket's slot is released once at disconnect."""
|
||||
return 0 if kwargs.get("call_type") in _CONNECTION_SCOPED_CALL_TYPES else 1
|
||||
if kwargs.get("call_type") in _CONNECTION_SCOPED_CALL_TYPES or _is_websocket_passthrough_row(kwargs):
|
||||
return 0
|
||||
return 1
|
||||
|
||||
|
||||
class CacheObject(TypedDict):
|
||||
|
|
|
|||
|
|
@ -2705,6 +2705,12 @@ _NO_CLIENT_FRAME_REQUESTS: Final = _ClientFrameRequests(
|
|||
starts_turn=False, lane=None, turn_model=None, models=(), unverifiable_model_refusal=None
|
||||
)
|
||||
_FORWARD_WITHOUT_TURN: Final = _ForwardClientFrame(starts_turn=False, turn=None, lane=None)
|
||||
_UNREADABLE_CLIENT_FRAME: Final = _RefuseClientFrame(
|
||||
refusal=WebsocketFrameModelRefused(
|
||||
message="The gateway cannot parse this frame, so it cannot check which model it asks for.",
|
||||
error_type="invalid_request_error",
|
||||
)
|
||||
)
|
||||
_TURN_STARTING_FRAME: Final = "response.create"
|
||||
_FRAME_TEXT_WORTH_PARSING: Final = (f'"{_TURN_STARTING_FRAME}"', '"model"', '"prompt"', '"tools"', "\\")
|
||||
_TURN_MODEL_PLACEMENTS: Final = (("model",), ("response", "model"), ("session", "model"))
|
||||
|
|
@ -2725,6 +2731,16 @@ _UNNAMED_TOOL_MODEL_REFUSAL: Final = (
|
|||
"The gateway cannot check which model an unnamed one runs."
|
||||
)
|
||||
_FIRST_SENTENCE: Final = re.compile(r".*?[.!?](?=\s|$)")
|
||||
_LANE_NAME: Final = re.compile(r"[A-Za-z0-9_.-]{1,256}")
|
||||
_UNROUTABLE_LANE_REFUSAL: Final = _RefuseClientFrame(
|
||||
refusal=WebsocketFrameModelRefused(
|
||||
message=(
|
||||
"This stream_id is not a lane name the provider accepts. "
|
||||
"The provider's error for it names no lane, so the gateway cannot tell which turn it refuses."
|
||||
),
|
||||
error_type="invalid_request_error",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _may_name_a_model(text_data: str) -> bool:
|
||||
|
|
@ -2736,6 +2752,11 @@ def _frame_lane(frame: Mapping[str, object]) -> str | None:
|
|||
return stream_id if isinstance(stream_id, str) else None
|
||||
|
||||
|
||||
def _names_an_unroutable_lane(frame: Mapping[str, object]) -> bool:
|
||||
stream_id: Final = frame.get("stream_id")
|
||||
return stream_id is not None and not (isinstance(stream_id, str) and _LANE_NAME.fullmatch(stream_id) is not None)
|
||||
|
||||
|
||||
def _frames_of_lane(lane: str | None) -> Callable[[Mapping[str, object]], bool]:
|
||||
def in_lane(frame: Mapping[str, object]) -> bool:
|
||||
return _frame_lane(frame) == lane
|
||||
|
|
@ -2795,12 +2816,17 @@ def _unverifiable_model_refusal(frame: Mapping[str, object], tools: Sequence[Map
|
|||
return None
|
||||
|
||||
|
||||
def _client_frame_requests(frame_data: str | bytes) -> _ClientFrameRequests:
|
||||
def _client_frame_requests(frame_data: str | bytes) -> _ClientFrameRequests | _RefuseClientFrame:
|
||||
if isinstance(frame_data, str) and not _may_name_a_model(frame_data):
|
||||
return _NO_CLIENT_FRAME_REQUESTS
|
||||
frame: Final = _json_object_frame(frame_data)
|
||||
try:
|
||||
frame: Final = _json_object_frame(frame_data)
|
||||
except (RecursionError, ValueError):
|
||||
return _UNREADABLE_CLIENT_FRAME
|
||||
if frame is None:
|
||||
return _NO_CLIENT_FRAME_REQUESTS
|
||||
if _names_an_unroutable_lane(frame):
|
||||
return _UNROUTABLE_LANE_REFUSAL
|
||||
turn_models: Final = _placed_models(frame, _TURN_MODEL_PLACEMENTS)
|
||||
tools: Final = _placed_tools(frame)
|
||||
tool_models: Final = (model for model in (tool.get("model") for tool in tools) if isinstance(model, str))
|
||||
|
|
@ -2827,6 +2853,8 @@ async def _client_frame_verdict(
|
|||
if gate is None:
|
||||
return _FORWARD_WITHOUT_TURN
|
||||
requests: Final = _client_frame_requests(frame_data)
|
||||
if isinstance(requests, _RefuseClientFrame):
|
||||
return requests
|
||||
verdicts: Final = tuple([await gate(websocket, model) for model in requests.models])
|
||||
refusal: Final = next((verdict for verdict in verdicts if isinstance(verdict, WebsocketFrameModelRefused)), None)
|
||||
if refusal is not None:
|
||||
|
|
@ -3535,6 +3563,7 @@ async def websocket_passthrough_request(
|
|||
if _client_socket_is_open(websocket):
|
||||
await websocket.close(code=1011, reason="WebSocket passthrough error")
|
||||
finally:
|
||||
await proxy_logging_obj.arelease_responses_websocket_slot(user_api_key_dict)
|
||||
if _client_socket_is_open(websocket):
|
||||
await websocket.close()
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.proxy._lazy_features import LazyFeature, attach_lazy_features
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
from litellm.proxy.hooks.parallel_request_limiter import PROXY_MaxParallelRequestsHandler
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import router as llm_passthrough_router
|
||||
|
|
@ -7395,6 +7396,7 @@ async def test_websocket_passthrough_forwards_non_ascii_first_frame():
|
|||
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
|
||||
mock_proxy_logging.post_call_success_hook = AsyncMock()
|
||||
mock_proxy_logging.post_call_failure_hook = AsyncMock()
|
||||
mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock()
|
||||
mock_worker.ensure_initialized_and_enqueue = MagicMock(
|
||||
side_effect=lambda async_coroutine: async_coroutine.close()
|
||||
)
|
||||
|
|
@ -7447,6 +7449,7 @@ async def test_websocket_passthrough_propagates_active_trace_context(
|
|||
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
|
||||
mock_proxy_logging.post_call_success_hook = AsyncMock()
|
||||
mock_proxy_logging.post_call_failure_hook = AsyncMock()
|
||||
mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock()
|
||||
mock_worker = MagicMock()
|
||||
mock_worker.ensure_initialized_and_enqueue = MagicMock(side_effect=lambda async_coroutine: async_coroutine.close())
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging)
|
||||
|
|
@ -7513,6 +7516,7 @@ async def test_websocket_passthrough_never_forwards_caller_credentials_upstream(
|
|||
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
|
||||
mock_proxy_logging.post_call_success_hook = AsyncMock()
|
||||
mock_proxy_logging.post_call_failure_hook = AsyncMock()
|
||||
mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock()
|
||||
mock_worker = MagicMock()
|
||||
mock_worker.ensure_initialized_and_enqueue = MagicMock(side_effect=lambda async_coroutine: async_coroutine.close())
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging)
|
||||
|
|
@ -7606,6 +7610,7 @@ def _patched_websocket_passthrough_environment(upstream_ws):
|
|||
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
|
||||
mock_proxy_logging.post_call_success_hook = AsyncMock()
|
||||
mock_proxy_logging.post_call_failure_hook = AsyncMock()
|
||||
mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock()
|
||||
mock_worker.ensure_initialized_and_enqueue = MagicMock(
|
||||
side_effect=lambda async_coroutine: async_coroutine.close()
|
||||
)
|
||||
|
|
@ -8558,6 +8563,105 @@ async def test_websocket_passthrough_holds_the_keys_parallel_slot_until_the_sess
|
|||
assert (while_open, after_close) == ([False], True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"models, upstream_refuses, while_open",
|
||||
[
|
||||
((), False, [False]),
|
||||
(("gpt-5.4-mini",), False, [False]),
|
||||
(("gpt-5.4-mini", "gpt-5.4"), False, [False]),
|
||||
((), True, []),
|
||||
],
|
||||
ids=["no-turn", "one-turn", "two-turns", "upstream-refused"],
|
||||
)
|
||||
async def test_websocket_passthrough_frees_one_legacy_parallel_slot_only_when_the_session_ends(
|
||||
models: tuple[str, ...], upstream_refuses: bool, while_open: list[bool]
|
||||
):
|
||||
limiter: Final = PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(DualCache()), clock=lambda: datetime(2026, 1, 31, 23, 59, 30)
|
||||
)
|
||||
proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache())
|
||||
proxy_logging_obj.proxy_hook_mapping["parallel_request_limiter"] = limiter
|
||||
key: Final = UserAPIKeyAuth(api_key=hash_token("sk-legacy-socket"), max_parallel_requests=2)
|
||||
logged_rows: Final[list[asyncio.Task[None]]] = []
|
||||
seen_while_open: Final[list[bool]] = []
|
||||
|
||||
async def admit(user_api_key_dict: UserAPIKeyAuth, data: Mapping[str, object], call_type: str) -> None:
|
||||
await limiter.async_pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict, cache=DualCache(), data=data, call_type=call_type
|
||||
)
|
||||
|
||||
async def log_row(**row: object) -> None:
|
||||
await limiter.async_log_success_event(
|
||||
kwargs=row, response_obj=None, start_time=row["start_time"], end_time=row["end_time"]
|
||||
)
|
||||
|
||||
async def opens_another_session() -> bool:
|
||||
await asyncio.gather(*logged_rows)
|
||||
try:
|
||||
await admit(key, {}, "pass_through_endpoint")
|
||||
except ProxyRateLimitError:
|
||||
return False
|
||||
return True
|
||||
|
||||
client: Final = _turn_taking_client(
|
||||
[json.dumps({"type": "response.create", "model": model}) for model in models],
|
||||
answered_by='"response.completed"',
|
||||
)
|
||||
take_frame: Final = client.receive
|
||||
|
||||
async def receive() -> Mapping[str, str | bytes]:
|
||||
message: Final = await take_frame()
|
||||
if message["type"] == "websocket.disconnect":
|
||||
seen_while_open.append(await opens_another_session())
|
||||
return message
|
||||
|
||||
client.receive = receive
|
||||
await admit(key, {}, "pass_through_endpoint")
|
||||
|
||||
with ExitStack() as stack:
|
||||
proxy_logging: Final = stack.enter_context(
|
||||
_patched_websocket_passthrough_environment(
|
||||
ScriptedUpstreamWebSocket(replies=[_responses_turn(model) for model in models])
|
||||
)
|
||||
)
|
||||
worker: Final = stack.enter_context(
|
||||
patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.GLOBAL_LOGGING_WORKER")
|
||||
)
|
||||
stack.enter_context(patch(SUCCESS_HANDLER, new_callable=AsyncMock, side_effect=log_row))
|
||||
if upstream_refuses:
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.connect",
|
||||
side_effect=OSError("upstream refused the socket"),
|
||||
)
|
||||
)
|
||||
proxy_logging.pre_call_hook = AsyncMock(side_effect=admit)
|
||||
proxy_logging.arelease_responses_websocket_slot = proxy_logging_obj.arelease_responses_websocket_slot
|
||||
worker.ensure_initialized_and_enqueue = MagicMock(
|
||||
side_effect=lambda async_coroutine: logged_rows.append(asyncio.create_task(async_coroutine))
|
||||
)
|
||||
await asyncio.wait_for(
|
||||
websocket_passthrough_request(
|
||||
websocket=client,
|
||||
target="wss://api.openai.com/v1/responses",
|
||||
custom_headers={"Authorization": "Bearer sk-provider"},
|
||||
user_api_key_dict=key,
|
||||
forward_headers=False,
|
||||
endpoint="/openai_passthrough/v1/responses",
|
||||
accept_websocket=False,
|
||||
client_frame_model_gate=RecordingFrameModelGate(),
|
||||
upstream_frame_kind=openai_websocket_frame_kind,
|
||||
walkout_turn_drain_seconds=0.05,
|
||||
),
|
||||
timeout=10,
|
||||
)
|
||||
after_close: Final = (await opens_another_session(), await opens_another_session())
|
||||
|
||||
assert len(logged_rows) == len(models)
|
||||
assert (seen_while_open, after_close) == (while_open, (True, False))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_charges_pipelined_turns_to_the_frame_that_asked_for_each():
|
||||
"""
|
||||
|
|
@ -8687,6 +8791,28 @@ async def test_websocket_passthrough_charges_turns_on_parallel_lanes_to_the_fram
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream_id", ["", 7])
|
||||
async def test_websocket_passthrough_refuses_a_frame_on_a_lane_the_provider_error_cannot_name(stream_id):
|
||||
"""
|
||||
The provider refuses a frame with an invalid stream_id through an error that
|
||||
names no lane (OpenAI WebSocket mode guide, "Errors to handle", read
|
||||
2026-11-02, which also says an empty stream_id is invalid). Relayed, that
|
||||
error would refuse a turn waiting on the default lane instead, and that turn
|
||||
would reach spend without the groups its own frame asked for.
|
||||
"""
|
||||
gate: Final = RecordingFrameModelGate()
|
||||
allowed: Final = json.dumps({"type": "response.create", "model": "gpt-5.4-mini"})
|
||||
|
||||
relayed: Final = await _relay_client_frames(
|
||||
[allowed, json.dumps({"type": "response.create", "stream_id": stream_id, "model": "gpt-5.4-mini"})], gate
|
||||
)
|
||||
|
||||
assert [sent.args[0] for sent in relayed.upstream.send.await_args_list] == [allowed]
|
||||
assert json.loads(relayed.client.send_text.await_args.args[0])["type"] == "error"
|
||||
assert relayed.client.close.await_args.kwargs["code"] == 1008
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_charges_realtime_turns_to_the_session_the_key_opened():
|
||||
"""
|
||||
|
|
@ -9224,7 +9350,20 @@ async def test_websocket_passthrough_keeps_the_session_update_a_clip_ran_under_u
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_still_bills_the_answer_in_flight_when_a_later_frame_is_refused():
|
||||
@pytest.mark.parametrize(
|
||||
"later_frame",
|
||||
[
|
||||
json.dumps({"type": "response.create", "model": "gpt-5.4"}),
|
||||
'{"type": "response.create", "model": "gpt-5.4-mini", "metadata": ' + "[" * 100_000 + "]" * 100_000 + "}",
|
||||
(
|
||||
'{"type": "response.create", "model": "gpt-5.4-mini", "seed": '
|
||||
+ "9" * (sys.get_int_max_str_digits() + 1)
|
||||
+ "}"
|
||||
),
|
||||
],
|
||||
ids=["names_a_denied_model", "nests_too_deep_to_parse", "holds_a_number_too_long_to_parse"],
|
||||
)
|
||||
async def test_websocket_passthrough_still_bills_the_answer_in_flight_when_a_later_frame_is_refused(later_frame: str):
|
||||
"""
|
||||
A refused frame closes the client while the provider may still be sending the
|
||||
answer to an earlier, allowed frame. That answer cannot go to a closed client,
|
||||
|
|
@ -9233,10 +9372,7 @@ async def test_websocket_passthrough_still_bills_the_answer_in_flight_when_a_lat
|
|||
"""
|
||||
gate: Final = RecordingFrameModelGate(MappingProxyType({"gpt-5.4": KEY_MODEL_REFUSAL}))
|
||||
client: Final = _turn_taking_client(
|
||||
[
|
||||
json.dumps({"type": "response.create", "model": "gpt-5.4-mini"}),
|
||||
json.dumps({"type": "response.create", "model": "gpt-5.4"}),
|
||||
],
|
||||
[json.dumps({"type": "response.create", "model": "gpt-5.4-mini"}), later_frame],
|
||||
answered_by='"response.created"',
|
||||
)
|
||||
refused: Final = asyncio.Event()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue