diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index cd01d15d834..f9e85c31d46 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -13,7 +13,7 @@ from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.exceptions import RateLimitType from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs +from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_parent_otel_span_from_kwargs from litellm.proxy._types import CommonProxyErrors, CurrentItemRateLimit, UserAPIKeyAuth from litellm.proxy.auth.auth_utils import ( get_key_model_rpm_limit, @@ -53,9 +53,16 @@ def _response_total_tokens(kwargs: Mapping[str, object], response_obj: object) - _CONNECTION_SCOPED_CALL_TYPES: Final = frozenset({CallTypes.aresponses_websocket.value}) +def _is_websocket_passthrough_row(kwargs: Mapping[str, object]) -> bool: + passthrough_payload: Final = as_str_mapping(kwargs.get("passthrough_logging_payload")) + return passthrough_payload is not None and passthrough_payload.get("request_method") == "WEBSOCKET" + + def _slots_released(kwargs: Mapping[str, object]) -> int: """A websocket turn callback releases nothing; the socket's slot is released once at disconnect.""" - return 0 if kwargs.get("call_type") in _CONNECTION_SCOPED_CALL_TYPES else 1 + if kwargs.get("call_type") in _CONNECTION_SCOPED_CALL_TYPES or _is_websocket_passthrough_row(kwargs): + return 0 + return 1 class CacheObject(TypedDict): diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 8a3d2deb992..7f5983d5799 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2705,6 +2705,12 @@ _NO_CLIENT_FRAME_REQUESTS: Final = _ClientFrameRequests( starts_turn=False, lane=None, turn_model=None, models=(), unverifiable_model_refusal=None ) _FORWARD_WITHOUT_TURN: Final = _ForwardClientFrame(starts_turn=False, turn=None, lane=None) +_UNREADABLE_CLIENT_FRAME: Final = _RefuseClientFrame( + refusal=WebsocketFrameModelRefused( + message="The gateway cannot parse this frame, so it cannot check which model it asks for.", + error_type="invalid_request_error", + ) +) _TURN_STARTING_FRAME: Final = "response.create" _FRAME_TEXT_WORTH_PARSING: Final = (f'"{_TURN_STARTING_FRAME}"', '"model"', '"prompt"', '"tools"', "\\") _TURN_MODEL_PLACEMENTS: Final = (("model",), ("response", "model"), ("session", "model")) @@ -2725,6 +2731,16 @@ _UNNAMED_TOOL_MODEL_REFUSAL: Final = ( "The gateway cannot check which model an unnamed one runs." ) _FIRST_SENTENCE: Final = re.compile(r".*?[.!?](?=\s|$)") +_LANE_NAME: Final = re.compile(r"[A-Za-z0-9_.-]{1,256}") +_UNROUTABLE_LANE_REFUSAL: Final = _RefuseClientFrame( + refusal=WebsocketFrameModelRefused( + message=( + "This stream_id is not a lane name the provider accepts. " + "The provider's error for it names no lane, so the gateway cannot tell which turn it refuses." + ), + error_type="invalid_request_error", + ) +) def _may_name_a_model(text_data: str) -> bool: @@ -2736,6 +2752,11 @@ def _frame_lane(frame: Mapping[str, object]) -> str | None: return stream_id if isinstance(stream_id, str) else None +def _names_an_unroutable_lane(frame: Mapping[str, object]) -> bool: + stream_id: Final = frame.get("stream_id") + return stream_id is not None and not (isinstance(stream_id, str) and _LANE_NAME.fullmatch(stream_id) is not None) + + def _frames_of_lane(lane: str | None) -> Callable[[Mapping[str, object]], bool]: def in_lane(frame: Mapping[str, object]) -> bool: return _frame_lane(frame) == lane @@ -2795,12 +2816,17 @@ def _unverifiable_model_refusal(frame: Mapping[str, object], tools: Sequence[Map return None -def _client_frame_requests(frame_data: str | bytes) -> _ClientFrameRequests: +def _client_frame_requests(frame_data: str | bytes) -> _ClientFrameRequests | _RefuseClientFrame: if isinstance(frame_data, str) and not _may_name_a_model(frame_data): return _NO_CLIENT_FRAME_REQUESTS - frame: Final = _json_object_frame(frame_data) + try: + frame: Final = _json_object_frame(frame_data) + except (RecursionError, ValueError): + return _UNREADABLE_CLIENT_FRAME if frame is None: return _NO_CLIENT_FRAME_REQUESTS + if _names_an_unroutable_lane(frame): + return _UNROUTABLE_LANE_REFUSAL turn_models: Final = _placed_models(frame, _TURN_MODEL_PLACEMENTS) tools: Final = _placed_tools(frame) tool_models: Final = (model for model in (tool.get("model") for tool in tools) if isinstance(model, str)) @@ -2827,6 +2853,8 @@ async def _client_frame_verdict( if gate is None: return _FORWARD_WITHOUT_TURN requests: Final = _client_frame_requests(frame_data) + if isinstance(requests, _RefuseClientFrame): + return requests verdicts: Final = tuple([await gate(websocket, model) for model in requests.models]) refusal: Final = next((verdict for verdict in verdicts if isinstance(verdict, WebsocketFrameModelRefused)), None) if refusal is not None: @@ -3535,6 +3563,7 @@ async def websocket_passthrough_request( if _client_socket_is_open(websocket): await websocket.close(code=1011, reason="WebSocket passthrough error") finally: + await proxy_logging_obj.arelease_responses_websocket_slot(user_api_key_dict) if _client_socket_is_open(websocket): await websocket.close() diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index a04d0241f17..824f04aee75 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -41,6 +41,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.proxy._lazy_features import LazyFeature, attach_lazy_features from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError +from litellm.proxy.hooks.parallel_request_limiter import PROXY_MaxParallelRequestsHandler from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import router as llm_passthrough_router @@ -7395,6 +7396,7 @@ async def test_websocket_passthrough_forwards_non_ascii_first_frame(): mock_proxy_logging.pre_call_hook = AsyncMock(return_value={}) mock_proxy_logging.post_call_success_hook = AsyncMock() mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock() mock_worker.ensure_initialized_and_enqueue = MagicMock( side_effect=lambda async_coroutine: async_coroutine.close() ) @@ -7447,6 +7449,7 @@ async def test_websocket_passthrough_propagates_active_trace_context( mock_proxy_logging.pre_call_hook = AsyncMock(return_value={}) mock_proxy_logging.post_call_success_hook = AsyncMock() mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock() mock_worker = MagicMock() mock_worker.ensure_initialized_and_enqueue = MagicMock(side_effect=lambda async_coroutine: async_coroutine.close()) monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging) @@ -7513,6 +7516,7 @@ async def test_websocket_passthrough_never_forwards_caller_credentials_upstream( mock_proxy_logging.pre_call_hook = AsyncMock(return_value={}) mock_proxy_logging.post_call_success_hook = AsyncMock() mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock() mock_worker = MagicMock() mock_worker.ensure_initialized_and_enqueue = MagicMock(side_effect=lambda async_coroutine: async_coroutine.close()) monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging) @@ -7606,6 +7610,7 @@ def _patched_websocket_passthrough_environment(upstream_ws): mock_proxy_logging.pre_call_hook = AsyncMock(return_value={}) mock_proxy_logging.post_call_success_hook = AsyncMock() mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.arelease_responses_websocket_slot = AsyncMock() mock_worker.ensure_initialized_and_enqueue = MagicMock( side_effect=lambda async_coroutine: async_coroutine.close() ) @@ -8558,6 +8563,105 @@ async def test_websocket_passthrough_holds_the_keys_parallel_slot_until_the_sess assert (while_open, after_close) == ([False], True) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "models, upstream_refuses, while_open", + [ + ((), False, [False]), + (("gpt-5.4-mini",), False, [False]), + (("gpt-5.4-mini", "gpt-5.4"), False, [False]), + ((), True, []), + ], + ids=["no-turn", "one-turn", "two-turns", "upstream-refused"], +) +async def test_websocket_passthrough_frees_one_legacy_parallel_slot_only_when_the_session_ends( + models: tuple[str, ...], upstream_refuses: bool, while_open: list[bool] +): + limiter: Final = PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()), clock=lambda: datetime(2026, 1, 31, 23, 59, 30) + ) + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.proxy_hook_mapping["parallel_request_limiter"] = limiter + key: Final = UserAPIKeyAuth(api_key=hash_token("sk-legacy-socket"), max_parallel_requests=2) + logged_rows: Final[list[asyncio.Task[None]]] = [] + seen_while_open: Final[list[bool]] = [] + + async def admit(user_api_key_dict: UserAPIKeyAuth, data: Mapping[str, object], call_type: str) -> None: + await limiter.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, cache=DualCache(), data=data, call_type=call_type + ) + + async def log_row(**row: object) -> None: + await limiter.async_log_success_event( + kwargs=row, response_obj=None, start_time=row["start_time"], end_time=row["end_time"] + ) + + async def opens_another_session() -> bool: + await asyncio.gather(*logged_rows) + try: + await admit(key, {}, "pass_through_endpoint") + except ProxyRateLimitError: + return False + return True + + client: Final = _turn_taking_client( + [json.dumps({"type": "response.create", "model": model}) for model in models], + answered_by='"response.completed"', + ) + take_frame: Final = client.receive + + async def receive() -> Mapping[str, str | bytes]: + message: Final = await take_frame() + if message["type"] == "websocket.disconnect": + seen_while_open.append(await opens_another_session()) + return message + + client.receive = receive + await admit(key, {}, "pass_through_endpoint") + + with ExitStack() as stack: + proxy_logging: Final = stack.enter_context( + _patched_websocket_passthrough_environment( + ScriptedUpstreamWebSocket(replies=[_responses_turn(model) for model in models]) + ) + ) + worker: Final = stack.enter_context( + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.GLOBAL_LOGGING_WORKER") + ) + stack.enter_context(patch(SUCCESS_HANDLER, new_callable=AsyncMock, side_effect=log_row)) + if upstream_refuses: + stack.enter_context( + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.connect", + side_effect=OSError("upstream refused the socket"), + ) + ) + proxy_logging.pre_call_hook = AsyncMock(side_effect=admit) + proxy_logging.arelease_responses_websocket_slot = proxy_logging_obj.arelease_responses_websocket_slot + worker.ensure_initialized_and_enqueue = MagicMock( + side_effect=lambda async_coroutine: logged_rows.append(asyncio.create_task(async_coroutine)) + ) + await asyncio.wait_for( + websocket_passthrough_request( + websocket=client, + target="wss://api.openai.com/v1/responses", + custom_headers={"Authorization": "Bearer sk-provider"}, + user_api_key_dict=key, + forward_headers=False, + endpoint="/openai_passthrough/v1/responses", + accept_websocket=False, + client_frame_model_gate=RecordingFrameModelGate(), + upstream_frame_kind=openai_websocket_frame_kind, + walkout_turn_drain_seconds=0.05, + ), + timeout=10, + ) + after_close: Final = (await opens_another_session(), await opens_another_session()) + + assert len(logged_rows) == len(models) + assert (seen_while_open, after_close) == (while_open, (True, False)) + + @pytest.mark.asyncio async def test_websocket_passthrough_charges_pipelined_turns_to_the_frame_that_asked_for_each(): """ @@ -8687,6 +8791,28 @@ async def test_websocket_passthrough_charges_turns_on_parallel_lanes_to_the_fram ] +@pytest.mark.asyncio +@pytest.mark.parametrize("stream_id", ["", 7]) +async def test_websocket_passthrough_refuses_a_frame_on_a_lane_the_provider_error_cannot_name(stream_id): + """ + The provider refuses a frame with an invalid stream_id through an error that + names no lane (OpenAI WebSocket mode guide, "Errors to handle", read + 2026-11-02, which also says an empty stream_id is invalid). Relayed, that + error would refuse a turn waiting on the default lane instead, and that turn + would reach spend without the groups its own frame asked for. + """ + gate: Final = RecordingFrameModelGate() + allowed: Final = json.dumps({"type": "response.create", "model": "gpt-5.4-mini"}) + + relayed: Final = await _relay_client_frames( + [allowed, json.dumps({"type": "response.create", "stream_id": stream_id, "model": "gpt-5.4-mini"})], gate + ) + + assert [sent.args[0] for sent in relayed.upstream.send.await_args_list] == [allowed] + assert json.loads(relayed.client.send_text.await_args.args[0])["type"] == "error" + assert relayed.client.close.await_args.kwargs["code"] == 1008 + + @pytest.mark.asyncio async def test_websocket_passthrough_charges_realtime_turns_to_the_session_the_key_opened(): """ @@ -9224,7 +9350,20 @@ async def test_websocket_passthrough_keeps_the_session_update_a_clip_ran_under_u @pytest.mark.asyncio -async def test_websocket_passthrough_still_bills_the_answer_in_flight_when_a_later_frame_is_refused(): +@pytest.mark.parametrize( + "later_frame", + [ + json.dumps({"type": "response.create", "model": "gpt-5.4"}), + '{"type": "response.create", "model": "gpt-5.4-mini", "metadata": ' + "[" * 100_000 + "]" * 100_000 + "}", + ( + '{"type": "response.create", "model": "gpt-5.4-mini", "seed": ' + + "9" * (sys.get_int_max_str_digits() + 1) + + "}" + ), + ], + ids=["names_a_denied_model", "nests_too_deep_to_parse", "holds_a_number_too_long_to_parse"], +) +async def test_websocket_passthrough_still_bills_the_answer_in_flight_when_a_later_frame_is_refused(later_frame: str): """ A refused frame closes the client while the provider may still be sending the answer to an earlier, allowed frame. That answer cannot go to a closed client, @@ -9233,10 +9372,7 @@ async def test_websocket_passthrough_still_bills_the_answer_in_flight_when_a_lat """ gate: Final = RecordingFrameModelGate(MappingProxyType({"gpt-5.4": KEY_MODEL_REFUSAL})) client: Final = _turn_taking_client( - [ - json.dumps({"type": "response.create", "model": "gpt-5.4-mini"}), - json.dumps({"type": "response.create", "model": "gpt-5.4"}), - ], + [json.dumps({"type": "response.create", "model": "gpt-5.4-mini"}), later_frame], answered_by='"response.created"', ) refused: Final = asyncio.Event()