fix(passthrough): keep reading a websocket turn the client left before OpenAI started it

A client that sends response.create and hangs up before OpenAI answers with
response.created made the walkout checks see no turn in flight, so the gateway
closed the provider socket and the turn OpenAI still ran and billed never
reached spend. The relay now counts every forwarded create until the provider
starts or refuses it, and keeps draining while one is owed
This commit is contained in:
mateo-berri 2026-10-31 17:41:02 -07:00
parent f414a42ecf
commit f2d430f555
2 changed files with 185 additions and 19 deletions

View file

@ -2878,12 +2878,14 @@ class _WebsocketTurnLedger:
self.session_opened = False
self.started = started
self.logged_turns = 0
self.unstarted_turns = 0
def current(self, lane: str | None) -> _TurnAuthorization:
oldest: Final = next(iter(self.queued_turns.get(lane, ())), None)
return self.opening if oldest is None else oldest
def queue_turn(self, lane: str | None, turn: _TurnAuthorization | None) -> None:
self.unstarted_turns += 1
if not self.session_opened:
self.queued_turns = MappingProxyType({**self.queued_turns, lane: (*self.queued_turns.get(lane, ()), turn)})
@ -2896,7 +2898,11 @@ class _WebsocketTurnLedger:
self.started = end_time
self._forget_turn(lane, 0)
def start_turn(self) -> None:
self.unstarted_turns = max(0, self.unstarted_turns - 1)
def drop_refused_turn(self, lane: str | None, lane_is_running_a_turn: bool) -> None:
self.start_turn()
self._forget_turn(lane, 1 if lane_is_running_a_turn else 0)
def _forget_turn(self, lane: str | None, position: int) -> None:
@ -2923,6 +2929,16 @@ def _turn_in_flight(
)
def _turn_owed(
upstream_frame_kind: WebsocketUpstreamFrameClassifier | None,
unlogged_frames: Sequence[Mapping[str, object]],
ledger: _WebsocketTurnLedger,
) -> bool:
return _turn_in_flight(upstream_frame_kind, unlogged_frames) or (
upstream_frame_kind is not None and ledger.unstarted_turns > 0
)
def _lanes_in_flight(
upstream_frame_kind: WebsocketUpstreamFrameClassifier, unlogged_frames: Sequence[Mapping[str, object]]
) -> tuple[str | None, ...]:
@ -3323,7 +3339,7 @@ async def websocket_passthrough_request(
_RefuseClientFrame,
):
break
if not _turn_in_flight(upstream_frame_kind, websocket_messages):
if not _turn_owed(upstream_frame_kind, websocket_messages, ledger):
await upstream_ws.close()
except asyncio.CancelledError:
raise
@ -3362,6 +3378,8 @@ async def websocket_passthrough_request(
match None if upstream_frame_kind is None else upstream_frame_kind(message_data):
case "opens_session":
ledger.open_session()
case "starts_turn":
ledger.start_turn()
case "updates_session" if upstream_frame_kind is not None:
websocket_messages[:] = _without_earlier_session_updates(
upstream_frame_kind, websocket_messages, message_data
@ -3381,7 +3399,7 @@ async def websocket_passthrough_request(
async def relay_upstream_frame(upstream_message: str | bytes) -> None:
record_upstream_frame(upstream_message)
if not _client_socket_is_open(websocket):
if not _turn_in_flight(upstream_frame_kind, websocket_messages):
if not _turn_owed(upstream_frame_kind, websocket_messages, ledger):
await upstream_ws.close()
return
try:
@ -3390,7 +3408,7 @@ async def websocket_passthrough_request(
else:
await websocket.send_text(upstream_message)
except WebSocketDisconnect:
if not _turn_in_flight(upstream_frame_kind, websocket_messages):
if not _turn_owed(upstream_frame_kind, websocket_messages, ledger):
raise
async def forward_upstream_to_client() -> Close | None:
@ -3421,7 +3439,7 @@ async def websocket_passthrough_request(
tasks: Final = (client_task, upstream_task)
await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
if client_task.done() and _turn_in_flight(upstream_frame_kind, websocket_messages):
if client_task.done() and _turn_owed(upstream_frame_kind, websocket_messages, ledger):
await asyncio.wait((upstream_task,), timeout=walkout_turn_drain_seconds)
done: Final = tuple(task for task in tasks if task.done())
pending: Final = tuple(task for task in tasks if not task.done())

View file

@ -51,6 +51,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS,
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
STORED_PROMPT_PROBE_MODEL,
WALKOUT_TURN_DRAIN_SECONDS,
HttpPassThroughEndpointHelpers,
InitPassThroughEndpointHelpers,
SafeRouteAdder,
@ -8797,10 +8798,19 @@ async def test_websocket_passthrough_charges_each_lane_a_client_walked_out_of_to
class UpstreamFinishingTheTurnAfterTheClientLeft:
"""Starts a turn for each client frame and sends the rest of it only after the client left, and stops once closed"""
"""Starts a turn for each client frame, unless it only answers once the client left, sends the rest of the session
once the client left, and stops once closed"""
def __init__(self, client_left: asyncio.Event, started: str, rest_of_the_turn: Sequence[str]):
def __init__(
self,
client_left: asyncio.Event,
started: str | None,
rest_of_the_turn: Sequence[str],
opening: Sequence[str] = (),
):
self._frames: asyncio.Queue[str | None] = asyncio.Queue()
for frame in opening:
self._frames.put_nowait(frame)
self._client_left = client_left
self._started = started
self._rest_of_the_turn = rest_of_the_turn
@ -8814,8 +8824,10 @@ class UpstreamFinishingTheTurnAfterTheClientLeft:
self._frames.put_nowait(None)
async def _start_turn(self, frame: str | bytes) -> None:
self._frames.put_nowait(self._started)
self._finishing.append(asyncio.create_task(self._finish_once_the_client_left()))
if self._started is not None:
self._frames.put_nowait(self._started)
if not self._finishing:
self._finishing.append(asyncio.create_task(self._finish_once_the_client_left()))
async def _finish_once_the_client_left(self) -> None:
await self._client_left.wait()
@ -8845,13 +8857,42 @@ async def test_websocket_passthrough_bills_the_turn_a_client_walked_out_of_once_
bills it, so the gateway keeps reading until the turn ends and logs its usage,
whether it reads the hang-up first or finds it by failing to relay a frame.
"""
client_left: Final = asyncio.Event()
client: Final = _walking_out_client(
[json.dumps({"type": "response.create", "model": "gpt-5.4"})],
'"response.created"',
None,
client_left,
hang_up_read_before_the_next_send,
)
upstream_ws: Final = UpstreamFinishingTheTurnAfterTheClientLeft(
client_left,
_responses_turn_started("gpt-5.4"),
(json.dumps({"type": "response.output_text.delta", "delta": "Hi"}), _responses_turn_ended("gpt-5.4")),
)
relayed: Final = await _relay_turns(client, upstream_ws, RecordingFrameModelGate(), walkout_turn_drain_seconds=5)
assert [
[json.loads(frame)["type"] if isinstance(frame, str) else frame["type"] for frame in logged.kwargs["response_body"]]
for logged in relayed.success_handler.call_args_list
] == [["response.created", "response.output_text.delta", "response.completed"]]
upstream_ws.close.assert_awaited()
def _walking_out_client(
frames: Sequence[str],
answered_by: str,
answers_due: Sequence[int] | None,
client_left: asyncio.Event,
hang_up_read_before_the_next_send: bool = True,
) -> MagicMock:
"""A turn-taking client that sets client_left as it hangs up, which the gateway either reads at once or only finds
when its next send to the client fails"""
from starlette.websockets import WebSocketDisconnect, WebSocketState
client_left: Final = asyncio.Event()
send_failed: Final = asyncio.Event()
client: Final = _turn_taking_client(
[json.dumps({"type": "response.create", "model": "gpt-5.4"})], answered_by='"response.created"'
)
client: Final = _turn_taking_client(frames, answered_by, answers_due)
receive: Final = client.receive
send_text: Final = client.send_text
@ -8872,21 +8913,128 @@ async def test_websocket_passthrough_bills_the_turn_a_client_walked_out_of_once_
client.receive = receive_until_leaving
client.send_text = AsyncMock(side_effect=send_text_until_leaving)
return client
@pytest.mark.asyncio
async def test_websocket_passthrough_bills_the_turn_a_client_walked_out_of_before_the_provider_started_it():
"""
A client can ask for a turn and leave before the provider says it started.
The provider still runs and bills that turn, so the gateway keeps reading
until it ends and logs its usage.
"""
client_left: Final = asyncio.Event()
client: Final = _walking_out_client(
[json.dumps({"type": "response.create", "model": "gpt-5.4"})], '"response.created"', (0, 0), client_left
)
upstream_ws: Final = UpstreamFinishingTheTurnAfterTheClientLeft(
client_left,
_responses_turn_started("gpt-5.4"),
(json.dumps({"type": "response.output_text.delta", "delta": "Hi"}), _responses_turn_ended("gpt-5.4")),
client_left, None, (_responses_turn_started("gpt-5.4"), _responses_turn_ended("gpt-5.4"))
)
relayed: Final = await _relay_turns(client, upstream_ws, RecordingFrameModelGate(), walkout_turn_drain_seconds=5)
assert [
[json.loads(frame)["type"] if isinstance(frame, str) else frame["type"] for frame in logged.kwargs["response_body"]]
for logged in relayed.success_handler.call_args_list
] == [["response.created", "response.output_text.delta", "response.completed"]]
assert _logged_frame_types(relayed) == [["response.created", "response.completed"]]
upstream_ws.close.assert_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("hang_up_read_before_the_next_send", [True, False], ids=["read_first", "found_by_a_send"])
async def test_websocket_passthrough_bills_a_realtime_turn_a_client_walked_out_of_while_the_provider_stored_its_input(
hang_up_read_before_the_next_send: bool,
):
"""
A realtime client can add an item, ask for a turn, and leave at once. The
provider confirms the item before it starts the turn, and that confirmation
must not end the session while the turn it was asked for is still owed.
"""
client_left: Final = asyncio.Event()
client: Final = _walking_out_client(
[
json.dumps({"type": "conversation.item.create", "item": {"type": "message", "role": "user"}}),
json.dumps({"type": "response.create"}),
],
'"session.created"',
(1, 1, 1),
client_left,
hang_up_read_before_the_next_send,
)
upstream_ws: Final = UpstreamFinishingTheTurnAfterTheClientLeft(
client_left,
None,
(
json.dumps({"type": "conversation.item.created", "item": {"type": "message", "role": "user"}}),
json.dumps({"type": "response.created", "response": {"status": "in_progress"}}),
REALTIME_RESPONSE_DONE,
),
opening=(REALTIME_SESSION_CREATED,),
)
relayed: Final = await _relay_turns(
client, upstream_ws, RecordingFrameModelGate(), "/openai/v1/realtime", walkout_turn_drain_seconds=5
)
assert _logged_frame_types(relayed) == [
["session.created", "conversation.item.created", "response.created", "response.done"]
]
@pytest.mark.asyncio
async def test_websocket_passthrough_bills_a_realtime_turn_a_client_walked_out_of_after_the_provider_started_one_itself():
"""
A realtime provider can start a turn on its own when voice detection hears
the user stop. That turn answers no client frame, so a turn the client asks
for next and leaves before it starts is still owed and still billed.
"""
realtime_response_created: Final = json.dumps({"type": "response.created", "response": {"status": "in_progress"}})
client_left: Final = asyncio.Event()
client: Final = _walking_out_client(
[json.dumps({"type": "response.create"})], '"response.done"', (1, 1), client_left
)
upstream_ws: Final = UpstreamFinishingTheTurnAfterTheClientLeft(
client_left,
None,
(realtime_response_created, REALTIME_RESPONSE_DONE),
opening=(REALTIME_SESSION_CREATED, realtime_response_created, REALTIME_RESPONSE_DONE),
)
relayed: Final = await _relay_turns(
client, upstream_ws, RecordingFrameModelGate(), "/openai/v1/realtime", walkout_turn_drain_seconds=5
)
assert _logged_frame_types(relayed) == [
["session.created", "response.created", "response.done"],
["session.created", "response.created", "response.done"],
]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"answer",
[_responses_turn("gpt-5.4"), [_responses_error("previous_response_not_found")]],
ids=["finished", "refused"],
)
async def test_websocket_passthrough_lets_go_of_the_provider_at_once_when_a_client_leaves_owing_no_turn(
answer: Sequence[str],
):
"""
Once every turn a client asked for has ended or been refused, nothing more
can be billed, so a client leaving closes the provider socket right away
instead of holding it for the walkout drain.
"""
client: Final = _turn_taking_client(
[json.dumps({"type": "response.create", "model": "gpt-5.4"})],
answered_by=json.dumps(json.loads(answer[-1])["type"]),
)
upstream_ws: Final = ScriptedUpstreamWebSocket(replies=[answer])
relayed: Final = await _relay_turns(
client, upstream_ws, RecordingFrameModelGate(), walkout_turn_drain_seconds=WALKOUT_TURN_DRAIN_SECONDS
)
upstream_ws.close.assert_awaited()
assert len(relayed.success_handler.call_args_list) == 1
@pytest.mark.asyncio
async def test_websocket_passthrough_logs_no_extra_row_for_what_the_provider_sends_after_the_last_turn():
"""