mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(passthrough): keep reading a websocket turn the client left before OpenAI started it
A client that sends response.create and hangs up before OpenAI answers with response.created made the walkout checks see no turn in flight, so the gateway closed the provider socket and the turn OpenAI still ran and billed never reached spend. The relay now counts every forwarded create until the provider starts or refuses it, and keeps draining while one is owed
This commit is contained in:
parent
f414a42ecf
commit
f2d430f555
2 changed files with 185 additions and 19 deletions
|
|
@ -2878,12 +2878,14 @@ class _WebsocketTurnLedger:
|
|||
self.session_opened = False
|
||||
self.started = started
|
||||
self.logged_turns = 0
|
||||
self.unstarted_turns = 0
|
||||
|
||||
def current(self, lane: str | None) -> _TurnAuthorization:
|
||||
oldest: Final = next(iter(self.queued_turns.get(lane, ())), None)
|
||||
return self.opening if oldest is None else oldest
|
||||
|
||||
def queue_turn(self, lane: str | None, turn: _TurnAuthorization | None) -> None:
|
||||
self.unstarted_turns += 1
|
||||
if not self.session_opened:
|
||||
self.queued_turns = MappingProxyType({**self.queued_turns, lane: (*self.queued_turns.get(lane, ()), turn)})
|
||||
|
||||
|
|
@ -2896,7 +2898,11 @@ class _WebsocketTurnLedger:
|
|||
self.started = end_time
|
||||
self._forget_turn(lane, 0)
|
||||
|
||||
def start_turn(self) -> None:
|
||||
self.unstarted_turns = max(0, self.unstarted_turns - 1)
|
||||
|
||||
def drop_refused_turn(self, lane: str | None, lane_is_running_a_turn: bool) -> None:
|
||||
self.start_turn()
|
||||
self._forget_turn(lane, 1 if lane_is_running_a_turn else 0)
|
||||
|
||||
def _forget_turn(self, lane: str | None, position: int) -> None:
|
||||
|
|
@ -2923,6 +2929,16 @@ def _turn_in_flight(
|
|||
)
|
||||
|
||||
|
||||
def _turn_owed(
|
||||
upstream_frame_kind: WebsocketUpstreamFrameClassifier | None,
|
||||
unlogged_frames: Sequence[Mapping[str, object]],
|
||||
ledger: _WebsocketTurnLedger,
|
||||
) -> bool:
|
||||
return _turn_in_flight(upstream_frame_kind, unlogged_frames) or (
|
||||
upstream_frame_kind is not None and ledger.unstarted_turns > 0
|
||||
)
|
||||
|
||||
|
||||
def _lanes_in_flight(
|
||||
upstream_frame_kind: WebsocketUpstreamFrameClassifier, unlogged_frames: Sequence[Mapping[str, object]]
|
||||
) -> tuple[str | None, ...]:
|
||||
|
|
@ -3323,7 +3339,7 @@ async def websocket_passthrough_request(
|
|||
_RefuseClientFrame,
|
||||
):
|
||||
break
|
||||
if not _turn_in_flight(upstream_frame_kind, websocket_messages):
|
||||
if not _turn_owed(upstream_frame_kind, websocket_messages, ledger):
|
||||
await upstream_ws.close()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
|
|
@ -3362,6 +3378,8 @@ async def websocket_passthrough_request(
|
|||
match None if upstream_frame_kind is None else upstream_frame_kind(message_data):
|
||||
case "opens_session":
|
||||
ledger.open_session()
|
||||
case "starts_turn":
|
||||
ledger.start_turn()
|
||||
case "updates_session" if upstream_frame_kind is not None:
|
||||
websocket_messages[:] = _without_earlier_session_updates(
|
||||
upstream_frame_kind, websocket_messages, message_data
|
||||
|
|
@ -3381,7 +3399,7 @@ async def websocket_passthrough_request(
|
|||
async def relay_upstream_frame(upstream_message: str | bytes) -> None:
|
||||
record_upstream_frame(upstream_message)
|
||||
if not _client_socket_is_open(websocket):
|
||||
if not _turn_in_flight(upstream_frame_kind, websocket_messages):
|
||||
if not _turn_owed(upstream_frame_kind, websocket_messages, ledger):
|
||||
await upstream_ws.close()
|
||||
return
|
||||
try:
|
||||
|
|
@ -3390,7 +3408,7 @@ async def websocket_passthrough_request(
|
|||
else:
|
||||
await websocket.send_text(upstream_message)
|
||||
except WebSocketDisconnect:
|
||||
if not _turn_in_flight(upstream_frame_kind, websocket_messages):
|
||||
if not _turn_owed(upstream_frame_kind, websocket_messages, ledger):
|
||||
raise
|
||||
|
||||
async def forward_upstream_to_client() -> Close | None:
|
||||
|
|
@ -3421,7 +3439,7 @@ async def websocket_passthrough_request(
|
|||
tasks: Final = (client_task, upstream_task)
|
||||
|
||||
await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
|
||||
if client_task.done() and _turn_in_flight(upstream_frame_kind, websocket_messages):
|
||||
if client_task.done() and _turn_owed(upstream_frame_kind, websocket_messages, ledger):
|
||||
await asyncio.wait((upstream_task,), timeout=walkout_turn_drain_seconds)
|
||||
done: Final = tuple(task for task in tasks if task.done())
|
||||
pending: Final = tuple(task for task in tasks if not task.done())
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
|||
DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS,
|
||||
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
|
||||
STORED_PROMPT_PROBE_MODEL,
|
||||
WALKOUT_TURN_DRAIN_SECONDS,
|
||||
HttpPassThroughEndpointHelpers,
|
||||
InitPassThroughEndpointHelpers,
|
||||
SafeRouteAdder,
|
||||
|
|
@ -8797,10 +8798,19 @@ async def test_websocket_passthrough_charges_each_lane_a_client_walked_out_of_to
|
|||
|
||||
|
||||
class UpstreamFinishingTheTurnAfterTheClientLeft:
|
||||
"""Starts a turn for each client frame and sends the rest of it only after the client left, and stops once closed"""
|
||||
"""Starts a turn for each client frame, unless it only answers once the client left, sends the rest of the session
|
||||
once the client left, and stops once closed"""
|
||||
|
||||
def __init__(self, client_left: asyncio.Event, started: str, rest_of_the_turn: Sequence[str]):
|
||||
def __init__(
|
||||
self,
|
||||
client_left: asyncio.Event,
|
||||
started: str | None,
|
||||
rest_of_the_turn: Sequence[str],
|
||||
opening: Sequence[str] = (),
|
||||
):
|
||||
self._frames: asyncio.Queue[str | None] = asyncio.Queue()
|
||||
for frame in opening:
|
||||
self._frames.put_nowait(frame)
|
||||
self._client_left = client_left
|
||||
self._started = started
|
||||
self._rest_of_the_turn = rest_of_the_turn
|
||||
|
|
@ -8814,8 +8824,10 @@ class UpstreamFinishingTheTurnAfterTheClientLeft:
|
|||
self._frames.put_nowait(None)
|
||||
|
||||
async def _start_turn(self, frame: str | bytes) -> None:
|
||||
self._frames.put_nowait(self._started)
|
||||
self._finishing.append(asyncio.create_task(self._finish_once_the_client_left()))
|
||||
if self._started is not None:
|
||||
self._frames.put_nowait(self._started)
|
||||
if not self._finishing:
|
||||
self._finishing.append(asyncio.create_task(self._finish_once_the_client_left()))
|
||||
|
||||
async def _finish_once_the_client_left(self) -> None:
|
||||
await self._client_left.wait()
|
||||
|
|
@ -8845,13 +8857,42 @@ async def test_websocket_passthrough_bills_the_turn_a_client_walked_out_of_once_
|
|||
bills it, so the gateway keeps reading until the turn ends and logs its usage,
|
||||
whether it reads the hang-up first or finds it by failing to relay a frame.
|
||||
"""
|
||||
client_left: Final = asyncio.Event()
|
||||
client: Final = _walking_out_client(
|
||||
[json.dumps({"type": "response.create", "model": "gpt-5.4"})],
|
||||
'"response.created"',
|
||||
None,
|
||||
client_left,
|
||||
hang_up_read_before_the_next_send,
|
||||
)
|
||||
upstream_ws: Final = UpstreamFinishingTheTurnAfterTheClientLeft(
|
||||
client_left,
|
||||
_responses_turn_started("gpt-5.4"),
|
||||
(json.dumps({"type": "response.output_text.delta", "delta": "Hi"}), _responses_turn_ended("gpt-5.4")),
|
||||
)
|
||||
|
||||
relayed: Final = await _relay_turns(client, upstream_ws, RecordingFrameModelGate(), walkout_turn_drain_seconds=5)
|
||||
|
||||
assert [
|
||||
[json.loads(frame)["type"] if isinstance(frame, str) else frame["type"] for frame in logged.kwargs["response_body"]]
|
||||
for logged in relayed.success_handler.call_args_list
|
||||
] == [["response.created", "response.output_text.delta", "response.completed"]]
|
||||
upstream_ws.close.assert_awaited()
|
||||
|
||||
|
||||
def _walking_out_client(
|
||||
frames: Sequence[str],
|
||||
answered_by: str,
|
||||
answers_due: Sequence[int] | None,
|
||||
client_left: asyncio.Event,
|
||||
hang_up_read_before_the_next_send: bool = True,
|
||||
) -> MagicMock:
|
||||
"""A turn-taking client that sets client_left as it hangs up, which the gateway either reads at once or only finds
|
||||
when its next send to the client fails"""
|
||||
from starlette.websockets import WebSocketDisconnect, WebSocketState
|
||||
|
||||
client_left: Final = asyncio.Event()
|
||||
send_failed: Final = asyncio.Event()
|
||||
client: Final = _turn_taking_client(
|
||||
[json.dumps({"type": "response.create", "model": "gpt-5.4"})], answered_by='"response.created"'
|
||||
)
|
||||
client: Final = _turn_taking_client(frames, answered_by, answers_due)
|
||||
receive: Final = client.receive
|
||||
send_text: Final = client.send_text
|
||||
|
||||
|
|
@ -8872,21 +8913,128 @@ async def test_websocket_passthrough_bills_the_turn_a_client_walked_out_of_once_
|
|||
|
||||
client.receive = receive_until_leaving
|
||||
client.send_text = AsyncMock(side_effect=send_text_until_leaving)
|
||||
return client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_bills_the_turn_a_client_walked_out_of_before_the_provider_started_it():
|
||||
"""
|
||||
A client can ask for a turn and leave before the provider says it started.
|
||||
The provider still runs and bills that turn, so the gateway keeps reading
|
||||
until it ends and logs its usage.
|
||||
"""
|
||||
client_left: Final = asyncio.Event()
|
||||
client: Final = _walking_out_client(
|
||||
[json.dumps({"type": "response.create", "model": "gpt-5.4"})], '"response.created"', (0, 0), client_left
|
||||
)
|
||||
upstream_ws: Final = UpstreamFinishingTheTurnAfterTheClientLeft(
|
||||
client_left,
|
||||
_responses_turn_started("gpt-5.4"),
|
||||
(json.dumps({"type": "response.output_text.delta", "delta": "Hi"}), _responses_turn_ended("gpt-5.4")),
|
||||
client_left, None, (_responses_turn_started("gpt-5.4"), _responses_turn_ended("gpt-5.4"))
|
||||
)
|
||||
|
||||
relayed: Final = await _relay_turns(client, upstream_ws, RecordingFrameModelGate(), walkout_turn_drain_seconds=5)
|
||||
|
||||
assert [
|
||||
[json.loads(frame)["type"] if isinstance(frame, str) else frame["type"] for frame in logged.kwargs["response_body"]]
|
||||
for logged in relayed.success_handler.call_args_list
|
||||
] == [["response.created", "response.output_text.delta", "response.completed"]]
|
||||
assert _logged_frame_types(relayed) == [["response.created", "response.completed"]]
|
||||
upstream_ws.close.assert_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("hang_up_read_before_the_next_send", [True, False], ids=["read_first", "found_by_a_send"])
|
||||
async def test_websocket_passthrough_bills_a_realtime_turn_a_client_walked_out_of_while_the_provider_stored_its_input(
|
||||
hang_up_read_before_the_next_send: bool,
|
||||
):
|
||||
"""
|
||||
A realtime client can add an item, ask for a turn, and leave at once. The
|
||||
provider confirms the item before it starts the turn, and that confirmation
|
||||
must not end the session while the turn it was asked for is still owed.
|
||||
"""
|
||||
client_left: Final = asyncio.Event()
|
||||
client: Final = _walking_out_client(
|
||||
[
|
||||
json.dumps({"type": "conversation.item.create", "item": {"type": "message", "role": "user"}}),
|
||||
json.dumps({"type": "response.create"}),
|
||||
],
|
||||
'"session.created"',
|
||||
(1, 1, 1),
|
||||
client_left,
|
||||
hang_up_read_before_the_next_send,
|
||||
)
|
||||
upstream_ws: Final = UpstreamFinishingTheTurnAfterTheClientLeft(
|
||||
client_left,
|
||||
None,
|
||||
(
|
||||
json.dumps({"type": "conversation.item.created", "item": {"type": "message", "role": "user"}}),
|
||||
json.dumps({"type": "response.created", "response": {"status": "in_progress"}}),
|
||||
REALTIME_RESPONSE_DONE,
|
||||
),
|
||||
opening=(REALTIME_SESSION_CREATED,),
|
||||
)
|
||||
|
||||
relayed: Final = await _relay_turns(
|
||||
client, upstream_ws, RecordingFrameModelGate(), "/openai/v1/realtime", walkout_turn_drain_seconds=5
|
||||
)
|
||||
|
||||
assert _logged_frame_types(relayed) == [
|
||||
["session.created", "conversation.item.created", "response.created", "response.done"]
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_bills_a_realtime_turn_a_client_walked_out_of_after_the_provider_started_one_itself():
|
||||
"""
|
||||
A realtime provider can start a turn on its own when voice detection hears
|
||||
the user stop. That turn answers no client frame, so a turn the client asks
|
||||
for next and leaves before it starts is still owed and still billed.
|
||||
"""
|
||||
realtime_response_created: Final = json.dumps({"type": "response.created", "response": {"status": "in_progress"}})
|
||||
client_left: Final = asyncio.Event()
|
||||
client: Final = _walking_out_client(
|
||||
[json.dumps({"type": "response.create"})], '"response.done"', (1, 1), client_left
|
||||
)
|
||||
upstream_ws: Final = UpstreamFinishingTheTurnAfterTheClientLeft(
|
||||
client_left,
|
||||
None,
|
||||
(realtime_response_created, REALTIME_RESPONSE_DONE),
|
||||
opening=(REALTIME_SESSION_CREATED, realtime_response_created, REALTIME_RESPONSE_DONE),
|
||||
)
|
||||
|
||||
relayed: Final = await _relay_turns(
|
||||
client, upstream_ws, RecordingFrameModelGate(), "/openai/v1/realtime", walkout_turn_drain_seconds=5
|
||||
)
|
||||
|
||||
assert _logged_frame_types(relayed) == [
|
||||
["session.created", "response.created", "response.done"],
|
||||
["session.created", "response.created", "response.done"],
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"answer",
|
||||
[_responses_turn("gpt-5.4"), [_responses_error("previous_response_not_found")]],
|
||||
ids=["finished", "refused"],
|
||||
)
|
||||
async def test_websocket_passthrough_lets_go_of_the_provider_at_once_when_a_client_leaves_owing_no_turn(
|
||||
answer: Sequence[str],
|
||||
):
|
||||
"""
|
||||
Once every turn a client asked for has ended or been refused, nothing more
|
||||
can be billed, so a client leaving closes the provider socket right away
|
||||
instead of holding it for the walkout drain.
|
||||
"""
|
||||
client: Final = _turn_taking_client(
|
||||
[json.dumps({"type": "response.create", "model": "gpt-5.4"})],
|
||||
answered_by=json.dumps(json.loads(answer[-1])["type"]),
|
||||
)
|
||||
upstream_ws: Final = ScriptedUpstreamWebSocket(replies=[answer])
|
||||
|
||||
relayed: Final = await _relay_turns(
|
||||
client, upstream_ws, RecordingFrameModelGate(), walkout_turn_drain_seconds=WALKOUT_TURN_DRAIN_SECONDS
|
||||
)
|
||||
|
||||
upstream_ws.close.assert_awaited()
|
||||
assert len(relayed.success_handler.call_args_list) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_logs_no_extra_row_for_what_the_provider_sends_after_the_last_turn():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue