From f414a42ecfe33f1c748bf417d68b9e6aa9533ba8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:09:21 -0700 Subject: [PATCH] fix(passthrough): keep the session update a buffered clip ran under until its answer is billed A transcription that completes while an answer streams waits in the buffer for that answer's response.done. A session.updated in between used to drop the earlier update, so the clip was priced at the opening configuration. An earlier update now stays while an unbilled clip follows it, and drops at the next update once that clip is billed. --- .../pass_through_endpoints.py | 14 ++++++- .../test_pass_through_endpoints.py | 37 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 1bc817ad348..efd18dd8799 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -10,7 +10,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, M from dataclasses import dataclass from datetime import datetime from functools import reduce -from itertools import chain, count, groupby +from itertools import chain, count, groupby, takewhile from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, TypedDict, TypeVar, cast from urllib.parse import parse_qs, urlencode, urlparse @@ -2752,7 +2752,17 @@ def _without_earlier_session_updates( frames: Sequence[_FrameT], latest_update: _FrameT, ) -> tuple[_FrameT, ...]: - return tuple(frame for frame in frames if frame is latest_update or upstream_frame_kind(frame) != "updates_session") + return tuple( + frame + for index, frame in enumerate(frames) + if frame is latest_update + or upstream_frame_kind(frame) != "updates_session" + or _unbilled_usage_ran_under(map(upstream_frame_kind, frames[index + 1 :])) + ) + + +def _unbilled_usage_ran_under(later_kinds: Iterable[WebsocketUpstreamFrameKind]) -> bool: + return "bills_usage" in takewhile(lambda kind: kind != "updates_session", later_kinds) def _nested_frame(frame: Mapping[str, object] | None, key: str) -> Mapping[str, object] | None: diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 2d98c8fabd0..9ff143ac058 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -9030,6 +9030,43 @@ async def test_websocket_passthrough_carries_only_the_latest_session_update_into assert len(relayed.success_handler.call_args_list) == 2 +@pytest.mark.asyncio +async def test_websocket_passthrough_keeps_the_session_update_a_clip_ran_under_until_its_answer_is_billed(): + client: Final = _turn_taking_client( + [ + _transcription_session_update("session.update", "gpt-transcribe-a"), + json.dumps({"type": "response.create"}), + _transcription_session_update("session.update", "gpt-transcribe-b"), + ], + answered_by='"type"', + answers_due=(1, 2, 4, 6), + ) + upstream_ws: Final = ScriptedUpstreamWebSocket( + REALTIME_SESSION_CREATED, + replies=[ + [_transcription_session_update("session.updated", "gpt-transcribe-a")], + [ + json.dumps({"type": "response.created", "response": {"status": "in_progress"}}), + REALTIME_TRANSCRIPTION_COMPLETED, + ], + [_transcription_session_update("session.updated", "gpt-transcribe-b"), REALTIME_RESPONSE_DONE], + ], + ) + + relayed: Final = await _relay_turns(client, upstream_ws, RecordingFrameModelGate(), "/openai/v1/realtime") + + assert _logged_frame_types(relayed) == [ + [ + "session.created", + "session.updated", + "response.created", + TRANSCRIPTION_COMPLETED, + "session.updated", + "response.done", + ] + ] + + @pytest.mark.asyncio async def test_websocket_passthrough_still_bills_the_answer_in_flight_when_a_later_frame_is_refused(): """