mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(passthrough): keep the session update a buffered clip ran under until its answer is billed
A transcription that completes while an answer streams waits in the buffer for that answer's response.done. A session.updated in between used to drop the earlier update, so the clip was priced at the opening configuration. An earlier update now stays while an unbilled clip follows it, and drops at the next update once that clip is billed.
This commit is contained in:
parent
3e8aefd633
commit
f414a42ecf
2 changed files with 49 additions and 2 deletions
|
|
@ -10,7 +10,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, M
|
|||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from functools import reduce
|
||||
from itertools import chain, count, groupby
|
||||
from itertools import chain, count, groupby, takewhile
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, TypedDict, TypeVar, cast
|
||||
from urllib.parse import parse_qs, urlencode, urlparse
|
||||
|
|
@ -2752,7 +2752,17 @@ def _without_earlier_session_updates(
|
|||
frames: Sequence[_FrameT],
|
||||
latest_update: _FrameT,
|
||||
) -> tuple[_FrameT, ...]:
|
||||
return tuple(frame for frame in frames if frame is latest_update or upstream_frame_kind(frame) != "updates_session")
|
||||
return tuple(
|
||||
frame
|
||||
for index, frame in enumerate(frames)
|
||||
if frame is latest_update
|
||||
or upstream_frame_kind(frame) != "updates_session"
|
||||
or _unbilled_usage_ran_under(map(upstream_frame_kind, frames[index + 1 :]))
|
||||
)
|
||||
|
||||
|
||||
def _unbilled_usage_ran_under(later_kinds: Iterable[WebsocketUpstreamFrameKind]) -> bool:
|
||||
return "bills_usage" in takewhile(lambda kind: kind != "updates_session", later_kinds)
|
||||
|
||||
|
||||
def _nested_frame(frame: Mapping[str, object] | None, key: str) -> Mapping[str, object] | None:
|
||||
|
|
|
|||
|
|
@ -9030,6 +9030,43 @@ async def test_websocket_passthrough_carries_only_the_latest_session_update_into
|
|||
assert len(relayed.success_handler.call_args_list) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_keeps_the_session_update_a_clip_ran_under_until_its_answer_is_billed():
|
||||
client: Final = _turn_taking_client(
|
||||
[
|
||||
_transcription_session_update("session.update", "gpt-transcribe-a"),
|
||||
json.dumps({"type": "response.create"}),
|
||||
_transcription_session_update("session.update", "gpt-transcribe-b"),
|
||||
],
|
||||
answered_by='"type"',
|
||||
answers_due=(1, 2, 4, 6),
|
||||
)
|
||||
upstream_ws: Final = ScriptedUpstreamWebSocket(
|
||||
REALTIME_SESSION_CREATED,
|
||||
replies=[
|
||||
[_transcription_session_update("session.updated", "gpt-transcribe-a")],
|
||||
[
|
||||
json.dumps({"type": "response.created", "response": {"status": "in_progress"}}),
|
||||
REALTIME_TRANSCRIPTION_COMPLETED,
|
||||
],
|
||||
[_transcription_session_update("session.updated", "gpt-transcribe-b"), REALTIME_RESPONSE_DONE],
|
||||
],
|
||||
)
|
||||
|
||||
relayed: Final = await _relay_turns(client, upstream_ws, RecordingFrameModelGate(), "/openai/v1/realtime")
|
||||
|
||||
assert _logged_frame_types(relayed) == [
|
||||
[
|
||||
"session.created",
|
||||
"session.updated",
|
||||
"response.created",
|
||||
TRANSCRIPTION_COMPLETED,
|
||||
"session.updated",
|
||||
"response.done",
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_still_bills_the_answer_in_flight_when_a_later_frame_is_refused():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue