mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(realtime): count a relay that fails on its own as an abnormal end
This commit is contained in:
parent
fb6ad8db2f
commit
cedad9ddf9
2 changed files with 65 additions and 5 deletions
|
|
@ -72,6 +72,7 @@ class BackendClose:
|
|||
class ClientLoopExit(Enum):
|
||||
CLIENT_DISCONNECTED = auto()
|
||||
BACKEND_CLOSED = auto()
|
||||
RELAY_FAILED = auto()
|
||||
|
||||
|
||||
def backend_close_from(error: "ConnectionClosed") -> BackendClose:
|
||||
|
|
@ -2021,14 +2022,17 @@ class RealTimeStreaming:
|
|||
return ClientLoopExit.BACKEND_CLOSED
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Error in client ack messages: %s", e)
|
||||
return ClientLoopExit.CLIENT_DISCONNECTED
|
||||
return ClientLoopExit.RELAY_FAILED
|
||||
|
||||
async def bidirectional_forward(self) -> None:
|
||||
forward_task: Final = asyncio.create_task(self.backend_to_client_send_messages())
|
||||
client_task: Final = asyncio.create_task(self.client_ack_messages())
|
||||
try:
|
||||
await asyncio.wait((forward_task, client_task), return_when=asyncio.FIRST_COMPLETED)
|
||||
if client_task.done() and client_task.result() is ClientLoopExit.CLIENT_DISCONNECTED:
|
||||
client_exit: Final = client_task.result() if client_task.done() else None
|
||||
if client_exit is ClientLoopExit.RELAY_FAILED and self.logging_obj:
|
||||
self.logging_obj.model_call_details[REALTIME_SESSION_ENDED_ABNORMALLY_KEY] = True
|
||||
if client_exit in (ClientLoopExit.CLIENT_DISCONNECTED, ClientLoopExit.RELAY_FAILED):
|
||||
return
|
||||
close: Final = await forward_task
|
||||
if not close.ended_cleanly and self.logging_obj:
|
||||
|
|
|
|||
|
|
@ -578,11 +578,20 @@ def _realtime_session_logging() -> LiteLLMLogging:
|
|||
)
|
||||
|
||||
|
||||
async def _relay_upstream(litellm_logging_obj: LiteLLMLogging, *upstream_frames: bytes | Exception) -> None:
|
||||
async def _relay_upstream(
|
||||
litellm_logging_obj: LiteLLMLogging,
|
||||
*upstream_frames: bytes | Exception,
|
||||
client_ws: MagicMock | None = None,
|
||||
backend_send_error: Exception | None = None,
|
||||
upstream_stays_open: bool = False,
|
||||
) -> None:
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.recv = AsyncMock(side_effect=list(upstream_frames))
|
||||
backend_ws.recv = AsyncMock(
|
||||
side_effect=_frames_then_silence(*upstream_frames) if upstream_stays_open else list(upstream_frames)
|
||||
)
|
||||
backend_ws.send = AsyncMock(side_effect=backend_send_error)
|
||||
relay: Final = RealTimeStreaming(
|
||||
_realtime_client_ws(),
|
||||
client_ws or _realtime_client_ws(),
|
||||
backend_ws,
|
||||
litellm_logging_obj,
|
||||
model="gpt-realtime-2.1",
|
||||
|
|
@ -612,6 +621,49 @@ async def _upstream_close_before_any_frame(litellm_logging_obj: LiteLLMLogging)
|
|||
await _relay_upstream(litellm_logging_obj, ConnectionClosed(Close(1008, "model not found"), None))
|
||||
|
||||
|
||||
def _frames_then_silence(*frames: bytes | str) -> Callable[[], Awaitable[bytes | str]]:
|
||||
pending: Final = iter(frames)
|
||||
|
||||
async def _next_frame() -> bytes | str:
|
||||
frame: Final = next(pending, None)
|
||||
if frame is None:
|
||||
await asyncio.Event().wait()
|
||||
raise AssertionError("unreachable")
|
||||
return frame
|
||||
|
||||
return _next_frame
|
||||
|
||||
|
||||
def _client_ws_sending(*client_frames: str) -> MagicMock:
|
||||
client_ws: Final = _realtime_client_ws()
|
||||
client_ws.receive_text = AsyncMock(side_effect=_frames_then_silence(*client_frames))
|
||||
return client_ws
|
||||
|
||||
|
||||
_CLIENT_TEXT_ITEM: Final = json.dumps(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hello"}]},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def _relay_write_to_upstream_fails(litellm_logging_obj: LiteLLMLogging) -> None:
|
||||
await _relay_upstream(
|
||||
litellm_logging_obj,
|
||||
_REALTIME_SESSION_CREATED,
|
||||
client_ws=_client_ws_sending(_CLIENT_TEXT_ITEM),
|
||||
backend_send_error=OSError("broken pipe"),
|
||||
upstream_stays_open=True,
|
||||
)
|
||||
|
||||
|
||||
async def _client_hangs_up_after_a_frame(litellm_logging_obj: LiteLLMLogging) -> None:
|
||||
client_ws: Final = _realtime_client_ws()
|
||||
client_ws.receive_text = AsyncMock(side_effect=ConnectionClosed(Close(1000, ""), None))
|
||||
await _relay_upstream(litellm_logging_obj, _REALTIME_SESSION_CREATED, client_ws=client_ws, upstream_stays_open=True)
|
||||
|
||||
|
||||
async def _refused_handshake(litellm_logging_obj: LiteLLMLogging) -> None:
|
||||
await close_after_upstream_handshake_refusal(_realtime_client_ws(), 403)
|
||||
|
||||
|
|
@ -625,6 +677,8 @@ async def _refused_handshake(litellm_logging_obj: LiteLLMLogging) -> None:
|
|||
(_refused_handshake, 0),
|
||||
(_upstream_internal_error_after_a_frame, 0),
|
||||
(_upstream_drop_without_close_frame_after_a_frame, 0),
|
||||
(_relay_write_to_upstream_fails, 0),
|
||||
(_client_hangs_up_after_a_frame, 1),
|
||||
],
|
||||
ids=[
|
||||
"clean-session",
|
||||
|
|
@ -632,6 +686,8 @@ async def _refused_handshake(litellm_logging_obj: LiteLLMLogging) -> None:
|
|||
"refused-handshake",
|
||||
"upstream-1011-after-a-frame",
|
||||
"upstream-1006-after-a-frame",
|
||||
"relay-write-to-upstream-fails",
|
||||
"client-hangs-up-after-a-frame",
|
||||
],
|
||||
)
|
||||
async def test_realtime_session_counts_a_router_success_only_when_it_ended_without_a_failure(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue