From 8a3248de0d0ddf3153b7c2f6611a20ad390fcf9f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:42:11 -0700 Subject: [PATCH] fix(bedrock): fail a dropped Nova Sonic session right away and log it as failed When Bedrock's output side ends mid-session, by a drop or a modeled stream error, the client side no longer spends the close timeout sending close messages into the dead input stream, so the error reaches the app right after the drop. A session the app ends itself still sends Bedrock its close messages and closes the input The replay refusal now leads with the committed cause, so the error event and the cut 1011 close reason both start with it. It stays a 400 and is tagged as a failure outside the selected deployment, so router retries and fallbacks of a dropped session no longer add extra failure counts A clean realtime session now counts once toward its deployment's cooldown success counter, so cooldown reads the real failure rate instead of 100 percent. Realtime tpm and rpm counting is unchanged The spend row of a session the provider ended now reads failure with its error information, while the events already forwarded are still billed. A session the app ended keeps logging success --- litellm/llms/bedrock/realtime/handler.py | 137 ++++++++++++----- litellm/router.py | 9 ++ .../realtime/test_bedrock_realtime_handler.py | 140 ++++++++++++++---- .../test_router/test_router_helper_utils.py | 62 ++++++++ 4 files changed, 283 insertions(+), 65 deletions(-) diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 9f5c56fd106..a62122ee5ee 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -9,11 +9,12 @@ import contextlib import importlib.metadata import json from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from types import MappingProxyType from typing import Final, NoReturn, Protocol, runtime_checkable from pydantic import JsonValue, TypeAdapter +from typing_extensions import TypeIs # noqa: TID251 # narrows request metadata without copying it import litellm from litellm._logging import redact_string, verbose_proxy_logger @@ -34,6 +35,7 @@ from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeE from litellm.types.llms.bedrock import AwsAuthParams from litellm.types.llms.openai import OpenAIRealtimeEvents from litellm.types.realtime import RealtimeResponseTransformInput +from litellm.types.utils import StandardLoggingPayloadErrorInformation from ..base_aws_llm import BaseAWSLLM, run_aws_signing from ..common_utils import BedrockError @@ -165,6 +167,12 @@ class _BridgeOutcome: client_disconnected: bool +@dataclass(frozen=True, slots=True) +class SessionEndSignals: + input_closing: asyncio.Event = field(default_factory=asyncio.Event) + output_failed: asyncio.Event = field(default_factory=asyncio.Event) + + async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message: str | None) -> AsyncIterator[str]: if initial_message is not None: yield initial_message @@ -184,14 +192,18 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None: """A fallback attempt on a websocket whose Bedrock stream already took input would silently start over.""" if scope is None or not _is_session_committed(scope): return + from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure + committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY) - raise BedrockError( + refusal: Final = BedrockError( status_code=400, message=( - "Bedrock realtime session already committed to a provider stream; it cannot be replayed" - + (f". The committed stream failed with: {committed_failure}" if committed_failure else "") + (f"{committed_failure}. " if committed_failure else "") + + "Bedrock realtime session already committed to a provider stream; it cannot be replayed" ), ) + mark_deployment_selection_failure(refusal) + raise refusal def _pending_opening_message(scope: Mapping[str, object]) -> str | None: @@ -201,6 +213,36 @@ def _pending_opening_message(scope: Mapping[str, object]) -> str | None: return pending if isinstance(pending, str) else None +def _is_str_keyed_dict( + value: object, +) -> TypeIs[dict[str, object]]: # guard-ok: request kwargs have str keys # mutable-ok: the spend row reads it + return isinstance(value, dict) + + +def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseException) -> None: + """The spend row of a session the provider ended reads failure while it still bills the forwarded events.""" + litellm_params: Final[object] = logging_obj.model_call_details.get("litellm_params") + metadata_candidates: Final = ( + (litellm_params.get("litellm_metadata"), litellm_params.get("metadata")) + if _is_str_keyed_dict(litellm_params) + else () + ) + spend_log_metadata: Final = next( + (metadata for metadata in metadata_candidates if _is_str_keyed_dict(metadata) and metadata), None + ) + if spend_log_metadata is None: + return + status_code: Final[object] = getattr(error, "status_code", None) + error_information: Final[StandardLoggingPayloadErrorInformation] = { + "error_code": "" if status_code is None else str(status_code), + "error_class": type(error).__name__, + "error_message": redact_string(str(error)), + "llm_provider": "bedrock", + } + spend_log_metadata["status"] = "failure" + spend_log_metadata["error_information"] = error_information + + def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn: error: Final = _as_bedrock_error(failure) verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error))) @@ -279,6 +321,25 @@ async def _reject_session_update( return True +async def _close_bedrock_input( + bedrock_stream: BedrockBidirectionalStream, + close_messages: Sequence[str], + send_to_bedrock: Callable[[str], Awaitable[None]], + timeout: float, +) -> None: + async def flush_and_close() -> None: + for close_message in close_messages: + with contextlib.suppress(Exception): + await send_to_bedrock(close_message) + with contextlib.suppress(Exception): + await bedrock_stream.input_stream.close() + + try: + await asyncio.wait_for(flush_and_close(), timeout) + except (asyncio.TimeoutError, TimeoutError): + verbose_proxy_logger.debug("Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", timeout) + + class BedrockRealtime(BaseAWSLLM): """Handler for Bedrock Nova Sonic realtime speech-to-speech API.""" @@ -445,7 +506,14 @@ class BedrockRealtime(BaseAWSLLM): if _should_log_event(leftover_event) ), ) + if outcome.provider_failure is not None and outcome.client_disconnected: + verbose_proxy_logger.debug( + "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure + ) + provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure if logged_events: + if provider_failure is not None: + _record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure)) GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( logging_obj.dispatch_success_handlers( list(logged_events), @@ -454,14 +522,9 @@ class BedrockRealtime(BaseAWSLLM): ) logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True - if outcome.provider_failure is None: + if provider_failure is None: return - if outcome.client_disconnected: - verbose_proxy_logger.debug( - "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure - ) - return - _raise_provider_failure(websocket.scope, outcome.provider_failure) + _raise_provider_failure(websocket.scope, provider_failure) async def _bridge( self, @@ -475,19 +538,23 @@ class BedrockRealtime(BaseAWSLLM): ) -> _BridgeOutcome: """Run both forwarding directions until the client leaves or either side fails.""" logged: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events forwarded before a failure are still spend - input_closing: Final = asyncio.Event() + signals: Final = SessionEndSignals() async def collect_logged_events() -> None: - async for event in self._forward_bedrock_to_client( - bedrock_stream, - websocket, - transformation_config, - model, - logging_obj, - session_state, - input_closing=input_closing, - ): - logged.append(event) + try: + async for event in self._forward_bedrock_to_client( + bedrock_stream, + websocket, + transformation_config, + model, + logging_obj, + session_state, + signals=signals, + ): + logged.append(event) + except Exception: + signals.output_failed.set() + raise client_task: Final = asyncio.create_task( self._forward_client_to_bedrock( @@ -498,7 +565,7 @@ class BedrockRealtime(BaseAWSLLM): session_state, logging_obj, initial_message, - input_closing=input_closing, + signals=signals, ) ) bedrock_task: Final = asyncio.create_task(collect_logged_events()) @@ -533,7 +600,7 @@ class BedrockRealtime(BaseAWSLLM): logging_obj: LiteLLMLogging | None = None, initial_message: str | None = None, *, - input_closing: asyncio.Event, + signals: SessionEndSignals, ) -> None: """Forward messages from client WebSocket to Bedrock stream. @@ -581,21 +648,11 @@ class BedrockRealtime(BaseAWSLLM): client_ws, transformation_config, model, logging_obj, parsed_client_message ): break - input_closing.set() + signals.input_closing.set() finally: - - async def close_bedrock_input() -> None: - for close_message in transformation_config.session_close_messages(): - with contextlib.suppress(Exception): - await send_to_bedrock(close_message) - with contextlib.suppress(Exception): - await bedrock_stream.input_stream.close() - - try: - await asyncio.wait_for(close_bedrock_input(), self._close_timeout) - except (asyncio.TimeoutError, TimeoutError): - verbose_proxy_logger.debug( - "Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", self._close_timeout + if not signals.output_failed.is_set(): + await _close_bedrock_input( + bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout ) async def _forward_bedrock_to_client( @@ -607,7 +664,7 @@ class BedrockRealtime(BaseAWSLLM): logging_obj: LiteLLMLogging, session_state: RealtimeResponseTransformInput, *, - input_closing: asyncio.Event, + signals: SessionEndSignals, ) -> AsyncIterator[OpenAIRealtimeEvents]: """Forward messages from Bedrock to the client, yielding the ones to record for spend logging. @@ -628,7 +685,7 @@ class BedrockRealtime(BaseAWSLLM): result = await output[1].receive() if result is None: - if not input_closing.is_set(): + if not signals.input_closing.is_set(): raise BedrockError( status_code=503, message="Bedrock realtime stream ended before the session closed", diff --git a/litellm/router.py b/litellm/router.py index 02ad6275b82..0dc798be380 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6478,6 +6478,15 @@ class Router: raise safeguard_refusal_error(model=model, stop_details=refusal_details) self.success_calls[model_name] += 1 + realtime_session_deployment_id: Final = ( + _MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id") + if generic_function_name == "_arealtime" + else None + ) + if realtime_session_deployment_id is not None: + increment_deployment_successes_for_current_minute( + litellm_router_instance=self, deployment_id=str(realtime_session_deployment_id) + ) verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 329ea9dd336..9767f294af7 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -1,7 +1,9 @@ import asyncio import json import sys +import time import types +from datetime import datetime, timezone from types import SimpleNamespace from typing import Final from unittest.mock import MagicMock @@ -9,13 +11,18 @@ from unittest.mock import MagicMock import pytest import litellm +from litellm import Router from litellm.constants import ( + BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY, BEDROCK_REALTIME_SDK_SUPPORTED_RANGE, + BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) +from litellm.litellm_core_utils.realtime_errors import websocket_close_reason from litellm.llms.bedrock.common_utils import BedrockError -from litellm.llms.bedrock.realtime.handler import BedrockRealtime +from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals, raise_if_session_committed +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig from litellm.realtime_api import main as realtime_main from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault @@ -36,10 +43,16 @@ def _isolate_host_aws_config(monkeypatch, tmp_path): monkeypatch.delenv(env_var, raising=False) -def _closing_input() -> asyncio.Event: - closing: Final = asyncio.Event() - closing.set() - return closing +def _closing_input() -> SessionEndSignals: + signals: Final = SessionEndSignals() + signals.input_closing.set() + return signals + + +def _spend_row(logging_obj): + now: Final = datetime.now(timezone.utc) + row: Final = get_logging_payload(kwargs=logging_obj.model_call_details, response_obj=None, start_time=now, end_time=now) + return row["status"], json.loads(row["metadata"])["error_information"] class FakePayloadPart: @@ -484,7 +497,7 @@ class TestBedrockRealtimeHandler: ) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] @@ -500,7 +513,7 @@ class TestBedrockRealtimeHandler: stream = FakeBedrockStream() await handler._forward_client_to_bedrock( - DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) assert stream.input_stream.sent == [] @@ -517,7 +530,7 @@ class TestBedrockRealtimeHandler: with pytest.raises(RuntimeError, match="bedrock send failed"): await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) assert stream.input_stream.closed @@ -533,7 +546,7 @@ class TestBedrockRealtimeHandler: await asyncio.wait_for( handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ), 2, ) @@ -550,7 +563,7 @@ class TestBedrockRealtimeHandler: ) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] @@ -580,7 +593,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, - input_closing=_closing_input(), + signals=_closing_input(), ) ] @@ -612,7 +625,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, - input_closing=_closing_input(), + signals=_closing_input(), ) ] @@ -684,7 +697,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", MagicMock(), {}, - input_closing=_closing_input(), + signals=_closing_input(), ): pass @@ -745,7 +758,7 @@ class TestBedrockRealtimeSessionLifecycle: ) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals() ) acked = [json.loads(message) for message in client_ws.sent_to_client] @@ -761,7 +774,7 @@ class TestBedrockRealtimeSessionLifecycle: client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})]) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) assert client_ws.sent_to_client == [] @@ -786,7 +799,7 @@ class TestBedrockRealtimeSessionLifecycle: client_ws = DisconnectingClientWS([update([tool]), update([other_tool]), update([tool])]) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals() ) acked: Final = [json.loads(message) for message in client_ws.sent_to_client] @@ -942,9 +955,6 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] - flushed = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] - assert [next(iter(event["event"])) for event in flushed][-2:] == ["promptEnd", "sessionEnd"] - assert stream.input_stream.closed with pytest.raises(BedrockError) as replay: await handler.async_realtime( @@ -1046,12 +1056,14 @@ class TestBedrockRealtimeProviderFailurePropagation: websocket: Final = ConnectedClientWS([self.SESSION_UPDATE]) stream: Final = ScriptedBedrockStream(self.TEXT_TURN) stub_aws_sdk_client["streams"] = [stream] + logging_obj: Final = spend_dispatch["logging_obj"] + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed-key"}} with pytest.raises(BedrockError) as failure: await handler.async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, - logging_obj=spend_dispatch["logging_obj"], + logging_obj=logging_obj, **self.AWS_PARAMS, ) @@ -1060,18 +1072,93 @@ class TestBedrockRealtimeProviderFailurePropagation: assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] - assert _sent_event_names(stream.input_stream)[-2:] == ["promptEnd", "sessionEnd"] + status, error_information = _spend_row(logging_obj) + assert status == "failure", "the Logs page must show the dropped session as failed while it bills its turn" + assert error_information["error_code"] == "503" + assert error_information["error_message"] == failure.value.message with pytest.raises(BedrockError) as replay: await handler.async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, - logging_obj=spend_dispatch["logging_obj"], + logging_obj=logging_obj, **self.AWS_PARAMS, ) assert replay.value.status_code == 400, "a router retry must not restart the dropped session" - assert failure.value.message in replay.value.message + assert str(replay.value).startswith(failure.value.message), "the app's error event leads with the drop" + assert websocket_close_reason(str(replay.value), fallback="Internal server error").startswith( + failure.value.message + ), "the length-capped 1011 close reason must still carry the drop" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_failed_bedrock_output_reaches_the_app_without_waiting_on_the_input_close( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type) + stream.input_stream = HangingCloseInputStream() + stub_aws_sdk_client["streams"] = [stream] + started: Final = time.monotonic() + + with pytest.raises(BedrockError) as failure: + await asyncio.wait_for( + BedrockRealtime(close_timeout=30).async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ), + timeout=5, + ) + + assert failure.value.status_code == status_code + assert time.monotonic() - started < 1, "the app must not wait on closing a Bedrock input that already failed" + assert stub_aws_sdk_client["client_closed"], "closing the SDK client still tears the dead stream down" + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + + @pytest.mark.asyncio + async def test_replay_refusals_do_not_count_toward_deployment_cooldown(self): + router: Final = Router( + model_list=[ + { + "model_name": "nova-sonic", + "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("sonic-a", "sonic-b") + ] + ) + with pytest.raises(BedrockError) as refusal: + raise_if_session_committed( + { + BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: True, + BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: "Bedrock realtime stream ended before the session closed", + } + ) + drop: Final = BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed") + + def cooled_down_after(exception: Exception, deployment_id: str) -> bool: + return router.deployment_callback_on_failure( + kwargs={ + "exception": exception, + "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": deployment_id}}, + }, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + + refusals_then_drop: Final = [cooled_down_after(error, "sonic-a") for error in (*[refusal.value] * 4, drop)] + drops_only: Final = [cooled_down_after(drop, "sonic-b") for _ in range(5)] + + assert not any(refusals_then_drop), "one drop retried four times is one deployment failure, not five" + assert drops_only[-1], "five real drops in a minute still cool the deployment down" @pytest.mark.asyncio async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( @@ -1080,6 +1167,9 @@ class TestBedrockRealtimeProviderFailurePropagation: stream: Final = SessionBoundBedrockStream(self.TEXT_TURN, input_stream_type=SlowClosingInputStream) stub_aws_sdk_client["streams"] = [stream] websocket: Final = RealtimeClientWS([self.SESSION_UPDATE]) + spend_dispatch["logging_obj"].model_call_details["litellm_params"] = { + "litellm_metadata": {"user_api_key": "hashed-key"} + } await asyncio.wait_for( BedrockRealtime(close_timeout=30).async_realtime( @@ -1097,6 +1187,7 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + assert _spend_row(spend_dispatch["logging_obj"]) == ("success", None) @pytest.mark.asyncio async def test_bedrock_output_ending_after_the_input_side_failed_still_fails_the_session( @@ -1167,7 +1258,7 @@ class TestBedrockRealtimeProviderFailurePropagation: "amazon.nova-sonic-v1:0", {}, FakeLogging(), - input_closing=asyncio.Event(), + signals=SessionEndSignals(), ) assert client_ws.sent_to_client == [] @@ -1313,7 +1404,6 @@ class TestBedrockRealtimeSdkLifecycle: ) assert stub_aws_sdk_client["client_closed"] - assert stub_aws_sdk_client["input_closed_before_client_close"] @pytest.mark.asyncio async def test_client_without_close_completes_session(self, monkeypatch): diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 513136ae150..3edb5f74454 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -538,6 +538,68 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( assert await router.cache.async_get_cache(key=tpm_key) == 17 +@pytest.mark.asyncio +@pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)]) +async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served( + clean_sessions: int, cooled_down: bool +) -> None: + import time + + router: Final = Router( + model_list=[ + { + "model_name": "nova-sonic", + "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0", "order": order}, + "model_info": {"id": deployment_id}, + } + for order, deployment_id in ((1, "sonic-a"), (2, "sonic-b")) + ], + num_retries=0, + ) + drop: Final = litellm.ServiceUnavailableError( + message="Bedrock realtime stream ended before the session closed", + llm_provider="bedrock", + model="amazon.nova-sonic-v1:0", + ) + + def realtime_session(failure: Exception | None): + async def _arealtime(**kwargs: object) -> None: + await router.deployment_callback_on_success( + kwargs={ + "call_type": "_arealtime", + "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": kwargs["model_info"]}, + "standard_logging_object": create_standard_logging_payload(), + }, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + if failure is not None: + raise failure + + return _arealtime + + for _ in range(clean_sessions): + await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(None)) + + async def drop_session() -> bool: + with pytest.raises(litellm.ServiceUnavailableError): + await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(drop)) + return router.deployment_callback_on_failure( + kwargs={ + "exception": drop, + "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": "sonic-a"}}, + }, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + + drop_cooldowns: Final = [await drop_session() for _ in range(5)] + + assert any(drop_cooldowns) is cooled_down + + async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]: routed: Final[dict[str, object]] = {}