diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 9f5c56fd106..a62122ee5ee 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -9,11 +9,12 @@ import contextlib import importlib.metadata import json from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from types import MappingProxyType from typing import Final, NoReturn, Protocol, runtime_checkable from pydantic import JsonValue, TypeAdapter +from typing_extensions import TypeIs # noqa: TID251 # narrows request metadata without copying it import litellm from litellm._logging import redact_string, verbose_proxy_logger @@ -34,6 +35,7 @@ from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeE from litellm.types.llms.bedrock import AwsAuthParams from litellm.types.llms.openai import OpenAIRealtimeEvents from litellm.types.realtime import RealtimeResponseTransformInput +from litellm.types.utils import StandardLoggingPayloadErrorInformation from ..base_aws_llm import BaseAWSLLM, run_aws_signing from ..common_utils import BedrockError @@ -165,6 +167,12 @@ class _BridgeOutcome: client_disconnected: bool +@dataclass(frozen=True, slots=True) +class SessionEndSignals: + input_closing: asyncio.Event = field(default_factory=asyncio.Event) + output_failed: asyncio.Event = field(default_factory=asyncio.Event) + + async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message: str | None) -> AsyncIterator[str]: if initial_message is not None: yield initial_message @@ -184,14 +192,18 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None: """A fallback attempt on a websocket whose Bedrock stream already took input would silently start over.""" if scope is None or not _is_session_committed(scope): return + from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure + committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY) - raise BedrockError( + refusal: Final = BedrockError( status_code=400, message=( - "Bedrock realtime session already committed to a provider stream; it cannot be replayed" - + (f". The committed stream failed with: {committed_failure}" if committed_failure else "") + (f"{committed_failure}. " if committed_failure else "") + + "Bedrock realtime session already committed to a provider stream; it cannot be replayed" ), ) + mark_deployment_selection_failure(refusal) + raise refusal def _pending_opening_message(scope: Mapping[str, object]) -> str | None: @@ -201,6 +213,36 @@ def _pending_opening_message(scope: Mapping[str, object]) -> str | None: return pending if isinstance(pending, str) else None +def _is_str_keyed_dict( + value: object, +) -> TypeIs[dict[str, object]]: # guard-ok: request kwargs have str keys # mutable-ok: the spend row reads it + return isinstance(value, dict) + + +def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseException) -> None: + """The spend row of a session the provider ended reads failure while it still bills the forwarded events.""" + litellm_params: Final[object] = logging_obj.model_call_details.get("litellm_params") + metadata_candidates: Final = ( + (litellm_params.get("litellm_metadata"), litellm_params.get("metadata")) + if _is_str_keyed_dict(litellm_params) + else () + ) + spend_log_metadata: Final = next( + (metadata for metadata in metadata_candidates if _is_str_keyed_dict(metadata) and metadata), None + ) + if spend_log_metadata is None: + return + status_code: Final[object] = getattr(error, "status_code", None) + error_information: Final[StandardLoggingPayloadErrorInformation] = { + "error_code": "" if status_code is None else str(status_code), + "error_class": type(error).__name__, + "error_message": redact_string(str(error)), + "llm_provider": "bedrock", + } + spend_log_metadata["status"] = "failure" + spend_log_metadata["error_information"] = error_information + + def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn: error: Final = _as_bedrock_error(failure) verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error))) @@ -279,6 +321,25 @@ async def _reject_session_update( return True +async def _close_bedrock_input( + bedrock_stream: BedrockBidirectionalStream, + close_messages: Sequence[str], + send_to_bedrock: Callable[[str], Awaitable[None]], + timeout: float, +) -> None: + async def flush_and_close() -> None: + for close_message in close_messages: + with contextlib.suppress(Exception): + await send_to_bedrock(close_message) + with contextlib.suppress(Exception): + await bedrock_stream.input_stream.close() + + try: + await asyncio.wait_for(flush_and_close(), timeout) + except (asyncio.TimeoutError, TimeoutError): + verbose_proxy_logger.debug("Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", timeout) + + class BedrockRealtime(BaseAWSLLM): """Handler for Bedrock Nova Sonic realtime speech-to-speech API.""" @@ -445,7 +506,14 @@ class BedrockRealtime(BaseAWSLLM): if _should_log_event(leftover_event) ), ) + if outcome.provider_failure is not None and outcome.client_disconnected: + verbose_proxy_logger.debug( + "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure + ) + provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure if logged_events: + if provider_failure is not None: + _record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure)) GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( logging_obj.dispatch_success_handlers( list(logged_events), @@ -454,14 +522,9 @@ class BedrockRealtime(BaseAWSLLM): ) logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True - if outcome.provider_failure is None: + if provider_failure is None: return - if outcome.client_disconnected: - verbose_proxy_logger.debug( - "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure - ) - return - _raise_provider_failure(websocket.scope, outcome.provider_failure) + _raise_provider_failure(websocket.scope, provider_failure) async def _bridge( self, @@ -475,19 +538,23 @@ class BedrockRealtime(BaseAWSLLM): ) -> _BridgeOutcome: """Run both forwarding directions until the client leaves or either side fails.""" logged: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events forwarded before a failure are still spend - input_closing: Final = asyncio.Event() + signals: Final = SessionEndSignals() async def collect_logged_events() -> None: - async for event in self._forward_bedrock_to_client( - bedrock_stream, - websocket, - transformation_config, - model, - logging_obj, - session_state, - input_closing=input_closing, - ): - logged.append(event) + try: + async for event in self._forward_bedrock_to_client( + bedrock_stream, + websocket, + transformation_config, + model, + logging_obj, + session_state, + signals=signals, + ): + logged.append(event) + except Exception: + signals.output_failed.set() + raise client_task: Final = asyncio.create_task( self._forward_client_to_bedrock( @@ -498,7 +565,7 @@ class BedrockRealtime(BaseAWSLLM): session_state, logging_obj, initial_message, - input_closing=input_closing, + signals=signals, ) ) bedrock_task: Final = asyncio.create_task(collect_logged_events()) @@ -533,7 +600,7 @@ class BedrockRealtime(BaseAWSLLM): logging_obj: LiteLLMLogging | None = None, initial_message: str | None = None, *, - input_closing: asyncio.Event, + signals: SessionEndSignals, ) -> None: """Forward messages from client WebSocket to Bedrock stream. @@ -581,21 +648,11 @@ class BedrockRealtime(BaseAWSLLM): client_ws, transformation_config, model, logging_obj, parsed_client_message ): break - input_closing.set() + signals.input_closing.set() finally: - - async def close_bedrock_input() -> None: - for close_message in transformation_config.session_close_messages(): - with contextlib.suppress(Exception): - await send_to_bedrock(close_message) - with contextlib.suppress(Exception): - await bedrock_stream.input_stream.close() - - try: - await asyncio.wait_for(close_bedrock_input(), self._close_timeout) - except (asyncio.TimeoutError, TimeoutError): - verbose_proxy_logger.debug( - "Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", self._close_timeout + if not signals.output_failed.is_set(): + await _close_bedrock_input( + bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout ) async def _forward_bedrock_to_client( @@ -607,7 +664,7 @@ class BedrockRealtime(BaseAWSLLM): logging_obj: LiteLLMLogging, session_state: RealtimeResponseTransformInput, *, - input_closing: asyncio.Event, + signals: SessionEndSignals, ) -> AsyncIterator[OpenAIRealtimeEvents]: """Forward messages from Bedrock to the client, yielding the ones to record for spend logging. @@ -628,7 +685,7 @@ class BedrockRealtime(BaseAWSLLM): result = await output[1].receive() if result is None: - if not input_closing.is_set(): + if not signals.input_closing.is_set(): raise BedrockError( status_code=503, message="Bedrock realtime stream ended before the session closed", diff --git a/litellm/router.py b/litellm/router.py index 02ad6275b82..0dc798be380 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6478,6 +6478,15 @@ class Router: raise safeguard_refusal_error(model=model, stop_details=refusal_details) self.success_calls[model_name] += 1 + realtime_session_deployment_id: Final = ( + _MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id") + if generic_function_name == "_arealtime" + else None + ) + if realtime_session_deployment_id is not None: + increment_deployment_successes_for_current_minute( + litellm_router_instance=self, deployment_id=str(realtime_session_deployment_id) + ) verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 329ea9dd336..9767f294af7 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -1,7 +1,9 @@ import asyncio import json import sys +import time import types +from datetime import datetime, timezone from types import SimpleNamespace from typing import Final from unittest.mock import MagicMock @@ -9,13 +11,18 @@ from unittest.mock import MagicMock import pytest import litellm +from litellm import Router from litellm.constants import ( + BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY, BEDROCK_REALTIME_SDK_SUPPORTED_RANGE, + BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) +from litellm.litellm_core_utils.realtime_errors import websocket_close_reason from litellm.llms.bedrock.common_utils import BedrockError -from litellm.llms.bedrock.realtime.handler import BedrockRealtime +from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals, raise_if_session_committed +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig from litellm.realtime_api import main as realtime_main from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault @@ -36,10 +43,16 @@ def _isolate_host_aws_config(monkeypatch, tmp_path): monkeypatch.delenv(env_var, raising=False) -def _closing_input() -> asyncio.Event: - closing: Final = asyncio.Event() - closing.set() - return closing +def _closing_input() -> SessionEndSignals: + signals: Final = SessionEndSignals() + signals.input_closing.set() + return signals + + +def _spend_row(logging_obj): + now: Final = datetime.now(timezone.utc) + row: Final = get_logging_payload(kwargs=logging_obj.model_call_details, response_obj=None, start_time=now, end_time=now) + return row["status"], json.loads(row["metadata"])["error_information"] class FakePayloadPart: @@ -484,7 +497,7 @@ class TestBedrockRealtimeHandler: ) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] @@ -500,7 +513,7 @@ class TestBedrockRealtimeHandler: stream = FakeBedrockStream() await handler._forward_client_to_bedrock( - DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) assert stream.input_stream.sent == [] @@ -517,7 +530,7 @@ class TestBedrockRealtimeHandler: with pytest.raises(RuntimeError, match="bedrock send failed"): await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) assert stream.input_stream.closed @@ -533,7 +546,7 @@ class TestBedrockRealtimeHandler: await asyncio.wait_for( handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ), 2, ) @@ -550,7 +563,7 @@ class TestBedrockRealtimeHandler: ) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] @@ -580,7 +593,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, - input_closing=_closing_input(), + signals=_closing_input(), ) ] @@ -612,7 +625,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, - input_closing=_closing_input(), + signals=_closing_input(), ) ] @@ -684,7 +697,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", MagicMock(), {}, - input_closing=_closing_input(), + signals=_closing_input(), ): pass @@ -745,7 +758,7 @@ class TestBedrockRealtimeSessionLifecycle: ) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals() ) acked = [json.loads(message) for message in client_ws.sent_to_client] @@ -761,7 +774,7 @@ class TestBedrockRealtimeSessionLifecycle: client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})]) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) assert client_ws.sent_to_client == [] @@ -786,7 +799,7 @@ class TestBedrockRealtimeSessionLifecycle: client_ws = DisconnectingClientWS([update([tool]), update([other_tool]), update([tool])]) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals() ) acked: Final = [json.loads(message) for message in client_ws.sent_to_client] @@ -942,9 +955,6 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] - flushed = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] - assert [next(iter(event["event"])) for event in flushed][-2:] == ["promptEnd", "sessionEnd"] - assert stream.input_stream.closed with pytest.raises(BedrockError) as replay: await handler.async_realtime( @@ -1046,12 +1056,14 @@ class TestBedrockRealtimeProviderFailurePropagation: websocket: Final = ConnectedClientWS([self.SESSION_UPDATE]) stream: Final = ScriptedBedrockStream(self.TEXT_TURN) stub_aws_sdk_client["streams"] = [stream] + logging_obj: Final = spend_dispatch["logging_obj"] + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed-key"}} with pytest.raises(BedrockError) as failure: await handler.async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, - logging_obj=spend_dispatch["logging_obj"], + logging_obj=logging_obj, **self.AWS_PARAMS, ) @@ -1060,18 +1072,93 @@ class TestBedrockRealtimeProviderFailurePropagation: assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] - assert _sent_event_names(stream.input_stream)[-2:] == ["promptEnd", "sessionEnd"] + status, error_information = _spend_row(logging_obj) + assert status == "failure", "the Logs page must show the dropped session as failed while it bills its turn" + assert error_information["error_code"] == "503" + assert error_information["error_message"] == failure.value.message with pytest.raises(BedrockError) as replay: await handler.async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, - logging_obj=spend_dispatch["logging_obj"], + logging_obj=logging_obj, **self.AWS_PARAMS, ) assert replay.value.status_code == 400, "a router retry must not restart the dropped session" - assert failure.value.message in replay.value.message + assert str(replay.value).startswith(failure.value.message), "the app's error event leads with the drop" + assert websocket_close_reason(str(replay.value), fallback="Internal server error").startswith( + failure.value.message + ), "the length-capped 1011 close reason must still carry the drop" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_failed_bedrock_output_reaches_the_app_without_waiting_on_the_input_close( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type) + stream.input_stream = HangingCloseInputStream() + stub_aws_sdk_client["streams"] = [stream] + started: Final = time.monotonic() + + with pytest.raises(BedrockError) as failure: + await asyncio.wait_for( + BedrockRealtime(close_timeout=30).async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ), + timeout=5, + ) + + assert failure.value.status_code == status_code + assert time.monotonic() - started < 1, "the app must not wait on closing a Bedrock input that already failed" + assert stub_aws_sdk_client["client_closed"], "closing the SDK client still tears the dead stream down" + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + + @pytest.mark.asyncio + async def test_replay_refusals_do_not_count_toward_deployment_cooldown(self): + router: Final = Router( + model_list=[ + { + "model_name": "nova-sonic", + "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("sonic-a", "sonic-b") + ] + ) + with pytest.raises(BedrockError) as refusal: + raise_if_session_committed( + { + BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: True, + BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: "Bedrock realtime stream ended before the session closed", + } + ) + drop: Final = BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed") + + def cooled_down_after(exception: Exception, deployment_id: str) -> bool: + return router.deployment_callback_on_failure( + kwargs={ + "exception": exception, + "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": deployment_id}}, + }, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + + refusals_then_drop: Final = [cooled_down_after(error, "sonic-a") for error in (*[refusal.value] * 4, drop)] + drops_only: Final = [cooled_down_after(drop, "sonic-b") for _ in range(5)] + + assert not any(refusals_then_drop), "one drop retried four times is one deployment failure, not five" + assert drops_only[-1], "five real drops in a minute still cool the deployment down" @pytest.mark.asyncio async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( @@ -1080,6 +1167,9 @@ class TestBedrockRealtimeProviderFailurePropagation: stream: Final = SessionBoundBedrockStream(self.TEXT_TURN, input_stream_type=SlowClosingInputStream) stub_aws_sdk_client["streams"] = [stream] websocket: Final = RealtimeClientWS([self.SESSION_UPDATE]) + spend_dispatch["logging_obj"].model_call_details["litellm_params"] = { + "litellm_metadata": {"user_api_key": "hashed-key"} + } await asyncio.wait_for( BedrockRealtime(close_timeout=30).async_realtime( @@ -1097,6 +1187,7 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + assert _spend_row(spend_dispatch["logging_obj"]) == ("success", None) @pytest.mark.asyncio async def test_bedrock_output_ending_after_the_input_side_failed_still_fails_the_session( @@ -1167,7 +1258,7 @@ class TestBedrockRealtimeProviderFailurePropagation: "amazon.nova-sonic-v1:0", {}, FakeLogging(), - input_closing=asyncio.Event(), + signals=SessionEndSignals(), ) assert client_ws.sent_to_client == [] @@ -1313,7 +1404,6 @@ class TestBedrockRealtimeSdkLifecycle: ) assert stub_aws_sdk_client["client_closed"] - assert stub_aws_sdk_client["input_closed_before_client_close"] @pytest.mark.asyncio async def test_client_without_close_completes_session(self, monkeypatch): diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 513136ae150..3edb5f74454 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -538,6 +538,68 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( assert await router.cache.async_get_cache(key=tpm_key) == 17 +@pytest.mark.asyncio +@pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)]) +async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served( + clean_sessions: int, cooled_down: bool +) -> None: + import time + + router: Final = Router( + model_list=[ + { + "model_name": "nova-sonic", + "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0", "order": order}, + "model_info": {"id": deployment_id}, + } + for order, deployment_id in ((1, "sonic-a"), (2, "sonic-b")) + ], + num_retries=0, + ) + drop: Final = litellm.ServiceUnavailableError( + message="Bedrock realtime stream ended before the session closed", + llm_provider="bedrock", + model="amazon.nova-sonic-v1:0", + ) + + def realtime_session(failure: Exception | None): + async def _arealtime(**kwargs: object) -> None: + await router.deployment_callback_on_success( + kwargs={ + "call_type": "_arealtime", + "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": kwargs["model_info"]}, + "standard_logging_object": create_standard_logging_payload(), + }, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + if failure is not None: + raise failure + + return _arealtime + + for _ in range(clean_sessions): + await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(None)) + + async def drop_session() -> bool: + with pytest.raises(litellm.ServiceUnavailableError): + await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(drop)) + return router.deployment_callback_on_failure( + kwargs={ + "exception": drop, + "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": "sonic-a"}}, + }, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + + drop_cooldowns: Final = [await drop_session() for _ in range(5)] + + assert any(drop_cooldowns) is cooled_down + + async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]: routed: Final[dict[str, object]] = {}