fix(bedrock): fail a dropped Nova Sonic session right away and log it as failed

When Bedrock's output side ends mid-session, by a drop or a modeled stream error, the client side no longer spends the close timeout sending close messages into the dead input stream, so the error reaches the app right after the drop. A session the app ends itself still sends Bedrock its close messages and closes the input

The replay refusal now leads with the committed cause, so the error event and the cut 1011 close reason both start with it. It stays a 400 and is tagged as a failure outside the selected deployment, so router retries and fallbacks of a dropped session no longer add extra failure counts

A clean realtime session now counts once toward its deployment's cooldown success counter, so cooldown reads the real failure rate instead of 100 percent. Realtime tpm and rpm counting is unchanged

The spend row of a session the provider ended now reads failure with its error information, while the events already forwarded are still billed. A session the app ended keeps logging success
This commit is contained in:
mateo-berri 2026-10-29 21:42:11 -07:00
parent f05d6129df
commit 8a3248de0d
4 changed files with 283 additions and 65 deletions

View file

@ -9,11 +9,12 @@ import contextlib
import importlib.metadata
import json
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import Final, NoReturn, Protocol, runtime_checkable
from pydantic import JsonValue, TypeAdapter
from typing_extensions import TypeIs # noqa: TID251 # narrows request metadata without copying it
import litellm
from litellm._logging import redact_string, verbose_proxy_logger
@ -34,6 +35,7 @@ from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeE
from litellm.types.llms.bedrock import AwsAuthParams
from litellm.types.llms.openai import OpenAIRealtimeEvents
from litellm.types.realtime import RealtimeResponseTransformInput
from litellm.types.utils import StandardLoggingPayloadErrorInformation
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
from ..common_utils import BedrockError
@ -165,6 +167,12 @@ class _BridgeOutcome:
client_disconnected: bool
@dataclass(frozen=True, slots=True)
class SessionEndSignals:
input_closing: asyncio.Event = field(default_factory=asyncio.Event)
output_failed: asyncio.Event = field(default_factory=asyncio.Event)
async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message: str | None) -> AsyncIterator[str]:
if initial_message is not None:
yield initial_message
@ -184,14 +192,18 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None:
"""A fallback attempt on a websocket whose Bedrock stream already took input would silently start over."""
if scope is None or not _is_session_committed(scope):
return
from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure
committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY)
raise BedrockError(
refusal: Final = BedrockError(
status_code=400,
message=(
"Bedrock realtime session already committed to a provider stream; it cannot be replayed"
+ (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
(f"{committed_failure}. " if committed_failure else "")
+ "Bedrock realtime session already committed to a provider stream; it cannot be replayed"
),
)
mark_deployment_selection_failure(refusal)
raise refusal
def _pending_opening_message(scope: Mapping[str, object]) -> str | None:
@ -201,6 +213,36 @@ def _pending_opening_message(scope: Mapping[str, object]) -> str | None:
return pending if isinstance(pending, str) else None
def _is_str_keyed_dict(
value: object,
) -> TypeIs[dict[str, object]]: # guard-ok: request kwargs have str keys # mutable-ok: the spend row reads it
return isinstance(value, dict)
def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseException) -> None:
"""The spend row of a session the provider ended reads failure while it still bills the forwarded events."""
litellm_params: Final[object] = logging_obj.model_call_details.get("litellm_params")
metadata_candidates: Final = (
(litellm_params.get("litellm_metadata"), litellm_params.get("metadata"))
if _is_str_keyed_dict(litellm_params)
else ()
)
spend_log_metadata: Final = next(
(metadata for metadata in metadata_candidates if _is_str_keyed_dict(metadata) and metadata), None
)
if spend_log_metadata is None:
return
status_code: Final[object] = getattr(error, "status_code", None)
error_information: Final[StandardLoggingPayloadErrorInformation] = {
"error_code": "" if status_code is None else str(status_code),
"error_class": type(error).__name__,
"error_message": redact_string(str(error)),
"llm_provider": "bedrock",
}
spend_log_metadata["status"] = "failure"
spend_log_metadata["error_information"] = error_information
def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn:
error: Final = _as_bedrock_error(failure)
verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error)))
@ -279,6 +321,25 @@ async def _reject_session_update(
return True
async def _close_bedrock_input(
bedrock_stream: BedrockBidirectionalStream,
close_messages: Sequence[str],
send_to_bedrock: Callable[[str], Awaitable[None]],
timeout: float,
) -> None:
async def flush_and_close() -> None:
for close_message in close_messages:
with contextlib.suppress(Exception):
await send_to_bedrock(close_message)
with contextlib.suppress(Exception):
await bedrock_stream.input_stream.close()
try:
await asyncio.wait_for(flush_and_close(), timeout)
except (asyncio.TimeoutError, TimeoutError):
verbose_proxy_logger.debug("Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", timeout)
class BedrockRealtime(BaseAWSLLM):
"""Handler for Bedrock Nova Sonic realtime speech-to-speech API."""
@ -445,7 +506,14 @@ class BedrockRealtime(BaseAWSLLM):
if _should_log_event(leftover_event)
),
)
if outcome.provider_failure is not None and outcome.client_disconnected:
verbose_proxy_logger.debug(
"Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
)
provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure
if logged_events:
if provider_failure is not None:
_record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure))
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
logging_obj.dispatch_success_handlers(
list(logged_events),
@ -454,14 +522,9 @@ class BedrockRealtime(BaseAWSLLM):
)
logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True
if outcome.provider_failure is None:
if provider_failure is None:
return
if outcome.client_disconnected:
verbose_proxy_logger.debug(
"Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
)
return
_raise_provider_failure(websocket.scope, outcome.provider_failure)
_raise_provider_failure(websocket.scope, provider_failure)
async def _bridge(
self,
@ -475,19 +538,23 @@ class BedrockRealtime(BaseAWSLLM):
) -> _BridgeOutcome:
"""Run both forwarding directions until the client leaves or either side fails."""
logged: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events forwarded before a failure are still spend
input_closing: Final = asyncio.Event()
signals: Final = SessionEndSignals()
async def collect_logged_events() -> None:
async for event in self._forward_bedrock_to_client(
bedrock_stream,
websocket,
transformation_config,
model,
logging_obj,
session_state,
input_closing=input_closing,
):
logged.append(event)
try:
async for event in self._forward_bedrock_to_client(
bedrock_stream,
websocket,
transformation_config,
model,
logging_obj,
session_state,
signals=signals,
):
logged.append(event)
except Exception:
signals.output_failed.set()
raise
client_task: Final = asyncio.create_task(
self._forward_client_to_bedrock(
@ -498,7 +565,7 @@ class BedrockRealtime(BaseAWSLLM):
session_state,
logging_obj,
initial_message,
input_closing=input_closing,
signals=signals,
)
)
bedrock_task: Final = asyncio.create_task(collect_logged_events())
@ -533,7 +600,7 @@ class BedrockRealtime(BaseAWSLLM):
logging_obj: LiteLLMLogging | None = None,
initial_message: str | None = None,
*,
input_closing: asyncio.Event,
signals: SessionEndSignals,
) -> None:
"""Forward messages from client WebSocket to Bedrock stream.
@ -581,21 +648,11 @@ class BedrockRealtime(BaseAWSLLM):
client_ws, transformation_config, model, logging_obj, parsed_client_message
):
break
input_closing.set()
signals.input_closing.set()
finally:
async def close_bedrock_input() -> None:
for close_message in transformation_config.session_close_messages():
with contextlib.suppress(Exception):
await send_to_bedrock(close_message)
with contextlib.suppress(Exception):
await bedrock_stream.input_stream.close()
try:
await asyncio.wait_for(close_bedrock_input(), self._close_timeout)
except (asyncio.TimeoutError, TimeoutError):
verbose_proxy_logger.debug(
"Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", self._close_timeout
if not signals.output_failed.is_set():
await _close_bedrock_input(
bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout
)
async def _forward_bedrock_to_client(
@ -607,7 +664,7 @@ class BedrockRealtime(BaseAWSLLM):
logging_obj: LiteLLMLogging,
session_state: RealtimeResponseTransformInput,
*,
input_closing: asyncio.Event,
signals: SessionEndSignals,
) -> AsyncIterator[OpenAIRealtimeEvents]:
"""Forward messages from Bedrock to the client, yielding the ones to record for spend logging.
@ -628,7 +685,7 @@ class BedrockRealtime(BaseAWSLLM):
result = await output[1].receive()
if result is None:
if not input_closing.is_set():
if not signals.input_closing.is_set():
raise BedrockError(
status_code=503,
message="Bedrock realtime stream ended before the session closed",

View file

@ -6478,6 +6478,15 @@ class Router:
raise safeguard_refusal_error(model=model, stop_details=refusal_details)
self.success_calls[model_name] += 1
realtime_session_deployment_id: Final = (
_MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id")
if generic_function_name == "_arealtime"
else None
)
if realtime_session_deployment_id is not None:
increment_deployment_successes_for_current_minute(
litellm_router_instance=self, deployment_id=str(realtime_session_deployment_id)
)
verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
return response

View file

@ -1,7 +1,9 @@
import asyncio
import json
import sys
import time
import types
from datetime import datetime, timezone
from types import SimpleNamespace
from typing import Final
from unittest.mock import MagicMock
@ -9,13 +11,18 @@ from unittest.mock import MagicMock
import pytest
import litellm
from litellm import Router
from litellm.constants import (
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
WEBSOCKET_CLOSE_REASON_MAX_BYTES,
)
from litellm.litellm_core_utils.realtime_errors import websocket_close_reason
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.realtime.handler import BedrockRealtime
from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals, raise_if_session_committed
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
from litellm.realtime_api import main as realtime_main
from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
@ -36,10 +43,16 @@ def _isolate_host_aws_config(monkeypatch, tmp_path):
monkeypatch.delenv(env_var, raising=False)
def _closing_input() -> asyncio.Event:
closing: Final = asyncio.Event()
closing.set()
return closing
def _closing_input() -> SessionEndSignals:
signals: Final = SessionEndSignals()
signals.input_closing.set()
return signals
def _spend_row(logging_obj):
now: Final = datetime.now(timezone.utc)
row: Final = get_logging_payload(kwargs=logging_obj.model_call_details, response_obj=None, start_time=now, end_time=now)
return row["status"], json.loads(row["metadata"])["error_information"]
class FakePayloadPart:
@ -484,7 +497,7 @@ class TestBedrockRealtimeHandler:
)
await handler._forward_client_to_bedrock(
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
)
sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent]
@ -500,7 +513,7 @@ class TestBedrockRealtimeHandler:
stream = FakeBedrockStream()
await handler._forward_client_to_bedrock(
DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
)
assert stream.input_stream.sent == []
@ -517,7 +530,7 @@ class TestBedrockRealtimeHandler:
with pytest.raises(RuntimeError, match="bedrock send failed"):
await handler._forward_client_to_bedrock(
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
)
assert stream.input_stream.closed
@ -533,7 +546,7 @@ class TestBedrockRealtimeHandler:
await asyncio.wait_for(
handler._forward_client_to_bedrock(
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
),
2,
)
@ -550,7 +563,7 @@ class TestBedrockRealtimeHandler:
)
await handler._forward_client_to_bedrock(
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
)
sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent]
@ -580,7 +593,7 @@ class TestBedrockRealtimeHandler:
"amazon.nova-sonic-v1:0",
FakeLogging(),
{},
input_closing=_closing_input(),
signals=_closing_input(),
)
]
@ -612,7 +625,7 @@ class TestBedrockRealtimeHandler:
"amazon.nova-sonic-v1:0",
FakeLogging(),
{},
input_closing=_closing_input(),
signals=_closing_input(),
)
]
@ -684,7 +697,7 @@ class TestBedrockRealtimeHandler:
"amazon.nova-sonic-v1:0",
MagicMock(),
{},
input_closing=_closing_input(),
signals=_closing_input(),
):
pass
@ -745,7 +758,7 @@ class TestBedrockRealtimeSessionLifecycle:
)
await handler._forward_client_to_bedrock(
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event()
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals()
)
acked = [json.loads(message) for message in client_ws.sent_to_client]
@ -761,7 +774,7 @@ class TestBedrockRealtimeSessionLifecycle:
client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})])
await handler._forward_client_to_bedrock(
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
)
assert client_ws.sent_to_client == []
@ -786,7 +799,7 @@ class TestBedrockRealtimeSessionLifecycle:
client_ws = DisconnectingClientWS([update([tool]), update([other_tool]), update([tool])])
await handler._forward_client_to_bedrock(
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event()
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals()
)
acked: Final = [json.loads(message) for message in client_ws.sent_to_client]
@ -942,9 +955,6 @@ class TestBedrockRealtimeProviderFailurePropagation:
await spend_dispatch["coro"]
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client]
flushed = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent]
assert [next(iter(event["event"])) for event in flushed][-2:] == ["promptEnd", "sessionEnd"]
assert stream.input_stream.closed
with pytest.raises(BedrockError) as replay:
await handler.async_realtime(
@ -1046,12 +1056,14 @@ class TestBedrockRealtimeProviderFailurePropagation:
websocket: Final = ConnectedClientWS([self.SESSION_UPDATE])
stream: Final = ScriptedBedrockStream(self.TEXT_TURN)
stub_aws_sdk_client["streams"] = [stream]
logging_obj: Final = spend_dispatch["logging_obj"]
logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed-key"}}
with pytest.raises(BedrockError) as failure:
await handler.async_realtime(
model="amazon.nova-sonic-v1:0",
websocket=websocket,
logging_obj=spend_dispatch["logging_obj"],
logging_obj=logging_obj,
**self.AWS_PARAMS,
)
@ -1060,18 +1072,93 @@ class TestBedrockRealtimeProviderFailurePropagation:
assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client]
await spend_dispatch["coro"]
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
assert _sent_event_names(stream.input_stream)[-2:] == ["promptEnd", "sessionEnd"]
status, error_information = _spend_row(logging_obj)
assert status == "failure", "the Logs page must show the dropped session as failed while it bills its turn"
assert error_information["error_code"] == "503"
assert error_information["error_message"] == failure.value.message
with pytest.raises(BedrockError) as replay:
await handler.async_realtime(
model="amazon.nova-sonic-v1:0",
websocket=websocket,
logging_obj=spend_dispatch["logging_obj"],
logging_obj=logging_obj,
**self.AWS_PARAMS,
)
assert replay.value.status_code == 400, "a router retry must not restart the dropped session"
assert failure.value.message in replay.value.message
assert str(replay.value).startswith(failure.value.message), "the app's error event leads with the drop"
assert websocket_close_reason(str(replay.value), fallback="Internal server error").startswith(
failure.value.message
), "the length-capped 1011 close reason must still carry the drop"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("receiver_type", "status_code"),
[(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)],
ids=["output-ends", "output-raises"],
)
async def test_a_failed_bedrock_output_reaches_the_app_without_waiting_on_the_input_close(
self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int
):
stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type)
stream.input_stream = HangingCloseInputStream()
stub_aws_sdk_client["streams"] = [stream]
started: Final = time.monotonic()
with pytest.raises(BedrockError) as failure:
await asyncio.wait_for(
BedrockRealtime(close_timeout=30).async_realtime(
model="amazon.nova-sonic-v1:0",
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
logging_obj=spend_dispatch["logging_obj"],
**self.AWS_PARAMS,
),
timeout=5,
)
assert failure.value.status_code == status_code
assert time.monotonic() - started < 1, "the app must not wait on closing a Bedrock input that already failed"
assert stub_aws_sdk_client["client_closed"], "closing the SDK client still tears the dead stream down"
await spend_dispatch["coro"]
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
@pytest.mark.asyncio
async def test_replay_refusals_do_not_count_toward_deployment_cooldown(self):
router: Final = Router(
model_list=[
{
"model_name": "nova-sonic",
"litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0"},
"model_info": {"id": deployment_id},
}
for deployment_id in ("sonic-a", "sonic-b")
]
)
with pytest.raises(BedrockError) as refusal:
raise_if_session_committed(
{
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: True,
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: "Bedrock realtime stream ended before the session closed",
}
)
drop: Final = BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed")
def cooled_down_after(exception: Exception, deployment_id: str) -> bool:
return router.deployment_callback_on_failure(
kwargs={
"exception": exception,
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": deployment_id}},
},
completion_response=None,
start_time=time.time(),
end_time=time.time(),
)
refusals_then_drop: Final = [cooled_down_after(error, "sonic-a") for error in (*[refusal.value] * 4, drop)]
drops_only: Final = [cooled_down_after(drop, "sonic-b") for _ in range(5)]
assert not any(refusals_then_drop), "one drop retried four times is one deployment failure, not five"
assert drops_only[-1], "five real drops in a minute still cool the deployment down"
@pytest.mark.asyncio
async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish(
@ -1080,6 +1167,9 @@ class TestBedrockRealtimeProviderFailurePropagation:
stream: Final = SessionBoundBedrockStream(self.TEXT_TURN, input_stream_type=SlowClosingInputStream)
stub_aws_sdk_client["streams"] = [stream]
websocket: Final = RealtimeClientWS([self.SESSION_UPDATE])
spend_dispatch["logging_obj"].model_call_details["litellm_params"] = {
"litellm_metadata": {"user_api_key": "hashed-key"}
}
await asyncio.wait_for(
BedrockRealtime(close_timeout=30).async_realtime(
@ -1097,6 +1187,7 @@ class TestBedrockRealtimeProviderFailurePropagation:
await spend_dispatch["coro"]
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True
assert _spend_row(spend_dispatch["logging_obj"]) == ("success", None)
@pytest.mark.asyncio
async def test_bedrock_output_ending_after_the_input_side_failed_still_fails_the_session(
@ -1167,7 +1258,7 @@ class TestBedrockRealtimeProviderFailurePropagation:
"amazon.nova-sonic-v1:0",
{},
FakeLogging(),
input_closing=asyncio.Event(),
signals=SessionEndSignals(),
)
assert client_ws.sent_to_client == []
@ -1313,7 +1404,6 @@ class TestBedrockRealtimeSdkLifecycle:
)
assert stub_aws_sdk_client["client_closed"]
assert stub_aws_sdk_client["input_closed_before_client_close"]
@pytest.mark.asyncio
async def test_client_without_close_completes_session(self, monkeypatch):

View file

@ -538,6 +538,68 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn(
assert await router.cache.async_get_cache(key=tpm_key) == 17
@pytest.mark.asyncio
@pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)])
async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served(
clean_sessions: int, cooled_down: bool
) -> None:
import time
router: Final = Router(
model_list=[
{
"model_name": "nova-sonic",
"litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0", "order": order},
"model_info": {"id": deployment_id},
}
for order, deployment_id in ((1, "sonic-a"), (2, "sonic-b"))
],
num_retries=0,
)
drop: Final = litellm.ServiceUnavailableError(
message="Bedrock realtime stream ended before the session closed",
llm_provider="bedrock",
model="amazon.nova-sonic-v1:0",
)
def realtime_session(failure: Exception | None):
async def _arealtime(**kwargs: object) -> None:
await router.deployment_callback_on_success(
kwargs={
"call_type": "_arealtime",
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": kwargs["model_info"]},
"standard_logging_object": create_standard_logging_payload(),
},
completion_response=None,
start_time=time.time(),
end_time=time.time(),
)
if failure is not None:
raise failure
return _arealtime
for _ in range(clean_sessions):
await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(None))
async def drop_session() -> bool:
with pytest.raises(litellm.ServiceUnavailableError):
await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(drop))
return router.deployment_callback_on_failure(
kwargs={
"exception": drop,
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": "sonic-a"}},
},
completion_response=None,
start_time=time.time(),
end_time=time.time(),
)
drop_cooldowns: Final = [await drop_session() for _ in range(5)]
assert any(drop_cooldowns) is cooled_down
async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]:
routed: Final[dict[str, object]] = {}