mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(bedrock): fail a dropped Nova Sonic session right away and log it as failed
When Bedrock's output side ends mid-session, by a drop or a modeled stream error, the client side no longer spends the close timeout sending close messages into the dead input stream, so the error reaches the app right after the drop. A session the app ends itself still sends Bedrock its close messages and closes the input The replay refusal now leads with the committed cause, so the error event and the cut 1011 close reason both start with it. It stays a 400 and is tagged as a failure outside the selected deployment, so router retries and fallbacks of a dropped session no longer add extra failure counts A clean realtime session now counts once toward its deployment's cooldown success counter, so cooldown reads the real failure rate instead of 100 percent. Realtime tpm and rpm counting is unchanged The spend row of a session the provider ended now reads failure with its error information, while the events already forwarded are still billed. A session the app ended keeps logging success
This commit is contained in:
parent
f05d6129df
commit
8a3248de0d
4 changed files with 283 additions and 65 deletions
|
|
@ -9,11 +9,12 @@ import contextlib
|
|||
import importlib.metadata
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import Final, NoReturn, Protocol, runtime_checkable
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from typing_extensions import TypeIs # noqa: TID251 # narrows request metadata without copying it
|
||||
|
||||
import litellm
|
||||
from litellm._logging import redact_string, verbose_proxy_logger
|
||||
|
|
@ -34,6 +35,7 @@ from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeE
|
|||
from litellm.types.llms.bedrock import AwsAuthParams
|
||||
from litellm.types.llms.openai import OpenAIRealtimeEvents
|
||||
from litellm.types.realtime import RealtimeResponseTransformInput
|
||||
from litellm.types.utils import StandardLoggingPayloadErrorInformation
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
|
||||
from ..common_utils import BedrockError
|
||||
|
|
@ -165,6 +167,12 @@ class _BridgeOutcome:
|
|||
client_disconnected: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SessionEndSignals:
|
||||
input_closing: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
output_failed: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
|
||||
|
||||
async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message: str | None) -> AsyncIterator[str]:
|
||||
if initial_message is not None:
|
||||
yield initial_message
|
||||
|
|
@ -184,14 +192,18 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None:
|
|||
"""A fallback attempt on a websocket whose Bedrock stream already took input would silently start over."""
|
||||
if scope is None or not _is_session_committed(scope):
|
||||
return
|
||||
from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure
|
||||
|
||||
committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY)
|
||||
raise BedrockError(
|
||||
refusal: Final = BedrockError(
|
||||
status_code=400,
|
||||
message=(
|
||||
"Bedrock realtime session already committed to a provider stream; it cannot be replayed"
|
||||
+ (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
|
||||
(f"{committed_failure}. " if committed_failure else "")
|
||||
+ "Bedrock realtime session already committed to a provider stream; it cannot be replayed"
|
||||
),
|
||||
)
|
||||
mark_deployment_selection_failure(refusal)
|
||||
raise refusal
|
||||
|
||||
|
||||
def _pending_opening_message(scope: Mapping[str, object]) -> str | None:
|
||||
|
|
@ -201,6 +213,36 @@ def _pending_opening_message(scope: Mapping[str, object]) -> str | None:
|
|||
return pending if isinstance(pending, str) else None
|
||||
|
||||
|
||||
def _is_str_keyed_dict(
|
||||
value: object,
|
||||
) -> TypeIs[dict[str, object]]: # guard-ok: request kwargs have str keys # mutable-ok: the spend row reads it
|
||||
return isinstance(value, dict)
|
||||
|
||||
|
||||
def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseException) -> None:
|
||||
"""The spend row of a session the provider ended reads failure while it still bills the forwarded events."""
|
||||
litellm_params: Final[object] = logging_obj.model_call_details.get("litellm_params")
|
||||
metadata_candidates: Final = (
|
||||
(litellm_params.get("litellm_metadata"), litellm_params.get("metadata"))
|
||||
if _is_str_keyed_dict(litellm_params)
|
||||
else ()
|
||||
)
|
||||
spend_log_metadata: Final = next(
|
||||
(metadata for metadata in metadata_candidates if _is_str_keyed_dict(metadata) and metadata), None
|
||||
)
|
||||
if spend_log_metadata is None:
|
||||
return
|
||||
status_code: Final[object] = getattr(error, "status_code", None)
|
||||
error_information: Final[StandardLoggingPayloadErrorInformation] = {
|
||||
"error_code": "" if status_code is None else str(status_code),
|
||||
"error_class": type(error).__name__,
|
||||
"error_message": redact_string(str(error)),
|
||||
"llm_provider": "bedrock",
|
||||
}
|
||||
spend_log_metadata["status"] = "failure"
|
||||
spend_log_metadata["error_information"] = error_information
|
||||
|
||||
|
||||
def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn:
|
||||
error: Final = _as_bedrock_error(failure)
|
||||
verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error)))
|
||||
|
|
@ -279,6 +321,25 @@ async def _reject_session_update(
|
|||
return True
|
||||
|
||||
|
||||
async def _close_bedrock_input(
|
||||
bedrock_stream: BedrockBidirectionalStream,
|
||||
close_messages: Sequence[str],
|
||||
send_to_bedrock: Callable[[str], Awaitable[None]],
|
||||
timeout: float,
|
||||
) -> None:
|
||||
async def flush_and_close() -> None:
|
||||
for close_message in close_messages:
|
||||
with contextlib.suppress(Exception):
|
||||
await send_to_bedrock(close_message)
|
||||
with contextlib.suppress(Exception):
|
||||
await bedrock_stream.input_stream.close()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(flush_and_close(), timeout)
|
||||
except (asyncio.TimeoutError, TimeoutError):
|
||||
verbose_proxy_logger.debug("Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", timeout)
|
||||
|
||||
|
||||
class BedrockRealtime(BaseAWSLLM):
|
||||
"""Handler for Bedrock Nova Sonic realtime speech-to-speech API."""
|
||||
|
||||
|
|
@ -445,7 +506,14 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
if _should_log_event(leftover_event)
|
||||
),
|
||||
)
|
||||
if outcome.provider_failure is not None and outcome.client_disconnected:
|
||||
verbose_proxy_logger.debug(
|
||||
"Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
|
||||
)
|
||||
provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure
|
||||
if logged_events:
|
||||
if provider_failure is not None:
|
||||
_record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure))
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
logging_obj.dispatch_success_handlers(
|
||||
list(logged_events),
|
||||
|
|
@ -454,14 +522,9 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
)
|
||||
logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True
|
||||
|
||||
if outcome.provider_failure is None:
|
||||
if provider_failure is None:
|
||||
return
|
||||
if outcome.client_disconnected:
|
||||
verbose_proxy_logger.debug(
|
||||
"Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
|
||||
)
|
||||
return
|
||||
_raise_provider_failure(websocket.scope, outcome.provider_failure)
|
||||
_raise_provider_failure(websocket.scope, provider_failure)
|
||||
|
||||
async def _bridge(
|
||||
self,
|
||||
|
|
@ -475,19 +538,23 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
) -> _BridgeOutcome:
|
||||
"""Run both forwarding directions until the client leaves or either side fails."""
|
||||
logged: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events forwarded before a failure are still spend
|
||||
input_closing: Final = asyncio.Event()
|
||||
signals: Final = SessionEndSignals()
|
||||
|
||||
async def collect_logged_events() -> None:
|
||||
async for event in self._forward_bedrock_to_client(
|
||||
bedrock_stream,
|
||||
websocket,
|
||||
transformation_config,
|
||||
model,
|
||||
logging_obj,
|
||||
session_state,
|
||||
input_closing=input_closing,
|
||||
):
|
||||
logged.append(event)
|
||||
try:
|
||||
async for event in self._forward_bedrock_to_client(
|
||||
bedrock_stream,
|
||||
websocket,
|
||||
transformation_config,
|
||||
model,
|
||||
logging_obj,
|
||||
session_state,
|
||||
signals=signals,
|
||||
):
|
||||
logged.append(event)
|
||||
except Exception:
|
||||
signals.output_failed.set()
|
||||
raise
|
||||
|
||||
client_task: Final = asyncio.create_task(
|
||||
self._forward_client_to_bedrock(
|
||||
|
|
@ -498,7 +565,7 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
session_state,
|
||||
logging_obj,
|
||||
initial_message,
|
||||
input_closing=input_closing,
|
||||
signals=signals,
|
||||
)
|
||||
)
|
||||
bedrock_task: Final = asyncio.create_task(collect_logged_events())
|
||||
|
|
@ -533,7 +600,7 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
logging_obj: LiteLLMLogging | None = None,
|
||||
initial_message: str | None = None,
|
||||
*,
|
||||
input_closing: asyncio.Event,
|
||||
signals: SessionEndSignals,
|
||||
) -> None:
|
||||
"""Forward messages from client WebSocket to Bedrock stream.
|
||||
|
||||
|
|
@ -581,21 +648,11 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
client_ws, transformation_config, model, logging_obj, parsed_client_message
|
||||
):
|
||||
break
|
||||
input_closing.set()
|
||||
signals.input_closing.set()
|
||||
finally:
|
||||
|
||||
async def close_bedrock_input() -> None:
|
||||
for close_message in transformation_config.session_close_messages():
|
||||
with contextlib.suppress(Exception):
|
||||
await send_to_bedrock(close_message)
|
||||
with contextlib.suppress(Exception):
|
||||
await bedrock_stream.input_stream.close()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(close_bedrock_input(), self._close_timeout)
|
||||
except (asyncio.TimeoutError, TimeoutError):
|
||||
verbose_proxy_logger.debug(
|
||||
"Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", self._close_timeout
|
||||
if not signals.output_failed.is_set():
|
||||
await _close_bedrock_input(
|
||||
bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout
|
||||
)
|
||||
|
||||
async def _forward_bedrock_to_client(
|
||||
|
|
@ -607,7 +664,7 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
logging_obj: LiteLLMLogging,
|
||||
session_state: RealtimeResponseTransformInput,
|
||||
*,
|
||||
input_closing: asyncio.Event,
|
||||
signals: SessionEndSignals,
|
||||
) -> AsyncIterator[OpenAIRealtimeEvents]:
|
||||
"""Forward messages from Bedrock to the client, yielding the ones to record for spend logging.
|
||||
|
||||
|
|
@ -628,7 +685,7 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
result = await output[1].receive()
|
||||
|
||||
if result is None:
|
||||
if not input_closing.is_set():
|
||||
if not signals.input_closing.is_set():
|
||||
raise BedrockError(
|
||||
status_code=503,
|
||||
message="Bedrock realtime stream ended before the session closed",
|
||||
|
|
|
|||
|
|
@ -6478,6 +6478,15 @@ class Router:
|
|||
raise safeguard_refusal_error(model=model, stop_details=refusal_details)
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
realtime_session_deployment_id: Final = (
|
||||
_MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id")
|
||||
if generic_function_name == "_arealtime"
|
||||
else None
|
||||
)
|
||||
if realtime_session_deployment_id is not None:
|
||||
increment_deployment_successes_for_current_minute(
|
||||
litellm_router_instance=self, deployment_id=str(realtime_session_deployment_id)
|
||||
)
|
||||
verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
|
@ -9,13 +11,18 @@ from unittest.mock import MagicMock
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.constants import (
|
||||
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
|
||||
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
|
||||
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY,
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
WEBSOCKET_CLOSE_REASON_MAX_BYTES,
|
||||
)
|
||||
from litellm.litellm_core_utils.realtime_errors import websocket_close_reason
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.realtime.handler import BedrockRealtime
|
||||
from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals, raise_if_session_committed
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
|
||||
from litellm.realtime_api import main as realtime_main
|
||||
from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
|
||||
|
|
@ -36,10 +43,16 @@ def _isolate_host_aws_config(monkeypatch, tmp_path):
|
|||
monkeypatch.delenv(env_var, raising=False)
|
||||
|
||||
|
||||
def _closing_input() -> asyncio.Event:
|
||||
closing: Final = asyncio.Event()
|
||||
closing.set()
|
||||
return closing
|
||||
def _closing_input() -> SessionEndSignals:
|
||||
signals: Final = SessionEndSignals()
|
||||
signals.input_closing.set()
|
||||
return signals
|
||||
|
||||
|
||||
def _spend_row(logging_obj):
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
row: Final = get_logging_payload(kwargs=logging_obj.model_call_details, response_obj=None, start_time=now, end_time=now)
|
||||
return row["status"], json.loads(row["metadata"])["error_information"]
|
||||
|
||||
|
||||
class FakePayloadPart:
|
||||
|
|
@ -484,7 +497,7 @@ class TestBedrockRealtimeHandler:
|
|||
)
|
||||
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent]
|
||||
|
|
@ -500,7 +513,7 @@ class TestBedrockRealtimeHandler:
|
|||
stream = FakeBedrockStream()
|
||||
|
||||
await handler._forward_client_to_bedrock(
|
||||
DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
|
||||
DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
assert stream.input_stream.sent == []
|
||||
|
|
@ -517,7 +530,7 @@ class TestBedrockRealtimeHandler:
|
|||
|
||||
with pytest.raises(RuntimeError, match="bedrock send failed"):
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
assert stream.input_stream.closed
|
||||
|
|
@ -533,7 +546,7 @@ class TestBedrockRealtimeHandler:
|
|||
|
||||
await asyncio.wait_for(
|
||||
handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
),
|
||||
2,
|
||||
)
|
||||
|
|
@ -550,7 +563,7 @@ class TestBedrockRealtimeHandler:
|
|||
)
|
||||
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent]
|
||||
|
|
@ -580,7 +593,7 @@ class TestBedrockRealtimeHandler:
|
|||
"amazon.nova-sonic-v1:0",
|
||||
FakeLogging(),
|
||||
{},
|
||||
input_closing=_closing_input(),
|
||||
signals=_closing_input(),
|
||||
)
|
||||
]
|
||||
|
||||
|
|
@ -612,7 +625,7 @@ class TestBedrockRealtimeHandler:
|
|||
"amazon.nova-sonic-v1:0",
|
||||
FakeLogging(),
|
||||
{},
|
||||
input_closing=_closing_input(),
|
||||
signals=_closing_input(),
|
||||
)
|
||||
]
|
||||
|
||||
|
|
@ -684,7 +697,7 @@ class TestBedrockRealtimeHandler:
|
|||
"amazon.nova-sonic-v1:0",
|
||||
MagicMock(),
|
||||
{},
|
||||
input_closing=_closing_input(),
|
||||
signals=_closing_input(),
|
||||
):
|
||||
pass
|
||||
|
||||
|
|
@ -745,7 +758,7 @@ class TestBedrockRealtimeSessionLifecycle:
|
|||
)
|
||||
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event()
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
acked = [json.loads(message) for message in client_ws.sent_to_client]
|
||||
|
|
@ -761,7 +774,7 @@ class TestBedrockRealtimeSessionLifecycle:
|
|||
client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})])
|
||||
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event()
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
assert client_ws.sent_to_client == []
|
||||
|
|
@ -786,7 +799,7 @@ class TestBedrockRealtimeSessionLifecycle:
|
|||
client_ws = DisconnectingClientWS([update([tool]), update([other_tool]), update([tool])])
|
||||
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event()
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
acked: Final = [json.loads(message) for message in client_ws.sent_to_client]
|
||||
|
|
@ -942,9 +955,6 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
await spend_dispatch["coro"]
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client]
|
||||
flushed = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent]
|
||||
assert [next(iter(event["event"])) for event in flushed][-2:] == ["promptEnd", "sessionEnd"]
|
||||
assert stream.input_stream.closed
|
||||
|
||||
with pytest.raises(BedrockError) as replay:
|
||||
await handler.async_realtime(
|
||||
|
|
@ -1046,12 +1056,14 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
websocket: Final = ConnectedClientWS([self.SESSION_UPDATE])
|
||||
stream: Final = ScriptedBedrockStream(self.TEXT_TURN)
|
||||
stub_aws_sdk_client["streams"] = [stream]
|
||||
logging_obj: Final = spend_dispatch["logging_obj"]
|
||||
logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed-key"}}
|
||||
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await handler.async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=websocket,
|
||||
logging_obj=spend_dispatch["logging_obj"],
|
||||
logging_obj=logging_obj,
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
|
||||
|
|
@ -1060,18 +1072,93 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client]
|
||||
await spend_dispatch["coro"]
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
assert _sent_event_names(stream.input_stream)[-2:] == ["promptEnd", "sessionEnd"]
|
||||
status, error_information = _spend_row(logging_obj)
|
||||
assert status == "failure", "the Logs page must show the dropped session as failed while it bills its turn"
|
||||
assert error_information["error_code"] == "503"
|
||||
assert error_information["error_message"] == failure.value.message
|
||||
|
||||
with pytest.raises(BedrockError) as replay:
|
||||
await handler.async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=websocket,
|
||||
logging_obj=spend_dispatch["logging_obj"],
|
||||
logging_obj=logging_obj,
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
|
||||
assert replay.value.status_code == 400, "a router retry must not restart the dropped session"
|
||||
assert failure.value.message in replay.value.message
|
||||
assert str(replay.value).startswith(failure.value.message), "the app's error event leads with the drop"
|
||||
assert websocket_close_reason(str(replay.value), fallback="Internal server error").startswith(
|
||||
failure.value.message
|
||||
), "the length-capped 1011 close reason must still carry the drop"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("receiver_type", "status_code"),
|
||||
[(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)],
|
||||
ids=["output-ends", "output-raises"],
|
||||
)
|
||||
async def test_a_failed_bedrock_output_reaches_the_app_without_waiting_on_the_input_close(
|
||||
self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int
|
||||
):
|
||||
stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type)
|
||||
stream.input_stream = HangingCloseInputStream()
|
||||
stub_aws_sdk_client["streams"] = [stream]
|
||||
started: Final = time.monotonic()
|
||||
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await asyncio.wait_for(
|
||||
BedrockRealtime(close_timeout=30).async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
|
||||
logging_obj=spend_dispatch["logging_obj"],
|
||||
**self.AWS_PARAMS,
|
||||
),
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
assert failure.value.status_code == status_code
|
||||
assert time.monotonic() - started < 1, "the app must not wait on closing a Bedrock input that already failed"
|
||||
assert stub_aws_sdk_client["client_closed"], "closing the SDK client still tears the dead stream down"
|
||||
await spend_dispatch["coro"]
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replay_refusals_do_not_count_toward_deployment_cooldown(self):
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "nova-sonic",
|
||||
"litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0"},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for deployment_id in ("sonic-a", "sonic-b")
|
||||
]
|
||||
)
|
||||
with pytest.raises(BedrockError) as refusal:
|
||||
raise_if_session_committed(
|
||||
{
|
||||
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: True,
|
||||
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: "Bedrock realtime stream ended before the session closed",
|
||||
}
|
||||
)
|
||||
drop: Final = BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed")
|
||||
|
||||
def cooled_down_after(exception: Exception, deployment_id: str) -> bool:
|
||||
return router.deployment_callback_on_failure(
|
||||
kwargs={
|
||||
"exception": exception,
|
||||
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": deployment_id}},
|
||||
},
|
||||
completion_response=None,
|
||||
start_time=time.time(),
|
||||
end_time=time.time(),
|
||||
)
|
||||
|
||||
refusals_then_drop: Final = [cooled_down_after(error, "sonic-a") for error in (*[refusal.value] * 4, drop)]
|
||||
drops_only: Final = [cooled_down_after(drop, "sonic-b") for _ in range(5)]
|
||||
|
||||
assert not any(refusals_then_drop), "one drop retried four times is one deployment failure, not five"
|
||||
assert drops_only[-1], "five real drops in a minute still cool the deployment down"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish(
|
||||
|
|
@ -1080,6 +1167,9 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
stream: Final = SessionBoundBedrockStream(self.TEXT_TURN, input_stream_type=SlowClosingInputStream)
|
||||
stub_aws_sdk_client["streams"] = [stream]
|
||||
websocket: Final = RealtimeClientWS([self.SESSION_UPDATE])
|
||||
spend_dispatch["logging_obj"].model_call_details["litellm_params"] = {
|
||||
"litellm_metadata": {"user_api_key": "hashed-key"}
|
||||
}
|
||||
|
||||
await asyncio.wait_for(
|
||||
BedrockRealtime(close_timeout=30).async_realtime(
|
||||
|
|
@ -1097,6 +1187,7 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
await spend_dispatch["coro"]
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True
|
||||
assert _spend_row(spend_dispatch["logging_obj"]) == ("success", None)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_output_ending_after_the_input_side_failed_still_fails_the_session(
|
||||
|
|
@ -1167,7 +1258,7 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
"amazon.nova-sonic-v1:0",
|
||||
{},
|
||||
FakeLogging(),
|
||||
input_closing=asyncio.Event(),
|
||||
signals=SessionEndSignals(),
|
||||
)
|
||||
|
||||
assert client_ws.sent_to_client == []
|
||||
|
|
@ -1313,7 +1404,6 @@ class TestBedrockRealtimeSdkLifecycle:
|
|||
)
|
||||
|
||||
assert stub_aws_sdk_client["client_closed"]
|
||||
assert stub_aws_sdk_client["input_closed_before_client_close"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_without_close_completes_session(self, monkeypatch):
|
||||
|
|
|
|||
|
|
@ -538,6 +538,68 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn(
|
|||
assert await router.cache.async_get_cache(key=tpm_key) == 17
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)])
|
||||
async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served(
|
||||
clean_sessions: int, cooled_down: bool
|
||||
) -> None:
|
||||
import time
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "nova-sonic",
|
||||
"litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0", "order": order},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for order, deployment_id in ((1, "sonic-a"), (2, "sonic-b"))
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
drop: Final = litellm.ServiceUnavailableError(
|
||||
message="Bedrock realtime stream ended before the session closed",
|
||||
llm_provider="bedrock",
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
)
|
||||
|
||||
def realtime_session(failure: Exception | None):
|
||||
async def _arealtime(**kwargs: object) -> None:
|
||||
await router.deployment_callback_on_success(
|
||||
kwargs={
|
||||
"call_type": "_arealtime",
|
||||
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": kwargs["model_info"]},
|
||||
"standard_logging_object": create_standard_logging_payload(),
|
||||
},
|
||||
completion_response=None,
|
||||
start_time=time.time(),
|
||||
end_time=time.time(),
|
||||
)
|
||||
if failure is not None:
|
||||
raise failure
|
||||
|
||||
return _arealtime
|
||||
|
||||
for _ in range(clean_sessions):
|
||||
await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(None))
|
||||
|
||||
async def drop_session() -> bool:
|
||||
with pytest.raises(litellm.ServiceUnavailableError):
|
||||
await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(drop))
|
||||
return router.deployment_callback_on_failure(
|
||||
kwargs={
|
||||
"exception": drop,
|
||||
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": "sonic-a"}},
|
||||
},
|
||||
completion_response=None,
|
||||
start_time=time.time(),
|
||||
end_time=time.time(),
|
||||
)
|
||||
|
||||
drop_cooldowns: Final = [await drop_session() for _ in range(5)]
|
||||
|
||||
assert any(drop_cooldowns) is cooled_down
|
||||
|
||||
|
||||
async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]:
|
||||
routed: Final[dict[str, object]] = {}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue