mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge fb1ffe5ff5 into 60fc121ce8
This commit is contained in:
commit
b0df1af4dc
14 changed files with 1223 additions and 86 deletions
|
|
@ -451,6 +451,7 @@ OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset(
|
|||
{"response.completed", "response.incomplete", "response.failed"}
|
||||
)
|
||||
REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
|
||||
REALTIME_SESSION_ENDED_ABNORMALLY_KEY: Final = "realtime_session_ended_abnormally"
|
||||
|
||||
# SSL/TLS cipher configuration for faster handshakes
|
||||
# Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.constants import (
|
|||
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
|
||||
REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
REALTIME_SESSION_ENDED_ABNORMALLY_KEY,
|
||||
REALTIME_SESSION_FAILURE_LOGGED_KEY,
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
)
|
||||
|
|
@ -57,6 +58,10 @@ class BackendClose:
|
|||
code: int
|
||||
reason: str
|
||||
|
||||
@property
|
||||
def ended_cleanly(self) -> bool:
|
||||
return self.code == 1000
|
||||
|
||||
@property
|
||||
def message(self) -> str:
|
||||
if not self.reason:
|
||||
|
|
@ -2025,7 +2030,10 @@ class RealTimeStreaming:
|
|||
await asyncio.wait((forward_task, client_task), return_when=asyncio.FIRST_COMPLETED)
|
||||
if client_task.done() and client_task.result() is ClientLoopExit.CLIENT_DISCONNECTED:
|
||||
return
|
||||
await self._close_client(await forward_task)
|
||||
close: Final = await forward_task
|
||||
if not close.ended_cleanly and self.logging_obj:
|
||||
self.logging_obj.model_call_details[REALTIME_SESSION_ENDED_ABNORMALLY_KEY] = True
|
||||
await self._close_client(close)
|
||||
finally:
|
||||
forward_task.cancel()
|
||||
client_task.cancel()
|
||||
|
|
@ -2035,7 +2043,7 @@ class RealTimeStreaming:
|
|||
redacted_message: Final = redact_internal_details_from_client_message(close.message)
|
||||
redacted_reason: Final = redact_internal_details_from_client_message(close.reason)
|
||||
try:
|
||||
if close.code != 1000:
|
||||
if not close.ended_cleanly:
|
||||
await self.websocket.send_text(realtime_error_event(redacted_message, error_type="server_error"))
|
||||
await self.websocket.close(
|
||||
code=client_close_code(close.code),
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import contextlib
|
|||
import importlib.metadata
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import Final, NoReturn, Protocol, runtime_checkable
|
||||
|
||||
|
|
@ -34,12 +34,15 @@ from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateR
|
|||
from litellm.types.llms.bedrock import AwsAuthParams
|
||||
from litellm.types.llms.openai import OpenAIRealtimeEvents
|
||||
from litellm.types.realtime import RealtimeResponseTransformInput
|
||||
from litellm.types.utils import StandardLoggingPayloadErrorInformation
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
|
||||
from ..common_utils import BedrockError
|
||||
from .transformation import BedrockRealtimeConfig, requested_modalities
|
||||
|
||||
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
_REQUEST_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
_SPEND_LOG_METADATA_KEYS: Final = ("litellm_metadata", "metadata")
|
||||
|
||||
_EMPTY_JSON_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({})
|
||||
|
||||
|
|
@ -61,6 +64,10 @@ _BEDROCK_STREAM_ERROR_STATUS: Final[Mapping[str, int]] = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
def _stream_ended_before_session_closed() -> BedrockError:
|
||||
return BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed")
|
||||
|
||||
|
||||
def _as_bedrock_error(error: BaseException) -> BaseException:
|
||||
status_code: Final = _BEDROCK_STREAM_ERROR_STATUS.get(type(error).__name__)
|
||||
if status_code is None:
|
||||
|
|
@ -165,6 +172,12 @@ class _BridgeOutcome:
|
|||
client_disconnected: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SessionEndSignals:
|
||||
input_closing: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
output_failed: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
|
||||
|
||||
async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message: str | None) -> AsyncIterator[str]:
|
||||
if initial_message is not None:
|
||||
yield initial_message
|
||||
|
|
@ -188,8 +201,8 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None:
|
|||
raise BedrockError(
|
||||
status_code=400,
|
||||
message=(
|
||||
"Bedrock realtime session already committed to a provider stream; it cannot be replayed"
|
||||
+ (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
|
||||
(f"{committed_failure}. " if committed_failure else "")
|
||||
+ "Bedrock realtime session already committed to a provider stream; it cannot be replayed"
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -201,11 +214,48 @@ def _pending_opening_message(scope: Mapping[str, object]) -> str | None:
|
|||
return pending if isinstance(pending, str) else None
|
||||
|
||||
|
||||
def _spend_log_metadata(litellm_params: Mapping[str, object]) -> tuple[str, Mapping[str, object]] | None:
|
||||
return next(
|
||||
(
|
||||
(key, metadata)
|
||||
for key in _SPEND_LOG_METADATA_KEYS
|
||||
if (metadata := _REQUEST_MAPPING_ADAPTER.validate_python(litellm_params.get(key)))
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseException) -> None:
|
||||
"""The spend row of a session the provider ended reads failure while it still bills the forwarded events."""
|
||||
litellm_params: Final = _REQUEST_MAPPING_ADAPTER.validate_python(
|
||||
logging_obj.model_call_details.get("litellm_params")
|
||||
)
|
||||
if litellm_params is None:
|
||||
return
|
||||
spend_log_metadata: Final = _spend_log_metadata(litellm_params)
|
||||
if spend_log_metadata is None:
|
||||
return
|
||||
status_code: Final[object] = getattr(error, "status_code", None)
|
||||
error_information: Final[StandardLoggingPayloadErrorInformation] = {
|
||||
"error_code": "" if status_code is None else str(status_code),
|
||||
"error_class": type(error).__name__,
|
||||
"error_message": redact_string(str(error)),
|
||||
"llm_provider": "bedrock",
|
||||
}
|
||||
metadata_key, metadata = spend_log_metadata
|
||||
logging_obj.model_call_details["litellm_params"] = { # rebind-ok: the spend row reads the logging object
|
||||
**litellm_params,
|
||||
metadata_key: {**metadata, "status": "failure", "error_information": error_information},
|
||||
}
|
||||
|
||||
|
||||
def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn:
|
||||
error: Final = _as_bedrock_error(failure)
|
||||
verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error)))
|
||||
if _is_session_committed(scope):
|
||||
scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = redact_string(str(error))
|
||||
if error is failure:
|
||||
raise failure
|
||||
raise error from failure
|
||||
|
||||
|
||||
|
|
@ -267,6 +317,25 @@ async def _reject_session_update(
|
|||
return True
|
||||
|
||||
|
||||
async def _close_bedrock_input(
|
||||
bedrock_stream: BedrockBidirectionalStream,
|
||||
close_messages: Sequence[str],
|
||||
send_to_bedrock: Callable[[str], Awaitable[None]],
|
||||
timeout: float,
|
||||
) -> None:
|
||||
async def flush_and_close() -> None:
|
||||
for close_message in close_messages:
|
||||
with contextlib.suppress(Exception):
|
||||
await send_to_bedrock(close_message)
|
||||
with contextlib.suppress(Exception):
|
||||
await bedrock_stream.input_stream.close()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(flush_and_close(), timeout)
|
||||
except (asyncio.TimeoutError, TimeoutError):
|
||||
verbose_proxy_logger.debug("Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", timeout)
|
||||
|
||||
|
||||
class BedrockRealtime(BaseAWSLLM):
|
||||
"""Handler for Bedrock Nova Sonic realtime speech-to-speech API."""
|
||||
|
||||
|
|
@ -278,6 +347,18 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
super().__init__()
|
||||
self._sdk_version_lookup: Final = sdk_version_lookup
|
||||
self._close_timeout: Final = close_timeout
|
||||
self._failed_input_closes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs to running closes
|
||||
|
||||
def _close_failed_input_in_background(
|
||||
self,
|
||||
bedrock_stream: BedrockBidirectionalStream,
|
||||
send_to_bedrock: Callable[[str], Awaitable[None]],
|
||||
) -> None:
|
||||
close_task: Final = asyncio.create_task(
|
||||
_close_bedrock_input(bedrock_stream, (), send_to_bedrock, self._close_timeout)
|
||||
)
|
||||
self._failed_input_closes.add(close_task)
|
||||
close_task.add_done_callback(self._failed_input_closes.discard)
|
||||
|
||||
async def async_realtime(
|
||||
self,
|
||||
|
|
@ -433,23 +514,26 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
if _should_log_event(leftover_event)
|
||||
),
|
||||
)
|
||||
if outcome.provider_failure is not None and outcome.client_disconnected:
|
||||
verbose_proxy_logger.debug(
|
||||
"Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
|
||||
)
|
||||
provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure
|
||||
if logged_events:
|
||||
if provider_failure is not None:
|
||||
_record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure))
|
||||
usage_logging_obj: Final = logging_obj if provider_failure is None else logging_obj.attempt_scoped_copy()
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
logging_obj.dispatch_success_handlers(
|
||||
usage_logging_obj.dispatch_success_handlers(
|
||||
list(logged_events),
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
)
|
||||
logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True
|
||||
|
||||
if outcome.provider_failure is None:
|
||||
if provider_failure is None:
|
||||
return
|
||||
if outcome.client_disconnected:
|
||||
verbose_proxy_logger.debug(
|
||||
"Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
|
||||
)
|
||||
return
|
||||
_raise_provider_failure(websocket.scope, outcome.provider_failure)
|
||||
_raise_provider_failure(websocket.scope, provider_failure)
|
||||
|
||||
async def _bridge(
|
||||
self,
|
||||
|
|
@ -463,16 +547,34 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
) -> _BridgeOutcome:
|
||||
"""Run both forwarding directions until the client leaves or either side fails."""
|
||||
logged: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events forwarded before a failure are still spend
|
||||
signals: Final = SessionEndSignals()
|
||||
|
||||
async def collect_logged_events() -> None:
|
||||
async for event in self._forward_bedrock_to_client(
|
||||
bedrock_stream, websocket, transformation_config, model, logging_obj, session_state
|
||||
):
|
||||
logged.append(event)
|
||||
try:
|
||||
async for event in self._forward_bedrock_to_client(
|
||||
bedrock_stream,
|
||||
websocket,
|
||||
transformation_config,
|
||||
model,
|
||||
logging_obj,
|
||||
session_state,
|
||||
signals=signals,
|
||||
):
|
||||
logged.append(event)
|
||||
except Exception:
|
||||
signals.output_failed.set()
|
||||
raise
|
||||
|
||||
client_task: Final = asyncio.create_task(
|
||||
self._forward_client_to_bedrock(
|
||||
websocket, bedrock_stream, transformation_config, model, session_state, logging_obj, initial_message
|
||||
websocket,
|
||||
bedrock_stream,
|
||||
transformation_config,
|
||||
model,
|
||||
session_state,
|
||||
logging_obj,
|
||||
initial_message,
|
||||
signals=signals,
|
||||
)
|
||||
)
|
||||
bedrock_task: Final = asyncio.create_task(collect_logged_events())
|
||||
|
|
@ -506,6 +608,8 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
session_state: RealtimeResponseTransformInput,
|
||||
logging_obj: LiteLLMLogging | None = None,
|
||||
initial_message: str | None = None,
|
||||
*,
|
||||
signals: SessionEndSignals,
|
||||
) -> None:
|
||||
"""Forward messages from client WebSocket to Bedrock stream.
|
||||
|
||||
|
|
@ -521,7 +625,10 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
|
||||
async def send_to_bedrock(bedrock_message: str) -> None:
|
||||
event: Final = build_input_chunk(bedrock_message.encode("utf-8"))
|
||||
await bedrock_stream.input_stream.send(event)
|
||||
try:
|
||||
await bedrock_stream.input_stream.send(event)
|
||||
except Exception as e:
|
||||
raise _stream_ended_before_session_closed() from e
|
||||
verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200])
|
||||
|
||||
try:
|
||||
|
|
@ -553,20 +660,13 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
client_ws, transformation_config, model, logging_obj, parsed_client_message
|
||||
):
|
||||
break
|
||||
signals.input_closing.set()
|
||||
finally:
|
||||
|
||||
async def close_bedrock_input() -> None:
|
||||
for close_message in transformation_config.session_close_messages():
|
||||
with contextlib.suppress(Exception):
|
||||
await send_to_bedrock(close_message)
|
||||
with contextlib.suppress(Exception):
|
||||
await bedrock_stream.input_stream.close()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(close_bedrock_input(), self._close_timeout)
|
||||
except (asyncio.TimeoutError, TimeoutError):
|
||||
verbose_proxy_logger.debug(
|
||||
"Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", self._close_timeout
|
||||
if signals.output_failed.is_set():
|
||||
self._close_failed_input_in_background(bedrock_stream, send_to_bedrock)
|
||||
else:
|
||||
await _close_bedrock_input(
|
||||
bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout
|
||||
)
|
||||
|
||||
async def _forward_bedrock_to_client(
|
||||
|
|
@ -577,6 +677,8 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
model: str,
|
||||
logging_obj: LiteLLMLogging,
|
||||
session_state: RealtimeResponseTransformInput,
|
||||
*,
|
||||
signals: SessionEndSignals,
|
||||
) -> AsyncIterator[OpenAIRealtimeEvents]:
|
||||
"""Forward messages from Bedrock to the client, yielding the ones to record for spend logging.
|
||||
|
||||
|
|
@ -597,6 +699,8 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
result = await output[1].receive()
|
||||
|
||||
if result is None:
|
||||
if not signals.input_closing.is_set():
|
||||
raise _stream_ended_before_session_closed()
|
||||
verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended")
|
||||
with contextlib.suppress(Exception):
|
||||
await client_ws.close()
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm._logging import redact_secrets
|
|||
from litellm.constants import (
|
||||
OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS,
|
||||
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
|
||||
REALTIME_SESSION_ENDED_ABNORMALLY_KEY,
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
|
||||
)
|
||||
|
|
@ -316,12 +317,15 @@ async def _forward_live_backend_frames(
|
|||
backend_websocket: LiveBackendWebSocket,
|
||||
client_gone: asyncio.Event,
|
||||
event_retention: _LiveEventRetention,
|
||||
upstream_ended_abnormally: asyncio.Event,
|
||||
) -> None:
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
from websockets.exceptions import ConnectionClosed
|
||||
|
||||
event_loop: Final = asyncio.get_running_loop()
|
||||
async for message, close_deadline in _live_backend_messages(websocket, backend_websocket):
|
||||
async for message, close_deadline in _live_backend_messages(
|
||||
websocket, backend_websocket, upstream_ended_abnormally
|
||||
):
|
||||
if (event := _metering_event(message)) is not None:
|
||||
event_retention.retain(event)
|
||||
if client_gone.is_set():
|
||||
|
|
@ -341,6 +345,7 @@ async def _forward_live_backend_frames(
|
|||
async def _live_backend_messages(
|
||||
websocket: LiveClientWebSocket,
|
||||
backend_websocket: LiveBackendWebSocket,
|
||||
upstream_ended_abnormally: asyncio.Event,
|
||||
) -> AsyncIterator[tuple[str, float | None]]:
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
from websockets.exceptions import ConnectionClosed
|
||||
|
|
@ -350,6 +355,8 @@ async def _live_backend_messages(
|
|||
yield message, None
|
||||
except ConnectionClosed as error:
|
||||
upstream_close: Final = backend_close_from(error)
|
||||
if not upstream_close.ended_cleanly:
|
||||
upstream_ended_abnormally.set()
|
||||
with suppress(WebSocketDisconnect, RuntimeError, ConnectionClosed, OSError):
|
||||
await websocket.close(code=upstream_close.code, reason=upstream_close.reason)
|
||||
return
|
||||
|
|
@ -434,6 +441,7 @@ class OpenAILiveSessions(OpenAIRealtime):
|
|||
"complete_input_dict": {"session_start": rewritten_start},
|
||||
}
|
||||
event_retention: Final = _LiveEventRetention()
|
||||
session_ended_abnormally: Final = asyncio.Event()
|
||||
try:
|
||||
logging_obj.pre_call(input=None, api_key=api_key, additional_args=pre_call_args)
|
||||
async with self._websocket_connector(
|
||||
|
|
@ -448,7 +456,11 @@ class OpenAILiveSessions(OpenAIRealtime):
|
|||
websocket=websocket,
|
||||
backend_websocket=backend_websocket,
|
||||
event_retention=event_retention,
|
||||
upstream_ended_abnormally=session_ended_abnormally,
|
||||
)
|
||||
except Exception:
|
||||
session_ended_abnormally.set()
|
||||
raise
|
||||
finally:
|
||||
session_end_time: Final = self._clock()
|
||||
retained_events: Final = event_retention.results()
|
||||
|
|
@ -463,6 +475,10 @@ class OpenAILiveSessions(OpenAIRealtime):
|
|||
logging_obj.model_call_details[ # rebind-ok: flags the shared logging object as already logged
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY
|
||||
] = True
|
||||
if session_ended_abnormally.is_set():
|
||||
logging_obj.model_call_details[ # rebind-ok: flags the shared logging object for the router
|
||||
REALTIME_SESSION_ENDED_ABNORMALLY_KEY
|
||||
] = True
|
||||
except InvalidStatus as error:
|
||||
await close_after_upstream_handshake_refusal(websocket, error.response.status_code)
|
||||
except Exception as error: # noqa: BLE001 # any relay failure must still close the client socket with a redacted reason
|
||||
|
|
@ -484,6 +500,7 @@ class OpenAILiveSessions(OpenAIRealtime):
|
|||
websocket: LiveClientWebSocket,
|
||||
backend_websocket: LiveBackendWebSocket,
|
||||
event_retention: _LiveEventRetention,
|
||||
upstream_ended_abnormally: asyncio.Event,
|
||||
) -> None:
|
||||
from websockets.exceptions import ConnectionClosed
|
||||
|
||||
|
|
@ -503,6 +520,7 @@ class OpenAILiveSessions(OpenAIRealtime):
|
|||
backend_websocket=backend_websocket,
|
||||
client_gone=client_gone,
|
||||
event_retention=event_retention,
|
||||
upstream_ended_abnormally=upstream_ended_abnormally,
|
||||
)
|
||||
)
|
||||
client_gone_task: Final = asyncio.create_task(client_gone.wait())
|
||||
|
|
|
|||
|
|
@ -749,6 +749,17 @@ class RequestRateLimiterStash:
|
|||
tpm_limited_tags: frozenset[str] = field(default_factory=frozenset)
|
||||
socket_agent_windows: tuple[RateLimitDescriptor, ...] = ()
|
||||
|
||||
def clear_reserved_tokens(self) -> None:
|
||||
self.reserved_tokens = 0
|
||||
self.ptu_reserved_tokens = 0
|
||||
self.itpm_reserved_tokens = 0
|
||||
self.otpm_reserved_tokens = 0
|
||||
|
||||
def discard_released_reservation(self) -> None:
|
||||
if self.reservation_released:
|
||||
self.clear_reserved_tokens()
|
||||
self.reservation_released = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CounterRefund:
|
||||
|
|
@ -4238,6 +4249,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
requested_model=requested_model,
|
||||
)
|
||||
|
||||
stash.discard_released_reservation()
|
||||
if itpm_reserved > 0:
|
||||
itpm_scopes: Final = tuple(
|
||||
(d["key"], d["value"]) for d in io_token_descriptors if d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY
|
||||
|
|
@ -4597,6 +4609,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
# incremented so post-call reconciliation only applies
|
||||
# the (actual - reserved) delta to those — unreserved
|
||||
# scopes get charged the full actual usage instead.
|
||||
stash.discard_released_reservation()
|
||||
stash.reserved_tokens = estimated_tokens
|
||||
stash.ptu_reserved_tokens = ptu_estimated_tokens
|
||||
stash.ptu_reserved_window_identities = frozenset(
|
||||
|
|
@ -5402,7 +5415,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
total_tokens = self._aggregate_only_total_tokens(usage=_usage)
|
||||
|
||||
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs))
|
||||
reserved_tokens: Final = stash.reserved_tokens if stash is not None else 0
|
||||
open_reservation: Final = stash if stash is not None and not stash.reservation_released else None
|
||||
reserved_tokens: Final = open_reservation.reserved_tokens if open_reservation is not None else 0
|
||||
reserved_model: Final = stash.reserved_model if stash is not None else None
|
||||
reserved_scopes: Final[frozenset[tuple[str, str]]] = stash.reserved_scopes if stash is not None else frozenset()
|
||||
# Reconciliation must target the same model-scoped counter that the
|
||||
|
|
@ -5464,7 +5478,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
reconcile_model=reconcile_model,
|
||||
reserved_scopes=reserved_scopes,
|
||||
reserved_ceiling=stash.ptu_ceiling if stash is not None else None,
|
||||
reserved_tokens=stash.ptu_reserved_tokens if stash is not None else 0,
|
||||
reserved_tokens=open_reservation.ptu_reserved_tokens if open_reservation is not None else 0,
|
||||
reserved_windows=stash.ptu_reserved_window_identities if stash is not None else frozenset(),
|
||||
total_tokens=0 if ptu_skips_socket_usage else total_tokens,
|
||||
)
|
||||
|
|
@ -5584,6 +5598,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
pipeline_operations=io_token_operations,
|
||||
parent_otel_span=litellm_parent_otel_span,
|
||||
)
|
||||
if stash is not None:
|
||||
stash.clear_reserved_tokens()
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error in rate limit success event: %s", e)
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 #
|
|||
_sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
deployment_params_provider,
|
||||
get_request_model_access_groups,
|
||||
sanitize_error_information_for_spend_logs,
|
||||
sanitize_error_information_for_spend_logs, # pyright: ignore[reportUnusedImport] # backwards-compatible package export
|
||||
should_store_prompts_and_responses_in_spend_logs,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, capture_callback, captured_callback_kwargs
|
||||
|
|
@ -79,7 +79,6 @@ from litellm.types.utils import (
|
|||
CallTypes,
|
||||
LiteLLMBatch,
|
||||
StandardLoggingPayload,
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
)
|
||||
from litellm.utils import get_end_user_id_for_cost_tracking
|
||||
|
||||
|
|
@ -492,13 +491,6 @@ class _ProxyDBLogger(CustomLogger):
|
|||
# ``.get("traceback")`` / truthy checks, and the TypedDict marks
|
||||
# the field as optional, so omitting is type-safe.
|
||||
_error_information.pop("traceback", None)
|
||||
# Strip echoed request input + apply DB-size cap before storing in
|
||||
# the spend-log metadata column (LIT-2992). Result is never None
|
||||
# here because the input above is constructed non-None.
|
||||
_error_information = cast(
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
sanitize_error_information_for_spend_logs(_error_information, original_exception=original_exception),
|
||||
)
|
||||
_metadata["error_information"] = _error_information
|
||||
|
||||
_metadata = ( # rebind-ok: enriched identity is shared with the earlier immutable capture
|
||||
|
|
|
|||
|
|
@ -607,6 +607,7 @@ def get_logging_payload(
|
|||
kwargs = {}
|
||||
|
||||
rejected_as_unknown_model: Final = isinstance(response_obj, ProxyModelNotFoundError)
|
||||
failure_exception: Final = response_obj if isinstance(response_obj, BaseException) else None
|
||||
response_served_from_cache: Final = served_from_cache(response_obj)
|
||||
if response_obj is None:
|
||||
response_obj = {}
|
||||
|
|
@ -897,14 +898,11 @@ def get_logging_payload(
|
|||
additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens
|
||||
clean_metadata["additional_usage_values"] = additional_usage_values
|
||||
drop_requester_ip_address: Final = requester_ip_address_logging_disabled()
|
||||
error_information: Final = clean_metadata["error_information"]
|
||||
if drop_requester_ip_address:
|
||||
clean_metadata["requester_ip_address"] = None
|
||||
if drop_requester_ip_address and isinstance(error_information, Mapping):
|
||||
clean_metadata["error_information"] = cast( # cast-ok: same keys, only the str values are masked
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
_error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(error_information)),
|
||||
)
|
||||
clean_metadata["error_information"] = sanitize_error_information_for_spend_logs(
|
||||
clean_metadata["error_information"], original_exception=failure_exception
|
||||
)
|
||||
|
||||
if litellm.cache is None:
|
||||
cache_key = "Cache OFF"
|
||||
|
|
|
|||
|
|
@ -84,6 +84,8 @@ from litellm.constants import (
|
|||
MID_STREAM_FAILURE_LOG_WAIT_SECONDS,
|
||||
OUTPUT_TOKEN_CEILING_PARAMS,
|
||||
PROMPT_CACHE_PIN_CACHE_MAX_SIZE,
|
||||
REALTIME_SESSION_ENDED_ABNORMALLY_KEY,
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
|
||||
ROUTING_REQUEST_TAGS_METADATA_KEY,
|
||||
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
|
||||
|
|
@ -668,6 +670,7 @@ _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType
|
|||
_NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_CALL_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str])
|
||||
_RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
|
||||
|
|
@ -6865,6 +6868,22 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
raise safeguard_refusal_error(model=model, stop_details=refusal_details)
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
realtime_session_logging: Final = (
|
||||
_CALL_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_logging_obj")
|
||||
if generic_function_name == "_arealtime"
|
||||
else None
|
||||
)
|
||||
realtime_session_deployment_id: Final = (
|
||||
_MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id")
|
||||
if isinstance(realtime_session_logging, LiteLLMLogging)
|
||||
and realtime_session_logging.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True
|
||||
and realtime_session_logging.model_call_details.get(REALTIME_SESSION_ENDED_ABNORMALLY_KEY) is not True
|
||||
else None
|
||||
)
|
||||
if realtime_session_deployment_id is not None:
|
||||
increment_deployment_successes_for_current_minute(
|
||||
ctx=self._core_ctx, deployment_id=str(realtime_session_deployment_id)
|
||||
)
|
||||
verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -1,12 +1,16 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
|
|
@ -14,10 +18,15 @@ from litellm.constants import (
|
|||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
WEBSOCKET_CLOSE_REASON_MAX_BYTES,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.realtime_errors import websocket_close_reason
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.realtime.handler import BedrockRealtime
|
||||
from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
|
||||
from litellm.realtime_api import main as realtime_main
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.types.utils import StandardLoggingPayload, StandardLoggingPayloadErrorInformation
|
||||
from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
|
||||
|
||||
|
||||
|
|
@ -36,6 +45,66 @@ def _isolate_host_aws_config(monkeypatch, tmp_path):
|
|||
monkeypatch.delenv(env_var, raising=False)
|
||||
|
||||
|
||||
def _closing_input() -> SessionEndSignals:
|
||||
signals: Final = SessionEndSignals()
|
||||
signals.input_closing.set()
|
||||
return signals
|
||||
|
||||
|
||||
_SPEND_ROW_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_ERROR_INFORMATION_ADAPTER: Final = TypeAdapter(StandardLoggingPayloadErrorInformation | None)
|
||||
_BEDROCK_INPUT_CHUNK_ADAPTER: Final = TypeAdapter(Mapping[str, Mapping[str, object]])
|
||||
_STANDARD_LOGGING_PAYLOAD_ADAPTER: Final = TypeAdapter(StandardLoggingPayload)
|
||||
|
||||
|
||||
def _spend_row(
|
||||
logging_obj: "FakeLogging | LiteLLMLogging",
|
||||
) -> tuple[Literal["success", "failure"], StandardLoggingPayloadErrorInformation | None]:
|
||||
return _spend_row_of(logging_obj.model_call_details)
|
||||
|
||||
|
||||
def _spend_row_of(
|
||||
model_call_details: dict[str, object],
|
||||
) -> tuple[Literal["success", "failure"], StandardLoggingPayloadErrorInformation | None]:
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
row: Final = get_logging_payload(kwargs=model_call_details, response_obj=None, start_time=now, end_time=now)
|
||||
metadata: Final = _SPEND_ROW_METADATA_ADAPTER.validate_json(row["metadata"])
|
||||
return row["status"], _ERROR_INFORMATION_ADAPTER.validate_python(metadata.get("error_information"))
|
||||
|
||||
|
||||
class LoggedEventRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.events: Final[list[tuple[Literal["success", "failure"], dict[str, object]]]] = []
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
self.events.append(("success", dict(kwargs)))
|
||||
|
||||
async def async_log_failure_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
self.events.append(("failure", dict(kwargs)))
|
||||
|
||||
def payload(self, event: Literal["success", "failure"]) -> StandardLoggingPayload:
|
||||
(logged,) = (kwargs for logged_event, kwargs in self.events if logged_event == event)
|
||||
return _STANDARD_LOGGING_PAYLOAD_ADAPTER.validate_python(logged["standard_logging_object"])
|
||||
|
||||
|
||||
class HeldSuccessLogger(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.entered: Final = asyncio.Event()
|
||||
self.release: Final = asyncio.Event()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
self.entered.set()
|
||||
await self.release.wait()
|
||||
|
||||
|
||||
class FakePayloadPart:
|
||||
def __init__(self, bytes_):
|
||||
self.bytes_ = bytes_
|
||||
|
|
@ -85,6 +154,19 @@ class HangingCloseInputStream(FakeInputStream):
|
|||
await asyncio.Future()
|
||||
|
||||
|
||||
class BodyWriterInputStream(FakeInputStream):
|
||||
"""Models the SDK input stream, whose request-body writer task only finishes once the stream is closed"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._close_requested = asyncio.Event()
|
||||
self.body_writer = asyncio.get_running_loop().create_task(self._close_requested.wait())
|
||||
|
||||
async def close(self):
|
||||
await super().close()
|
||||
self._close_requested.set()
|
||||
|
||||
|
||||
class FakeBedrockStream:
|
||||
def __init__(self, input_stream=None):
|
||||
self.input_stream = input_stream if input_stream is not None else FakeInputStream()
|
||||
|
|
@ -124,6 +206,11 @@ class FakeLogging:
|
|||
self.litellm_trace_id = trace_id
|
||||
self.model_call_details = {}
|
||||
|
||||
def attempt_scoped_copy(self):
|
||||
attempt = copy.copy(self)
|
||||
attempt.model_call_details = dict(self.model_call_details)
|
||||
return attempt
|
||||
|
||||
|
||||
class DisconnectingClientWS:
|
||||
def __init__(self, messages):
|
||||
|
|
@ -159,12 +246,15 @@ class EndedBedrockStream:
|
|||
|
||||
|
||||
class RealtimeClientWS:
|
||||
def __init__(self):
|
||||
def __init__(self, messages=()):
|
||||
self.closed = False
|
||||
self.sent_to_client = []
|
||||
self.scope = {}
|
||||
self._messages = list(messages)
|
||||
|
||||
async def receive_text(self):
|
||||
if self._messages:
|
||||
return self._messages.pop(0)
|
||||
raise RuntimeError("client disconnected")
|
||||
|
||||
async def send_text(self, message):
|
||||
|
|
@ -178,8 +268,7 @@ class ConnectedClientWS(RealtimeClientWS):
|
|||
"""Client that sends its scripted messages and then stays connected until the server closes it"""
|
||||
|
||||
def __init__(self, messages):
|
||||
super().__init__()
|
||||
self._messages = list(messages)
|
||||
super().__init__(messages)
|
||||
self._closed_event = asyncio.Event()
|
||||
|
||||
async def receive_text(self):
|
||||
|
|
@ -251,6 +340,77 @@ class ScriptedBedrockStream:
|
|||
return (None, self._receiver)
|
||||
|
||||
|
||||
def _sent_event_names(input_stream: FakeInputStream) -> list[str]:
|
||||
return [
|
||||
next(iter(_BEDROCK_INPUT_CHUNK_ADAPTER.validate_json(chunk.value.bytes_)["event"]))
|
||||
for chunk in input_stream.sent
|
||||
]
|
||||
|
||||
|
||||
class SessionEndingInputStream(FakeInputStream):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.session_ended = asyncio.Event()
|
||||
|
||||
async def send(self, event):
|
||||
await super().send(event)
|
||||
if "sessionEnd" in json.loads(event.value.bytes_.decode("utf-8")).get("event", {}):
|
||||
self.session_ended.set()
|
||||
|
||||
async def close(self):
|
||||
await super().close()
|
||||
self.session_ended.set()
|
||||
|
||||
|
||||
class SlowClosingInputStream(SessionEndingInputStream):
|
||||
async def close(self):
|
||||
await asyncio.Event().wait()
|
||||
|
||||
|
||||
class OutputEndsAfterSessionEndReceiver(ScriptedBedrockReceiver):
|
||||
def __init__(self, payloads, session_ended):
|
||||
super().__init__(payloads)
|
||||
self._session_ended = session_ended
|
||||
|
||||
async def receive(self):
|
||||
if not self._payloads:
|
||||
await self._session_ended.wait()
|
||||
return await super().receive()
|
||||
|
||||
|
||||
class SessionBoundBedrockStream:
|
||||
def __init__(self, payloads=(), input_stream_type=SessionEndingInputStream):
|
||||
self.input_stream = input_stream_type()
|
||||
self._receiver = OutputEndsAfterSessionEndReceiver(payloads, self.input_stream.session_ended)
|
||||
|
||||
async def await_output(self):
|
||||
return (None, self._receiver)
|
||||
|
||||
|
||||
class ConnectionResetOnAudioInputStream(FakeInputStream):
|
||||
def __init__(self, connection_reset):
|
||||
super().__init__()
|
||||
self._connection_reset = connection_reset
|
||||
|
||||
async def send(self, event):
|
||||
await asyncio.sleep(0)
|
||||
if self._connection_reset.is_set() or "audioInput" in json.loads(event.value.bytes_.decode("utf-8"))["event"]:
|
||||
self._connection_reset.set()
|
||||
raise RuntimeError("bedrock connection reset")
|
||||
self.sent.append(event)
|
||||
|
||||
|
||||
class OutputEndsAfterConnectionResetReceiver(ScriptedBedrockReceiver):
|
||||
def __init__(self, payloads, connection_reset):
|
||||
super().__init__(payloads)
|
||||
self._connection_reset = connection_reset
|
||||
|
||||
async def receive(self):
|
||||
if not self._payloads:
|
||||
await self._connection_reset.wait()
|
||||
return await super().receive()
|
||||
|
||||
|
||||
class FakeAWSCredentialsIdentity:
|
||||
def __init__(self, access_key_id, secret_access_key, session_token=None):
|
||||
self.access_key_id = access_key_id
|
||||
|
|
@ -352,7 +512,7 @@ def stub_aws_sdk_client(monkeypatch):
|
|||
raise stream
|
||||
captured["open_stream"] = stream
|
||||
return stream
|
||||
stream = ScriptedBedrockStream(captured.get("scripted_payloads", []))
|
||||
stream = SessionBoundBedrockStream(captured.get("scripted_payloads", []))
|
||||
captured["open_stream"] = stream
|
||||
return stream
|
||||
|
||||
|
|
@ -407,7 +567,9 @@ class TestBedrockRealtimeHandler:
|
|||
[json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})]
|
||||
)
|
||||
|
||||
await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {})
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent]
|
||||
event_names = [next(iter(event["event"])) for event in sent_events]
|
||||
|
|
@ -422,7 +584,7 @@ class TestBedrockRealtimeHandler:
|
|||
stream = FakeBedrockStream()
|
||||
|
||||
await handler._forward_client_to_bedrock(
|
||||
DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}
|
||||
DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
assert stream.input_stream.sent == []
|
||||
|
|
@ -437,9 +599,13 @@ class TestBedrockRealtimeHandler:
|
|||
[json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})]
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="bedrock send failed"):
|
||||
await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {})
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
assert failure.value.status_code == 503
|
||||
assert str(failure.value.__cause__) == "bedrock send failed"
|
||||
assert stream.input_stream.closed
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -452,7 +618,10 @@ class TestBedrockRealtimeHandler:
|
|||
)
|
||||
|
||||
await asyncio.wait_for(
|
||||
handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}), 2
|
||||
handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
),
|
||||
2,
|
||||
)
|
||||
|
||||
assert not stream.input_stream.closed
|
||||
|
|
@ -466,7 +635,9 @@ class TestBedrockRealtimeHandler:
|
|||
[json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})]
|
||||
)
|
||||
|
||||
await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {})
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent]
|
||||
event_names = [next(iter(event["event"])) for event in sent_events]
|
||||
|
|
@ -495,6 +666,7 @@ class TestBedrockRealtimeHandler:
|
|||
"amazon.nova-sonic-v1:0",
|
||||
FakeLogging(),
|
||||
{},
|
||||
signals=_closing_input(),
|
||||
)
|
||||
]
|
||||
|
||||
|
|
@ -526,6 +698,7 @@ class TestBedrockRealtimeHandler:
|
|||
"amazon.nova-sonic-v1:0",
|
||||
FakeLogging(),
|
||||
{},
|
||||
signals=_closing_input(),
|
||||
)
|
||||
]
|
||||
|
||||
|
|
@ -597,6 +770,7 @@ class TestBedrockRealtimeHandler:
|
|||
"amazon.nova-sonic-v1:0",
|
||||
MagicMock(),
|
||||
{},
|
||||
signals=_closing_input(),
|
||||
):
|
||||
pass
|
||||
|
||||
|
|
@ -656,7 +830,9 @@ class TestBedrockRealtimeSessionLifecycle:
|
|||
[json.dumps({"type": "session.update", "session": {"instructions": "hi", "modalities": ["text"]}})]
|
||||
)
|
||||
|
||||
await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging())
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
acked = [json.loads(message) for message in client_ws.sent_to_client]
|
||||
updated = [event for event in acked if event["type"] == "session.updated"]
|
||||
|
|
@ -670,7 +846,9 @@ class TestBedrockRealtimeSessionLifecycle:
|
|||
stream = FakeBedrockStream()
|
||||
client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})])
|
||||
|
||||
await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {})
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
assert client_ws.sent_to_client == []
|
||||
|
||||
|
|
@ -693,7 +871,9 @@ class TestBedrockRealtimeSessionLifecycle:
|
|||
stream = FakeBedrockStream()
|
||||
client_ws = DisconnectingClientWS([update([tool]), update([other_tool]), update([tool])])
|
||||
|
||||
await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging())
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals()
|
||||
)
|
||||
|
||||
acked: Final = [json.loads(message) for message in client_ws.sent_to_client]
|
||||
assert [event["type"] for event in acked] == ["session.updated", "error", "session.updated"]
|
||||
|
|
@ -720,10 +900,10 @@ class TestBedrockRealtimeSessionLifecycle:
|
|||
async def test_malformed_setting_is_rejected_or_dropped_and_the_session_still_answers(
|
||||
self, stub_aws_sdk_client, session, drop_params, expected_errors
|
||||
):
|
||||
stream: Final = ScriptedBedrockStream([])
|
||||
stream: Final = SessionBoundBedrockStream()
|
||||
stub_aws_sdk_client["streams"] = [stream]
|
||||
user_text: Final = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}
|
||||
websocket: Final = ConnectedClientWS(
|
||||
websocket: Final = RealtimeClientWS(
|
||||
[
|
||||
json.dumps({"type": "session.update", "session": session}),
|
||||
json.dumps({"type": "conversation.item.create", "item": user_text}),
|
||||
|
|
@ -761,8 +941,8 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
@pytest.mark.asyncio
|
||||
async def test_readiness_failure_escapes_and_fallback_replays_session_update(self, stub_aws_sdk_client):
|
||||
handler = BedrockRealtime()
|
||||
websocket = ConnectedClientWS([self.SESSION_UPDATE])
|
||||
healthy_stream = ScriptedBedrockStream([])
|
||||
websocket = RealtimeClientWS([self.SESSION_UPDATE])
|
||||
healthy_stream = SessionBoundBedrockStream()
|
||||
eager_failure = ServiceUnavailableException("fault injected before the stream was returned")
|
||||
stub_aws_sdk_client["streams"] = [UnavailableBedrockStream(), eager_failure, healthy_stream]
|
||||
|
||||
|
|
@ -848,9 +1028,6 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
await spend_dispatch["coro"]
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client]
|
||||
flushed = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent]
|
||||
assert [next(iter(event["event"])) for event in flushed][-2:] == ["promptEnd", "sessionEnd"]
|
||||
assert stream.input_stream.closed
|
||||
|
||||
with pytest.raises(BedrockError) as replay:
|
||||
await handler.async_realtime(
|
||||
|
|
@ -877,8 +1054,8 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
@pytest.mark.asyncio
|
||||
async def test_readiness_failure_on_an_input_first_start_replays_the_input(self, stub_aws_sdk_client):
|
||||
handler = BedrockRealtime()
|
||||
websocket = ConnectedClientWS([self.USER_TEXT])
|
||||
healthy_stream = ScriptedBedrockStream([])
|
||||
websocket = RealtimeClientWS([self.USER_TEXT])
|
||||
healthy_stream = SessionBoundBedrockStream()
|
||||
stub_aws_sdk_client["streams"] = [UnavailableBedrockStream(), healthy_stream]
|
||||
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
|
|
@ -907,7 +1084,7 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
[self.SESSION_UPDATE, json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})]
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="bedrock input stream reset"):
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await BedrockRealtime().async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=websocket,
|
||||
|
|
@ -915,6 +1092,8 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
**self.AWS_PARAMS,
|
||||
)
|
||||
|
||||
assert failure.value.status_code == 503
|
||||
assert str(failure.value.__cause__) == "bedrock input stream reset", "the proxy log keeps the reset's cause"
|
||||
assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client]
|
||||
await spend_dispatch["coro"]
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
|
|
@ -923,10 +1102,10 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
async def test_success_dispatch_stamps_the_ownership_marker_only_when_spend_was_logged(
|
||||
self, stub_aws_sdk_client, spend_dispatch
|
||||
):
|
||||
stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)]
|
||||
stub_aws_sdk_client["streams"] = [SessionBoundBedrockStream(self.TEXT_TURN)]
|
||||
await BedrockRealtime().async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
|
||||
websocket=RealtimeClientWS([self.SESSION_UPDATE]),
|
||||
logging_obj=spend_dispatch["logging_obj"],
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
|
|
@ -935,15 +1114,315 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True
|
||||
|
||||
idle_logging = FakeLogging()
|
||||
stub_aws_sdk_client["streams"] = [ScriptedBedrockStream([])]
|
||||
stub_aws_sdk_client["streams"] = [SessionBoundBedrockStream()]
|
||||
await BedrockRealtime().async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
|
||||
websocket=RealtimeClientWS([self.SESSION_UPDATE]),
|
||||
logging_obj=idle_logging,
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
assert REALTIME_SESSION_SUCCESS_LOGGED_KEY not in idle_logging.model_call_details
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_stream_ending_mid_session_fails_the_session_and_keeps_partial_spend(
|
||||
self, stub_aws_sdk_client, spend_dispatch
|
||||
):
|
||||
handler: Final = BedrockRealtime()
|
||||
websocket: Final = ConnectedClientWS([self.SESSION_UPDATE])
|
||||
stream: Final = ScriptedBedrockStream(self.TEXT_TURN)
|
||||
stub_aws_sdk_client["streams"] = [stream]
|
||||
logging_obj: Final = spend_dispatch["logging_obj"]
|
||||
request_metadata: Final = {"user_api_key": "hashed-key"}
|
||||
logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": request_metadata}
|
||||
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await handler.async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=websocket,
|
||||
logging_obj=logging_obj,
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
|
||||
assert failure.value.status_code == 503
|
||||
assert not websocket.closed, "the proxy route owns the client-facing error event and 1011 close"
|
||||
assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client]
|
||||
await spend_dispatch["coro"]
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
status, error_information = _spend_row(logging_obj)
|
||||
assert status == "failure", "the Logs page must show the dropped session as failed while it bills its turn"
|
||||
assert error_information is not None
|
||||
assert error_information.get("error_code") == "503"
|
||||
assert error_information.get("error_message") == failure.value.message
|
||||
assert request_metadata == {"user_api_key": "hashed-key"}, "the request's own metadata stays as it was sent"
|
||||
|
||||
with pytest.raises(BedrockError) as replay:
|
||||
await handler.async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=websocket,
|
||||
logging_obj=logging_obj,
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
|
||||
assert replay.value.status_code == 400, "a router retry must not restart the dropped session"
|
||||
assert str(replay.value).startswith(failure.value.message), "the app's error event leads with the drop"
|
||||
assert websocket_close_reason(str(replay.value), fallback="Internal server error").startswith(
|
||||
failure.value.message
|
||||
), "the length-capped 1011 close reason must still carry the drop"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("receiver_type", "status_code"),
|
||||
[(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)],
|
||||
ids=["output-ends", "output-raises"],
|
||||
)
|
||||
async def test_a_failed_bedrock_output_reaches_the_app_without_waiting_on_the_input_close(
|
||||
self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int
|
||||
):
|
||||
stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type)
|
||||
stream.input_stream = HangingCloseInputStream()
|
||||
stub_aws_sdk_client["streams"] = [stream]
|
||||
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await asyncio.wait_for(
|
||||
BedrockRealtime(close_timeout=30).async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
|
||||
logging_obj=spend_dispatch["logging_obj"],
|
||||
**self.AWS_PARAMS,
|
||||
),
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
assert failure.value.status_code == status_code
|
||||
assert stub_aws_sdk_client["client_closed"], "closing the SDK client still tears the dead stream down"
|
||||
await spend_dispatch["coro"]
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("receiver_type", "status_code"),
|
||||
[(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)],
|
||||
ids=["output-ends", "output-raises"],
|
||||
)
|
||||
async def test_a_failed_bedrock_output_leaves_no_pending_input_writer(
|
||||
self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int
|
||||
):
|
||||
stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type)
|
||||
stream.input_stream = BodyWriterInputStream()
|
||||
stub_aws_sdk_client["streams"] = [stream]
|
||||
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await BedrockRealtime().async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
|
||||
logging_obj=spend_dispatch["logging_obj"],
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
|
||||
assert failure.value.status_code == status_code
|
||||
await asyncio.wait_for(stream.input_stream.body_writer, timeout=1)
|
||||
await spend_dispatch["coro"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_dropped_session_logs_one_success_usage_record_and_a_failed_spend_row(
|
||||
self, stub_aws_sdk_client, spend_dispatch
|
||||
):
|
||||
stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)]
|
||||
recorder: Final = LoggedEventRecorder()
|
||||
logging_obj: Final = LiteLLMLogging(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="_arealtime",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="nova-sonic-drop",
|
||||
function_id="nova-sonic-drop",
|
||||
dynamic_async_success_callbacks=[recorder],
|
||||
)
|
||||
logging_obj.update_from_kwargs(
|
||||
kwargs={"litellm_metadata": {"user_api_key": "hashed-key"}},
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
optional_params={},
|
||||
litellm_params={"_arealtime": True},
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await BedrockRealtime().async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
|
||||
logging_obj=logging_obj,
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
await spend_dispatch["coro"]
|
||||
|
||||
assert failure.value.status_code == 503
|
||||
((logged_event, usage_kwargs),) = recorder.events
|
||||
assert logged_event == "success"
|
||||
usage_record: Final = recorder.payload("success")
|
||||
assert usage_record["status"] == "success", "the router's failure event is each logger's one error record"
|
||||
assert [event["type"] for event in usage_record["response"]["results"]] == ["response.done"]
|
||||
assert logging_obj.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True
|
||||
status, error_information = _spend_row_of(usage_kwargs)
|
||||
assert status == "failure"
|
||||
assert error_information is not None
|
||||
assert error_information.get("error_code") == "503"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_dropped_session_bills_its_usage_to_loggers_while_the_failure_event_runs(
|
||||
self, stub_aws_sdk_client, spend_dispatch
|
||||
):
|
||||
usage_event: Final = json.dumps(
|
||||
{"event": {"usageEvent": {"totalInputTokens": 1000, "totalOutputTokens": 50, "totalTokens": 1050}}}
|
||||
)
|
||||
stub_aws_sdk_client["streams"] = [ScriptedBedrockStream((*self.TEXT_TURN, usage_event))]
|
||||
held: Final = HeldSuccessLogger()
|
||||
recorder: Final = LoggedEventRecorder()
|
||||
logging_obj: Final = LiteLLMLogging(
|
||||
model="amazon.nova-2-5-sonic",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="_arealtime",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="nova-sonic-drop-order",
|
||||
function_id="nova-sonic-drop-order",
|
||||
dynamic_async_success_callbacks=[held, recorder],
|
||||
dynamic_async_failure_callbacks=[recorder],
|
||||
)
|
||||
logging_obj.update_from_kwargs(
|
||||
kwargs={"litellm_metadata": {"user_api_key": "hashed-key"}},
|
||||
model="amazon.nova-2-5-sonic",
|
||||
optional_params={},
|
||||
litellm_params={"_arealtime": True},
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await BedrockRealtime().async_realtime(
|
||||
model="amazon.nova-2-5-sonic",
|
||||
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
|
||||
logging_obj=logging_obj,
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
success_dispatch: Final = asyncio.create_task(spend_dispatch["coro"])
|
||||
await asyncio.wait_for(held.entered.wait(), timeout=5)
|
||||
logging_obj.failure_handler(failure.value, "", logging_obj.start_time, datetime.now(), for_async_request=True)
|
||||
await logging_obj.async_failure_handler(failure.value, "", logging_obj.start_time, datetime.now())
|
||||
held.release.set()
|
||||
await success_dispatch
|
||||
|
||||
usage_record: Final = recorder.payload("success")
|
||||
assert recorder.payload("failure")["status"] == "failure"
|
||||
assert usage_record["status"] == "success"
|
||||
assert usage_record["total_tokens"] == 1050
|
||||
assert usage_record["response_cost"] > 0
|
||||
((_, usage_kwargs),) = (logged for logged in recorder.events if logged[0] == "success")
|
||||
status, _ = _spend_row_of(usage_kwargs)
|
||||
assert status == "failure"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_dropped_session_without_request_metadata_still_fails_and_bills_its_turn(
|
||||
self, stub_aws_sdk_client, spend_dispatch
|
||||
):
|
||||
stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)]
|
||||
logging_obj: Final = spend_dispatch["logging_obj"]
|
||||
logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": None, "metadata": {}}
|
||||
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await BedrockRealtime().async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
|
||||
logging_obj=logging_obj,
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
await spend_dispatch["coro"]
|
||||
|
||||
assert failure.value.status_code == 503
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("receiver_type", "status_code"),
|
||||
[(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)],
|
||||
ids=["output-ends", "output-raises"],
|
||||
)
|
||||
async def test_a_session_bedrock_fails_before_any_logged_event_dispatches_no_success(
|
||||
self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int
|
||||
):
|
||||
stub_aws_sdk_client["streams"] = [ScriptedBedrockStream((), receiver_type=receiver_type)]
|
||||
logging_obj: Final = spend_dispatch["logging_obj"]
|
||||
logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed-key"}}
|
||||
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await BedrockRealtime().async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
|
||||
logging_obj=logging_obj,
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
|
||||
assert failure.value.status_code == status_code
|
||||
assert "coro" not in spend_dispatch, "a failed session with no usage gets no success event, as on base"
|
||||
assert REALTIME_SESSION_SUCCESS_LOGGED_KEY not in logging_obj.model_call_details
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish(
|
||||
self, stub_aws_sdk_client, spend_dispatch
|
||||
):
|
||||
stream: Final = SessionBoundBedrockStream(self.TEXT_TURN, input_stream_type=SlowClosingInputStream)
|
||||
stub_aws_sdk_client["streams"] = [stream]
|
||||
websocket: Final = RealtimeClientWS([self.SESSION_UPDATE])
|
||||
spend_dispatch["logging_obj"].model_call_details["litellm_params"] = {
|
||||
"litellm_metadata": {"user_api_key": "hashed-key"}
|
||||
}
|
||||
|
||||
await asyncio.wait_for(
|
||||
BedrockRealtime(close_timeout=30).async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=websocket,
|
||||
logging_obj=spend_dispatch["logging_obj"],
|
||||
**self.AWS_PARAMS,
|
||||
),
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
assert _sent_event_names(stream.input_stream)[-1] == "sessionEnd"
|
||||
assert not stream.input_stream.closed, "the input close must still be pending when the output ends"
|
||||
assert websocket.closed
|
||||
await spend_dispatch["coro"]
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True
|
||||
assert _spend_row(spend_dispatch["logging_obj"]) == ("success", None)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_output_ending_after_the_input_side_failed_still_fails_the_session(
|
||||
self, stub_aws_sdk_client, spend_dispatch
|
||||
):
|
||||
connection_reset: Final = asyncio.Event()
|
||||
stream: Final = ScriptedBedrockStream(
|
||||
self.TEXT_TURN,
|
||||
receiver_type=lambda payloads: OutputEndsAfterConnectionResetReceiver(payloads, connection_reset),
|
||||
)
|
||||
stream.input_stream = ConnectionResetOnAudioInputStream(connection_reset)
|
||||
stub_aws_sdk_client["streams"] = [stream]
|
||||
websocket: Final = ConnectedClientWS([self.SESSION_UPDATE, self.USER_AUDIO])
|
||||
|
||||
with pytest.raises(BedrockError) as failure:
|
||||
await BedrockRealtime().async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=websocket,
|
||||
logging_obj=spend_dispatch["logging_obj"],
|
||||
**self.AWS_PARAMS,
|
||||
)
|
||||
|
||||
assert (failure.value.status_code, failure.value.message) == (
|
||||
503,
|
||||
"Bedrock realtime stream ended before the session closed",
|
||||
), "the app sees one cause whichever side of the stream reports the reset first"
|
||||
assert not websocket.closed, "the proxy route owns the client-facing error event and 1011 close"
|
||||
await spend_dispatch["coro"]
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_failure_after_client_disconnect_is_not_a_provider_failure(self, stub_aws_sdk_client):
|
||||
stream = ScriptedBedrockStream([], receiver_type=BreakingBedrockReceiver)
|
||||
|
|
@ -982,7 +1461,13 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
|
||||
with pytest.raises(ServiceUnavailableException):
|
||||
await handler._forward_client_to_bedrock(
|
||||
client_ws, stream, BedrockRealtimeConfig(), "amazon.nova-sonic-v1:0", {}, FakeLogging()
|
||||
client_ws,
|
||||
stream,
|
||||
BedrockRealtimeConfig(),
|
||||
"amazon.nova-sonic-v1:0",
|
||||
{},
|
||||
FakeLogging(),
|
||||
signals=SessionEndSignals(),
|
||||
)
|
||||
|
||||
assert client_ws.sent_to_client == []
|
||||
|
|
@ -1128,7 +1613,6 @@ class TestBedrockRealtimeSdkLifecycle:
|
|||
)
|
||||
|
||||
assert stub_aws_sdk_client["client_closed"]
|
||||
assert stub_aws_sdk_client["input_closed_before_client_close"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_without_close_completes_session(self, monkeypatch):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import Coroutine
|
||||
from collections.abc import Callable, Coroutine
|
||||
from datetime import datetime
|
||||
from typing import Final, NoReturn, cast
|
||||
|
||||
|
|
@ -13,11 +13,13 @@ from websockets.frames import Close
|
|||
from websockets.http11 import Response
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.constants import REALTIME_SESSION_SUCCESS_LOGGED_KEY, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
from litellm.cost_calculator import handle_realtime_stream_cost_calculation
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.openai.live import handler as live_handler
|
||||
from litellm.llms.openai.live.handler import LiveSessionStart, OpenAILiveSessions, parse_live_session_start
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute
|
||||
from litellm.types.utils import LiteLLMRealtimeStreamLoggingObject, Usage
|
||||
|
||||
_LIVE_EVENT_FRAME_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
|
|
@ -1206,3 +1208,91 @@ async def test_async_live_session_reports_redacted_server_error_and_bounds_close
|
|||
finally:
|
||||
if worker.coroutine is not None:
|
||||
await worker.coroutine
|
||||
|
||||
|
||||
_LIVE_SESSION_STARTED: Final = '{"type":"session.started","session":{"model":"gpt-live-1"}}'
|
||||
_LIVE_SESSION_CLOSED: Final = '{"type":"session.closed","usage":{"seconds":4}}'
|
||||
|
||||
|
||||
def _upstream_closing_the_session() -> _FakeUpstream:
|
||||
return _FakeUpstream(client_frames=(), upstream_frames=(_LIVE_SESSION_STARTED, _LIVE_SESSION_CLOSED))
|
||||
|
||||
|
||||
def _upstream_failing_after_a_frame() -> _FakeUpstream:
|
||||
return _FakeUpstream(
|
||||
client_frames=(),
|
||||
upstream_frames=(_LIVE_SESSION_STARTED,),
|
||||
close_error=ConnectionClosedError(Close(code=1011, reason="upstream failed"), None),
|
||||
)
|
||||
|
||||
|
||||
def _upstream_dropping_after_a_frame() -> _FakeUpstream:
|
||||
return _FakeUpstream(
|
||||
client_frames=(), upstream_frames=(_LIVE_SESSION_STARTED,), close_error=ConnectionClosedError(None, None)
|
||||
)
|
||||
|
||||
|
||||
def _upstream_failing_a_client_send() -> _FakeUpstream:
|
||||
return _FakeUpstream(
|
||||
client_frames=(),
|
||||
upstream_frames=(_LIVE_SESSION_STARTED,),
|
||||
close_response=_LIVE_SESSION_CLOSED,
|
||||
fail_on_send_type="conversation.item.create",
|
||||
send_error=OSError("upstream socket write failed"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("upstream_factory", "counted_successes"),
|
||||
[
|
||||
(_upstream_closing_the_session, 1),
|
||||
(_upstream_failing_after_a_frame, 0),
|
||||
(_upstream_dropping_after_a_frame, 0),
|
||||
(_upstream_failing_a_client_send, 0),
|
||||
],
|
||||
ids=["session-closed", "upstream-1011-after-a-frame", "upstream-1006-after-a-frame", "upstream-send-failed"],
|
||||
)
|
||||
async def test_live_session_counts_a_router_success_only_when_it_ended_cleanly(
|
||||
upstream_factory: Callable[[], _FakeUpstream], counted_successes: int
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "live-voice", "litellm_params": {"model": "openai/gpt-live-1"}, "model_info": {"id": "live-a"}}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
worker: Final = _FakeLoggingWorker()
|
||||
|
||||
async def _arealtime(litellm_logging_obj: Logging, **kwargs: object) -> None:
|
||||
await OpenAILiveSessions(
|
||||
logging_worker=worker, websocket_connector=_FakeConnector(upstream_factory())
|
||||
).async_live_session(
|
||||
model="gpt-live-1",
|
||||
websocket=_FakeClientWebSocket(messages=('{"type":"conversation.item.create"}',)),
|
||||
logging_obj=litellm_logging_obj,
|
||||
session_start={"type": "session.start", "session": {"model": "gpt-live-1"}},
|
||||
api_base="https://api.openai.com/",
|
||||
api_key="live-test-key",
|
||||
)
|
||||
|
||||
await asyncio.wait_for(
|
||||
router._ageneric_api_call_with_fallbacks(
|
||||
model="live-voice",
|
||||
original_function=_arealtime,
|
||||
litellm_logging_obj=Logging(
|
||||
model="gpt-live-1",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="_arealtime",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="live-session",
|
||||
function_id="live-session",
|
||||
),
|
||||
),
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
if worker.coroutine is not None:
|
||||
worker.coroutine.close()
|
||||
assert get_deployment_successes_for_current_minute(router, "live-a") == counted_successes
|
||||
|
|
|
|||
|
|
@ -3711,6 +3711,41 @@ async def test_failure_event_refunds_reservation_for_input_only_estimate_v3():
|
|||
assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure_first", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
("response_obj", "used_tokens"),
|
||||
[
|
||||
([], 0),
|
||||
(ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)), 30),
|
||||
],
|
||||
)
|
||||
async def test_a_request_logging_a_failure_and_a_success_settles_its_reservation_once_v3(
|
||||
failure_first: bool, response_obj: object, used_tokens: int
|
||||
) -> None:
|
||||
api_key: Final = hash_token("sk-failure-and-success")
|
||||
local_cache: Final = DualCache()
|
||||
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache))
|
||||
tokens_key: Final = handler.create_rate_limit_keys(key="api_key", value=api_key, rate_limit_type="tokens")
|
||||
await _reserve_tpm_for_owner_call(handler, local_cache, api_key, "dropped-call")
|
||||
kwargs: Final = {
|
||||
"litellm_call_id": "dropped-call",
|
||||
"standard_logging_object": {"metadata": {"user_api_key_hash": api_key}},
|
||||
}
|
||||
now: Final = datetime.now()
|
||||
log_failure: Final = partial(
|
||||
handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now
|
||||
)
|
||||
log_success: Final = partial(
|
||||
handler.async_log_success_event, kwargs=dict(kwargs), response_obj=response_obj, start_time=now, end_time=now
|
||||
)
|
||||
|
||||
for log_event in (log_failure, log_success) if failure_first else (log_success, log_failure):
|
||||
await log_event()
|
||||
|
||||
assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == used_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_settles_reservation_at_recovered_partial_usage_v3():
|
||||
"""
|
||||
|
|
@ -4198,6 +4233,125 @@ async def test_failure_event_settles_project_itpm_otpm_at_recovered_partial_usag
|
|||
assert int(await local_cache.async_get_cache(key=otpm_key) or 0) == 7
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure_first", [True, False])
|
||||
async def test_a_request_logging_a_failure_and_a_success_settles_its_project_itpm_otpm_once_v3(
|
||||
failure_first: bool,
|
||||
) -> None:
|
||||
api_key: Final = hash_token("sk-failure-and-success-project-io")
|
||||
local_cache: Final = DualCache()
|
||||
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache))
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key=api_key,
|
||||
project_id="proj-dropped",
|
||||
project_metadata={
|
||||
"model_itpm_limit": {"gpt-4o-mini": 10_000},
|
||||
"model_otpm_limit": {"gpt-4o-mini": 10_000},
|
||||
},
|
||||
),
|
||||
cache=local_cache,
|
||||
data={
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_tokens": 50,
|
||||
"litellm_call_id": "dropped-project-call",
|
||||
},
|
||||
call_type="completion",
|
||||
)
|
||||
itpm_key: Final = handler.create_rate_limit_keys(
|
||||
key="model_per_project_itpm", value="proj-dropped:gpt-4o-mini", rate_limit_type="tokens"
|
||||
)
|
||||
otpm_key: Final = handler.create_rate_limit_keys(
|
||||
key="model_per_project_otpm", value="proj-dropped:gpt-4o-mini", rate_limit_type="tokens"
|
||||
)
|
||||
kwargs: Final = {
|
||||
"litellm_call_id": "dropped-project-call",
|
||||
"model": "gpt-4o-mini",
|
||||
"standard_logging_object": {"metadata": {"user_api_key_hash": api_key}},
|
||||
}
|
||||
now: Final = datetime.now()
|
||||
log_failure: Final = partial(
|
||||
handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now
|
||||
)
|
||||
log_success: Final = partial(
|
||||
handler.async_log_success_event,
|
||||
kwargs=dict(kwargs),
|
||||
response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)),
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
)
|
||||
|
||||
for log_event in (log_failure, log_success) if failure_first else (log_success, log_failure):
|
||||
await log_event()
|
||||
|
||||
assert int(await local_cache.async_get_cache(key=itpm_key) or 0) == 20
|
||||
assert int(await local_cache.async_get_cache(key=otpm_key) or 0) == 7
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_reservation_made_after_a_released_one_in_the_same_context_settles_at_its_usage_v3() -> None:
|
||||
api_key: Final = hash_token("sk-second-reservation")
|
||||
local_cache: Final = DualCache()
|
||||
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache))
|
||||
user_api_key_dict: Final = UserAPIKeyAuth(
|
||||
api_key=api_key,
|
||||
tpm_limit=10_000,
|
||||
project_id="proj-second",
|
||||
project_metadata={
|
||||
"model_itpm_limit": {"gpt-4o-mini": 10_000},
|
||||
"model_otpm_limit": {"gpt-4o-mini": 10_000},
|
||||
},
|
||||
)
|
||||
now: Final = datetime.now()
|
||||
|
||||
async def reserve(call_id: str) -> None:
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
cache=local_cache,
|
||||
data={
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_tokens": 50,
|
||||
"litellm_call_id": call_id,
|
||||
},
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
def logged(call_id: str) -> dict[str, object]:
|
||||
return {
|
||||
"litellm_call_id": call_id,
|
||||
"model": "gpt-4o-mini",
|
||||
"standard_logging_object": {"metadata": {"user_api_key_hash": api_key}},
|
||||
}
|
||||
|
||||
await reserve("first-call")
|
||||
await handler.async_log_failure_event(kwargs=logged("first-call"), response_obj=None, start_time=now, end_time=now)
|
||||
await reserve("second-call")
|
||||
await handler.async_log_success_event(
|
||||
kwargs=logged("second-call"),
|
||||
response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)),
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
)
|
||||
|
||||
counters: Final = {
|
||||
"tokens": ("api_key", api_key),
|
||||
"itpm": ("model_per_project_itpm", "proj-second:gpt-4o-mini"),
|
||||
"otpm": ("model_per_project_otpm", "proj-second:gpt-4o-mini"),
|
||||
}
|
||||
settled: Final = {
|
||||
name: int(
|
||||
await local_cache.async_get_cache(
|
||||
key=handler.create_rate_limit_keys(key=key, value=value, rate_limit_type="tokens")
|
||||
)
|
||||
or 0
|
||||
)
|
||||
for name, (key, value) in counters.items()
|
||||
}
|
||||
assert settled == {"tokens": 27, "itpm": 20, "otpm": 7}
|
||||
|
||||
# ----------------------- Per-MCP-server rate limiting (v3) -----------------------
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3491,6 +3491,40 @@ async def test_sidecar_ignores_an_undecodable_event(): # test-quality-ok: a dis
|
|||
mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited()
|
||||
|
||||
|
||||
def _spend_row_error_information(update_database_kwargs: Mapping[str, object]) -> dict[str, object]:
|
||||
row: Final = get_logging_payload(
|
||||
kwargs=cast(dict, update_database_kwargs["kwargs"]),
|
||||
response_obj=update_database_kwargs["completion_response"],
|
||||
start_time=cast(datetime, update_database_kwargs["start_time"]),
|
||||
end_time=cast(datetime, update_database_kwargs["end_time"]),
|
||||
)
|
||||
return json.loads(row["metadata"])["error_information"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_hook_spend_row_truncates_a_long_provider_error_exactly_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "100")
|
||||
writer: Final = MagicMock(spec=DBSpendUpdateWriter)
|
||||
writer.update_database = AsyncMock()
|
||||
logger: Final = ProxyDBLogger(spend_writer=lambda: writer)
|
||||
provider_error: Final = "upstream failure " + "x" * 1000
|
||||
|
||||
await logger.async_post_call_failure_hook(
|
||||
request_data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]},
|
||||
original_exception=litellm.InternalServerError(message=provider_error, llm_provider="openai", model="gpt-4o"),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"),
|
||||
)
|
||||
|
||||
stored_message: Final = _spend_row_error_information(writer.update_database.call_args.kwargs)["error_message"]
|
||||
assert isinstance(stored_message, str)
|
||||
assert stored_message.count("skipped") == 1
|
||||
kept_chars: Final = len(stored_message.split("... (")[0]) + len(stored_message.rsplit(") ...", 1)[1])
|
||||
skipped_chars: Final = int(stored_message.split("skipped ", 1)[1].split(" chars", 1)[0])
|
||||
assert kept_chars + skipped_chars > len(provider_error)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_model_rejection():
|
||||
raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes"
|
||||
|
|
@ -3504,9 +3538,7 @@ async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_
|
|||
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"),
|
||||
)
|
||||
|
||||
error_information: Final = writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][
|
||||
"error_information"
|
||||
]
|
||||
error_information: Final = _spend_row_error_information(writer.update_database.call_args.kwargs)
|
||||
assert "medical records" not in json.dumps(error_information)
|
||||
assert (
|
||||
error_information["error_message"]
|
||||
|
|
|
|||
|
|
@ -7469,6 +7469,46 @@ def test_failure_error_information_masks_ip_addresses_in_long_text_and_at_its_tr
|
|||
assert sanitized["error_code"] == "500"
|
||||
|
||||
|
||||
def test_spend_row_sanitizes_the_session_failure_a_realtime_success_row_carries(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", False)
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "200")
|
||||
provider_error: Final = 'stream ended {"messages": [{"role": "user", "content": "private-prompt"}]} ' + "x" * 2000
|
||||
now: Final = datetime.datetime.now(timezone.utc)
|
||||
|
||||
row: Final = get_logging_payload(
|
||||
kwargs={
|
||||
"model": "bedrock/amazon.nova-sonic-v1:0",
|
||||
"call_type": "_arealtime",
|
||||
"litellm_params": {
|
||||
"metadata": {
|
||||
"user_api_key": "hashed-key",
|
||||
"status": "failure",
|
||||
"error_information": {
|
||||
"error_code": "503",
|
||||
"error_class": "BedrockError",
|
||||
"error_message": provider_error,
|
||||
"llm_provider": "bedrock",
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
response_obj=None,
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
)
|
||||
|
||||
stored: Final = json.loads(row["metadata"])["error_information"]
|
||||
assert row["status"] == "failure"
|
||||
assert "private-prompt" not in stored["error_message"]
|
||||
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored["error_message"]
|
||||
assert len(stored["error_message"]) < len(provider_error)
|
||||
assert stored["error_class"] == "BedrockError"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"address", ["203.0.113.9", "2001:db8::9", "1:2:3:4:5:6:7:8", "1:2:3:4:5:6:7::", "::1:2:3:4:5:6:7", "fe80::1"]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@ import functools
|
|||
import gc
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
import weakref
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from types import MethodType
|
||||
from typing import Final
|
||||
|
|
@ -13,6 +14,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from websockets.exceptions import ConnectionClosed
|
||||
from websockets.frames import Close
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
|
|
@ -21,8 +24,12 @@ from litellm.caching.in_memory_cache import InMemoryCache
|
|||
from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.realtime_errors import close_after_upstream_handshake_refusal
|
||||
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute
|
||||
from litellm.types.litellm_params import RoutingStrategyName
|
||||
from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -540,6 +547,180 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn(
|
|||
assert await router.cache.async_get_cache(key=tpm_key) == 17
|
||||
|
||||
|
||||
class _DiscardingLoggingWorker:
|
||||
def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, object, None]) -> None:
|
||||
async_coroutine.close()
|
||||
|
||||
|
||||
async def _client_never_sends() -> str:
|
||||
await asyncio.Event().wait()
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
def _realtime_client_ws() -> MagicMock:
|
||||
client_ws: Final = MagicMock()
|
||||
client_ws.headers = {}
|
||||
client_ws.receive_text = AsyncMock(side_effect=_client_never_sends)
|
||||
client_ws.send_text = AsyncMock()
|
||||
client_ws.close = AsyncMock()
|
||||
return client_ws
|
||||
|
||||
|
||||
def _realtime_session_logging() -> LiteLLMLogging:
|
||||
return LiteLLMLogging(
|
||||
model="gpt-realtime-2.1",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="_arealtime",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id=str(uuid.uuid4()),
|
||||
function_id="realtime-session",
|
||||
)
|
||||
|
||||
|
||||
async def _relay_upstream(litellm_logging_obj: LiteLLMLogging, *upstream_frames: bytes | Exception) -> None:
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.recv = AsyncMock(side_effect=list(upstream_frames))
|
||||
relay: Final = RealTimeStreaming(
|
||||
_realtime_client_ws(),
|
||||
backend_ws,
|
||||
litellm_logging_obj,
|
||||
model="gpt-realtime-2.1",
|
||||
logging_worker=_DiscardingLoggingWorker(),
|
||||
)
|
||||
await asyncio.wait_for(relay.bidirectional_forward(), timeout=2)
|
||||
|
||||
|
||||
_REALTIME_SESSION_CREATED: Final = json.dumps({"type": "session.created", "session": {"id": "sess_1"}}).encode()
|
||||
|
||||
|
||||
async def _clean_realtime_session(litellm_logging_obj: LiteLLMLogging) -> None:
|
||||
await _relay_upstream(litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(Close(1000, ""), None))
|
||||
|
||||
|
||||
async def _upstream_internal_error_after_a_frame(litellm_logging_obj: LiteLLMLogging) -> None:
|
||||
await _relay_upstream(
|
||||
litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(Close(1011, "upstream failed"), None)
|
||||
)
|
||||
|
||||
|
||||
async def _upstream_drop_without_close_frame_after_a_frame(litellm_logging_obj: LiteLLMLogging) -> None:
|
||||
await _relay_upstream(litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(None, None))
|
||||
|
||||
|
||||
async def _upstream_close_before_any_frame(litellm_logging_obj: LiteLLMLogging) -> None:
|
||||
await _relay_upstream(litellm_logging_obj, ConnectionClosed(Close(1008, "model not found"), None))
|
||||
|
||||
|
||||
async def _refused_handshake(litellm_logging_obj: LiteLLMLogging) -> None:
|
||||
await close_after_upstream_handshake_refusal(_realtime_client_ws(), 403)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("realtime_session", "counted_successes"),
|
||||
[
|
||||
(_clean_realtime_session, 1),
|
||||
(_upstream_close_before_any_frame, 0),
|
||||
(_refused_handshake, 0),
|
||||
(_upstream_internal_error_after_a_frame, 0),
|
||||
(_upstream_drop_without_close_frame_after_a_frame, 0),
|
||||
],
|
||||
ids=[
|
||||
"clean-session",
|
||||
"upstream-close-before-any-frame",
|
||||
"refused-handshake",
|
||||
"upstream-1011-after-a-frame",
|
||||
"upstream-1006-after-a-frame",
|
||||
],
|
||||
)
|
||||
async def test_realtime_session_counts_a_router_success_only_when_it_ended_without_a_failure(
|
||||
realtime_session: Callable[[LiteLLMLogging], Awaitable[None]], counted_successes: int
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "realtime-voice",
|
||||
"litellm_params": {"model": "openai/gpt-realtime-2.1"},
|
||||
"model_info": {"id": "realtime-a"},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None:
|
||||
await realtime_session(litellm_logging_obj)
|
||||
|
||||
await router._ageneric_api_call_with_fallbacks(
|
||||
model="realtime-voice", original_function=_arealtime, litellm_logging_obj=_realtime_session_logging()
|
||||
)
|
||||
|
||||
assert get_deployment_successes_for_current_minute(router, "realtime-a") == counted_successes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("realtime_model", ["bedrock/amazon.nova-sonic-v1:0", "openai/gpt-realtime-2.1"])
|
||||
@pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)])
|
||||
async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served(
|
||||
realtime_model: str, clean_sessions: int, cooled_down: bool
|
||||
) -> None:
|
||||
import time
|
||||
|
||||
provider, _, provider_model = realtime_model.partition("/")
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "realtime-voice",
|
||||
"litellm_params": {"model": realtime_model, "order": order},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for order, deployment_id in ((1, "realtime-a"), (2, "realtime-b"))
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
drop: Final = litellm.ServiceUnavailableError(
|
||||
message="realtime provider stream ended before the session closed",
|
||||
llm_provider=provider,
|
||||
model=provider_model,
|
||||
)
|
||||
|
||||
def realtime_session(failure: Exception | None):
|
||||
async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None:
|
||||
if failure is not None:
|
||||
raise failure
|
||||
await _clean_realtime_session(litellm_logging_obj)
|
||||
|
||||
return _arealtime
|
||||
|
||||
for _ in range(clean_sessions):
|
||||
await router._ageneric_api_call_with_fallbacks(
|
||||
model="realtime-voice",
|
||||
original_function=realtime_session(None),
|
||||
litellm_logging_obj=_realtime_session_logging(),
|
||||
)
|
||||
|
||||
async def drop_session() -> bool:
|
||||
with pytest.raises(litellm.ServiceUnavailableError):
|
||||
await router._ageneric_api_call_with_fallbacks(
|
||||
model="realtime-voice",
|
||||
original_function=realtime_session(drop),
|
||||
litellm_logging_obj=_realtime_session_logging(),
|
||||
)
|
||||
return router.deployment_callback_on_failure(
|
||||
kwargs={
|
||||
"exception": drop,
|
||||
"litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": {"id": "realtime-a"}},
|
||||
},
|
||||
completion_response=None,
|
||||
start_time=time.time(),
|
||||
end_time=time.time(),
|
||||
)
|
||||
|
||||
drop_cooldowns: Final = [await drop_session() for _ in range(5)]
|
||||
|
||||
assert any(drop_cooldowns) is cooled_down
|
||||
|
||||
|
||||
async def _kwargs_routed_to(router: Router, model: str, function_name: str, **request: object) -> dict[str, object]:
|
||||
routed: Final[dict[str, object]] = {}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue