From d271d221fdeba042faf7f437b4381e8de422147f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:29:38 -0700 Subject: [PATCH] fix(bedrock): count realtime router successes only for sessions that logged one, and log first-turn drops The router counts a realtime success only when the session's success handlers ran, so a refused handshake or an upstream close before any frame no longer offsets its failure. The success event's payload status stays success again, so each logger gets one failure record per dropped session, while the spend row still reads failure from the metadata the Bedrock handler records. A committed Bedrock session that fails before any logged event now gets one zero-cost Logs row. After an output failure the input stream is closed in the background, so the SDK body writer finishes without delaying the client's error. Every input send failure reports the same 503 stream-ended cause as an output drop --- litellm/litellm_core_utils/litellm_logging.py | 27 +--- litellm/llms/bedrock/realtime/handler.py | 37 +++++- litellm/router.py | 10 +- .../test_litellm_logging.py | 67 ---------- .../realtime/test_bedrock_realtime_handler.py | 98 +++++++++++++- .../test_router/test_router_helper_utils.py | 121 ++++++++++++++++-- 6 files changed, 247 insertions(+), 113 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index b340455de98..7f4b069ded6 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -7478,25 +7478,10 @@ def get_standard_logging_object_payload( router_model_id=logging_obj.get_router_model_id(), ) - metadata_fields: Final = cast(Mapping[str, object], metadata) # cast-ok: merged request metadata is str-keyed - recorded_error: Final = metadata_fields.get("error_information") - recorded_failure: Final = ( - cast(StandardLoggingPayloadErrorInformation, recorded_error) # cast-ok: a reserved key only litellm writes - if isinstance(recorded_error, dict) - and status == "success" - and call_type == CallTypes.arealtime.value - and metadata_fields.get("status") == "failure" - else None - ) - payload_status: Final[StandardLoggingPayloadStatus] = "failure" if recorded_failure is not None else status - error_information, error_str = ( - (recorded_failure, error_str or recorded_failure.get("error_message")) - if recorded_failure is not None - else StandardLoggingPayloadSetup.get_error_information_for_logging_payload( - metadata=metadata, - original_exception=original_exception, - error_str=error_str, - ) + error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( + metadata=metadata, + original_exception=original_exception, + error_str=error_str, ) ## get final response object ## @@ -7587,9 +7572,9 @@ def get_standard_logging_object_payload( call_type=call_type or "", cache_hit=cache_hit, stream=stream, - status=payload_status, + status=status, status_fields=_get_status_fields( - status=payload_status, + status=status, guardrail_information=metadata.get("standard_logging_guardrail_information", None), error_str=error_str, ), diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index b447aacd993..e6b34487cbf 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -63,6 +63,10 @@ _BEDROCK_STREAM_ERROR_STATUS: Final[Mapping[str, int]] = MappingProxyType( ) +def _stream_ended_before_session_closed() -> BedrockError: + return BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed") + + def _as_bedrock_error(error: BaseException) -> BaseException: status_code: Final = _BEDROCK_STREAM_ERROR_STATUS.get(type(error).__name__) if status_code is None: @@ -244,6 +248,8 @@ def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseExc verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error))) if _is_session_committed(scope): scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = redact_string(str(error)) + if error is failure: + raise failure raise error from failure @@ -347,6 +353,18 @@ class BedrockRealtime(BaseAWSLLM): super().__init__() self._sdk_version_lookup: Final = sdk_version_lookup self._close_timeout: Final = close_timeout + self._failed_input_closes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs to running closes + + def _close_failed_input_in_background( + self, + bedrock_stream: BedrockBidirectionalStream, + send_to_bedrock: Callable[[str], Awaitable[None]], + ) -> None: + close_task: Final = asyncio.create_task( + _close_bedrock_input(bedrock_stream, (), send_to_bedrock, self._close_timeout) + ) + self._failed_input_closes.add(close_task) + close_task.add_done_callback(self._failed_input_closes.discard) async def async_realtime( self, @@ -507,7 +525,10 @@ class BedrockRealtime(BaseAWSLLM): "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure ) provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure - if logged_events: + provider_ended_committed_session: Final = provider_failure is not None and _is_session_committed( + websocket.scope + ) + if logged_events or provider_ended_committed_session: if provider_failure is not None: _record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure)) GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( @@ -612,7 +633,10 @@ class BedrockRealtime(BaseAWSLLM): async def send_to_bedrock(bedrock_message: str) -> None: event: Final = build_input_chunk(bedrock_message.encode("utf-8")) - await bedrock_stream.input_stream.send(event) + try: + await bedrock_stream.input_stream.send(event) + except Exception as e: + raise _stream_ended_before_session_closed() from e verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200]) try: @@ -646,7 +670,9 @@ class BedrockRealtime(BaseAWSLLM): break signals.input_closing.set() finally: - if not signals.output_failed.is_set(): + if signals.output_failed.is_set(): + self._close_failed_input_in_background(bedrock_stream, send_to_bedrock) + else: await _close_bedrock_input( bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout ) @@ -682,10 +708,7 @@ class BedrockRealtime(BaseAWSLLM): if result is None: if not signals.input_closing.is_set(): - raise BedrockError( - status_code=503, - message="Bedrock realtime stream ended before the session closed", - ) + raise _stream_ended_before_session_closed() verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended") with contextlib.suppress(Exception): await client_ws.close() diff --git a/litellm/router.py b/litellm/router.py index 0dc798be380..af3d6c7505d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -98,6 +98,7 @@ from litellm.constants import ( MID_STREAM_FAILURE_LOG_WAIT_SECONDS, OUTPUT_TOKEN_CEILING_PARAMS, PROMPT_CACHE_PIN_CACHE_MAX_SIZE, + REALTIME_SESSION_SUCCESS_LOGGED_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, RUNTIME_UPDATABLE_ROUTER_SETTINGS, @@ -649,6 +650,7 @@ _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType _NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_CALL_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) @@ -6478,9 +6480,15 @@ class Router: raise safeguard_refusal_error(model=model, stop_details=refusal_details) self.success_calls[model_name] += 1 + realtime_session_logging: Final = ( + _CALL_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_logging_obj") + if generic_function_name == "_arealtime" + else None + ) realtime_session_deployment_id: Final = ( _MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id") - if generic_function_name == "_arealtime" + if isinstance(realtime_session_logging, LiteLLMLogging) + and realtime_session_logging.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True else None ) if realtime_session_deployment_id is not None: diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 868fe529815..299088a6c4f 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6511,73 +6511,6 @@ def test_get_error_information_for_logging_payload_client_disconnect(): assert error_str is None -_SESSION_FAILURE: Final = { - "error_code": "503", - "error_class": "BedrockError", - "error_message": "Bedrock realtime stream ended before the session closed", - "llm_provider": "bedrock", -} - - -@pytest.mark.parametrize("metadata_bucket", ["metadata", "litellm_metadata"]) -def test_realtime_success_payload_reports_the_session_failure_its_metadata_recorded( - logging_obj: LitellmLogging, metadata_bucket: str -) -> None: - from datetime import datetime - - from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload - - now: Final = datetime(2026, 1, 1) - payload: Final = get_standard_logging_object_payload( - kwargs={ - "model": "bedrock/amazon.nova-sonic-v1:0", - "call_type": "_arealtime", - "litellm_params": {metadata_bucket: {"status": "failure", "error_information": dict(_SESSION_FAILURE)}}, - }, - init_response_obj={}, - start_time=now, - end_time=now, - logging_obj=logging_obj, - status="success", - ) - - assert payload is not None - assert payload["status"] == "failure" - assert payload["status_fields"]["llm_api_status"] == "failure" - assert payload["error_information"] == _SESSION_FAILURE - assert payload["error_str"] == _SESSION_FAILURE["error_message"] - - -@pytest.mark.parametrize( - ("call_type", "metadata"), - [ - ("_arealtime", {"status": "failure"}), - ("_arealtime", {"error_information": dict(_SESSION_FAILURE)}), - ("acompletion", {"status": "failure", "error_information": dict(_SESSION_FAILURE)}), - ], -) -def test_success_payload_stays_success_without_a_recorded_realtime_session_failure( - logging_obj: LitellmLogging, call_type: str, metadata: dict[str, object] -) -> None: - from datetime import datetime - - from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload - - now: Final = datetime(2026, 1, 1) - payload: Final = get_standard_logging_object_payload( - kwargs={"model": "gpt-4o", "messages": [], "call_type": call_type, "litellm_params": {"metadata": metadata}}, - init_response_obj={}, - start_time=now, - end_time=now, - logging_obj=logging_obj, - status="success", - ) - - assert payload is not None - assert payload["status"] == "success" - assert payload["error_information"] != _SESSION_FAILURE - - def test_get_error_information_proxy_exception_preserves_message(): """ProxyException keeps its text in ``.message`` (str() was empty pre-fix), so error_information must still surface the message and code.""" diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 9e4ff7f05af..a075e89f626 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -16,6 +16,7 @@ from litellm.constants import ( REALTIME_SESSION_SUCCESS_LOGGED_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.realtime_errors import websocket_close_reason from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals @@ -101,6 +102,19 @@ class HangingCloseInputStream(FakeInputStream): await asyncio.Future() +class BodyWriterInputStream(FakeInputStream): + """Models the SDK input stream, whose request-body writer task only finishes once the stream is closed""" + + def __init__(self): + super().__init__() + self._close_requested = asyncio.Event() + self.body_writer = asyncio.get_running_loop().create_task(self._close_requested.wait()) + + async def close(self): + await super().close() + self._close_requested.set() + + class FakeBedrockStream: def __init__(self, input_stream=None): self.input_stream = input_stream if input_stream is not None else FakeInputStream() @@ -525,11 +539,13 @@ class TestBedrockRealtimeHandler: [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] ) - with pytest.raises(RuntimeError, match="bedrock send failed"): + with pytest.raises(BedrockError) as failure: await handler._forward_client_to_bedrock( client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) + assert failure.value.status_code == 503 + assert str(failure.value.__cause__) == "bedrock send failed" assert stream.input_stream.closed @pytest.mark.asyncio @@ -1008,7 +1024,7 @@ class TestBedrockRealtimeProviderFailurePropagation: [self.SESSION_UPDATE, json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})] ) - with pytest.raises(RuntimeError, match="bedrock input stream reset"): + with pytest.raises(BedrockError) as failure: await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, @@ -1016,6 +1032,8 @@ class TestBedrockRealtimeProviderFailurePropagation: **self.AWS_PARAMS, ) + assert failure.value.status_code == 503 + assert str(failure.value.__cause__) == "bedrock input stream reset", "the proxy log keeps the reset's cause" assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] @@ -1119,6 +1137,76 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_failed_bedrock_output_leaves_no_pending_input_writer( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type) + stream.input_stream = BodyWriterInputStream() + stub_aws_sdk_client["streams"] = [stream] + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ) + + assert failure.value.status_code == status_code + await asyncio.wait_for(stream.input_stream.body_writer, timeout=1) + await spend_dispatch["coro"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("bedrock_payloads", "logged_event_types"), + [(TEXT_TURN, ["response.done"]), ((), [])], + ids=["after-a-turn", "before-any-event"], + ) + async def test_a_dropped_session_logs_one_success_usage_record_and_a_failed_spend_row( + self, stub_aws_sdk_client, spend_dispatch, bedrock_payloads, logged_event_types: list[str] + ): + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(bedrock_payloads)] + logging_obj: Final = LiteLLMLogging( + model="amazon.nova-sonic-v1:0", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id="nova-sonic-drop", + function_id="nova-sonic-drop", + ) + logging_obj.update_from_kwargs( + kwargs={"litellm_metadata": {"user_api_key": "hashed-key"}}, + model="amazon.nova-sonic-v1:0", + optional_params={}, + litellm_params={"_arealtime": True}, + custom_llm_provider="bedrock", + ) + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + await spend_dispatch["coro"] + + assert failure.value.status_code == 503 + usage_record: Final = logging_obj.model_call_details["standard_logging_object"] + assert usage_record["status"] == "success", "the router's failure event is each logger's one error record" + assert [event["type"] for event in usage_record["response"]["results"]] == logged_event_types + assert usage_record["response_cost"] == 0, "no usage event streamed, so nothing is billed" + assert logging_obj.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + status, error_information = _spend_row(logging_obj) + assert (status, error_information["error_code"]) == ("failure", "503") + @pytest.mark.asyncio async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( self, stub_aws_sdk_client, spend_dispatch @@ -1161,7 +1249,7 @@ class TestBedrockRealtimeProviderFailurePropagation: stub_aws_sdk_client["streams"] = [stream] websocket: Final = ConnectedClientWS([self.SESSION_UPDATE, self.USER_AUDIO]) - with pytest.raises((BedrockError, RuntimeError)): + with pytest.raises(BedrockError) as failure: await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, @@ -1169,6 +1257,10 @@ class TestBedrockRealtimeProviderFailurePropagation: **self.AWS_PARAMS, ) + assert (failure.value.status_code, failure.value.message) == ( + 503, + "Bedrock realtime stream ended before the session closed", + ), "the app sees one cause whichever side of the stream reports the reset first" assert not websocket.closed, "the proxy route owns the client-facing error event and 1011 close" await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 17e954ccd9d..50e9d47bbc7 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -3,14 +3,17 @@ import functools import gc import json import os +import uuid import weakref -from collections.abc import Iterator, Mapping +from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping from datetime import datetime, timezone from types import MethodType from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +from websockets.exceptions import ConnectionClosed +from websockets.frames import Close import litellm from litellm import Router @@ -19,8 +22,12 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.realtime_errors import close_after_upstream_handshake_refusal +from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook +from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute from litellm.types.litellm_params import RoutingStrategyName from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo from litellm.types.utils import ( @@ -538,6 +545,93 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( assert await router.cache.async_get_cache(key=tpm_key) == 17 +class _DiscardingLoggingWorker: + def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, object, None]) -> None: + async_coroutine.close() + + +async def _client_never_sends() -> str: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + +def _realtime_client_ws() -> MagicMock: + client_ws: Final = MagicMock() + client_ws.headers = {} + client_ws.receive_text = AsyncMock(side_effect=_client_never_sends) + client_ws.send_text = AsyncMock() + client_ws.close = AsyncMock() + return client_ws + + +def _realtime_session_logging() -> LiteLLMLogging: + return LiteLLMLogging( + model="gpt-realtime-2.1", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id=str(uuid.uuid4()), + function_id="realtime-session", + ) + + +async def _relay_upstream(litellm_logging_obj: LiteLLMLogging, *upstream_frames: bytes | Exception) -> None: + backend_ws: Final = MagicMock() + backend_ws.recv = AsyncMock(side_effect=list(upstream_frames)) + relay: Final = RealTimeStreaming( + _realtime_client_ws(), + backend_ws, + litellm_logging_obj, + model="gpt-realtime-2.1", + logging_worker=_DiscardingLoggingWorker(), + ) + await asyncio.wait_for(relay.bidirectional_forward(), timeout=2) + + +async def _clean_realtime_session(litellm_logging_obj: LiteLLMLogging) -> None: + session_created: Final = json.dumps({"type": "session.created", "session": {"id": "sess_1"}}).encode() + await _relay_upstream(litellm_logging_obj, session_created, ConnectionClosed(Close(1000, ""), None)) + + +async def _upstream_close_before_any_frame(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream(litellm_logging_obj, ConnectionClosed(Close(1008, "model not found"), None)) + + +async def _refused_handshake(litellm_logging_obj: LiteLLMLogging) -> None: + await close_after_upstream_handshake_refusal(_realtime_client_ws(), 403) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("realtime_session", "counted_successes"), + [(_clean_realtime_session, 1), (_upstream_close_before_any_frame, 0), (_refused_handshake, 0)], + ids=["clean-session", "upstream-close-before-any-frame", "refused-handshake"], +) +async def test_realtime_session_counts_a_router_success_only_when_it_ended_without_a_failure( + realtime_session: Callable[[LiteLLMLogging], Awaitable[None]], counted_successes: int +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "realtime-voice", + "litellm_params": {"model": "openai/gpt-realtime-2.1"}, + "model_info": {"id": "realtime-a"}, + } + ], + num_retries=0, + ) + + async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None: + await realtime_session(litellm_logging_obj) + + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", original_function=_arealtime, litellm_logging_obj=_realtime_session_logging() + ) + + assert get_deployment_successes_for_current_minute(router, "realtime-a") == counted_successes + + @pytest.mark.asyncio @pytest.mark.parametrize("realtime_model", ["bedrock/amazon.nova-sonic-v1:0", "openai/gpt-realtime-2.1"]) @pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)]) @@ -565,28 +659,27 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy ) def realtime_session(failure: Exception | None): - async def _arealtime(**kwargs: object) -> None: - await router.deployment_callback_on_success( - kwargs={ - "call_type": "_arealtime", - "litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": kwargs["model_info"]}, - "standard_logging_object": create_standard_logging_payload(), - }, - completion_response=None, - start_time=time.time(), - end_time=time.time(), - ) + async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None: if failure is not None: raise failure + await _clean_realtime_session(litellm_logging_obj) return _arealtime for _ in range(clean_sessions): - await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(None)) + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", + original_function=realtime_session(None), + litellm_logging_obj=_realtime_session_logging(), + ) async def drop_session() -> bool: with pytest.raises(litellm.ServiceUnavailableError): - await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(drop)) + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", + original_function=realtime_session(drop), + litellm_logging_obj=_realtime_session_logging(), + ) return router.deployment_callback_on_failure( kwargs={ "exception": drop,