From f05d6129dfcec9ca8eb17454a7dba06cd8da4a3b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:08:47 -0700 Subject: [PATCH 1/8] fix(bedrock): fail a Nova Sonic realtime session whose Bedrock stream drops instead of closing 1000 The Bedrock SDK ends the output stream the same way for a dropped connection as for a session that sent sessionEnd, so the handler treated a dropped stream as a clean finish. The client got a 1000 close with no error event and the router logged 200 OK with no retry An output end now counts as clean only after the client left and the handler began closing the Bedrock input. An earlier end raises a 503 BedrockError, so the client gets an error event and a 1011 close, the failure handlers run, partial spend is still logged, and a retry of a committed session is still refused --- litellm/llms/bedrock/realtime/handler.py | 28 ++- .../realtime/test_bedrock_realtime_handler.py | 231 ++++++++++++++++-- 2 files changed, 234 insertions(+), 25 deletions(-) diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index e9b2f26e977..9f5c56fd106 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -475,16 +475,30 @@ class BedrockRealtime(BaseAWSLLM): ) -> _BridgeOutcome: """Run both forwarding directions until the client leaves or either side fails.""" logged: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events forwarded before a failure are still spend + input_closing: Final = asyncio.Event() async def collect_logged_events() -> None: async for event in self._forward_bedrock_to_client( - bedrock_stream, websocket, transformation_config, model, logging_obj, session_state + bedrock_stream, + websocket, + transformation_config, + model, + logging_obj, + session_state, + input_closing=input_closing, ): logged.append(event) client_task: Final = asyncio.create_task( self._forward_client_to_bedrock( - websocket, bedrock_stream, transformation_config, model, session_state, logging_obj, initial_message + websocket, + bedrock_stream, + transformation_config, + model, + session_state, + logging_obj, + initial_message, + input_closing=input_closing, ) ) bedrock_task: Final = asyncio.create_task(collect_logged_events()) @@ -518,6 +532,8 @@ class BedrockRealtime(BaseAWSLLM): session_state: RealtimeResponseTransformInput, logging_obj: LiteLLMLogging | None = None, initial_message: str | None = None, + *, + input_closing: asyncio.Event, ) -> None: """Forward messages from client WebSocket to Bedrock stream. @@ -565,6 +581,7 @@ class BedrockRealtime(BaseAWSLLM): client_ws, transformation_config, model, logging_obj, parsed_client_message ): break + input_closing.set() finally: async def close_bedrock_input() -> None: @@ -589,6 +606,8 @@ class BedrockRealtime(BaseAWSLLM): model: str, logging_obj: LiteLLMLogging, session_state: RealtimeResponseTransformInput, + *, + input_closing: asyncio.Event, ) -> AsyncIterator[OpenAIRealtimeEvents]: """Forward messages from Bedrock to the client, yielding the ones to record for spend logging. @@ -609,6 +628,11 @@ class BedrockRealtime(BaseAWSLLM): result = await output[1].receive() if result is None: + if not input_closing.is_set(): + raise BedrockError( + status_code=503, + message="Bedrock realtime stream ended before the session closed", + ) verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended") with contextlib.suppress(Exception): await client_ws.close() diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 2ac31b6887d..329ea9dd336 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -36,6 +36,12 @@ def _isolate_host_aws_config(monkeypatch, tmp_path): monkeypatch.delenv(env_var, raising=False) +def _closing_input() -> asyncio.Event: + closing: Final = asyncio.Event() + closing.set() + return closing + + class FakePayloadPart: def __init__(self, bytes_): self.bytes_ = bytes_ @@ -159,12 +165,15 @@ class EndedBedrockStream: class RealtimeClientWS: - def __init__(self): + def __init__(self, messages=()): self.closed = False self.sent_to_client = [] self.scope = {} + self._messages = list(messages) async def receive_text(self): + if self._messages: + return self._messages.pop(0) raise RuntimeError("client disconnected") async def send_text(self, message): @@ -178,8 +187,7 @@ class ConnectedClientWS(RealtimeClientWS): """Client that sends its scripted messages and then stays connected until the server closes it""" def __init__(self, messages): - super().__init__() - self._messages = list(messages) + super().__init__(messages) self._closed_event = asyncio.Event() async def receive_text(self): @@ -251,6 +259,74 @@ class ScriptedBedrockStream: return (None, self._receiver) +def _sent_event_names(input_stream): + return [next(iter(json.loads(chunk.value.bytes_.decode("utf-8"))["event"])) for chunk in input_stream.sent] + + +class SessionEndingInputStream(FakeInputStream): + def __init__(self): + super().__init__() + self.session_ended = asyncio.Event() + + async def send(self, event): + await super().send(event) + if "sessionEnd" in json.loads(event.value.bytes_.decode("utf-8")).get("event", {}): + self.session_ended.set() + + async def close(self): + await super().close() + self.session_ended.set() + + +class SlowClosingInputStream(SessionEndingInputStream): + async def close(self): + await asyncio.Event().wait() + + +class OutputEndsAfterSessionEndReceiver(ScriptedBedrockReceiver): + def __init__(self, payloads, session_ended): + super().__init__(payloads) + self._session_ended = session_ended + + async def receive(self): + if not self._payloads: + await self._session_ended.wait() + return await super().receive() + + +class SessionBoundBedrockStream: + def __init__(self, payloads=(), input_stream_type=SessionEndingInputStream): + self.input_stream = input_stream_type() + self._receiver = OutputEndsAfterSessionEndReceiver(payloads, self.input_stream.session_ended) + + async def await_output(self): + return (None, self._receiver) + + +class ConnectionResetOnAudioInputStream(FakeInputStream): + def __init__(self, connection_reset): + super().__init__() + self._connection_reset = connection_reset + + async def send(self, event): + await asyncio.sleep(0) + if self._connection_reset.is_set() or "audioInput" in json.loads(event.value.bytes_.decode("utf-8"))["event"]: + self._connection_reset.set() + raise RuntimeError("bedrock connection reset") + self.sent.append(event) + + +class OutputEndsAfterConnectionResetReceiver(ScriptedBedrockReceiver): + def __init__(self, payloads, connection_reset): + super().__init__(payloads) + self._connection_reset = connection_reset + + async def receive(self): + if not self._payloads: + await self._connection_reset.wait() + return await super().receive() + + class FakeAWSCredentialsIdentity: def __init__(self, access_key_id, secret_access_key, session_token=None): self.access_key_id = access_key_id @@ -352,7 +428,7 @@ def stub_aws_sdk_client(monkeypatch): raise stream captured["open_stream"] = stream return stream - stream = ScriptedBedrockStream(captured.get("scripted_payloads", [])) + stream = SessionBoundBedrockStream(captured.get("scripted_payloads", [])) captured["open_stream"] = stream return stream @@ -407,7 +483,9 @@ class TestBedrockRealtimeHandler: [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] ) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] event_names = [next(iter(event["event"])) for event in sent_events] @@ -422,7 +500,7 @@ class TestBedrockRealtimeHandler: stream = FakeBedrockStream() await handler._forward_client_to_bedrock( - DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {} + DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() ) assert stream.input_stream.sent == [] @@ -438,7 +516,9 @@ class TestBedrockRealtimeHandler: ) with pytest.raises(RuntimeError, match="bedrock send failed"): - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + ) assert stream.input_stream.closed @@ -452,7 +532,10 @@ class TestBedrockRealtimeHandler: ) await asyncio.wait_for( - handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}), 2 + handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + ), + 2, ) assert not stream.input_stream.closed @@ -466,7 +549,9 @@ class TestBedrockRealtimeHandler: [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] ) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] event_names = [next(iter(event["event"])) for event in sent_events] @@ -495,6 +580,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, + input_closing=_closing_input(), ) ] @@ -526,6 +612,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, + input_closing=_closing_input(), ) ] @@ -597,6 +684,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", MagicMock(), {}, + input_closing=_closing_input(), ): pass @@ -656,7 +744,9 @@ class TestBedrockRealtimeSessionLifecycle: [json.dumps({"type": "session.update", "session": {"instructions": "hi", "modalities": ["text"]}})] ) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging()) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event() + ) acked = [json.loads(message) for message in client_ws.sent_to_client] updated = [event for event in acked if event["type"] == "session.updated"] @@ -670,7 +760,9 @@ class TestBedrockRealtimeSessionLifecycle: stream = FakeBedrockStream() client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})]) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + ) assert client_ws.sent_to_client == [] @@ -693,7 +785,9 @@ class TestBedrockRealtimeSessionLifecycle: stream = FakeBedrockStream() client_ws = DisconnectingClientWS([update([tool]), update([other_tool]), update([tool])]) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging()) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event() + ) acked: Final = [json.loads(message) for message in client_ws.sent_to_client] assert [event["type"] for event in acked] == ["session.updated", "error", "session.updated"] @@ -720,10 +814,10 @@ class TestBedrockRealtimeSessionLifecycle: async def test_malformed_setting_is_rejected_or_dropped_and_the_session_still_answers( self, stub_aws_sdk_client, session, drop_params, expected_errors ): - stream: Final = ScriptedBedrockStream([]) + stream: Final = SessionBoundBedrockStream() stub_aws_sdk_client["streams"] = [stream] user_text: Final = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]} - websocket: Final = ConnectedClientWS( + websocket: Final = RealtimeClientWS( [ json.dumps({"type": "session.update", "session": session}), json.dumps({"type": "conversation.item.create", "item": user_text}), @@ -761,8 +855,8 @@ class TestBedrockRealtimeProviderFailurePropagation: @pytest.mark.asyncio async def test_readiness_failure_escapes_and_fallback_replays_session_update(self, stub_aws_sdk_client): handler = BedrockRealtime() - websocket = ConnectedClientWS([self.SESSION_UPDATE]) - healthy_stream = ScriptedBedrockStream([]) + websocket = RealtimeClientWS([self.SESSION_UPDATE]) + healthy_stream = SessionBoundBedrockStream() eager_failure = ServiceUnavailableException("fault injected before the stream was returned") stub_aws_sdk_client["streams"] = [UnavailableBedrockStream(), eager_failure, healthy_stream] @@ -877,8 +971,8 @@ class TestBedrockRealtimeProviderFailurePropagation: @pytest.mark.asyncio async def test_readiness_failure_on_an_input_first_start_replays_the_input(self, stub_aws_sdk_client): handler = BedrockRealtime() - websocket = ConnectedClientWS([self.USER_TEXT]) - healthy_stream = ScriptedBedrockStream([]) + websocket = RealtimeClientWS([self.USER_TEXT]) + healthy_stream = SessionBoundBedrockStream() stub_aws_sdk_client["streams"] = [UnavailableBedrockStream(), healthy_stream] with pytest.raises(BedrockError) as failure: @@ -923,10 +1017,10 @@ class TestBedrockRealtimeProviderFailurePropagation: async def test_success_dispatch_stamps_the_ownership_marker_only_when_spend_was_logged( self, stub_aws_sdk_client, spend_dispatch ): - stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)] + stub_aws_sdk_client["streams"] = [SessionBoundBedrockStream(self.TEXT_TURN)] await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", - websocket=ConnectedClientWS([self.SESSION_UPDATE]), + websocket=RealtimeClientWS([self.SESSION_UPDATE]), logging_obj=spend_dispatch["logging_obj"], **self.AWS_PARAMS, ) @@ -935,15 +1029,100 @@ class TestBedrockRealtimeProviderFailurePropagation: assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True idle_logging = FakeLogging() - stub_aws_sdk_client["streams"] = [ScriptedBedrockStream([])] + stub_aws_sdk_client["streams"] = [SessionBoundBedrockStream()] await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", - websocket=ConnectedClientWS([self.SESSION_UPDATE]), + websocket=RealtimeClientWS([self.SESSION_UPDATE]), logging_obj=idle_logging, **self.AWS_PARAMS, ) assert REALTIME_SESSION_SUCCESS_LOGGED_KEY not in idle_logging.model_call_details + @pytest.mark.asyncio + async def test_bedrock_stream_ending_mid_session_fails_the_session_and_keeps_partial_spend( + self, stub_aws_sdk_client, spend_dispatch + ): + handler: Final = BedrockRealtime() + websocket: Final = ConnectedClientWS([self.SESSION_UPDATE]) + stream: Final = ScriptedBedrockStream(self.TEXT_TURN) + stub_aws_sdk_client["streams"] = [stream] + + with pytest.raises(BedrockError) as failure: + await handler.async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ) + + assert failure.value.status_code == 503 + assert not websocket.closed, "the proxy route owns the client-facing error event and 1011 close" + assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + assert _sent_event_names(stream.input_stream)[-2:] == ["promptEnd", "sessionEnd"] + + with pytest.raises(BedrockError) as replay: + await handler.async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ) + + assert replay.value.status_code == 400, "a router retry must not restart the dropped session" + assert failure.value.message in replay.value.message + + @pytest.mark.asyncio + async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( + self, stub_aws_sdk_client, spend_dispatch + ): + stream: Final = SessionBoundBedrockStream(self.TEXT_TURN, input_stream_type=SlowClosingInputStream) + stub_aws_sdk_client["streams"] = [stream] + websocket: Final = RealtimeClientWS([self.SESSION_UPDATE]) + + await asyncio.wait_for( + BedrockRealtime(close_timeout=30).async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ), + timeout=5, + ) + + assert _sent_event_names(stream.input_stream)[-1] == "sessionEnd" + assert not stream.input_stream.closed, "the input close must still be pending when the output ends" + assert websocket.closed + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + + @pytest.mark.asyncio + async def test_bedrock_output_ending_after_the_input_side_failed_still_fails_the_session( + self, stub_aws_sdk_client, spend_dispatch + ): + connection_reset: Final = asyncio.Event() + stream: Final = ScriptedBedrockStream( + self.TEXT_TURN, + receiver_type=lambda payloads: OutputEndsAfterConnectionResetReceiver(payloads, connection_reset), + ) + stream.input_stream = ConnectionResetOnAudioInputStream(connection_reset) + stub_aws_sdk_client["streams"] = [stream] + websocket: Final = ConnectedClientWS([self.SESSION_UPDATE, self.USER_AUDIO]) + + with pytest.raises((BedrockError, RuntimeError)): + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ) + + assert not websocket.closed, "the proxy route owns the client-facing error event and 1011 close" + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + @pytest.mark.asyncio async def test_stream_failure_after_client_disconnect_is_not_a_provider_failure(self, stub_aws_sdk_client): stream = ScriptedBedrockStream([], receiver_type=BreakingBedrockReceiver) @@ -982,7 +1161,13 @@ class TestBedrockRealtimeProviderFailurePropagation: with pytest.raises(ServiceUnavailableException): await handler._forward_client_to_bedrock( - client_ws, stream, BedrockRealtimeConfig(), "amazon.nova-sonic-v1:0", {}, FakeLogging() + client_ws, + stream, + BedrockRealtimeConfig(), + "amazon.nova-sonic-v1:0", + {}, + FakeLogging(), + input_closing=asyncio.Event(), ) assert client_ws.sent_to_client == [] From 8a3248de0d0ddf3153b7c2f6611a20ad390fcf9f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:42:11 -0700 Subject: [PATCH 2/8] fix(bedrock): fail a dropped Nova Sonic session right away and log it as failed When Bedrock's output side ends mid-session, by a drop or a modeled stream error, the client side no longer spends the close timeout sending close messages into the dead input stream, so the error reaches the app right after the drop. A session the app ends itself still sends Bedrock its close messages and closes the input The replay refusal now leads with the committed cause, so the error event and the cut 1011 close reason both start with it. It stays a 400 and is tagged as a failure outside the selected deployment, so router retries and fallbacks of a dropped session no longer add extra failure counts A clean realtime session now counts once toward its deployment's cooldown success counter, so cooldown reads the real failure rate instead of 100 percent. Realtime tpm and rpm counting is unchanged The spend row of a session the provider ended now reads failure with its error information, while the events already forwarded are still billed. A session the app ended keeps logging success --- litellm/llms/bedrock/realtime/handler.py | 137 ++++++++++++----- litellm/router.py | 9 ++ .../realtime/test_bedrock_realtime_handler.py | 140 ++++++++++++++---- .../test_router/test_router_helper_utils.py | 62 ++++++++ 4 files changed, 283 insertions(+), 65 deletions(-) diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 9f5c56fd106..a62122ee5ee 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -9,11 +9,12 @@ import contextlib import importlib.metadata import json from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from types import MappingProxyType from typing import Final, NoReturn, Protocol, runtime_checkable from pydantic import JsonValue, TypeAdapter +from typing_extensions import TypeIs # noqa: TID251 # narrows request metadata without copying it import litellm from litellm._logging import redact_string, verbose_proxy_logger @@ -34,6 +35,7 @@ from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeE from litellm.types.llms.bedrock import AwsAuthParams from litellm.types.llms.openai import OpenAIRealtimeEvents from litellm.types.realtime import RealtimeResponseTransformInput +from litellm.types.utils import StandardLoggingPayloadErrorInformation from ..base_aws_llm import BaseAWSLLM, run_aws_signing from ..common_utils import BedrockError @@ -165,6 +167,12 @@ class _BridgeOutcome: client_disconnected: bool +@dataclass(frozen=True, slots=True) +class SessionEndSignals: + input_closing: asyncio.Event = field(default_factory=asyncio.Event) + output_failed: asyncio.Event = field(default_factory=asyncio.Event) + + async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message: str | None) -> AsyncIterator[str]: if initial_message is not None: yield initial_message @@ -184,14 +192,18 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None: """A fallback attempt on a websocket whose Bedrock stream already took input would silently start over.""" if scope is None or not _is_session_committed(scope): return + from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure + committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY) - raise BedrockError( + refusal: Final = BedrockError( status_code=400, message=( - "Bedrock realtime session already committed to a provider stream; it cannot be replayed" - + (f". The committed stream failed with: {committed_failure}" if committed_failure else "") + (f"{committed_failure}. " if committed_failure else "") + + "Bedrock realtime session already committed to a provider stream; it cannot be replayed" ), ) + mark_deployment_selection_failure(refusal) + raise refusal def _pending_opening_message(scope: Mapping[str, object]) -> str | None: @@ -201,6 +213,36 @@ def _pending_opening_message(scope: Mapping[str, object]) -> str | None: return pending if isinstance(pending, str) else None +def _is_str_keyed_dict( + value: object, +) -> TypeIs[dict[str, object]]: # guard-ok: request kwargs have str keys # mutable-ok: the spend row reads it + return isinstance(value, dict) + + +def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseException) -> None: + """The spend row of a session the provider ended reads failure while it still bills the forwarded events.""" + litellm_params: Final[object] = logging_obj.model_call_details.get("litellm_params") + metadata_candidates: Final = ( + (litellm_params.get("litellm_metadata"), litellm_params.get("metadata")) + if _is_str_keyed_dict(litellm_params) + else () + ) + spend_log_metadata: Final = next( + (metadata for metadata in metadata_candidates if _is_str_keyed_dict(metadata) and metadata), None + ) + if spend_log_metadata is None: + return + status_code: Final[object] = getattr(error, "status_code", None) + error_information: Final[StandardLoggingPayloadErrorInformation] = { + "error_code": "" if status_code is None else str(status_code), + "error_class": type(error).__name__, + "error_message": redact_string(str(error)), + "llm_provider": "bedrock", + } + spend_log_metadata["status"] = "failure" + spend_log_metadata["error_information"] = error_information + + def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn: error: Final = _as_bedrock_error(failure) verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error))) @@ -279,6 +321,25 @@ async def _reject_session_update( return True +async def _close_bedrock_input( + bedrock_stream: BedrockBidirectionalStream, + close_messages: Sequence[str], + send_to_bedrock: Callable[[str], Awaitable[None]], + timeout: float, +) -> None: + async def flush_and_close() -> None: + for close_message in close_messages: + with contextlib.suppress(Exception): + await send_to_bedrock(close_message) + with contextlib.suppress(Exception): + await bedrock_stream.input_stream.close() + + try: + await asyncio.wait_for(flush_and_close(), timeout) + except (asyncio.TimeoutError, TimeoutError): + verbose_proxy_logger.debug("Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", timeout) + + class BedrockRealtime(BaseAWSLLM): """Handler for Bedrock Nova Sonic realtime speech-to-speech API.""" @@ -445,7 +506,14 @@ class BedrockRealtime(BaseAWSLLM): if _should_log_event(leftover_event) ), ) + if outcome.provider_failure is not None and outcome.client_disconnected: + verbose_proxy_logger.debug( + "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure + ) + provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure if logged_events: + if provider_failure is not None: + _record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure)) GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( logging_obj.dispatch_success_handlers( list(logged_events), @@ -454,14 +522,9 @@ class BedrockRealtime(BaseAWSLLM): ) logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True - if outcome.provider_failure is None: + if provider_failure is None: return - if outcome.client_disconnected: - verbose_proxy_logger.debug( - "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure - ) - return - _raise_provider_failure(websocket.scope, outcome.provider_failure) + _raise_provider_failure(websocket.scope, provider_failure) async def _bridge( self, @@ -475,19 +538,23 @@ class BedrockRealtime(BaseAWSLLM): ) -> _BridgeOutcome: """Run both forwarding directions until the client leaves or either side fails.""" logged: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events forwarded before a failure are still spend - input_closing: Final = asyncio.Event() + signals: Final = SessionEndSignals() async def collect_logged_events() -> None: - async for event in self._forward_bedrock_to_client( - bedrock_stream, - websocket, - transformation_config, - model, - logging_obj, - session_state, - input_closing=input_closing, - ): - logged.append(event) + try: + async for event in self._forward_bedrock_to_client( + bedrock_stream, + websocket, + transformation_config, + model, + logging_obj, + session_state, + signals=signals, + ): + logged.append(event) + except Exception: + signals.output_failed.set() + raise client_task: Final = asyncio.create_task( self._forward_client_to_bedrock( @@ -498,7 +565,7 @@ class BedrockRealtime(BaseAWSLLM): session_state, logging_obj, initial_message, - input_closing=input_closing, + signals=signals, ) ) bedrock_task: Final = asyncio.create_task(collect_logged_events()) @@ -533,7 +600,7 @@ class BedrockRealtime(BaseAWSLLM): logging_obj: LiteLLMLogging | None = None, initial_message: str | None = None, *, - input_closing: asyncio.Event, + signals: SessionEndSignals, ) -> None: """Forward messages from client WebSocket to Bedrock stream. @@ -581,21 +648,11 @@ class BedrockRealtime(BaseAWSLLM): client_ws, transformation_config, model, logging_obj, parsed_client_message ): break - input_closing.set() + signals.input_closing.set() finally: - - async def close_bedrock_input() -> None: - for close_message in transformation_config.session_close_messages(): - with contextlib.suppress(Exception): - await send_to_bedrock(close_message) - with contextlib.suppress(Exception): - await bedrock_stream.input_stream.close() - - try: - await asyncio.wait_for(close_bedrock_input(), self._close_timeout) - except (asyncio.TimeoutError, TimeoutError): - verbose_proxy_logger.debug( - "Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", self._close_timeout + if not signals.output_failed.is_set(): + await _close_bedrock_input( + bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout ) async def _forward_bedrock_to_client( @@ -607,7 +664,7 @@ class BedrockRealtime(BaseAWSLLM): logging_obj: LiteLLMLogging, session_state: RealtimeResponseTransformInput, *, - input_closing: asyncio.Event, + signals: SessionEndSignals, ) -> AsyncIterator[OpenAIRealtimeEvents]: """Forward messages from Bedrock to the client, yielding the ones to record for spend logging. @@ -628,7 +685,7 @@ class BedrockRealtime(BaseAWSLLM): result = await output[1].receive() if result is None: - if not input_closing.is_set(): + if not signals.input_closing.is_set(): raise BedrockError( status_code=503, message="Bedrock realtime stream ended before the session closed", diff --git a/litellm/router.py b/litellm/router.py index 02ad6275b82..0dc798be380 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6478,6 +6478,15 @@ class Router: raise safeguard_refusal_error(model=model, stop_details=refusal_details) self.success_calls[model_name] += 1 + realtime_session_deployment_id: Final = ( + _MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id") + if generic_function_name == "_arealtime" + else None + ) + if realtime_session_deployment_id is not None: + increment_deployment_successes_for_current_minute( + litellm_router_instance=self, deployment_id=str(realtime_session_deployment_id) + ) verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 329ea9dd336..9767f294af7 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -1,7 +1,9 @@ import asyncio import json import sys +import time import types +from datetime import datetime, timezone from types import SimpleNamespace from typing import Final from unittest.mock import MagicMock @@ -9,13 +11,18 @@ from unittest.mock import MagicMock import pytest import litellm +from litellm import Router from litellm.constants import ( + BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY, BEDROCK_REALTIME_SDK_SUPPORTED_RANGE, + BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) +from litellm.litellm_core_utils.realtime_errors import websocket_close_reason from litellm.llms.bedrock.common_utils import BedrockError -from litellm.llms.bedrock.realtime.handler import BedrockRealtime +from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals, raise_if_session_committed +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig from litellm.realtime_api import main as realtime_main from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault @@ -36,10 +43,16 @@ def _isolate_host_aws_config(monkeypatch, tmp_path): monkeypatch.delenv(env_var, raising=False) -def _closing_input() -> asyncio.Event: - closing: Final = asyncio.Event() - closing.set() - return closing +def _closing_input() -> SessionEndSignals: + signals: Final = SessionEndSignals() + signals.input_closing.set() + return signals + + +def _spend_row(logging_obj): + now: Final = datetime.now(timezone.utc) + row: Final = get_logging_payload(kwargs=logging_obj.model_call_details, response_obj=None, start_time=now, end_time=now) + return row["status"], json.loads(row["metadata"])["error_information"] class FakePayloadPart: @@ -484,7 +497,7 @@ class TestBedrockRealtimeHandler: ) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] @@ -500,7 +513,7 @@ class TestBedrockRealtimeHandler: stream = FakeBedrockStream() await handler._forward_client_to_bedrock( - DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) assert stream.input_stream.sent == [] @@ -517,7 +530,7 @@ class TestBedrockRealtimeHandler: with pytest.raises(RuntimeError, match="bedrock send failed"): await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) assert stream.input_stream.closed @@ -533,7 +546,7 @@ class TestBedrockRealtimeHandler: await asyncio.wait_for( handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ), 2, ) @@ -550,7 +563,7 @@ class TestBedrockRealtimeHandler: ) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] @@ -580,7 +593,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, - input_closing=_closing_input(), + signals=_closing_input(), ) ] @@ -612,7 +625,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, - input_closing=_closing_input(), + signals=_closing_input(), ) ] @@ -684,7 +697,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", MagicMock(), {}, - input_closing=_closing_input(), + signals=_closing_input(), ): pass @@ -745,7 +758,7 @@ class TestBedrockRealtimeSessionLifecycle: ) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals() ) acked = [json.loads(message) for message in client_ws.sent_to_client] @@ -761,7 +774,7 @@ class TestBedrockRealtimeSessionLifecycle: client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})]) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) assert client_ws.sent_to_client == [] @@ -786,7 +799,7 @@ class TestBedrockRealtimeSessionLifecycle: client_ws = DisconnectingClientWS([update([tool]), update([other_tool]), update([tool])]) await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), input_closing=asyncio.Event() + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals() ) acked: Final = [json.loads(message) for message in client_ws.sent_to_client] @@ -942,9 +955,6 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] - flushed = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] - assert [next(iter(event["event"])) for event in flushed][-2:] == ["promptEnd", "sessionEnd"] - assert stream.input_stream.closed with pytest.raises(BedrockError) as replay: await handler.async_realtime( @@ -1046,12 +1056,14 @@ class TestBedrockRealtimeProviderFailurePropagation: websocket: Final = ConnectedClientWS([self.SESSION_UPDATE]) stream: Final = ScriptedBedrockStream(self.TEXT_TURN) stub_aws_sdk_client["streams"] = [stream] + logging_obj: Final = spend_dispatch["logging_obj"] + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed-key"}} with pytest.raises(BedrockError) as failure: await handler.async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, - logging_obj=spend_dispatch["logging_obj"], + logging_obj=logging_obj, **self.AWS_PARAMS, ) @@ -1060,18 +1072,93 @@ class TestBedrockRealtimeProviderFailurePropagation: assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] - assert _sent_event_names(stream.input_stream)[-2:] == ["promptEnd", "sessionEnd"] + status, error_information = _spend_row(logging_obj) + assert status == "failure", "the Logs page must show the dropped session as failed while it bills its turn" + assert error_information["error_code"] == "503" + assert error_information["error_message"] == failure.value.message with pytest.raises(BedrockError) as replay: await handler.async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, - logging_obj=spend_dispatch["logging_obj"], + logging_obj=logging_obj, **self.AWS_PARAMS, ) assert replay.value.status_code == 400, "a router retry must not restart the dropped session" - assert failure.value.message in replay.value.message + assert str(replay.value).startswith(failure.value.message), "the app's error event leads with the drop" + assert websocket_close_reason(str(replay.value), fallback="Internal server error").startswith( + failure.value.message + ), "the length-capped 1011 close reason must still carry the drop" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_failed_bedrock_output_reaches_the_app_without_waiting_on_the_input_close( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type) + stream.input_stream = HangingCloseInputStream() + stub_aws_sdk_client["streams"] = [stream] + started: Final = time.monotonic() + + with pytest.raises(BedrockError) as failure: + await asyncio.wait_for( + BedrockRealtime(close_timeout=30).async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ), + timeout=5, + ) + + assert failure.value.status_code == status_code + assert time.monotonic() - started < 1, "the app must not wait on closing a Bedrock input that already failed" + assert stub_aws_sdk_client["client_closed"], "closing the SDK client still tears the dead stream down" + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + + @pytest.mark.asyncio + async def test_replay_refusals_do_not_count_toward_deployment_cooldown(self): + router: Final = Router( + model_list=[ + { + "model_name": "nova-sonic", + "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("sonic-a", "sonic-b") + ] + ) + with pytest.raises(BedrockError) as refusal: + raise_if_session_committed( + { + BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: True, + BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: "Bedrock realtime stream ended before the session closed", + } + ) + drop: Final = BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed") + + def cooled_down_after(exception: Exception, deployment_id: str) -> bool: + return router.deployment_callback_on_failure( + kwargs={ + "exception": exception, + "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": deployment_id}}, + }, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + + refusals_then_drop: Final = [cooled_down_after(error, "sonic-a") for error in (*[refusal.value] * 4, drop)] + drops_only: Final = [cooled_down_after(drop, "sonic-b") for _ in range(5)] + + assert not any(refusals_then_drop), "one drop retried four times is one deployment failure, not five" + assert drops_only[-1], "five real drops in a minute still cool the deployment down" @pytest.mark.asyncio async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( @@ -1080,6 +1167,9 @@ class TestBedrockRealtimeProviderFailurePropagation: stream: Final = SessionBoundBedrockStream(self.TEXT_TURN, input_stream_type=SlowClosingInputStream) stub_aws_sdk_client["streams"] = [stream] websocket: Final = RealtimeClientWS([self.SESSION_UPDATE]) + spend_dispatch["logging_obj"].model_call_details["litellm_params"] = { + "litellm_metadata": {"user_api_key": "hashed-key"} + } await asyncio.wait_for( BedrockRealtime(close_timeout=30).async_realtime( @@ -1097,6 +1187,7 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + assert _spend_row(spend_dispatch["logging_obj"]) == ("success", None) @pytest.mark.asyncio async def test_bedrock_output_ending_after_the_input_side_failed_still_fails_the_session( @@ -1167,7 +1258,7 @@ class TestBedrockRealtimeProviderFailurePropagation: "amazon.nova-sonic-v1:0", {}, FakeLogging(), - input_closing=asyncio.Event(), + signals=SessionEndSignals(), ) assert client_ws.sent_to_client == [] @@ -1313,7 +1404,6 @@ class TestBedrockRealtimeSdkLifecycle: ) assert stub_aws_sdk_client["client_closed"] - assert stub_aws_sdk_client["input_closed_before_client_close"] @pytest.mark.asyncio async def test_client_without_close_completes_session(self, monkeypatch): diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 513136ae150..3edb5f74454 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -538,6 +538,68 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( assert await router.cache.async_get_cache(key=tpm_key) == 17 +@pytest.mark.asyncio +@pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)]) +async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served( + clean_sessions: int, cooled_down: bool +) -> None: + import time + + router: Final = Router( + model_list=[ + { + "model_name": "nova-sonic", + "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0", "order": order}, + "model_info": {"id": deployment_id}, + } + for order, deployment_id in ((1, "sonic-a"), (2, "sonic-b")) + ], + num_retries=0, + ) + drop: Final = litellm.ServiceUnavailableError( + message="Bedrock realtime stream ended before the session closed", + llm_provider="bedrock", + model="amazon.nova-sonic-v1:0", + ) + + def realtime_session(failure: Exception | None): + async def _arealtime(**kwargs: object) -> None: + await router.deployment_callback_on_success( + kwargs={ + "call_type": "_arealtime", + "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": kwargs["model_info"]}, + "standard_logging_object": create_standard_logging_payload(), + }, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + if failure is not None: + raise failure + + return _arealtime + + for _ in range(clean_sessions): + await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(None)) + + async def drop_session() -> bool: + with pytest.raises(litellm.ServiceUnavailableError): + await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(drop)) + return router.deployment_callback_on_failure( + kwargs={ + "exception": drop, + "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": "sonic-a"}}, + }, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + + drop_cooldowns: Final = [await drop_session() for _ in range(5)] + + assert any(drop_cooldowns) is cooled_down + + async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]: routed: Final[dict[str, object]] = {} From edb9aaeee46eef91b05b3c88dc36c47c8063fcdb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:35:25 -0700 Subject: [PATCH 3/8] fix(bedrock): agree on a dropped realtime session's failure across loggers and the spend row A realtime session's success event now reports status failure, with the same error_information the spend row stores, when the session recorded a provider failure in its metadata. Usage already streamed stays billed and the router still counts one cooldown failure The spend row builder now runs every row's error_information through sanitize_error_information_for_spend_logs (DB size cap plus prompt redaction), so the realtime failure rows get the same treatment as the proxy failure hook, which no longer sanitizes on its own to avoid truncating twice Drop the selection-failure tag on the committed-session refusal, since the shared logging object already keeps a retried refusal out of cooldown, and pin the router's realtime success counting with an OpenAI realtime deployment --- litellm/litellm_core_utils/litellm_logging.py | 27 ++++++-- litellm/llms/bedrock/realtime/handler.py | 6 +- .../proxy/hooks/proxy_track_cost_callback.py | 10 +-- .../spend_tracking/spend_tracking_utils.py | 10 ++- .../test_litellm_logging.py | 67 +++++++++++++++++++ .../realtime/test_bedrock_realtime_handler.py | 43 +----------- .../hooks/test_proxy_track_cost_callback.py | 38 ++++++++++- .../test_spend_tracking_utils.py | 40 +++++++++++ .../test_router/test_router_helper_utils.py | 24 ++++--- 9 files changed, 183 insertions(+), 82 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7f4b069ded6..b340455de98 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -7478,10 +7478,25 @@ def get_standard_logging_object_payload( router_model_id=logging_obj.get_router_model_id(), ) - error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( - metadata=metadata, - original_exception=original_exception, - error_str=error_str, + metadata_fields: Final = cast(Mapping[str, object], metadata) # cast-ok: merged request metadata is str-keyed + recorded_error: Final = metadata_fields.get("error_information") + recorded_failure: Final = ( + cast(StandardLoggingPayloadErrorInformation, recorded_error) # cast-ok: a reserved key only litellm writes + if isinstance(recorded_error, dict) + and status == "success" + and call_type == CallTypes.arealtime.value + and metadata_fields.get("status") == "failure" + else None + ) + payload_status: Final[StandardLoggingPayloadStatus] = "failure" if recorded_failure is not None else status + error_information, error_str = ( + (recorded_failure, error_str or recorded_failure.get("error_message")) + if recorded_failure is not None + else StandardLoggingPayloadSetup.get_error_information_for_logging_payload( + metadata=metadata, + original_exception=original_exception, + error_str=error_str, + ) ) ## get final response object ## @@ -7572,9 +7587,9 @@ def get_standard_logging_object_payload( call_type=call_type or "", cache_hit=cache_hit, stream=stream, - status=status, + status=payload_status, status_fields=_get_status_fields( - status=status, + status=payload_status, guardrail_information=metadata.get("standard_logging_guardrail_information", None), error_str=error_str, ), diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index a62122ee5ee..b447aacd993 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -192,18 +192,14 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None: """A fallback attempt on a websocket whose Bedrock stream already took input would silently start over.""" if scope is None or not _is_session_committed(scope): return - from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure - committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY) - refusal: Final = BedrockError( + raise BedrockError( status_code=400, message=( (f"{committed_failure}. " if committed_failure else "") + "Bedrock realtime session already committed to a provider stream; it cannot be replayed" ), ) - mark_deployment_selection_failure(refusal) - raise refusal def _pending_opening_message(scope: Mapping[str, object]) -> str | None: diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index c11b59a809a..e19c5612ef0 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -58,7 +58,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # _sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export deployment_params_provider, get_request_model_access_groups, - sanitize_error_information_for_spend_logs, + sanitize_error_information_for_spend_logs, # pyright: ignore[reportUnusedImport] # backwards-compatible package export should_store_prompts_and_responses_in_spend_logs, ) from litellm.proxy.utils import ( @@ -72,7 +72,6 @@ from litellm.types.utils import ( CallTypes, LiteLLMBatch, StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, ) from litellm.utils import get_end_user_id_for_cost_tracking @@ -429,13 +428,6 @@ class _ProxyDBLogger(CustomLogger): # ``.get("traceback")`` / truthy checks, and the TypedDict marks # the field as optional, so omitting is type-safe. _error_information.pop("traceback", None) - # Strip echoed request input + apply DB-size cap before storing in - # the spend-log metadata column (LIT-2992). Result is never None - # here because the input above is constructed non-None. - _error_information = cast( - StandardLoggingPayloadErrorInformation, - sanitize_error_information_for_spend_logs(_error_information, original_exception=original_exception), - ) _metadata["error_information"] = _error_information _metadata = await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled( # rebind-ok: pre-existing rebinding on a rename-only line diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..b740a21bc46 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -582,6 +582,7 @@ def get_logging_payload( kwargs = {} rejected_as_unknown_model: Final = isinstance(response_obj, ProxyModelNotFoundError) + failure_exception: Final = response_obj if isinstance(response_obj, BaseException) else None response_served_from_cache: Final = served_from_cache(response_obj) if response_obj is None: response_obj = {} @@ -851,14 +852,11 @@ def get_logging_payload( additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens clean_metadata["additional_usage_values"] = additional_usage_values drop_requester_ip_address: Final = requester_ip_address_logging_disabled() - error_information: Final = clean_metadata["error_information"] if drop_requester_ip_address: clean_metadata["requester_ip_address"] = None - if drop_requester_ip_address and isinstance(error_information, Mapping): - clean_metadata["error_information"] = cast( # cast-ok: same keys, only the str values are masked - StandardLoggingPayloadErrorInformation, - _error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(error_information)), - ) + clean_metadata["error_information"] = sanitize_error_information_for_spend_logs( + clean_metadata["error_information"], original_exception=failure_exception + ) if litellm.cache is None: cache_key = "Cache OFF" diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 299088a6c4f..868fe529815 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6511,6 +6511,73 @@ def test_get_error_information_for_logging_payload_client_disconnect(): assert error_str is None +_SESSION_FAILURE: Final = { + "error_code": "503", + "error_class": "BedrockError", + "error_message": "Bedrock realtime stream ended before the session closed", + "llm_provider": "bedrock", +} + + +@pytest.mark.parametrize("metadata_bucket", ["metadata", "litellm_metadata"]) +def test_realtime_success_payload_reports_the_session_failure_its_metadata_recorded( + logging_obj: LitellmLogging, metadata_bucket: str +) -> None: + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + now: Final = datetime(2026, 1, 1) + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": "bedrock/amazon.nova-sonic-v1:0", + "call_type": "_arealtime", + "litellm_params": {metadata_bucket: {"status": "failure", "error_information": dict(_SESSION_FAILURE)}}, + }, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["status"] == "failure" + assert payload["status_fields"]["llm_api_status"] == "failure" + assert payload["error_information"] == _SESSION_FAILURE + assert payload["error_str"] == _SESSION_FAILURE["error_message"] + + +@pytest.mark.parametrize( + ("call_type", "metadata"), + [ + ("_arealtime", {"status": "failure"}), + ("_arealtime", {"error_information": dict(_SESSION_FAILURE)}), + ("acompletion", {"status": "failure", "error_information": dict(_SESSION_FAILURE)}), + ], +) +def test_success_payload_stays_success_without_a_recorded_realtime_session_failure( + logging_obj: LitellmLogging, call_type: str, metadata: dict[str, object] +) -> None: + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + now: Final = datetime(2026, 1, 1) + payload: Final = get_standard_logging_object_payload( + kwargs={"model": "gpt-4o", "messages": [], "call_type": call_type, "litellm_params": {"metadata": metadata}}, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["status"] == "success" + assert payload["error_information"] != _SESSION_FAILURE + + def test_get_error_information_proxy_exception_preserves_message(): """ProxyException keeps its text in ``.message`` (str() was empty pre-fix), so error_information must still surface the message and code.""" diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 9767f294af7..9e4ff7f05af 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -11,17 +11,14 @@ from unittest.mock import MagicMock import pytest import litellm -from litellm import Router from litellm.constants import ( - BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY, BEDROCK_REALTIME_SDK_SUPPORTED_RANGE, - BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) from litellm.litellm_core_utils.realtime_errors import websocket_close_reason from litellm.llms.bedrock.common_utils import BedrockError -from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals, raise_if_session_committed +from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig from litellm.realtime_api import main as realtime_main @@ -1122,44 +1119,6 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] - @pytest.mark.asyncio - async def test_replay_refusals_do_not_count_toward_deployment_cooldown(self): - router: Final = Router( - model_list=[ - { - "model_name": "nova-sonic", - "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0"}, - "model_info": {"id": deployment_id}, - } - for deployment_id in ("sonic-a", "sonic-b") - ] - ) - with pytest.raises(BedrockError) as refusal: - raise_if_session_committed( - { - BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: True, - BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: "Bedrock realtime stream ended before the session closed", - } - ) - drop: Final = BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed") - - def cooled_down_after(exception: Exception, deployment_id: str) -> bool: - return router.deployment_callback_on_failure( - kwargs={ - "exception": exception, - "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": deployment_id}}, - }, - completion_response=None, - start_time=time.time(), - end_time=time.time(), - ) - - refusals_then_drop: Final = [cooled_down_after(error, "sonic-a") for error in (*[refusal.value] * 4, drop)] - drops_only: Final = [cooled_down_after(drop, "sonic-b") for _ in range(5)] - - assert not any(refusals_then_drop), "one drop retried four times is one deployment failure, not five" - assert drops_only[-1], "five real drops in a minute still cool the deployment down" - @pytest.mark.asyncio async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( self, stub_aws_sdk_client, spend_dispatch diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 4b5957a1237..e8a203b8637 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -3213,6 +3213,40 @@ async def test_sidecar_ignores_an_undecodable_event(): # test-quality-ok: a dis mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited() +def _spend_row_error_information(update_database_kwargs: Mapping[str, object]) -> dict[str, object]: + row: Final = get_logging_payload( + kwargs=cast(dict, update_database_kwargs["kwargs"]), + response_obj=update_database_kwargs["completion_response"], + start_time=cast(datetime, update_database_kwargs["start_time"]), + end_time=cast(datetime, update_database_kwargs["end_time"]), + ) + return json.loads(row["metadata"])["error_information"] + + +@pytest.mark.asyncio +async def test_failure_hook_spend_row_truncates_a_long_provider_error_exactly_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "100") + writer: Final = MagicMock(spec=DBSpendUpdateWriter) + writer.update_database = AsyncMock() + logger: Final = ProxyDBLogger(spend_writer=lambda: writer) + provider_error: Final = "upstream failure " + "x" * 1000 + + await logger.async_post_call_failure_hook( + request_data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=litellm.InternalServerError(message=provider_error, llm_provider="openai", model="gpt-4o"), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), + ) + + stored_message: Final = _spend_row_error_information(writer.update_database.call_args.kwargs)["error_message"] + assert isinstance(stored_message, str) + assert stored_message.count("skipped") == 1 + kept_chars: Final = len(stored_message.split("... (")[0]) + len(stored_message.rsplit(") ...", 1)[1]) + skipped_chars: Final = int(stored_message.split("skipped ", 1)[1].split(" chars", 1)[0]) + assert kept_chars + skipped_chars > len(provider_error) + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_model_rejection(): raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" @@ -3226,9 +3260,7 @@ async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_ user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), ) - error_information: Final = writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][ - "error_information" - ] + error_information: Final = _spend_row_error_information(writer.update_database.call_args.kwargs) assert "medical records" not in json.dumps(error_information) assert ( error_information["error_message"] diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 86d064050eb..bc31bcb2692 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -7326,6 +7326,46 @@ def test_failure_error_information_masks_ip_addresses_in_long_text_and_at_its_tr assert sanitized["error_code"] == "500" +def test_spend_row_sanitizes_the_session_failure_a_realtime_success_row_carries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.proxy_server import general_settings + + monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", False) + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "200") + provider_error: Final = 'stream ended {"messages": [{"role": "user", "content": "private-prompt"}]} ' + "x" * 2000 + now: Final = datetime.datetime.now(timezone.utc) + + row: Final = get_logging_payload( + kwargs={ + "model": "bedrock/amazon.nova-sonic-v1:0", + "call_type": "_arealtime", + "litellm_params": { + "metadata": { + "user_api_key": "hashed-key", + "status": "failure", + "error_information": { + "error_code": "503", + "error_class": "BedrockError", + "error_message": provider_error, + "llm_provider": "bedrock", + }, + } + }, + }, + response_obj=None, + start_time=now, + end_time=now, + ) + + stored: Final = json.loads(row["metadata"])["error_information"] + assert row["status"] == "failure" + assert "private-prompt" not in stored["error_message"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored["error_message"] + assert len(stored["error_message"]) < len(provider_error) + assert stored["error_class"] == "BedrockError" + + @pytest.mark.parametrize( "address", ["203.0.113.9", "2001:db8::9", "1:2:3:4:5:6:7:8", "1:2:3:4:5:6:7::", "::1:2:3:4:5:6:7", "fe80::1"] ) diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 3edb5f74454..17e954ccd9d 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -539,27 +539,29 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( @pytest.mark.asyncio +@pytest.mark.parametrize("realtime_model", ["bedrock/amazon.nova-sonic-v1:0", "openai/gpt-realtime-2.1"]) @pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)]) async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served( - clean_sessions: int, cooled_down: bool + realtime_model: str, clean_sessions: int, cooled_down: bool ) -> None: import time + provider, _, provider_model = realtime_model.partition("/") router: Final = Router( model_list=[ { - "model_name": "nova-sonic", - "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0", "order": order}, + "model_name": "realtime-voice", + "litellm_params": {"model": realtime_model, "order": order}, "model_info": {"id": deployment_id}, } - for order, deployment_id in ((1, "sonic-a"), (2, "sonic-b")) + for order, deployment_id in ((1, "realtime-a"), (2, "realtime-b")) ], num_retries=0, ) drop: Final = litellm.ServiceUnavailableError( - message="Bedrock realtime stream ended before the session closed", - llm_provider="bedrock", - model="amazon.nova-sonic-v1:0", + message="realtime provider stream ended before the session closed", + llm_provider=provider, + model=provider_model, ) def realtime_session(failure: Exception | None): @@ -567,7 +569,7 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy await router.deployment_callback_on_success( kwargs={ "call_type": "_arealtime", - "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": kwargs["model_info"]}, + "litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": kwargs["model_info"]}, "standard_logging_object": create_standard_logging_payload(), }, completion_response=None, @@ -580,15 +582,15 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy return _arealtime for _ in range(clean_sessions): - await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(None)) + await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(None)) async def drop_session() -> bool: with pytest.raises(litellm.ServiceUnavailableError): - await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(drop)) + await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(drop)) return router.deployment_callback_on_failure( kwargs={ "exception": drop, - "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": "sonic-a"}}, + "litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": {"id": "realtime-a"}}, }, completion_response=None, start_time=time.time(), From d271d221fdeba042faf7f437b4381e8de422147f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:29:38 -0700 Subject: [PATCH 4/8] fix(bedrock): count realtime router successes only for sessions that logged one, and log first-turn drops The router counts a realtime success only when the session's success handlers ran, so a refused handshake or an upstream close before any frame no longer offsets its failure. The success event's payload status stays success again, so each logger gets one failure record per dropped session, while the spend row still reads failure from the metadata the Bedrock handler records. A committed Bedrock session that fails before any logged event now gets one zero-cost Logs row. After an output failure the input stream is closed in the background, so the SDK body writer finishes without delaying the client's error. Every input send failure reports the same 503 stream-ended cause as an output drop --- litellm/litellm_core_utils/litellm_logging.py | 27 +--- litellm/llms/bedrock/realtime/handler.py | 37 +++++- litellm/router.py | 10 +- .../test_litellm_logging.py | 67 ---------- .../realtime/test_bedrock_realtime_handler.py | 98 +++++++++++++- .../test_router/test_router_helper_utils.py | 121 ++++++++++++++++-- 6 files changed, 247 insertions(+), 113 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index b340455de98..7f4b069ded6 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -7478,25 +7478,10 @@ def get_standard_logging_object_payload( router_model_id=logging_obj.get_router_model_id(), ) - metadata_fields: Final = cast(Mapping[str, object], metadata) # cast-ok: merged request metadata is str-keyed - recorded_error: Final = metadata_fields.get("error_information") - recorded_failure: Final = ( - cast(StandardLoggingPayloadErrorInformation, recorded_error) # cast-ok: a reserved key only litellm writes - if isinstance(recorded_error, dict) - and status == "success" - and call_type == CallTypes.arealtime.value - and metadata_fields.get("status") == "failure" - else None - ) - payload_status: Final[StandardLoggingPayloadStatus] = "failure" if recorded_failure is not None else status - error_information, error_str = ( - (recorded_failure, error_str or recorded_failure.get("error_message")) - if recorded_failure is not None - else StandardLoggingPayloadSetup.get_error_information_for_logging_payload( - metadata=metadata, - original_exception=original_exception, - error_str=error_str, - ) + error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( + metadata=metadata, + original_exception=original_exception, + error_str=error_str, ) ## get final response object ## @@ -7587,9 +7572,9 @@ def get_standard_logging_object_payload( call_type=call_type or "", cache_hit=cache_hit, stream=stream, - status=payload_status, + status=status, status_fields=_get_status_fields( - status=payload_status, + status=status, guardrail_information=metadata.get("standard_logging_guardrail_information", None), error_str=error_str, ), diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index b447aacd993..e6b34487cbf 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -63,6 +63,10 @@ _BEDROCK_STREAM_ERROR_STATUS: Final[Mapping[str, int]] = MappingProxyType( ) +def _stream_ended_before_session_closed() -> BedrockError: + return BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed") + + def _as_bedrock_error(error: BaseException) -> BaseException: status_code: Final = _BEDROCK_STREAM_ERROR_STATUS.get(type(error).__name__) if status_code is None: @@ -244,6 +248,8 @@ def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseExc verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error))) if _is_session_committed(scope): scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = redact_string(str(error)) + if error is failure: + raise failure raise error from failure @@ -347,6 +353,18 @@ class BedrockRealtime(BaseAWSLLM): super().__init__() self._sdk_version_lookup: Final = sdk_version_lookup self._close_timeout: Final = close_timeout + self._failed_input_closes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs to running closes + + def _close_failed_input_in_background( + self, + bedrock_stream: BedrockBidirectionalStream, + send_to_bedrock: Callable[[str], Awaitable[None]], + ) -> None: + close_task: Final = asyncio.create_task( + _close_bedrock_input(bedrock_stream, (), send_to_bedrock, self._close_timeout) + ) + self._failed_input_closes.add(close_task) + close_task.add_done_callback(self._failed_input_closes.discard) async def async_realtime( self, @@ -507,7 +525,10 @@ class BedrockRealtime(BaseAWSLLM): "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure ) provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure - if logged_events: + provider_ended_committed_session: Final = provider_failure is not None and _is_session_committed( + websocket.scope + ) + if logged_events or provider_ended_committed_session: if provider_failure is not None: _record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure)) GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( @@ -612,7 +633,10 @@ class BedrockRealtime(BaseAWSLLM): async def send_to_bedrock(bedrock_message: str) -> None: event: Final = build_input_chunk(bedrock_message.encode("utf-8")) - await bedrock_stream.input_stream.send(event) + try: + await bedrock_stream.input_stream.send(event) + except Exception as e: + raise _stream_ended_before_session_closed() from e verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200]) try: @@ -646,7 +670,9 @@ class BedrockRealtime(BaseAWSLLM): break signals.input_closing.set() finally: - if not signals.output_failed.is_set(): + if signals.output_failed.is_set(): + self._close_failed_input_in_background(bedrock_stream, send_to_bedrock) + else: await _close_bedrock_input( bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout ) @@ -682,10 +708,7 @@ class BedrockRealtime(BaseAWSLLM): if result is None: if not signals.input_closing.is_set(): - raise BedrockError( - status_code=503, - message="Bedrock realtime stream ended before the session closed", - ) + raise _stream_ended_before_session_closed() verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended") with contextlib.suppress(Exception): await client_ws.close() diff --git a/litellm/router.py b/litellm/router.py index 0dc798be380..af3d6c7505d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -98,6 +98,7 @@ from litellm.constants import ( MID_STREAM_FAILURE_LOG_WAIT_SECONDS, OUTPUT_TOKEN_CEILING_PARAMS, PROMPT_CACHE_PIN_CACHE_MAX_SIZE, + REALTIME_SESSION_SUCCESS_LOGGED_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, RUNTIME_UPDATABLE_ROUTER_SETTINGS, @@ -649,6 +650,7 @@ _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType _NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_CALL_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) @@ -6478,9 +6480,15 @@ class Router: raise safeguard_refusal_error(model=model, stop_details=refusal_details) self.success_calls[model_name] += 1 + realtime_session_logging: Final = ( + _CALL_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_logging_obj") + if generic_function_name == "_arealtime" + else None + ) realtime_session_deployment_id: Final = ( _MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id") - if generic_function_name == "_arealtime" + if isinstance(realtime_session_logging, LiteLLMLogging) + and realtime_session_logging.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True else None ) if realtime_session_deployment_id is not None: diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 868fe529815..299088a6c4f 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6511,73 +6511,6 @@ def test_get_error_information_for_logging_payload_client_disconnect(): assert error_str is None -_SESSION_FAILURE: Final = { - "error_code": "503", - "error_class": "BedrockError", - "error_message": "Bedrock realtime stream ended before the session closed", - "llm_provider": "bedrock", -} - - -@pytest.mark.parametrize("metadata_bucket", ["metadata", "litellm_metadata"]) -def test_realtime_success_payload_reports_the_session_failure_its_metadata_recorded( - logging_obj: LitellmLogging, metadata_bucket: str -) -> None: - from datetime import datetime - - from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload - - now: Final = datetime(2026, 1, 1) - payload: Final = get_standard_logging_object_payload( - kwargs={ - "model": "bedrock/amazon.nova-sonic-v1:0", - "call_type": "_arealtime", - "litellm_params": {metadata_bucket: {"status": "failure", "error_information": dict(_SESSION_FAILURE)}}, - }, - init_response_obj={}, - start_time=now, - end_time=now, - logging_obj=logging_obj, - status="success", - ) - - assert payload is not None - assert payload["status"] == "failure" - assert payload["status_fields"]["llm_api_status"] == "failure" - assert payload["error_information"] == _SESSION_FAILURE - assert payload["error_str"] == _SESSION_FAILURE["error_message"] - - -@pytest.mark.parametrize( - ("call_type", "metadata"), - [ - ("_arealtime", {"status": "failure"}), - ("_arealtime", {"error_information": dict(_SESSION_FAILURE)}), - ("acompletion", {"status": "failure", "error_information": dict(_SESSION_FAILURE)}), - ], -) -def test_success_payload_stays_success_without_a_recorded_realtime_session_failure( - logging_obj: LitellmLogging, call_type: str, metadata: dict[str, object] -) -> None: - from datetime import datetime - - from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload - - now: Final = datetime(2026, 1, 1) - payload: Final = get_standard_logging_object_payload( - kwargs={"model": "gpt-4o", "messages": [], "call_type": call_type, "litellm_params": {"metadata": metadata}}, - init_response_obj={}, - start_time=now, - end_time=now, - logging_obj=logging_obj, - status="success", - ) - - assert payload is not None - assert payload["status"] == "success" - assert payload["error_information"] != _SESSION_FAILURE - - def test_get_error_information_proxy_exception_preserves_message(): """ProxyException keeps its text in ``.message`` (str() was empty pre-fix), so error_information must still surface the message and code.""" diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 9e4ff7f05af..a075e89f626 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -16,6 +16,7 @@ from litellm.constants import ( REALTIME_SESSION_SUCCESS_LOGGED_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.realtime_errors import websocket_close_reason from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals @@ -101,6 +102,19 @@ class HangingCloseInputStream(FakeInputStream): await asyncio.Future() +class BodyWriterInputStream(FakeInputStream): + """Models the SDK input stream, whose request-body writer task only finishes once the stream is closed""" + + def __init__(self): + super().__init__() + self._close_requested = asyncio.Event() + self.body_writer = asyncio.get_running_loop().create_task(self._close_requested.wait()) + + async def close(self): + await super().close() + self._close_requested.set() + + class FakeBedrockStream: def __init__(self, input_stream=None): self.input_stream = input_stream if input_stream is not None else FakeInputStream() @@ -525,11 +539,13 @@ class TestBedrockRealtimeHandler: [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] ) - with pytest.raises(RuntimeError, match="bedrock send failed"): + with pytest.raises(BedrockError) as failure: await handler._forward_client_to_bedrock( client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) + assert failure.value.status_code == 503 + assert str(failure.value.__cause__) == "bedrock send failed" assert stream.input_stream.closed @pytest.mark.asyncio @@ -1008,7 +1024,7 @@ class TestBedrockRealtimeProviderFailurePropagation: [self.SESSION_UPDATE, json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})] ) - with pytest.raises(RuntimeError, match="bedrock input stream reset"): + with pytest.raises(BedrockError) as failure: await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, @@ -1016,6 +1032,8 @@ class TestBedrockRealtimeProviderFailurePropagation: **self.AWS_PARAMS, ) + assert failure.value.status_code == 503 + assert str(failure.value.__cause__) == "bedrock input stream reset", "the proxy log keeps the reset's cause" assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] @@ -1119,6 +1137,76 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_failed_bedrock_output_leaves_no_pending_input_writer( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type) + stream.input_stream = BodyWriterInputStream() + stub_aws_sdk_client["streams"] = [stream] + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ) + + assert failure.value.status_code == status_code + await asyncio.wait_for(stream.input_stream.body_writer, timeout=1) + await spend_dispatch["coro"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("bedrock_payloads", "logged_event_types"), + [(TEXT_TURN, ["response.done"]), ((), [])], + ids=["after-a-turn", "before-any-event"], + ) + async def test_a_dropped_session_logs_one_success_usage_record_and_a_failed_spend_row( + self, stub_aws_sdk_client, spend_dispatch, bedrock_payloads, logged_event_types: list[str] + ): + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(bedrock_payloads)] + logging_obj: Final = LiteLLMLogging( + model="amazon.nova-sonic-v1:0", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id="nova-sonic-drop", + function_id="nova-sonic-drop", + ) + logging_obj.update_from_kwargs( + kwargs={"litellm_metadata": {"user_api_key": "hashed-key"}}, + model="amazon.nova-sonic-v1:0", + optional_params={}, + litellm_params={"_arealtime": True}, + custom_llm_provider="bedrock", + ) + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + await spend_dispatch["coro"] + + assert failure.value.status_code == 503 + usage_record: Final = logging_obj.model_call_details["standard_logging_object"] + assert usage_record["status"] == "success", "the router's failure event is each logger's one error record" + assert [event["type"] for event in usage_record["response"]["results"]] == logged_event_types + assert usage_record["response_cost"] == 0, "no usage event streamed, so nothing is billed" + assert logging_obj.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + status, error_information = _spend_row(logging_obj) + assert (status, error_information["error_code"]) == ("failure", "503") + @pytest.mark.asyncio async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( self, stub_aws_sdk_client, spend_dispatch @@ -1161,7 +1249,7 @@ class TestBedrockRealtimeProviderFailurePropagation: stub_aws_sdk_client["streams"] = [stream] websocket: Final = ConnectedClientWS([self.SESSION_UPDATE, self.USER_AUDIO]) - with pytest.raises((BedrockError, RuntimeError)): + with pytest.raises(BedrockError) as failure: await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, @@ -1169,6 +1257,10 @@ class TestBedrockRealtimeProviderFailurePropagation: **self.AWS_PARAMS, ) + assert (failure.value.status_code, failure.value.message) == ( + 503, + "Bedrock realtime stream ended before the session closed", + ), "the app sees one cause whichever side of the stream reports the reset first" assert not websocket.closed, "the proxy route owns the client-facing error event and 1011 close" await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 17e954ccd9d..50e9d47bbc7 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -3,14 +3,17 @@ import functools import gc import json import os +import uuid import weakref -from collections.abc import Iterator, Mapping +from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping from datetime import datetime, timezone from types import MethodType from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +from websockets.exceptions import ConnectionClosed +from websockets.frames import Close import litellm from litellm import Router @@ -19,8 +22,12 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.realtime_errors import close_after_upstream_handshake_refusal +from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook +from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute from litellm.types.litellm_params import RoutingStrategyName from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo from litellm.types.utils import ( @@ -538,6 +545,93 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( assert await router.cache.async_get_cache(key=tpm_key) == 17 +class _DiscardingLoggingWorker: + def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, object, None]) -> None: + async_coroutine.close() + + +async def _client_never_sends() -> str: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + +def _realtime_client_ws() -> MagicMock: + client_ws: Final = MagicMock() + client_ws.headers = {} + client_ws.receive_text = AsyncMock(side_effect=_client_never_sends) + client_ws.send_text = AsyncMock() + client_ws.close = AsyncMock() + return client_ws + + +def _realtime_session_logging() -> LiteLLMLogging: + return LiteLLMLogging( + model="gpt-realtime-2.1", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id=str(uuid.uuid4()), + function_id="realtime-session", + ) + + +async def _relay_upstream(litellm_logging_obj: LiteLLMLogging, *upstream_frames: bytes | Exception) -> None: + backend_ws: Final = MagicMock() + backend_ws.recv = AsyncMock(side_effect=list(upstream_frames)) + relay: Final = RealTimeStreaming( + _realtime_client_ws(), + backend_ws, + litellm_logging_obj, + model="gpt-realtime-2.1", + logging_worker=_DiscardingLoggingWorker(), + ) + await asyncio.wait_for(relay.bidirectional_forward(), timeout=2) + + +async def _clean_realtime_session(litellm_logging_obj: LiteLLMLogging) -> None: + session_created: Final = json.dumps({"type": "session.created", "session": {"id": "sess_1"}}).encode() + await _relay_upstream(litellm_logging_obj, session_created, ConnectionClosed(Close(1000, ""), None)) + + +async def _upstream_close_before_any_frame(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream(litellm_logging_obj, ConnectionClosed(Close(1008, "model not found"), None)) + + +async def _refused_handshake(litellm_logging_obj: LiteLLMLogging) -> None: + await close_after_upstream_handshake_refusal(_realtime_client_ws(), 403) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("realtime_session", "counted_successes"), + [(_clean_realtime_session, 1), (_upstream_close_before_any_frame, 0), (_refused_handshake, 0)], + ids=["clean-session", "upstream-close-before-any-frame", "refused-handshake"], +) +async def test_realtime_session_counts_a_router_success_only_when_it_ended_without_a_failure( + realtime_session: Callable[[LiteLLMLogging], Awaitable[None]], counted_successes: int +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "realtime-voice", + "litellm_params": {"model": "openai/gpt-realtime-2.1"}, + "model_info": {"id": "realtime-a"}, + } + ], + num_retries=0, + ) + + async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None: + await realtime_session(litellm_logging_obj) + + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", original_function=_arealtime, litellm_logging_obj=_realtime_session_logging() + ) + + assert get_deployment_successes_for_current_minute(router, "realtime-a") == counted_successes + + @pytest.mark.asyncio @pytest.mark.parametrize("realtime_model", ["bedrock/amazon.nova-sonic-v1:0", "openai/gpt-realtime-2.1"]) @pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)]) @@ -565,28 +659,27 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy ) def realtime_session(failure: Exception | None): - async def _arealtime(**kwargs: object) -> None: - await router.deployment_callback_on_success( - kwargs={ - "call_type": "_arealtime", - "litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": kwargs["model_info"]}, - "standard_logging_object": create_standard_logging_payload(), - }, - completion_response=None, - start_time=time.time(), - end_time=time.time(), - ) + async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None: if failure is not None: raise failure + await _clean_realtime_session(litellm_logging_obj) return _arealtime for _ in range(clean_sessions): - await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(None)) + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", + original_function=realtime_session(None), + litellm_logging_obj=_realtime_session_logging(), + ) async def drop_session() -> bool: with pytest.raises(litellm.ServiceUnavailableError): - await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(drop)) + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", + original_function=realtime_session(drop), + litellm_logging_obj=_realtime_session_logging(), + ) return router.deployment_callback_on_failure( kwargs={ "exception": drop, From abd825dc67581c8ec53ce622856cfa32b364ba0a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:00:11 -0700 Subject: [PATCH 5/8] fix(realtime): count a router success only for a session that ended cleanly A realtime session the upstream closes with 1011 or 1006 after its first frame now marks itself as ended abnormally where that close is seen, in the shared relay and in the OpenAI Live handler, so the router no longer counts it as a cooldown success. The success callbacks still run, so the proxy keeps the budget and parallel-slot settlement it had A Bedrock session that fails before Bedrock sends any usage now logs no success event, as on base. The spend-row failure fields go into a new metadata mapping instead of the request's own dict, validated with a pydantic TypeAdapter instead of an unchecked TypeIs guard. The handler tests drop a wall-clock bound and type their helpers --- litellm/constants.py | 1 + .../litellm_core_utils/realtime_streaming.py | 12 ++- litellm/llms/bedrock/realtime/handler.py | 41 +++---- litellm/llms/openai/live/handler.py | 20 +++- litellm/router.py | 2 + .../realtime/test_bedrock_realtime_handler.py | 101 ++++++++++++++---- tests/unit/llms/openai/live/test_handler.py | 92 +++++++++++++++- .../test_router/test_router_helper_utils.py | 32 +++++- 8 files changed, 252 insertions(+), 49 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index c586a5f767a..fe852f4e733 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -448,6 +448,7 @@ OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset( {"response.completed", "response.incomplete", "response.failed"} ) REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged" +REALTIME_SESSION_ENDED_ABNORMALLY_KEY: Final = "realtime_session_ended_abnormally" # SSL/TLS cipher configuration for faster handshakes # Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 4db22e9d5bf..95a93f7e38d 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -19,6 +19,7 @@ from litellm.constants import ( REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS, REALTIME_LAZY_SETUP_DEADLINE_SECONDS, REALTIME_METERED_USAGE_KEY, + REALTIME_SESSION_ENDED_ABNORMALLY_KEY, REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) @@ -57,6 +58,10 @@ class BackendClose: code: int reason: str + @property + def ended_cleanly(self) -> bool: + return self.code == 1000 + @property def message(self) -> str: if not self.reason: @@ -2023,7 +2028,10 @@ class RealTimeStreaming: await asyncio.wait((forward_task, client_task), return_when=asyncio.FIRST_COMPLETED) if client_task.done() and client_task.result() is ClientLoopExit.CLIENT_DISCONNECTED: return - await self._close_client(await forward_task) + close: Final = await forward_task + if not close.ended_cleanly and self.logging_obj: + self.logging_obj.model_call_details[REALTIME_SESSION_ENDED_ABNORMALLY_KEY] = True + await self._close_client(close) finally: forward_task.cancel() client_task.cancel() @@ -2033,7 +2041,7 @@ class RealTimeStreaming: redacted_message: Final = redact_internal_details_from_client_message(close.message) redacted_reason: Final = redact_internal_details_from_client_message(close.reason) try: - if close.code != 1000: + if not close.ended_cleanly: await self.websocket.send_text(realtime_error_event(redacted_message, error_type="server_error")) await self.websocket.close( code=client_close_code(close.code), diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index aa83722ef7b..9dcfcf45875 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -14,7 +14,6 @@ from types import MappingProxyType from typing import Final, NoReturn, Protocol, runtime_checkable from pydantic import JsonValue, TypeAdapter -from typing_extensions import TypeIs # noqa: TID251 # narrows request metadata without copying it import litellm from litellm._logging import redact_string, verbose_proxy_logger @@ -42,6 +41,8 @@ from ..common_utils import BedrockError from .transformation import BedrockRealtimeConfig, requested_modalities _CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_REQUEST_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) +_SPEND_LOG_METADATA_KEYS: Final = ("litellm_metadata", "metadata") _EMPTY_JSON_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({}) @@ -213,23 +214,25 @@ def _pending_opening_message(scope: Mapping[str, object]) -> str | None: return pending if isinstance(pending, str) else None -def _is_str_keyed_dict( - value: object, -) -> TypeIs[dict[str, object]]: # guard-ok: request kwargs have str keys # mutable-ok: the spend row reads it - return isinstance(value, dict) +def _spend_log_metadata(litellm_params: Mapping[str, object]) -> tuple[str, Mapping[str, object]] | None: + return next( + ( + (key, metadata) + for key in _SPEND_LOG_METADATA_KEYS + if (metadata := _REQUEST_MAPPING_ADAPTER.validate_python(litellm_params.get(key))) + ), + None, + ) def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseException) -> None: """The spend row of a session the provider ended reads failure while it still bills the forwarded events.""" - litellm_params: Final[object] = logging_obj.model_call_details.get("litellm_params") - metadata_candidates: Final = ( - (litellm_params.get("litellm_metadata"), litellm_params.get("metadata")) - if _is_str_keyed_dict(litellm_params) - else () - ) - spend_log_metadata: Final = next( - (metadata for metadata in metadata_candidates if _is_str_keyed_dict(metadata) and metadata), None + litellm_params: Final = _REQUEST_MAPPING_ADAPTER.validate_python( + logging_obj.model_call_details.get("litellm_params") ) + if litellm_params is None: + return + spend_log_metadata: Final = _spend_log_metadata(litellm_params) if spend_log_metadata is None: return status_code: Final[object] = getattr(error, "status_code", None) @@ -239,8 +242,11 @@ def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseExcepti "error_message": redact_string(str(error)), "llm_provider": "bedrock", } - spend_log_metadata["status"] = "failure" - spend_log_metadata["error_information"] = error_information + metadata_key, metadata = spend_log_metadata + logging_obj.model_call_details["litellm_params"] = { # rebind-ok: the spend row reads the logging object + **litellm_params, + metadata_key: {**metadata, "status": "failure", "error_information": error_information}, + } def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn: @@ -513,10 +519,7 @@ class BedrockRealtime(BaseAWSLLM): "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure ) provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure - provider_ended_committed_session: Final = provider_failure is not None and _is_session_committed( - websocket.scope - ) - if logged_events or provider_ended_committed_session: + if logged_events: if provider_failure is not None: _record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure)) GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( diff --git a/litellm/llms/openai/live/handler.py b/litellm/llms/openai/live/handler.py index 297a147a288..5b899ac4162 100644 --- a/litellm/llms/openai/live/handler.py +++ b/litellm/llms/openai/live/handler.py @@ -18,6 +18,7 @@ from litellm._logging import redact_secrets from litellm.constants import ( OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS, OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, + REALTIME_SESSION_ENDED_ABNORMALLY_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ) @@ -316,12 +317,15 @@ async def _forward_live_backend_frames( backend_websocket: LiveBackendWebSocket, client_gone: asyncio.Event, event_retention: _LiveEventRetention, + upstream_ended_abnormally: asyncio.Event, ) -> None: from starlette.websockets import WebSocketDisconnect from websockets.exceptions import ConnectionClosed event_loop: Final = asyncio.get_running_loop() - async for message, close_deadline in _live_backend_messages(websocket, backend_websocket): + async for message, close_deadline in _live_backend_messages( + websocket, backend_websocket, upstream_ended_abnormally + ): if (event := _metering_event(message)) is not None: event_retention.retain(event) if client_gone.is_set(): @@ -341,6 +345,7 @@ async def _forward_live_backend_frames( async def _live_backend_messages( websocket: LiveClientWebSocket, backend_websocket: LiveBackendWebSocket, + upstream_ended_abnormally: asyncio.Event, ) -> AsyncIterator[tuple[str, float | None]]: from starlette.websockets import WebSocketDisconnect from websockets.exceptions import ConnectionClosed @@ -350,6 +355,8 @@ async def _live_backend_messages( yield message, None except ConnectionClosed as error: upstream_close: Final = backend_close_from(error) + if not upstream_close.ended_cleanly: + upstream_ended_abnormally.set() with suppress(WebSocketDisconnect, RuntimeError, ConnectionClosed, OSError): await websocket.close(code=upstream_close.code, reason=upstream_close.reason) return @@ -434,6 +441,7 @@ class OpenAILiveSessions(OpenAIRealtime): "complete_input_dict": {"session_start": rewritten_start}, } event_retention: Final = _LiveEventRetention() + session_ended_abnormally: Final = asyncio.Event() try: logging_obj.pre_call(input=None, api_key=api_key, additional_args=pre_call_args) async with self._websocket_connector( @@ -448,7 +456,11 @@ class OpenAILiveSessions(OpenAIRealtime): websocket=websocket, backend_websocket=backend_websocket, event_retention=event_retention, + upstream_ended_abnormally=session_ended_abnormally, ) + except Exception: + session_ended_abnormally.set() + raise finally: session_end_time: Final = self._clock() retained_events: Final = event_retention.results() @@ -463,6 +475,10 @@ class OpenAILiveSessions(OpenAIRealtime): logging_obj.model_call_details[ # rebind-ok: flags the shared logging object as already logged REALTIME_SESSION_SUCCESS_LOGGED_KEY ] = True + if session_ended_abnormally.is_set(): + logging_obj.model_call_details[ # rebind-ok: flags the shared logging object for the router + REALTIME_SESSION_ENDED_ABNORMALLY_KEY + ] = True except InvalidStatus as error: await close_after_upstream_handshake_refusal(websocket, error.response.status_code) except Exception as error: # noqa: BLE001 # any relay failure must still close the client socket with a redacted reason @@ -484,6 +500,7 @@ class OpenAILiveSessions(OpenAIRealtime): websocket: LiveClientWebSocket, backend_websocket: LiveBackendWebSocket, event_retention: _LiveEventRetention, + upstream_ended_abnormally: asyncio.Event, ) -> None: from websockets.exceptions import ConnectionClosed @@ -503,6 +520,7 @@ class OpenAILiveSessions(OpenAIRealtime): backend_websocket=backend_websocket, client_gone=client_gone, event_retention=event_retention, + upstream_ended_abnormally=upstream_ended_abnormally, ) ) client_gone_task: Final = asyncio.create_task(client_gone.wait()) diff --git a/litellm/router.py b/litellm/router.py index 347b7c6079c..ae2f7c6367c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -97,6 +97,7 @@ from litellm.constants import ( MID_STREAM_FAILURE_LOG_WAIT_SECONDS, OUTPUT_TOKEN_CEILING_PARAMS, PROMPT_CACHE_PIN_CACHE_MAX_SIZE, + REALTIME_SESSION_ENDED_ABNORMALLY_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, @@ -6686,6 +6687,7 @@ class Router(DeprecatedRouterMembers): _MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id") if isinstance(realtime_session_logging, LiteLLMLogging) and realtime_session_logging.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + and realtime_session_logging.model_call_details.get(REALTIME_SESSION_ENDED_ABNORMALLY_KEY) is not True else None ) if realtime_session_deployment_id is not None: diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index a075e89f626..ef9af6f8a69 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -1,14 +1,15 @@ import asyncio import json import sys -import time import types +from collections.abc import Mapping from datetime import datetime, timezone from types import SimpleNamespace -from typing import Final +from typing import Final, Literal from unittest.mock import MagicMock import pytest +from pydantic import TypeAdapter import litellm from litellm.constants import ( @@ -23,6 +24,7 @@ from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSig from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig from litellm.realtime_api import main as realtime_main +from litellm.types.utils import StandardLoggingPayloadErrorInformation from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault @@ -47,10 +49,20 @@ def _closing_input() -> SessionEndSignals: return signals -def _spend_row(logging_obj): +_SPEND_ROW_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_ERROR_INFORMATION_ADAPTER: Final = TypeAdapter(StandardLoggingPayloadErrorInformation | None) +_BEDROCK_INPUT_CHUNK_ADAPTER: Final = TypeAdapter(Mapping[str, Mapping[str, object]]) + + +def _spend_row( + logging_obj: "FakeLogging | LiteLLMLogging", +) -> tuple[Literal["success", "failure"], StandardLoggingPayloadErrorInformation | None]: now: Final = datetime.now(timezone.utc) - row: Final = get_logging_payload(kwargs=logging_obj.model_call_details, response_obj=None, start_time=now, end_time=now) - return row["status"], json.loads(row["metadata"])["error_information"] + row: Final = get_logging_payload( + kwargs=logging_obj.model_call_details, response_obj=None, start_time=now, end_time=now + ) + metadata: Final = _SPEND_ROW_METADATA_ADAPTER.validate_json(row["metadata"]) + return row["status"], _ERROR_INFORMATION_ADAPTER.validate_python(metadata.get("error_information")) class FakePayloadPart: @@ -283,8 +295,11 @@ class ScriptedBedrockStream: return (None, self._receiver) -def _sent_event_names(input_stream): - return [next(iter(json.loads(chunk.value.bytes_.decode("utf-8"))["event"])) for chunk in input_stream.sent] +def _sent_event_names(input_stream: FakeInputStream) -> list[str]: + return [ + next(iter(_BEDROCK_INPUT_CHUNK_ADAPTER.validate_json(chunk.value.bytes_)["event"])) + for chunk in input_stream.sent + ] class SessionEndingInputStream(FakeInputStream): @@ -1072,7 +1087,8 @@ class TestBedrockRealtimeProviderFailurePropagation: stream: Final = ScriptedBedrockStream(self.TEXT_TURN) stub_aws_sdk_client["streams"] = [stream] logging_obj: Final = spend_dispatch["logging_obj"] - logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed-key"}} + request_metadata: Final = {"user_api_key": "hashed-key"} + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": request_metadata} with pytest.raises(BedrockError) as failure: await handler.async_realtime( @@ -1089,8 +1105,10 @@ class TestBedrockRealtimeProviderFailurePropagation: assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] status, error_information = _spend_row(logging_obj) assert status == "failure", "the Logs page must show the dropped session as failed while it bills its turn" - assert error_information["error_code"] == "503" - assert error_information["error_message"] == failure.value.message + assert error_information is not None + assert error_information.get("error_code") == "503" + assert error_information.get("error_message") == failure.value.message + assert request_metadata == {"user_api_key": "hashed-key"}, "the request's own metadata stays as it was sent" with pytest.raises(BedrockError) as replay: await handler.async_realtime( @@ -1118,7 +1136,6 @@ class TestBedrockRealtimeProviderFailurePropagation: stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type) stream.input_stream = HangingCloseInputStream() stub_aws_sdk_client["streams"] = [stream] - started: Final = time.monotonic() with pytest.raises(BedrockError) as failure: await asyncio.wait_for( @@ -1132,7 +1149,6 @@ class TestBedrockRealtimeProviderFailurePropagation: ) assert failure.value.status_code == status_code - assert time.monotonic() - started < 1, "the app must not wait on closing a Bedrock input that already failed" assert stub_aws_sdk_client["client_closed"], "closing the SDK client still tears the dead stream down" await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] @@ -1163,15 +1179,10 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] @pytest.mark.asyncio - @pytest.mark.parametrize( - ("bedrock_payloads", "logged_event_types"), - [(TEXT_TURN, ["response.done"]), ((), [])], - ids=["after-a-turn", "before-any-event"], - ) async def test_a_dropped_session_logs_one_success_usage_record_and_a_failed_spend_row( - self, stub_aws_sdk_client, spend_dispatch, bedrock_payloads, logged_event_types: list[str] + self, stub_aws_sdk_client, spend_dispatch ): - stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(bedrock_payloads)] + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)] logging_obj: Final = LiteLLMLogging( model="amazon.nova-sonic-v1:0", messages=[], @@ -1201,11 +1212,57 @@ class TestBedrockRealtimeProviderFailurePropagation: assert failure.value.status_code == 503 usage_record: Final = logging_obj.model_call_details["standard_logging_object"] assert usage_record["status"] == "success", "the router's failure event is each logger's one error record" - assert [event["type"] for event in usage_record["response"]["results"]] == logged_event_types - assert usage_record["response_cost"] == 0, "no usage event streamed, so nothing is billed" + assert [event["type"] for event in usage_record["response"]["results"]] == ["response.done"] assert logging_obj.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True status, error_information = _spend_row(logging_obj) - assert (status, error_information["error_code"]) == ("failure", "503") + assert status == "failure" + assert error_information is not None + assert error_information.get("error_code") == "503" + + @pytest.mark.asyncio + async def test_a_dropped_session_without_request_metadata_still_fails_and_bills_its_turn( + self, stub_aws_sdk_client, spend_dispatch + ): + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)] + logging_obj: Final = spend_dispatch["logging_obj"] + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": None, "metadata": {}} + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + await spend_dispatch["coro"] + + assert failure.value.status_code == 503 + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_session_bedrock_fails_before_any_logged_event_dispatches_no_success( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream((), receiver_type=receiver_type)] + logging_obj: Final = spend_dispatch["logging_obj"] + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed-key"}} + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + + assert failure.value.status_code == status_code + assert "coro" not in spend_dispatch, "a failed session with no usage gets no success event, as on base" + assert REALTIME_SESSION_SUCCESS_LOGGED_KEY not in logging_obj.model_call_details @pytest.mark.asyncio async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( diff --git a/tests/unit/llms/openai/live/test_handler.py b/tests/unit/llms/openai/live/test_handler.py index ec1130376d0..bb7cda99e79 100644 --- a/tests/unit/llms/openai/live/test_handler.py +++ b/tests/unit/llms/openai/live/test_handler.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import Coroutine +from collections.abc import Callable, Coroutine from datetime import datetime from typing import Final, NoReturn, cast @@ -13,11 +13,13 @@ from websockets.frames import Close from websockets.http11 import Response import litellm +from litellm import Router from litellm.constants import REALTIME_SESSION_SUCCESS_LOGGED_KEY, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.cost_calculator import handle_realtime_stream_cost_calculation from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.openai.live import handler as live_handler from litellm.llms.openai.live.handler import LiveSessionStart, OpenAILiveSessions, parse_live_session_start +from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute from litellm.types.utils import LiteLLMRealtimeStreamLoggingObject, Usage _LIVE_EVENT_FRAME_ADAPTER: Final = TypeAdapter(dict[str, object]) @@ -1206,3 +1208,91 @@ async def test_async_live_session_reports_redacted_server_error_and_bounds_close finally: if worker.coroutine is not None: await worker.coroutine + + +_LIVE_SESSION_STARTED: Final = '{"type":"session.started","session":{"model":"gpt-live-1"}}' +_LIVE_SESSION_CLOSED: Final = '{"type":"session.closed","usage":{"seconds":4}}' + + +def _upstream_closing_the_session() -> _FakeUpstream: + return _FakeUpstream(client_frames=(), upstream_frames=(_LIVE_SESSION_STARTED, _LIVE_SESSION_CLOSED)) + + +def _upstream_failing_after_a_frame() -> _FakeUpstream: + return _FakeUpstream( + client_frames=(), + upstream_frames=(_LIVE_SESSION_STARTED,), + close_error=ConnectionClosedError(Close(code=1011, reason="upstream failed"), None), + ) + + +def _upstream_dropping_after_a_frame() -> _FakeUpstream: + return _FakeUpstream( + client_frames=(), upstream_frames=(_LIVE_SESSION_STARTED,), close_error=ConnectionClosedError(None, None) + ) + + +def _upstream_failing_a_client_send() -> _FakeUpstream: + return _FakeUpstream( + client_frames=(), + upstream_frames=(_LIVE_SESSION_STARTED,), + close_response=_LIVE_SESSION_CLOSED, + fail_on_send_type="conversation.item.create", + send_error=OSError("upstream socket write failed"), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("upstream_factory", "counted_successes"), + [ + (_upstream_closing_the_session, 1), + (_upstream_failing_after_a_frame, 0), + (_upstream_dropping_after_a_frame, 0), + (_upstream_failing_a_client_send, 0), + ], + ids=["session-closed", "upstream-1011-after-a-frame", "upstream-1006-after-a-frame", "upstream-send-failed"], +) +async def test_live_session_counts_a_router_success_only_when_it_ended_cleanly( + upstream_factory: Callable[[], _FakeUpstream], counted_successes: int +) -> None: + router: Final = Router( + model_list=[ + {"model_name": "live-voice", "litellm_params": {"model": "openai/gpt-live-1"}, "model_info": {"id": "live-a"}} + ], + num_retries=0, + ) + worker: Final = _FakeLoggingWorker() + + async def _arealtime(litellm_logging_obj: Logging, **kwargs: object) -> None: + await OpenAILiveSessions( + logging_worker=worker, websocket_connector=_FakeConnector(upstream_factory()) + ).async_live_session( + model="gpt-live-1", + websocket=_FakeClientWebSocket(messages=('{"type":"conversation.item.create"}',)), + logging_obj=litellm_logging_obj, + session_start={"type": "session.start", "session": {"model": "gpt-live-1"}}, + api_base="https://api.openai.com/", + api_key="live-test-key", + ) + + await asyncio.wait_for( + router._ageneric_api_call_with_fallbacks( + model="live-voice", + original_function=_arealtime, + litellm_logging_obj=Logging( + model="gpt-live-1", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id="live-session", + function_id="live-session", + ), + ), + timeout=5, + ) + + if worker.coroutine is not None: + worker.coroutine.close() + assert get_deployment_successes_for_current_minute(router, "live-a") == counted_successes diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 07cb87419bc..d610ebc2c0b 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -591,9 +591,21 @@ async def _relay_upstream(litellm_logging_obj: LiteLLMLogging, *upstream_frames: await asyncio.wait_for(relay.bidirectional_forward(), timeout=2) +_REALTIME_SESSION_CREATED: Final = json.dumps({"type": "session.created", "session": {"id": "sess_1"}}).encode() + + async def _clean_realtime_session(litellm_logging_obj: LiteLLMLogging) -> None: - session_created: Final = json.dumps({"type": "session.created", "session": {"id": "sess_1"}}).encode() - await _relay_upstream(litellm_logging_obj, session_created, ConnectionClosed(Close(1000, ""), None)) + await _relay_upstream(litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(Close(1000, ""), None)) + + +async def _upstream_internal_error_after_a_frame(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream( + litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(Close(1011, "upstream failed"), None) + ) + + +async def _upstream_drop_without_close_frame_after_a_frame(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream(litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(None, None)) async def _upstream_close_before_any_frame(litellm_logging_obj: LiteLLMLogging) -> None: @@ -607,8 +619,20 @@ async def _refused_handshake(litellm_logging_obj: LiteLLMLogging) -> None: @pytest.mark.asyncio @pytest.mark.parametrize( ("realtime_session", "counted_successes"), - [(_clean_realtime_session, 1), (_upstream_close_before_any_frame, 0), (_refused_handshake, 0)], - ids=["clean-session", "upstream-close-before-any-frame", "refused-handshake"], + [ + (_clean_realtime_session, 1), + (_upstream_close_before_any_frame, 0), + (_refused_handshake, 0), + (_upstream_internal_error_after_a_frame, 0), + (_upstream_drop_without_close_frame_after_a_frame, 0), + ], + ids=[ + "clean-session", + "upstream-close-before-any-frame", + "refused-handshake", + "upstream-1011-after-a-frame", + "upstream-1006-after-a-frame", + ], ) async def test_realtime_session_counts_a_router_success_only_when_it_ended_without_a_failure( realtime_session: Callable[[LiteLLMLogging], Awaitable[None]], counted_successes: int From 6b744dd1db1cb656102782478a8e8c47868387f1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:04:33 -0700 Subject: [PATCH 6/8] fix(bedrock): log a dropped realtime session's usage from its own logging state The failure event replaced the shared standard_logging_object while the queued success dispatch was still running its callbacks, so loggers received $0 and 0 tokens for a billed dropped session. --- litellm/llms/bedrock/realtime/handler.py | 3 +- .../realtime/test_bedrock_realtime_handler.py | 107 +++++++++++++++++- 2 files changed, 103 insertions(+), 7 deletions(-) diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 9dcfcf45875..49939399ec8 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -522,8 +522,9 @@ class BedrockRealtime(BaseAWSLLM): if logged_events: if provider_failure is not None: _record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure)) + usage_logging_obj: Final = logging_obj if provider_failure is None else logging_obj.attempt_scoped_copy() GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( - logging_obj.dispatch_success_handlers( + usage_logging_obj.dispatch_success_handlers( list(logged_events), prefer_async_handlers=True, ) diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index ef9af6f8a69..654d417edb2 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -1,4 +1,5 @@ import asyncio +import copy import json import sys import types @@ -24,7 +25,8 @@ from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSig from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig from litellm.realtime_api import main as realtime_main -from litellm.types.utils import StandardLoggingPayloadErrorInformation +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import StandardLoggingPayload, StandardLoggingPayloadErrorInformation from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault @@ -52,19 +54,51 @@ def _closing_input() -> SessionEndSignals: _SPEND_ROW_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _ERROR_INFORMATION_ADAPTER: Final = TypeAdapter(StandardLoggingPayloadErrorInformation | None) _BEDROCK_INPUT_CHUNK_ADAPTER: Final = TypeAdapter(Mapping[str, Mapping[str, object]]) +_STANDARD_LOGGING_PAYLOAD_ADAPTER: Final = TypeAdapter(StandardLoggingPayload) def _spend_row( logging_obj: "FakeLogging | LiteLLMLogging", +) -> tuple[Literal["success", "failure"], StandardLoggingPayloadErrorInformation | None]: + return _spend_row_of(logging_obj.model_call_details) + + +def _spend_row_of( + model_call_details: dict[str, object], ) -> tuple[Literal["success", "failure"], StandardLoggingPayloadErrorInformation | None]: now: Final = datetime.now(timezone.utc) - row: Final = get_logging_payload( - kwargs=logging_obj.model_call_details, response_obj=None, start_time=now, end_time=now - ) + row: Final = get_logging_payload(kwargs=model_call_details, response_obj=None, start_time=now, end_time=now) metadata: Final = _SPEND_ROW_METADATA_ADAPTER.validate_json(row["metadata"]) return row["status"], _ERROR_INFORMATION_ADAPTER.validate_python(metadata.get("error_information")) +class LoggedEventRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.events: Final[list[tuple[Literal["success", "failure"], dict[str, object]]]] = [] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.events.append(("success", dict(kwargs))) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.events.append(("failure", dict(kwargs))) + + def payload(self, event: Literal["success", "failure"]) -> StandardLoggingPayload: + (logged,) = (kwargs for logged_event, kwargs in self.events if logged_event == event) + return _STANDARD_LOGGING_PAYLOAD_ADAPTER.validate_python(logged["standard_logging_object"]) + + +class HeldSuccessLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.entered: Final = asyncio.Event() + self.release: Final = asyncio.Event() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.entered.set() + await self.release.wait() + + class FakePayloadPart: def __init__(self, bytes_): self.bytes_ = bytes_ @@ -166,6 +200,11 @@ class FakeLogging: self.litellm_trace_id = trace_id self.model_call_details = {} + def attempt_scoped_copy(self): + attempt = copy.copy(self) + attempt.model_call_details = dict(self.model_call_details) + return attempt + class DisconnectingClientWS: def __init__(self, messages): @@ -1183,6 +1222,7 @@ class TestBedrockRealtimeProviderFailurePropagation: self, stub_aws_sdk_client, spend_dispatch ): stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)] + recorder: Final = LoggedEventRecorder() logging_obj: Final = LiteLLMLogging( model="amazon.nova-sonic-v1:0", messages=[], @@ -1191,6 +1231,7 @@ class TestBedrockRealtimeProviderFailurePropagation: start_time=datetime.now(), litellm_call_id="nova-sonic-drop", function_id="nova-sonic-drop", + dynamic_async_success_callbacks=[recorder], ) logging_obj.update_from_kwargs( kwargs={"litellm_metadata": {"user_api_key": "hashed-key"}}, @@ -1210,15 +1251,69 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert failure.value.status_code == 503 - usage_record: Final = logging_obj.model_call_details["standard_logging_object"] + ((logged_event, usage_kwargs),) = recorder.events + assert logged_event == "success" + usage_record: Final = recorder.payload("success") assert usage_record["status"] == "success", "the router's failure event is each logger's one error record" assert [event["type"] for event in usage_record["response"]["results"]] == ["response.done"] assert logging_obj.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True - status, error_information = _spend_row(logging_obj) + status, error_information = _spend_row_of(usage_kwargs) assert status == "failure" assert error_information is not None assert error_information.get("error_code") == "503" + @pytest.mark.asyncio + async def test_a_dropped_session_bills_its_usage_to_loggers_while_the_failure_event_runs( + self, stub_aws_sdk_client, spend_dispatch + ): + usage_event: Final = json.dumps( + {"event": {"usageEvent": {"totalInputTokens": 1000, "totalOutputTokens": 50, "totalTokens": 1050}}} + ) + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream((*self.TEXT_TURN, usage_event))] + held: Final = HeldSuccessLogger() + recorder: Final = LoggedEventRecorder() + logging_obj: Final = LiteLLMLogging( + model="amazon.nova-2-5-sonic", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id="nova-sonic-drop-order", + function_id="nova-sonic-drop-order", + dynamic_async_success_callbacks=[held, recorder], + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_from_kwargs( + kwargs={"litellm_metadata": {"user_api_key": "hashed-key"}}, + model="amazon.nova-2-5-sonic", + optional_params={}, + litellm_params={"_arealtime": True}, + custom_llm_provider="bedrock", + ) + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-2-5-sonic", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + success_dispatch: Final = asyncio.create_task(spend_dispatch["coro"]) + await asyncio.wait_for(held.entered.wait(), timeout=5) + logging_obj.failure_handler(failure.value, "", logging_obj.start_time, datetime.now(), for_async_request=True) + await logging_obj.async_failure_handler(failure.value, "", logging_obj.start_time, datetime.now()) + held.release.set() + await success_dispatch + + usage_record: Final = recorder.payload("success") + assert recorder.payload("failure")["status"] == "failure" + assert usage_record["status"] == "success" + assert usage_record["total_tokens"] == 1050 + assert usage_record["response_cost"] > 0 + ((_, usage_kwargs),) = (logged for logged in recorder.events if logged[0] == "success") + status, _ = _spend_row_of(usage_kwargs) + assert status == "failure" + @pytest.mark.asyncio async def test_a_dropped_session_without_request_metadata_still_fails_and_bills_its_turn( self, stub_aws_sdk_client, spend_dispatch From 852770cbd8a96a852b4f36d7c53818da172c8066 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:34:00 -0700 Subject: [PATCH 7/8] fix(proxy): settle a rate-limit reservation once when a request logs a failure and a success A dropped realtime session fires a failure event and then a success event for the same call. The v3 limiter refunded the TPM reservation on the failure and released it again on the success, so the key's TPM counter went negative. The success path now skips a reservation the failure already released and marks it settled for a later failure. The realtime handler test callbacks are typed --- .../hooks/parallel_request_limiter_v3.py | 8 +++-- .../realtime/test_bedrock_realtime_handler.py | 12 +++++-- .../hooks/test_parallel_request_limiter_v3.py | 35 +++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 15282679622..6d5ff93c5eb 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -5317,7 +5317,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): total_tokens = self._aggregate_only_total_tokens(usage=_usage) stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) - reserved_tokens: Final = stash.reserved_tokens if stash is not None else 0 + open_reservation: Final = stash if stash is not None and not stash.reservation_released else None + reserved_tokens: Final = open_reservation.reserved_tokens if open_reservation is not None else 0 reserved_model: Final = stash.reserved_model if stash is not None else None reserved_scopes: Final[frozenset[tuple[str, str]]] = stash.reserved_scopes if stash is not None else frozenset() # Reconciliation must target the same model-scoped counter that the @@ -5378,7 +5379,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): reconcile_model=reconcile_model, reserved_scopes=reserved_scopes, reserved_ceiling=stash.ptu_ceiling if stash is not None else None, - reserved_tokens=stash.ptu_reserved_tokens if stash is not None else 0, + reserved_tokens=open_reservation.ptu_reserved_tokens if open_reservation is not None else 0, reserved_windows=stash.ptu_reserved_window_identities if stash is not None else frozenset(), total_tokens=total_tokens, ) @@ -5511,6 +5512,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): pipeline_operations=io_token_operations, parent_otel_span=litellm_parent_otel_span, ) + if stash is not None: + stash.reserved_tokens = 0 + stash.ptu_reserved_tokens = 0 except Exception as e: verbose_proxy_logger.exception("Error in rate limit success event: %s", e) diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 654d417edb2..cf2f25faaa9 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -77,10 +77,14 @@ class LoggedEventRecorder(CustomLogger): super().__init__() self.events: Final[list[tuple[Literal["success", "failure"], dict[str, object]]]] = [] - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: self.events.append(("success", dict(kwargs))) - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: self.events.append(("failure", dict(kwargs))) def payload(self, event: Literal["success", "failure"]) -> StandardLoggingPayload: @@ -94,7 +98,9 @@ class HeldSuccessLogger(CustomLogger): self.entered: Final = asyncio.Event() self.release: Final = asyncio.Event() - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: self.entered.set() await self.release.wait() diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 3aad2bda9bf..30e557215d7 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3707,6 +3707,41 @@ async def test_failure_event_refunds_reservation_for_input_only_estimate_v3(): assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 0 +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_first", [True, False]) +@pytest.mark.parametrize( + ("response_obj", "used_tokens"), + [ + ([], 0), + (ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)), 30), + ], +) +async def test_a_request_logging_a_failure_and_a_success_settles_its_reservation_once_v3( + failure_first: bool, response_obj: object, used_tokens: int +) -> None: + api_key: Final = hash_token("sk-failure-and-success") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + tokens_key: Final = handler.create_rate_limit_keys(key="api_key", value=api_key, rate_limit_type="tokens") + await _reserve_tpm_for_owner_call(handler, local_cache, api_key, "dropped-call") + kwargs: Final = { + "litellm_call_id": "dropped-call", + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + log_failure: Final = partial( + handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now + ) + log_success: Final = partial( + handler.async_log_success_event, kwargs=dict(kwargs), response_obj=response_obj, start_time=now, end_time=now + ) + + for log_event in (log_failure, log_success) if failure_first else (log_success, log_failure): + await log_event() + + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == used_tokens + + @pytest.mark.asyncio async def test_post_call_failure_hook_settles_reservation_at_recovered_partial_usage_v3(): """ From 30515fd8a1b5981486dcdbb277923fb0413c8ecc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 03:08:40 -0700 Subject: [PATCH 8/8] fix(rate-limit): settle each token reservation once when a request logs both a failure and a success A dropped realtime session logs a success for its usage and then a failure. After a success the limiter now clears every reserved amount (TPM, PTU, ITPM, OTPM), so a later failure refunds nothing twice. A new reservation written in the same context after a released one now starts clean, so reserve, fail, reserve, succeed charges the second request's usage once instead of 78 TPM or 28 ITPM for 27 and 20 used --- .../hooks/parallel_request_limiter_v3.py | 16 ++- .../hooks/test_parallel_request_limiter_v3.py | 119 ++++++++++++++++++ 2 files changed, 133 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 6d5ff93c5eb..b9c1047cc52 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -746,6 +746,17 @@ class RequestRateLimiterStash: reservation_released: bool = False tpm_limited_tags: frozenset[str] = field(default_factory=frozenset) + def clear_reserved_tokens(self) -> None: + self.reserved_tokens = 0 + self.ptu_reserved_tokens = 0 + self.itpm_reserved_tokens = 0 + self.otpm_reserved_tokens = 0 + + def discard_released_reservation(self) -> None: + if self.reservation_released: + self.clear_reserved_tokens() + self.reservation_released = False + @dataclass(frozen=True, slots=True) class CounterRefund: @@ -4151,6 +4162,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): requested_model=requested_model, ) + stash.discard_released_reservation() if itpm_reserved > 0: itpm_scopes: Final = tuple( (d["key"], d["value"]) for d in io_token_descriptors if d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY @@ -4509,6 +4521,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # incremented so post-call reconciliation only applies # the (actual - reserved) delta to those — unreserved # scopes get charged the full actual usage instead. + stash.discard_released_reservation() stash.reserved_tokens = estimated_tokens stash.ptu_reserved_tokens = ptu_estimated_tokens stash.ptu_reserved_window_identities = frozenset( @@ -5513,8 +5526,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): parent_otel_span=litellm_parent_otel_span, ) if stash is not None: - stash.reserved_tokens = 0 - stash.ptu_reserved_tokens = 0 + stash.clear_reserved_tokens() except Exception as e: verbose_proxy_logger.exception("Error in rate limit success event: %s", e) diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 30e557215d7..068071269b5 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3834,6 +3834,125 @@ async def test_failure_event_settles_project_itpm_otpm_at_recovered_partial_usag assert int(await local_cache.async_get_cache(key=otpm_key) or 0) == 7 + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_first", [True, False]) +async def test_a_request_logging_a_failure_and_a_success_settles_its_project_itpm_otpm_once_v3( + failure_first: bool, +) -> None: + api_key: Final = hash_token("sk-failure-and-success-project-io") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + await handler.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + api_key=api_key, + project_id="proj-dropped", + project_metadata={ + "model_itpm_limit": {"gpt-4o-mini": 10_000}, + "model_otpm_limit": {"gpt-4o-mini": 10_000}, + }, + ), + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "dropped-project-call", + }, + call_type="completion", + ) + itpm_key: Final = handler.create_rate_limit_keys( + key="model_per_project_itpm", value="proj-dropped:gpt-4o-mini", rate_limit_type="tokens" + ) + otpm_key: Final = handler.create_rate_limit_keys( + key="model_per_project_otpm", value="proj-dropped:gpt-4o-mini", rate_limit_type="tokens" + ) + kwargs: Final = { + "litellm_call_id": "dropped-project-call", + "model": "gpt-4o-mini", + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + log_failure: Final = partial( + handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now + ) + log_success: Final = partial( + handler.async_log_success_event, + kwargs=dict(kwargs), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)), + start_time=now, + end_time=now, + ) + + for log_event in (log_failure, log_success) if failure_first else (log_success, log_failure): + await log_event() + + assert int(await local_cache.async_get_cache(key=itpm_key) or 0) == 20 + assert int(await local_cache.async_get_cache(key=otpm_key) or 0) == 7 + + +@pytest.mark.asyncio +async def test_a_reservation_made_after_a_released_one_in_the_same_context_settles_at_its_usage_v3() -> None: + api_key: Final = hash_token("sk-second-reservation") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + user_api_key_dict: Final = UserAPIKeyAuth( + api_key=api_key, + tpm_limit=10_000, + project_id="proj-second", + project_metadata={ + "model_itpm_limit": {"gpt-4o-mini": 10_000}, + "model_otpm_limit": {"gpt-4o-mini": 10_000}, + }, + ) + now: Final = datetime.now() + + async def reserve(call_id: str) -> None: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": call_id, + }, + call_type="completion", + ) + + def logged(call_id: str) -> dict[str, object]: + return { + "litellm_call_id": call_id, + "model": "gpt-4o-mini", + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + + await reserve("first-call") + await handler.async_log_failure_event(kwargs=logged("first-call"), response_obj=None, start_time=now, end_time=now) + await reserve("second-call") + await handler.async_log_success_event( + kwargs=logged("second-call"), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)), + start_time=now, + end_time=now, + ) + + counters: Final = { + "tokens": ("api_key", api_key), + "itpm": ("model_per_project_itpm", "proj-second:gpt-4o-mini"), + "otpm": ("model_per_project_otpm", "proj-second:gpt-4o-mini"), + } + settled: Final = { + name: int( + await local_cache.async_get_cache( + key=handler.create_rate_limit_keys(key=key, value=value, rate_limit_type="tokens") + ) + or 0 + ) + for name, (key, value) in counters.items() + } + assert settled == {"tokens": 27, "itpm": 20, "otpm": 7} + # ----------------------- Per-MCP-server rate limiting (v3) -----------------------