fix(bedrock): count realtime router successes only for sessions that logged one, and log first-turn drops

The router counts a realtime success only when the session's success handlers ran, so a refused handshake or an upstream close before any frame no longer offsets its failure. The success event's payload status stays success again, so each logger gets one failure record per dropped session, while the spend row still reads failure from the metadata the Bedrock handler records. A committed Bedrock session that fails before any logged event now gets one zero-cost Logs row. After an output failure the input stream is closed in the background, so the SDK body writer finishes without delaying the client's error. Every input send failure reports the same 503 stream-ended cause as an output drop
This commit is contained in:
mateo-berri 2026-10-30 07:29:38 -07:00
parent edb9aaeee4
commit d271d221fd
6 changed files with 247 additions and 113 deletions

View file

@ -7478,25 +7478,10 @@ def get_standard_logging_object_payload(
router_model_id=logging_obj.get_router_model_id(),
)
metadata_fields: Final = cast(Mapping[str, object], metadata) # cast-ok: merged request metadata is str-keyed
recorded_error: Final = metadata_fields.get("error_information")
recorded_failure: Final = (
cast(StandardLoggingPayloadErrorInformation, recorded_error) # cast-ok: a reserved key only litellm writes
if isinstance(recorded_error, dict)
and status == "success"
and call_type == CallTypes.arealtime.value
and metadata_fields.get("status") == "failure"
else None
)
payload_status: Final[StandardLoggingPayloadStatus] = "failure" if recorded_failure is not None else status
error_information, error_str = (
(recorded_failure, error_str or recorded_failure.get("error_message"))
if recorded_failure is not None
else StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata=metadata,
original_exception=original_exception,
error_str=error_str,
)
error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata=metadata,
original_exception=original_exception,
error_str=error_str,
)
## get final response object ##
@ -7587,9 +7572,9 @@ def get_standard_logging_object_payload(
call_type=call_type or "",
cache_hit=cache_hit,
stream=stream,
status=payload_status,
status=status,
status_fields=_get_status_fields(
status=payload_status,
status=status,
guardrail_information=metadata.get("standard_logging_guardrail_information", None),
error_str=error_str,
),

View file

@ -63,6 +63,10 @@ _BEDROCK_STREAM_ERROR_STATUS: Final[Mapping[str, int]] = MappingProxyType(
)
def _stream_ended_before_session_closed() -> BedrockError:
return BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed")
def _as_bedrock_error(error: BaseException) -> BaseException:
status_code: Final = _BEDROCK_STREAM_ERROR_STATUS.get(type(error).__name__)
if status_code is None:
@ -244,6 +248,8 @@ def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseExc
verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error)))
if _is_session_committed(scope):
scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = redact_string(str(error))
if error is failure:
raise failure
raise error from failure
@ -347,6 +353,18 @@ class BedrockRealtime(BaseAWSLLM):
super().__init__()
self._sdk_version_lookup: Final = sdk_version_lookup
self._close_timeout: Final = close_timeout
self._failed_input_closes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs to running closes
def _close_failed_input_in_background(
self,
bedrock_stream: BedrockBidirectionalStream,
send_to_bedrock: Callable[[str], Awaitable[None]],
) -> None:
close_task: Final = asyncio.create_task(
_close_bedrock_input(bedrock_stream, (), send_to_bedrock, self._close_timeout)
)
self._failed_input_closes.add(close_task)
close_task.add_done_callback(self._failed_input_closes.discard)
async def async_realtime(
self,
@ -507,7 +525,10 @@ class BedrockRealtime(BaseAWSLLM):
"Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
)
provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure
if logged_events:
provider_ended_committed_session: Final = provider_failure is not None and _is_session_committed(
websocket.scope
)
if logged_events or provider_ended_committed_session:
if provider_failure is not None:
_record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure))
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
@ -612,7 +633,10 @@ class BedrockRealtime(BaseAWSLLM):
async def send_to_bedrock(bedrock_message: str) -> None:
event: Final = build_input_chunk(bedrock_message.encode("utf-8"))
await bedrock_stream.input_stream.send(event)
try:
await bedrock_stream.input_stream.send(event)
except Exception as e:
raise _stream_ended_before_session_closed() from e
verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200])
try:
@ -646,7 +670,9 @@ class BedrockRealtime(BaseAWSLLM):
break
signals.input_closing.set()
finally:
if not signals.output_failed.is_set():
if signals.output_failed.is_set():
self._close_failed_input_in_background(bedrock_stream, send_to_bedrock)
else:
await _close_bedrock_input(
bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout
)
@ -682,10 +708,7 @@ class BedrockRealtime(BaseAWSLLM):
if result is None:
if not signals.input_closing.is_set():
raise BedrockError(
status_code=503,
message="Bedrock realtime stream ended before the session closed",
)
raise _stream_ended_before_session_closed()
verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended")
with contextlib.suppress(Exception):
await client_ws.close()

View file

@ -98,6 +98,7 @@ from litellm.constants import (
MID_STREAM_FAILURE_LOG_WAIT_SECONDS,
OUTPUT_TOKEN_CEILING_PARAMS,
PROMPT_CACHE_PIN_CACHE_MAX_SIZE,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
ROUTING_REQUEST_TAGS_METADATA_KEY,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
@ -649,6 +650,7 @@ _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType
_NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_CALL_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str])
_RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
@ -6478,9 +6480,15 @@ class Router:
raise safeguard_refusal_error(model=model, stop_details=refusal_details)
self.success_calls[model_name] += 1
realtime_session_logging: Final = (
_CALL_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_logging_obj")
if generic_function_name == "_arealtime"
else None
)
realtime_session_deployment_id: Final = (
_MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id")
if generic_function_name == "_arealtime"
if isinstance(realtime_session_logging, LiteLLMLogging)
and realtime_session_logging.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True
else None
)
if realtime_session_deployment_id is not None:

View file

@ -6511,73 +6511,6 @@ def test_get_error_information_for_logging_payload_client_disconnect():
assert error_str is None
_SESSION_FAILURE: Final = {
"error_code": "503",
"error_class": "BedrockError",
"error_message": "Bedrock realtime stream ended before the session closed",
"llm_provider": "bedrock",
}
@pytest.mark.parametrize("metadata_bucket", ["metadata", "litellm_metadata"])
def test_realtime_success_payload_reports_the_session_failure_its_metadata_recorded(
logging_obj: LitellmLogging, metadata_bucket: str
) -> None:
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
now: Final = datetime(2026, 1, 1)
payload: Final = get_standard_logging_object_payload(
kwargs={
"model": "bedrock/amazon.nova-sonic-v1:0",
"call_type": "_arealtime",
"litellm_params": {metadata_bucket: {"status": "failure", "error_information": dict(_SESSION_FAILURE)}},
},
init_response_obj={},
start_time=now,
end_time=now,
logging_obj=logging_obj,
status="success",
)
assert payload is not None
assert payload["status"] == "failure"
assert payload["status_fields"]["llm_api_status"] == "failure"
assert payload["error_information"] == _SESSION_FAILURE
assert payload["error_str"] == _SESSION_FAILURE["error_message"]
@pytest.mark.parametrize(
("call_type", "metadata"),
[
("_arealtime", {"status": "failure"}),
("_arealtime", {"error_information": dict(_SESSION_FAILURE)}),
("acompletion", {"status": "failure", "error_information": dict(_SESSION_FAILURE)}),
],
)
def test_success_payload_stays_success_without_a_recorded_realtime_session_failure(
logging_obj: LitellmLogging, call_type: str, metadata: dict[str, object]
) -> None:
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
now: Final = datetime(2026, 1, 1)
payload: Final = get_standard_logging_object_payload(
kwargs={"model": "gpt-4o", "messages": [], "call_type": call_type, "litellm_params": {"metadata": metadata}},
init_response_obj={},
start_time=now,
end_time=now,
logging_obj=logging_obj,
status="success",
)
assert payload is not None
assert payload["status"] == "success"
assert payload["error_information"] != _SESSION_FAILURE
def test_get_error_information_proxy_exception_preserves_message():
"""ProxyException keeps its text in ``.message`` (str() was empty pre-fix),
so error_information must still surface the message and code."""

View file

@ -16,6 +16,7 @@ from litellm.constants import (
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
WEBSOCKET_CLOSE_REASON_MAX_BYTES,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.realtime_errors import websocket_close_reason
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals
@ -101,6 +102,19 @@ class HangingCloseInputStream(FakeInputStream):
await asyncio.Future()
class BodyWriterInputStream(FakeInputStream):
"""Models the SDK input stream, whose request-body writer task only finishes once the stream is closed"""
def __init__(self):
super().__init__()
self._close_requested = asyncio.Event()
self.body_writer = asyncio.get_running_loop().create_task(self._close_requested.wait())
async def close(self):
await super().close()
self._close_requested.set()
class FakeBedrockStream:
def __init__(self, input_stream=None):
self.input_stream = input_stream if input_stream is not None else FakeInputStream()
@ -525,11 +539,13 @@ class TestBedrockRealtimeHandler:
[json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})]
)
with pytest.raises(RuntimeError, match="bedrock send failed"):
with pytest.raises(BedrockError) as failure:
await handler._forward_client_to_bedrock(
client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals()
)
assert failure.value.status_code == 503
assert str(failure.value.__cause__) == "bedrock send failed"
assert stream.input_stream.closed
@pytest.mark.asyncio
@ -1008,7 +1024,7 @@ class TestBedrockRealtimeProviderFailurePropagation:
[self.SESSION_UPDATE, json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})]
)
with pytest.raises(RuntimeError, match="bedrock input stream reset"):
with pytest.raises(BedrockError) as failure:
await BedrockRealtime().async_realtime(
model="amazon.nova-sonic-v1:0",
websocket=websocket,
@ -1016,6 +1032,8 @@ class TestBedrockRealtimeProviderFailurePropagation:
**self.AWS_PARAMS,
)
assert failure.value.status_code == 503
assert str(failure.value.__cause__) == "bedrock input stream reset", "the proxy log keeps the reset's cause"
assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client]
await spend_dispatch["coro"]
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
@ -1119,6 +1137,76 @@ class TestBedrockRealtimeProviderFailurePropagation:
await spend_dispatch["coro"]
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("receiver_type", "status_code"),
[(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)],
ids=["output-ends", "output-raises"],
)
async def test_a_failed_bedrock_output_leaves_no_pending_input_writer(
self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int
):
stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type)
stream.input_stream = BodyWriterInputStream()
stub_aws_sdk_client["streams"] = [stream]
with pytest.raises(BedrockError) as failure:
await BedrockRealtime().async_realtime(
model="amazon.nova-sonic-v1:0",
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
logging_obj=spend_dispatch["logging_obj"],
**self.AWS_PARAMS,
)
assert failure.value.status_code == status_code
await asyncio.wait_for(stream.input_stream.body_writer, timeout=1)
await spend_dispatch["coro"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("bedrock_payloads", "logged_event_types"),
[(TEXT_TURN, ["response.done"]), ((), [])],
ids=["after-a-turn", "before-any-event"],
)
async def test_a_dropped_session_logs_one_success_usage_record_and_a_failed_spend_row(
self, stub_aws_sdk_client, spend_dispatch, bedrock_payloads, logged_event_types: list[str]
):
stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(bedrock_payloads)]
logging_obj: Final = LiteLLMLogging(
model="amazon.nova-sonic-v1:0",
messages=[],
stream=False,
call_type="_arealtime",
start_time=datetime.now(),
litellm_call_id="nova-sonic-drop",
function_id="nova-sonic-drop",
)
logging_obj.update_from_kwargs(
kwargs={"litellm_metadata": {"user_api_key": "hashed-key"}},
model="amazon.nova-sonic-v1:0",
optional_params={},
litellm_params={"_arealtime": True},
custom_llm_provider="bedrock",
)
with pytest.raises(BedrockError) as failure:
await BedrockRealtime().async_realtime(
model="amazon.nova-sonic-v1:0",
websocket=ConnectedClientWS([self.SESSION_UPDATE]),
logging_obj=logging_obj,
**self.AWS_PARAMS,
)
await spend_dispatch["coro"]
assert failure.value.status_code == 503
usage_record: Final = logging_obj.model_call_details["standard_logging_object"]
assert usage_record["status"] == "success", "the router's failure event is each logger's one error record"
assert [event["type"] for event in usage_record["response"]["results"]] == logged_event_types
assert usage_record["response_cost"] == 0, "no usage event streamed, so nothing is billed"
assert logging_obj.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True
status, error_information = _spend_row(logging_obj)
assert (status, error_information["error_code"]) == ("failure", "503")
@pytest.mark.asyncio
async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish(
self, stub_aws_sdk_client, spend_dispatch
@ -1161,7 +1249,7 @@ class TestBedrockRealtimeProviderFailurePropagation:
stub_aws_sdk_client["streams"] = [stream]
websocket: Final = ConnectedClientWS([self.SESSION_UPDATE, self.USER_AUDIO])
with pytest.raises((BedrockError, RuntimeError)):
with pytest.raises(BedrockError) as failure:
await BedrockRealtime().async_realtime(
model="amazon.nova-sonic-v1:0",
websocket=websocket,
@ -1169,6 +1257,10 @@ class TestBedrockRealtimeProviderFailurePropagation:
**self.AWS_PARAMS,
)
assert (failure.value.status_code, failure.value.message) == (
503,
"Bedrock realtime stream ended before the session closed",
), "the app sees one cause whichever side of the stream reports the reset first"
assert not websocket.closed, "the proxy route owns the client-facing error event and 1011 close"
await spend_dispatch["coro"]
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]

View file

@ -3,14 +3,17 @@ import functools
import gc
import json
import os
import uuid
import weakref
from collections.abc import Iterator, Mapping
from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping
from datetime import datetime, timezone
from types import MethodType
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from websockets.exceptions import ConnectionClosed
from websockets.frames import Close
import litellm
from litellm import Router
@ -19,8 +22,12 @@ from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.realtime_errors import close_after_upstream_handshake_refusal
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute
from litellm.types.litellm_params import RoutingStrategyName
from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
from litellm.types.utils import (
@ -538,6 +545,93 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn(
assert await router.cache.async_get_cache(key=tpm_key) == 17
class _DiscardingLoggingWorker:
def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, object, None]) -> None:
async_coroutine.close()
async def _client_never_sends() -> str:
await asyncio.Event().wait()
raise AssertionError("unreachable")
def _realtime_client_ws() -> MagicMock:
client_ws: Final = MagicMock()
client_ws.headers = {}
client_ws.receive_text = AsyncMock(side_effect=_client_never_sends)
client_ws.send_text = AsyncMock()
client_ws.close = AsyncMock()
return client_ws
def _realtime_session_logging() -> LiteLLMLogging:
return LiteLLMLogging(
model="gpt-realtime-2.1",
messages=[],
stream=False,
call_type="_arealtime",
start_time=datetime.now(),
litellm_call_id=str(uuid.uuid4()),
function_id="realtime-session",
)
async def _relay_upstream(litellm_logging_obj: LiteLLMLogging, *upstream_frames: bytes | Exception) -> None:
backend_ws: Final = MagicMock()
backend_ws.recv = AsyncMock(side_effect=list(upstream_frames))
relay: Final = RealTimeStreaming(
_realtime_client_ws(),
backend_ws,
litellm_logging_obj,
model="gpt-realtime-2.1",
logging_worker=_DiscardingLoggingWorker(),
)
await asyncio.wait_for(relay.bidirectional_forward(), timeout=2)
async def _clean_realtime_session(litellm_logging_obj: LiteLLMLogging) -> None:
session_created: Final = json.dumps({"type": "session.created", "session": {"id": "sess_1"}}).encode()
await _relay_upstream(litellm_logging_obj, session_created, ConnectionClosed(Close(1000, ""), None))
async def _upstream_close_before_any_frame(litellm_logging_obj: LiteLLMLogging) -> None:
await _relay_upstream(litellm_logging_obj, ConnectionClosed(Close(1008, "model not found"), None))
async def _refused_handshake(litellm_logging_obj: LiteLLMLogging) -> None:
await close_after_upstream_handshake_refusal(_realtime_client_ws(), 403)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("realtime_session", "counted_successes"),
[(_clean_realtime_session, 1), (_upstream_close_before_any_frame, 0), (_refused_handshake, 0)],
ids=["clean-session", "upstream-close-before-any-frame", "refused-handshake"],
)
async def test_realtime_session_counts_a_router_success_only_when_it_ended_without_a_failure(
realtime_session: Callable[[LiteLLMLogging], Awaitable[None]], counted_successes: int
) -> None:
router: Final = Router(
model_list=[
{
"model_name": "realtime-voice",
"litellm_params": {"model": "openai/gpt-realtime-2.1"},
"model_info": {"id": "realtime-a"},
}
],
num_retries=0,
)
async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None:
await realtime_session(litellm_logging_obj)
await router._ageneric_api_call_with_fallbacks(
model="realtime-voice", original_function=_arealtime, litellm_logging_obj=_realtime_session_logging()
)
assert get_deployment_successes_for_current_minute(router, "realtime-a") == counted_successes
@pytest.mark.asyncio
@pytest.mark.parametrize("realtime_model", ["bedrock/amazon.nova-sonic-v1:0", "openai/gpt-realtime-2.1"])
@pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)])
@ -565,28 +659,27 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy
)
def realtime_session(failure: Exception | None):
async def _arealtime(**kwargs: object) -> None:
await router.deployment_callback_on_success(
kwargs={
"call_type": "_arealtime",
"litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": kwargs["model_info"]},
"standard_logging_object": create_standard_logging_payload(),
},
completion_response=None,
start_time=time.time(),
end_time=time.time(),
)
async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None:
if failure is not None:
raise failure
await _clean_realtime_session(litellm_logging_obj)
return _arealtime
for _ in range(clean_sessions):
await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(None))
await router._ageneric_api_call_with_fallbacks(
model="realtime-voice",
original_function=realtime_session(None),
litellm_logging_obj=_realtime_session_logging(),
)
async def drop_session() -> bool:
with pytest.raises(litellm.ServiceUnavailableError):
await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(drop))
await router._ageneric_api_call_with_fallbacks(
model="realtime-voice",
original_function=realtime_session(drop),
litellm_logging_obj=_realtime_session_logging(),
)
return router.deployment_callback_on_failure(
kwargs={
"exception": drop,