fix(bedrock): agree on a dropped realtime session's failure across loggers and the spend row

A realtime session's success event now reports status failure, with the same
error_information the spend row stores, when the session recorded a provider
failure in its metadata. Usage already streamed stays billed and the router
still counts one cooldown failure

The spend row builder now runs every row's error_information through
sanitize_error_information_for_spend_logs (DB size cap plus prompt redaction),
so the realtime failure rows get the same treatment as the proxy failure hook,
which no longer sanitizes on its own to avoid truncating twice

Drop the selection-failure tag on the committed-session refusal, since the
shared logging object already keeps a retried refusal out of cooldown, and pin
the router's realtime success counting with an OpenAI realtime deployment
This commit is contained in:
mateo-berri 2026-10-30 04:35:25 -07:00
parent 8a3248de0d
commit edb9aaeee4
9 changed files with 183 additions and 82 deletions

View file

@ -7478,10 +7478,25 @@ def get_standard_logging_object_payload(
router_model_id=logging_obj.get_router_model_id(),
)
error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata=metadata,
original_exception=original_exception,
error_str=error_str,
metadata_fields: Final = cast(Mapping[str, object], metadata) # cast-ok: merged request metadata is str-keyed
recorded_error: Final = metadata_fields.get("error_information")
recorded_failure: Final = (
cast(StandardLoggingPayloadErrorInformation, recorded_error) # cast-ok: a reserved key only litellm writes
if isinstance(recorded_error, dict)
and status == "success"
and call_type == CallTypes.arealtime.value
and metadata_fields.get("status") == "failure"
else None
)
payload_status: Final[StandardLoggingPayloadStatus] = "failure" if recorded_failure is not None else status
error_information, error_str = (
(recorded_failure, error_str or recorded_failure.get("error_message"))
if recorded_failure is not None
else StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata=metadata,
original_exception=original_exception,
error_str=error_str,
)
)
## get final response object ##
@ -7572,9 +7587,9 @@ def get_standard_logging_object_payload(
call_type=call_type or "",
cache_hit=cache_hit,
stream=stream,
status=status,
status=payload_status,
status_fields=_get_status_fields(
status=status,
status=payload_status,
guardrail_information=metadata.get("standard_logging_guardrail_information", None),
error_str=error_str,
),

View file

@ -192,18 +192,14 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None:
"""A fallback attempt on a websocket whose Bedrock stream already took input would silently start over."""
if scope is None or not _is_session_committed(scope):
return
from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure
committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY)
refusal: Final = BedrockError(
raise BedrockError(
status_code=400,
message=(
(f"{committed_failure}. " if committed_failure else "")
+ "Bedrock realtime session already committed to a provider stream; it cannot be replayed"
),
)
mark_deployment_selection_failure(refusal)
raise refusal
def _pending_opening_message(scope: Mapping[str, object]) -> str | None:

View file

@ -58,7 +58,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 #
_sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
deployment_params_provider,
get_request_model_access_groups,
sanitize_error_information_for_spend_logs,
sanitize_error_information_for_spend_logs, # pyright: ignore[reportUnusedImport] # backwards-compatible package export
should_store_prompts_and_responses_in_spend_logs,
)
from litellm.proxy.utils import (
@ -72,7 +72,6 @@ from litellm.types.utils import (
CallTypes,
LiteLLMBatch,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
from litellm.utils import get_end_user_id_for_cost_tracking
@ -429,13 +428,6 @@ class _ProxyDBLogger(CustomLogger):
# ``.get("traceback")`` / truthy checks, and the TypedDict marks
# the field as optional, so omitting is type-safe.
_error_information.pop("traceback", None)
# Strip echoed request input + apply DB-size cap before storing in
# the spend-log metadata column (LIT-2992). Result is never None
# here because the input above is constructed non-None.
_error_information = cast(
StandardLoggingPayloadErrorInformation,
sanitize_error_information_for_spend_logs(_error_information, original_exception=original_exception),
)
_metadata["error_information"] = _error_information
_metadata = await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled( # rebind-ok: pre-existing rebinding on a rename-only line

View file

@ -582,6 +582,7 @@ def get_logging_payload(
kwargs = {}
rejected_as_unknown_model: Final = isinstance(response_obj, ProxyModelNotFoundError)
failure_exception: Final = response_obj if isinstance(response_obj, BaseException) else None
response_served_from_cache: Final = served_from_cache(response_obj)
if response_obj is None:
response_obj = {}
@ -851,14 +852,11 @@ def get_logging_payload(
additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens
clean_metadata["additional_usage_values"] = additional_usage_values
drop_requester_ip_address: Final = requester_ip_address_logging_disabled()
error_information: Final = clean_metadata["error_information"]
if drop_requester_ip_address:
clean_metadata["requester_ip_address"] = None
if drop_requester_ip_address and isinstance(error_information, Mapping):
clean_metadata["error_information"] = cast( # cast-ok: same keys, only the str values are masked
StandardLoggingPayloadErrorInformation,
_error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(error_information)),
)
clean_metadata["error_information"] = sanitize_error_information_for_spend_logs(
clean_metadata["error_information"], original_exception=failure_exception
)
if litellm.cache is None:
cache_key = "Cache OFF"

View file

@ -6511,6 +6511,73 @@ def test_get_error_information_for_logging_payload_client_disconnect():
assert error_str is None
_SESSION_FAILURE: Final = {
"error_code": "503",
"error_class": "BedrockError",
"error_message": "Bedrock realtime stream ended before the session closed",
"llm_provider": "bedrock",
}
@pytest.mark.parametrize("metadata_bucket", ["metadata", "litellm_metadata"])
def test_realtime_success_payload_reports_the_session_failure_its_metadata_recorded(
logging_obj: LitellmLogging, metadata_bucket: str
) -> None:
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
now: Final = datetime(2026, 1, 1)
payload: Final = get_standard_logging_object_payload(
kwargs={
"model": "bedrock/amazon.nova-sonic-v1:0",
"call_type": "_arealtime",
"litellm_params": {metadata_bucket: {"status": "failure", "error_information": dict(_SESSION_FAILURE)}},
},
init_response_obj={},
start_time=now,
end_time=now,
logging_obj=logging_obj,
status="success",
)
assert payload is not None
assert payload["status"] == "failure"
assert payload["status_fields"]["llm_api_status"] == "failure"
assert payload["error_information"] == _SESSION_FAILURE
assert payload["error_str"] == _SESSION_FAILURE["error_message"]
@pytest.mark.parametrize(
("call_type", "metadata"),
[
("_arealtime", {"status": "failure"}),
("_arealtime", {"error_information": dict(_SESSION_FAILURE)}),
("acompletion", {"status": "failure", "error_information": dict(_SESSION_FAILURE)}),
],
)
def test_success_payload_stays_success_without_a_recorded_realtime_session_failure(
logging_obj: LitellmLogging, call_type: str, metadata: dict[str, object]
) -> None:
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
now: Final = datetime(2026, 1, 1)
payload: Final = get_standard_logging_object_payload(
kwargs={"model": "gpt-4o", "messages": [], "call_type": call_type, "litellm_params": {"metadata": metadata}},
init_response_obj={},
start_time=now,
end_time=now,
logging_obj=logging_obj,
status="success",
)
assert payload is not None
assert payload["status"] == "success"
assert payload["error_information"] != _SESSION_FAILURE
def test_get_error_information_proxy_exception_preserves_message():
"""ProxyException keeps its text in ``.message`` (str() was empty pre-fix),
so error_information must still surface the message and code."""

View file

@ -11,17 +11,14 @@ from unittest.mock import MagicMock
import pytest
import litellm
from litellm import Router
from litellm.constants import (
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
WEBSOCKET_CLOSE_REASON_MAX_BYTES,
)
from litellm.litellm_core_utils.realtime_errors import websocket_close_reason
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals, raise_if_session_committed
from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
from litellm.realtime_api import main as realtime_main
@ -1122,44 +1119,6 @@ class TestBedrockRealtimeProviderFailurePropagation:
await spend_dispatch["coro"]
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
@pytest.mark.asyncio
async def test_replay_refusals_do_not_count_toward_deployment_cooldown(self):
router: Final = Router(
model_list=[
{
"model_name": "nova-sonic",
"litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0"},
"model_info": {"id": deployment_id},
}
for deployment_id in ("sonic-a", "sonic-b")
]
)
with pytest.raises(BedrockError) as refusal:
raise_if_session_committed(
{
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: True,
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: "Bedrock realtime stream ended before the session closed",
}
)
drop: Final = BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed")
def cooled_down_after(exception: Exception, deployment_id: str) -> bool:
return router.deployment_callback_on_failure(
kwargs={
"exception": exception,
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": deployment_id}},
},
completion_response=None,
start_time=time.time(),
end_time=time.time(),
)
refusals_then_drop: Final = [cooled_down_after(error, "sonic-a") for error in (*[refusal.value] * 4, drop)]
drops_only: Final = [cooled_down_after(drop, "sonic-b") for _ in range(5)]
assert not any(refusals_then_drop), "one drop retried four times is one deployment failure, not five"
assert drops_only[-1], "five real drops in a minute still cool the deployment down"
@pytest.mark.asyncio
async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish(
self, stub_aws_sdk_client, spend_dispatch

View file

@ -3213,6 +3213,40 @@ async def test_sidecar_ignores_an_undecodable_event(): # test-quality-ok: a dis
mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited()
def _spend_row_error_information(update_database_kwargs: Mapping[str, object]) -> dict[str, object]:
row: Final = get_logging_payload(
kwargs=cast(dict, update_database_kwargs["kwargs"]),
response_obj=update_database_kwargs["completion_response"],
start_time=cast(datetime, update_database_kwargs["start_time"]),
end_time=cast(datetime, update_database_kwargs["end_time"]),
)
return json.loads(row["metadata"])["error_information"]
@pytest.mark.asyncio
async def test_failure_hook_spend_row_truncates_a_long_provider_error_exactly_once(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "100")
writer: Final = MagicMock(spec=DBSpendUpdateWriter)
writer.update_database = AsyncMock()
logger: Final = ProxyDBLogger(spend_writer=lambda: writer)
provider_error: Final = "upstream failure " + "x" * 1000
await logger.async_post_call_failure_hook(
request_data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]},
original_exception=litellm.InternalServerError(message=provider_error, llm_provider="openai", model="gpt-4o"),
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"),
)
stored_message: Final = _spend_row_error_information(writer.update_database.call_args.kwargs)["error_message"]
assert isinstance(stored_message, str)
assert stored_message.count("skipped") == 1
kept_chars: Final = len(stored_message.split("... (")[0]) + len(stored_message.rsplit(") ...", 1)[1])
skipped_chars: Final = int(stored_message.split("skipped ", 1)[1].split(" chars", 1)[0])
assert kept_chars + skipped_chars > len(provider_error)
@pytest.mark.asyncio
async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_model_rejection():
raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes"
@ -3226,9 +3260,7 @@ async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"),
)
error_information: Final = writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][
"error_information"
]
error_information: Final = _spend_row_error_information(writer.update_database.call_args.kwargs)
assert "medical records" not in json.dumps(error_information)
assert (
error_information["error_message"]

View file

@ -7326,6 +7326,46 @@ def test_failure_error_information_masks_ip_addresses_in_long_text_and_at_its_tr
assert sanitized["error_code"] == "500"
def test_spend_row_sanitizes_the_session_failure_a_realtime_success_row_carries(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy.proxy_server import general_settings
monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", False)
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "200")
provider_error: Final = 'stream ended {"messages": [{"role": "user", "content": "private-prompt"}]} ' + "x" * 2000
now: Final = datetime.datetime.now(timezone.utc)
row: Final = get_logging_payload(
kwargs={
"model": "bedrock/amazon.nova-sonic-v1:0",
"call_type": "_arealtime",
"litellm_params": {
"metadata": {
"user_api_key": "hashed-key",
"status": "failure",
"error_information": {
"error_code": "503",
"error_class": "BedrockError",
"error_message": provider_error,
"llm_provider": "bedrock",
},
}
},
},
response_obj=None,
start_time=now,
end_time=now,
)
stored: Final = json.loads(row["metadata"])["error_information"]
assert row["status"] == "failure"
assert "private-prompt" not in stored["error_message"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored["error_message"]
assert len(stored["error_message"]) < len(provider_error)
assert stored["error_class"] == "BedrockError"
@pytest.mark.parametrize(
"address", ["203.0.113.9", "2001:db8::9", "1:2:3:4:5:6:7:8", "1:2:3:4:5:6:7::", "::1:2:3:4:5:6:7", "fe80::1"]
)

View file

@ -539,27 +539,29 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn(
@pytest.mark.asyncio
@pytest.mark.parametrize("realtime_model", ["bedrock/amazon.nova-sonic-v1:0", "openai/gpt-realtime-2.1"])
@pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)])
async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served(
clean_sessions: int, cooled_down: bool
realtime_model: str, clean_sessions: int, cooled_down: bool
) -> None:
import time
provider, _, provider_model = realtime_model.partition("/")
router: Final = Router(
model_list=[
{
"model_name": "nova-sonic",
"litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0", "order": order},
"model_name": "realtime-voice",
"litellm_params": {"model": realtime_model, "order": order},
"model_info": {"id": deployment_id},
}
for order, deployment_id in ((1, "sonic-a"), (2, "sonic-b"))
for order, deployment_id in ((1, "realtime-a"), (2, "realtime-b"))
],
num_retries=0,
)
drop: Final = litellm.ServiceUnavailableError(
message="Bedrock realtime stream ended before the session closed",
llm_provider="bedrock",
model="amazon.nova-sonic-v1:0",
message="realtime provider stream ended before the session closed",
llm_provider=provider,
model=provider_model,
)
def realtime_session(failure: Exception | None):
@ -567,7 +569,7 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy
await router.deployment_callback_on_success(
kwargs={
"call_type": "_arealtime",
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": kwargs["model_info"]},
"litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": kwargs["model_info"]},
"standard_logging_object": create_standard_logging_payload(),
},
completion_response=None,
@ -580,15 +582,15 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy
return _arealtime
for _ in range(clean_sessions):
await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(None))
await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(None))
async def drop_session() -> bool:
with pytest.raises(litellm.ServiceUnavailableError):
await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(drop))
await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(drop))
return router.deployment_callback_on_failure(
kwargs={
"exception": drop,
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": "sonic-a"}},
"litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": {"id": "realtime-a"}},
},
completion_response=None,
start_time=time.time(),