From edb9aaeee46eef91b05b3c88dc36c47c8063fcdb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:35:25 -0700 Subject: [PATCH] fix(bedrock): agree on a dropped realtime session's failure across loggers and the spend row A realtime session's success event now reports status failure, with the same error_information the spend row stores, when the session recorded a provider failure in its metadata. Usage already streamed stays billed and the router still counts one cooldown failure The spend row builder now runs every row's error_information through sanitize_error_information_for_spend_logs (DB size cap plus prompt redaction), so the realtime failure rows get the same treatment as the proxy failure hook, which no longer sanitizes on its own to avoid truncating twice Drop the selection-failure tag on the committed-session refusal, since the shared logging object already keeps a retried refusal out of cooldown, and pin the router's realtime success counting with an OpenAI realtime deployment --- litellm/litellm_core_utils/litellm_logging.py | 27 ++++++-- litellm/llms/bedrock/realtime/handler.py | 6 +- .../proxy/hooks/proxy_track_cost_callback.py | 10 +-- .../spend_tracking/spend_tracking_utils.py | 10 ++- .../test_litellm_logging.py | 67 +++++++++++++++++++ .../realtime/test_bedrock_realtime_handler.py | 43 +----------- .../hooks/test_proxy_track_cost_callback.py | 38 ++++++++++- .../test_spend_tracking_utils.py | 40 +++++++++++ .../test_router/test_router_helper_utils.py | 24 ++++--- 9 files changed, 183 insertions(+), 82 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7f4b069ded6..b340455de98 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -7478,10 +7478,25 @@ def get_standard_logging_object_payload( router_model_id=logging_obj.get_router_model_id(), ) - error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( - metadata=metadata, - original_exception=original_exception, - error_str=error_str, + metadata_fields: Final = cast(Mapping[str, object], metadata) # cast-ok: merged request metadata is str-keyed + recorded_error: Final = metadata_fields.get("error_information") + recorded_failure: Final = ( + cast(StandardLoggingPayloadErrorInformation, recorded_error) # cast-ok: a reserved key only litellm writes + if isinstance(recorded_error, dict) + and status == "success" + and call_type == CallTypes.arealtime.value + and metadata_fields.get("status") == "failure" + else None + ) + payload_status: Final[StandardLoggingPayloadStatus] = "failure" if recorded_failure is not None else status + error_information, error_str = ( + (recorded_failure, error_str or recorded_failure.get("error_message")) + if recorded_failure is not None + else StandardLoggingPayloadSetup.get_error_information_for_logging_payload( + metadata=metadata, + original_exception=original_exception, + error_str=error_str, + ) ) ## get final response object ## @@ -7572,9 +7587,9 @@ def get_standard_logging_object_payload( call_type=call_type or "", cache_hit=cache_hit, stream=stream, - status=status, + status=payload_status, status_fields=_get_status_fields( - status=status, + status=payload_status, guardrail_information=metadata.get("standard_logging_guardrail_information", None), error_str=error_str, ), diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index a62122ee5ee..b447aacd993 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -192,18 +192,14 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None: """A fallback attempt on a websocket whose Bedrock stream already took input would silently start over.""" if scope is None or not _is_session_committed(scope): return - from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure - committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY) - refusal: Final = BedrockError( + raise BedrockError( status_code=400, message=( (f"{committed_failure}. " if committed_failure else "") + "Bedrock realtime session already committed to a provider stream; it cannot be replayed" ), ) - mark_deployment_selection_failure(refusal) - raise refusal def _pending_opening_message(scope: Mapping[str, object]) -> str | None: diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index c11b59a809a..e19c5612ef0 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -58,7 +58,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # _sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export deployment_params_provider, get_request_model_access_groups, - sanitize_error_information_for_spend_logs, + sanitize_error_information_for_spend_logs, # pyright: ignore[reportUnusedImport] # backwards-compatible package export should_store_prompts_and_responses_in_spend_logs, ) from litellm.proxy.utils import ( @@ -72,7 +72,6 @@ from litellm.types.utils import ( CallTypes, LiteLLMBatch, StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, ) from litellm.utils import get_end_user_id_for_cost_tracking @@ -429,13 +428,6 @@ class _ProxyDBLogger(CustomLogger): # ``.get("traceback")`` / truthy checks, and the TypedDict marks # the field as optional, so omitting is type-safe. _error_information.pop("traceback", None) - # Strip echoed request input + apply DB-size cap before storing in - # the spend-log metadata column (LIT-2992). Result is never None - # here because the input above is constructed non-None. - _error_information = cast( - StandardLoggingPayloadErrorInformation, - sanitize_error_information_for_spend_logs(_error_information, original_exception=original_exception), - ) _metadata["error_information"] = _error_information _metadata = await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled( # rebind-ok: pre-existing rebinding on a rename-only line diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..b740a21bc46 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -582,6 +582,7 @@ def get_logging_payload( kwargs = {} rejected_as_unknown_model: Final = isinstance(response_obj, ProxyModelNotFoundError) + failure_exception: Final = response_obj if isinstance(response_obj, BaseException) else None response_served_from_cache: Final = served_from_cache(response_obj) if response_obj is None: response_obj = {} @@ -851,14 +852,11 @@ def get_logging_payload( additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens clean_metadata["additional_usage_values"] = additional_usage_values drop_requester_ip_address: Final = requester_ip_address_logging_disabled() - error_information: Final = clean_metadata["error_information"] if drop_requester_ip_address: clean_metadata["requester_ip_address"] = None - if drop_requester_ip_address and isinstance(error_information, Mapping): - clean_metadata["error_information"] = cast( # cast-ok: same keys, only the str values are masked - StandardLoggingPayloadErrorInformation, - _error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(error_information)), - ) + clean_metadata["error_information"] = sanitize_error_information_for_spend_logs( + clean_metadata["error_information"], original_exception=failure_exception + ) if litellm.cache is None: cache_key = "Cache OFF" diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 299088a6c4f..868fe529815 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6511,6 +6511,73 @@ def test_get_error_information_for_logging_payload_client_disconnect(): assert error_str is None +_SESSION_FAILURE: Final = { + "error_code": "503", + "error_class": "BedrockError", + "error_message": "Bedrock realtime stream ended before the session closed", + "llm_provider": "bedrock", +} + + +@pytest.mark.parametrize("metadata_bucket", ["metadata", "litellm_metadata"]) +def test_realtime_success_payload_reports_the_session_failure_its_metadata_recorded( + logging_obj: LitellmLogging, metadata_bucket: str +) -> None: + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + now: Final = datetime(2026, 1, 1) + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": "bedrock/amazon.nova-sonic-v1:0", + "call_type": "_arealtime", + "litellm_params": {metadata_bucket: {"status": "failure", "error_information": dict(_SESSION_FAILURE)}}, + }, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["status"] == "failure" + assert payload["status_fields"]["llm_api_status"] == "failure" + assert payload["error_information"] == _SESSION_FAILURE + assert payload["error_str"] == _SESSION_FAILURE["error_message"] + + +@pytest.mark.parametrize( + ("call_type", "metadata"), + [ + ("_arealtime", {"status": "failure"}), + ("_arealtime", {"error_information": dict(_SESSION_FAILURE)}), + ("acompletion", {"status": "failure", "error_information": dict(_SESSION_FAILURE)}), + ], +) +def test_success_payload_stays_success_without_a_recorded_realtime_session_failure( + logging_obj: LitellmLogging, call_type: str, metadata: dict[str, object] +) -> None: + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + now: Final = datetime(2026, 1, 1) + payload: Final = get_standard_logging_object_payload( + kwargs={"model": "gpt-4o", "messages": [], "call_type": call_type, "litellm_params": {"metadata": metadata}}, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["status"] == "success" + assert payload["error_information"] != _SESSION_FAILURE + + def test_get_error_information_proxy_exception_preserves_message(): """ProxyException keeps its text in ``.message`` (str() was empty pre-fix), so error_information must still surface the message and code.""" diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 9767f294af7..9e4ff7f05af 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -11,17 +11,14 @@ from unittest.mock import MagicMock import pytest import litellm -from litellm import Router from litellm.constants import ( - BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY, BEDROCK_REALTIME_SDK_SUPPORTED_RANGE, - BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) from litellm.litellm_core_utils.realtime_errors import websocket_close_reason from litellm.llms.bedrock.common_utils import BedrockError -from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals, raise_if_session_committed +from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig from litellm.realtime_api import main as realtime_main @@ -1122,44 +1119,6 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] - @pytest.mark.asyncio - async def test_replay_refusals_do_not_count_toward_deployment_cooldown(self): - router: Final = Router( - model_list=[ - { - "model_name": "nova-sonic", - "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0"}, - "model_info": {"id": deployment_id}, - } - for deployment_id in ("sonic-a", "sonic-b") - ] - ) - with pytest.raises(BedrockError) as refusal: - raise_if_session_committed( - { - BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: True, - BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: "Bedrock realtime stream ended before the session closed", - } - ) - drop: Final = BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed") - - def cooled_down_after(exception: Exception, deployment_id: str) -> bool: - return router.deployment_callback_on_failure( - kwargs={ - "exception": exception, - "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": deployment_id}}, - }, - completion_response=None, - start_time=time.time(), - end_time=time.time(), - ) - - refusals_then_drop: Final = [cooled_down_after(error, "sonic-a") for error in (*[refusal.value] * 4, drop)] - drops_only: Final = [cooled_down_after(drop, "sonic-b") for _ in range(5)] - - assert not any(refusals_then_drop), "one drop retried four times is one deployment failure, not five" - assert drops_only[-1], "five real drops in a minute still cool the deployment down" - @pytest.mark.asyncio async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( self, stub_aws_sdk_client, spend_dispatch diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 4b5957a1237..e8a203b8637 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -3213,6 +3213,40 @@ async def test_sidecar_ignores_an_undecodable_event(): # test-quality-ok: a dis mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited() +def _spend_row_error_information(update_database_kwargs: Mapping[str, object]) -> dict[str, object]: + row: Final = get_logging_payload( + kwargs=cast(dict, update_database_kwargs["kwargs"]), + response_obj=update_database_kwargs["completion_response"], + start_time=cast(datetime, update_database_kwargs["start_time"]), + end_time=cast(datetime, update_database_kwargs["end_time"]), + ) + return json.loads(row["metadata"])["error_information"] + + +@pytest.mark.asyncio +async def test_failure_hook_spend_row_truncates_a_long_provider_error_exactly_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "100") + writer: Final = MagicMock(spec=DBSpendUpdateWriter) + writer.update_database = AsyncMock() + logger: Final = ProxyDBLogger(spend_writer=lambda: writer) + provider_error: Final = "upstream failure " + "x" * 1000 + + await logger.async_post_call_failure_hook( + request_data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=litellm.InternalServerError(message=provider_error, llm_provider="openai", model="gpt-4o"), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), + ) + + stored_message: Final = _spend_row_error_information(writer.update_database.call_args.kwargs)["error_message"] + assert isinstance(stored_message, str) + assert stored_message.count("skipped") == 1 + kept_chars: Final = len(stored_message.split("... (")[0]) + len(stored_message.rsplit(") ...", 1)[1]) + skipped_chars: Final = int(stored_message.split("skipped ", 1)[1].split(" chars", 1)[0]) + assert kept_chars + skipped_chars > len(provider_error) + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_model_rejection(): raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" @@ -3226,9 +3260,7 @@ async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_ user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), ) - error_information: Final = writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][ - "error_information" - ] + error_information: Final = _spend_row_error_information(writer.update_database.call_args.kwargs) assert "medical records" not in json.dumps(error_information) assert ( error_information["error_message"] diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 86d064050eb..bc31bcb2692 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -7326,6 +7326,46 @@ def test_failure_error_information_masks_ip_addresses_in_long_text_and_at_its_tr assert sanitized["error_code"] == "500" +def test_spend_row_sanitizes_the_session_failure_a_realtime_success_row_carries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.proxy_server import general_settings + + monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", False) + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "200") + provider_error: Final = 'stream ended {"messages": [{"role": "user", "content": "private-prompt"}]} ' + "x" * 2000 + now: Final = datetime.datetime.now(timezone.utc) + + row: Final = get_logging_payload( + kwargs={ + "model": "bedrock/amazon.nova-sonic-v1:0", + "call_type": "_arealtime", + "litellm_params": { + "metadata": { + "user_api_key": "hashed-key", + "status": "failure", + "error_information": { + "error_code": "503", + "error_class": "BedrockError", + "error_message": provider_error, + "llm_provider": "bedrock", + }, + } + }, + }, + response_obj=None, + start_time=now, + end_time=now, + ) + + stored: Final = json.loads(row["metadata"])["error_information"] + assert row["status"] == "failure" + assert "private-prompt" not in stored["error_message"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored["error_message"] + assert len(stored["error_message"]) < len(provider_error) + assert stored["error_class"] == "BedrockError" + + @pytest.mark.parametrize( "address", ["203.0.113.9", "2001:db8::9", "1:2:3:4:5:6:7:8", "1:2:3:4:5:6:7::", "::1:2:3:4:5:6:7", "fe80::1"] ) diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 3edb5f74454..17e954ccd9d 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -539,27 +539,29 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( @pytest.mark.asyncio +@pytest.mark.parametrize("realtime_model", ["bedrock/amazon.nova-sonic-v1:0", "openai/gpt-realtime-2.1"]) @pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)]) async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served( - clean_sessions: int, cooled_down: bool + realtime_model: str, clean_sessions: int, cooled_down: bool ) -> None: import time + provider, _, provider_model = realtime_model.partition("/") router: Final = Router( model_list=[ { - "model_name": "nova-sonic", - "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0", "order": order}, + "model_name": "realtime-voice", + "litellm_params": {"model": realtime_model, "order": order}, "model_info": {"id": deployment_id}, } - for order, deployment_id in ((1, "sonic-a"), (2, "sonic-b")) + for order, deployment_id in ((1, "realtime-a"), (2, "realtime-b")) ], num_retries=0, ) drop: Final = litellm.ServiceUnavailableError( - message="Bedrock realtime stream ended before the session closed", - llm_provider="bedrock", - model="amazon.nova-sonic-v1:0", + message="realtime provider stream ended before the session closed", + llm_provider=provider, + model=provider_model, ) def realtime_session(failure: Exception | None): @@ -567,7 +569,7 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy await router.deployment_callback_on_success( kwargs={ "call_type": "_arealtime", - "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": kwargs["model_info"]}, + "litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": kwargs["model_info"]}, "standard_logging_object": create_standard_logging_payload(), }, completion_response=None, @@ -580,15 +582,15 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy return _arealtime for _ in range(clean_sessions): - await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(None)) + await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(None)) async def drop_session() -> bool: with pytest.raises(litellm.ServiceUnavailableError): - await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(drop)) + await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(drop)) return router.deployment_callback_on_failure( kwargs={ "exception": drop, - "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": "sonic-a"}}, + "litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": {"id": "realtime-a"}}, }, completion_response=None, start_time=time.time(),