diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7f4b069ded6..b340455de98 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -7478,10 +7478,25 @@ def get_standard_logging_object_payload( router_model_id=logging_obj.get_router_model_id(), ) - error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( - metadata=metadata, - original_exception=original_exception, - error_str=error_str, + metadata_fields: Final = cast(Mapping[str, object], metadata) # cast-ok: merged request metadata is str-keyed + recorded_error: Final = metadata_fields.get("error_information") + recorded_failure: Final = ( + cast(StandardLoggingPayloadErrorInformation, recorded_error) # cast-ok: a reserved key only litellm writes + if isinstance(recorded_error, dict) + and status == "success" + and call_type == CallTypes.arealtime.value + and metadata_fields.get("status") == "failure" + else None + ) + payload_status: Final[StandardLoggingPayloadStatus] = "failure" if recorded_failure is not None else status + error_information, error_str = ( + (recorded_failure, error_str or recorded_failure.get("error_message")) + if recorded_failure is not None + else StandardLoggingPayloadSetup.get_error_information_for_logging_payload( + metadata=metadata, + original_exception=original_exception, + error_str=error_str, + ) ) ## get final response object ## @@ -7572,9 +7587,9 @@ def get_standard_logging_object_payload( call_type=call_type or "", cache_hit=cache_hit, stream=stream, - status=status, + status=payload_status, status_fields=_get_status_fields( - status=status, + status=payload_status, guardrail_information=metadata.get("standard_logging_guardrail_information", None), error_str=error_str, ), diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index a62122ee5ee..b447aacd993 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -192,18 +192,14 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None: """A fallback attempt on a websocket whose Bedrock stream already took input would silently start over.""" if scope is None or not _is_session_committed(scope): return - from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure - committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY) - refusal: Final = BedrockError( + raise BedrockError( status_code=400, message=( (f"{committed_failure}. " if committed_failure else "") + "Bedrock realtime session already committed to a provider stream; it cannot be replayed" ), ) - mark_deployment_selection_failure(refusal) - raise refusal def _pending_opening_message(scope: Mapping[str, object]) -> str | None: diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index c11b59a809a..e19c5612ef0 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -58,7 +58,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # _sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export deployment_params_provider, get_request_model_access_groups, - sanitize_error_information_for_spend_logs, + sanitize_error_information_for_spend_logs, # pyright: ignore[reportUnusedImport] # backwards-compatible package export should_store_prompts_and_responses_in_spend_logs, ) from litellm.proxy.utils import ( @@ -72,7 +72,6 @@ from litellm.types.utils import ( CallTypes, LiteLLMBatch, StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, ) from litellm.utils import get_end_user_id_for_cost_tracking @@ -429,13 +428,6 @@ class _ProxyDBLogger(CustomLogger): # ``.get("traceback")`` / truthy checks, and the TypedDict marks # the field as optional, so omitting is type-safe. _error_information.pop("traceback", None) - # Strip echoed request input + apply DB-size cap before storing in - # the spend-log metadata column (LIT-2992). Result is never None - # here because the input above is constructed non-None. - _error_information = cast( - StandardLoggingPayloadErrorInformation, - sanitize_error_information_for_spend_logs(_error_information, original_exception=original_exception), - ) _metadata["error_information"] = _error_information _metadata = await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled( # rebind-ok: pre-existing rebinding on a rename-only line diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..b740a21bc46 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -582,6 +582,7 @@ def get_logging_payload( kwargs = {} rejected_as_unknown_model: Final = isinstance(response_obj, ProxyModelNotFoundError) + failure_exception: Final = response_obj if isinstance(response_obj, BaseException) else None response_served_from_cache: Final = served_from_cache(response_obj) if response_obj is None: response_obj = {} @@ -851,14 +852,11 @@ def get_logging_payload( additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens clean_metadata["additional_usage_values"] = additional_usage_values drop_requester_ip_address: Final = requester_ip_address_logging_disabled() - error_information: Final = clean_metadata["error_information"] if drop_requester_ip_address: clean_metadata["requester_ip_address"] = None - if drop_requester_ip_address and isinstance(error_information, Mapping): - clean_metadata["error_information"] = cast( # cast-ok: same keys, only the str values are masked - StandardLoggingPayloadErrorInformation, - _error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(error_information)), - ) + clean_metadata["error_information"] = sanitize_error_information_for_spend_logs( + clean_metadata["error_information"], original_exception=failure_exception + ) if litellm.cache is None: cache_key = "Cache OFF" diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 299088a6c4f..868fe529815 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6511,6 +6511,73 @@ def test_get_error_information_for_logging_payload_client_disconnect(): assert error_str is None +_SESSION_FAILURE: Final = { + "error_code": "503", + "error_class": "BedrockError", + "error_message": "Bedrock realtime stream ended before the session closed", + "llm_provider": "bedrock", +} + + +@pytest.mark.parametrize("metadata_bucket", ["metadata", "litellm_metadata"]) +def test_realtime_success_payload_reports_the_session_failure_its_metadata_recorded( + logging_obj: LitellmLogging, metadata_bucket: str +) -> None: + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + now: Final = datetime(2026, 1, 1) + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": "bedrock/amazon.nova-sonic-v1:0", + "call_type": "_arealtime", + "litellm_params": {metadata_bucket: {"status": "failure", "error_information": dict(_SESSION_FAILURE)}}, + }, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["status"] == "failure" + assert payload["status_fields"]["llm_api_status"] == "failure" + assert payload["error_information"] == _SESSION_FAILURE + assert payload["error_str"] == _SESSION_FAILURE["error_message"] + + +@pytest.mark.parametrize( + ("call_type", "metadata"), + [ + ("_arealtime", {"status": "failure"}), + ("_arealtime", {"error_information": dict(_SESSION_FAILURE)}), + ("acompletion", {"status": "failure", "error_information": dict(_SESSION_FAILURE)}), + ], +) +def test_success_payload_stays_success_without_a_recorded_realtime_session_failure( + logging_obj: LitellmLogging, call_type: str, metadata: dict[str, object] +) -> None: + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + now: Final = datetime(2026, 1, 1) + payload: Final = get_standard_logging_object_payload( + kwargs={"model": "gpt-4o", "messages": [], "call_type": call_type, "litellm_params": {"metadata": metadata}}, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["status"] == "success" + assert payload["error_information"] != _SESSION_FAILURE + + def test_get_error_information_proxy_exception_preserves_message(): """ProxyException keeps its text in ``.message`` (str() was empty pre-fix), so error_information must still surface the message and code.""" diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 9767f294af7..9e4ff7f05af 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -11,17 +11,14 @@ from unittest.mock import MagicMock import pytest import litellm -from litellm import Router from litellm.constants import ( - BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY, BEDROCK_REALTIME_SDK_SUPPORTED_RANGE, - BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) from litellm.litellm_core_utils.realtime_errors import websocket_close_reason from litellm.llms.bedrock.common_utils import BedrockError -from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals, raise_if_session_committed +from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig from litellm.realtime_api import main as realtime_main @@ -1122,44 +1119,6 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] - @pytest.mark.asyncio - async def test_replay_refusals_do_not_count_toward_deployment_cooldown(self): - router: Final = Router( - model_list=[ - { - "model_name": "nova-sonic", - "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0"}, - "model_info": {"id": deployment_id}, - } - for deployment_id in ("sonic-a", "sonic-b") - ] - ) - with pytest.raises(BedrockError) as refusal: - raise_if_session_committed( - { - BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: True, - BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: "Bedrock realtime stream ended before the session closed", - } - ) - drop: Final = BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed") - - def cooled_down_after(exception: Exception, deployment_id: str) -> bool: - return router.deployment_callback_on_failure( - kwargs={ - "exception": exception, - "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": deployment_id}}, - }, - completion_response=None, - start_time=time.time(), - end_time=time.time(), - ) - - refusals_then_drop: Final = [cooled_down_after(error, "sonic-a") for error in (*[refusal.value] * 4, drop)] - drops_only: Final = [cooled_down_after(drop, "sonic-b") for _ in range(5)] - - assert not any(refusals_then_drop), "one drop retried four times is one deployment failure, not five" - assert drops_only[-1], "five real drops in a minute still cool the deployment down" - @pytest.mark.asyncio async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( self, stub_aws_sdk_client, spend_dispatch diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 4b5957a1237..e8a203b8637 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -3213,6 +3213,40 @@ async def test_sidecar_ignores_an_undecodable_event(): # test-quality-ok: a dis mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited() +def _spend_row_error_information(update_database_kwargs: Mapping[str, object]) -> dict[str, object]: + row: Final = get_logging_payload( + kwargs=cast(dict, update_database_kwargs["kwargs"]), + response_obj=update_database_kwargs["completion_response"], + start_time=cast(datetime, update_database_kwargs["start_time"]), + end_time=cast(datetime, update_database_kwargs["end_time"]), + ) + return json.loads(row["metadata"])["error_information"] + + +@pytest.mark.asyncio +async def test_failure_hook_spend_row_truncates_a_long_provider_error_exactly_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "100") + writer: Final = MagicMock(spec=DBSpendUpdateWriter) + writer.update_database = AsyncMock() + logger: Final = ProxyDBLogger(spend_writer=lambda: writer) + provider_error: Final = "upstream failure " + "x" * 1000 + + await logger.async_post_call_failure_hook( + request_data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=litellm.InternalServerError(message=provider_error, llm_provider="openai", model="gpt-4o"), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), + ) + + stored_message: Final = _spend_row_error_information(writer.update_database.call_args.kwargs)["error_message"] + assert isinstance(stored_message, str) + assert stored_message.count("skipped") == 1 + kept_chars: Final = len(stored_message.split("... (")[0]) + len(stored_message.rsplit(") ...", 1)[1]) + skipped_chars: Final = int(stored_message.split("skipped ", 1)[1].split(" chars", 1)[0]) + assert kept_chars + skipped_chars > len(provider_error) + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_model_rejection(): raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" @@ -3226,9 +3260,7 @@ async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_ user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), ) - error_information: Final = writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][ - "error_information" - ] + error_information: Final = _spend_row_error_information(writer.update_database.call_args.kwargs) assert "medical records" not in json.dumps(error_information) assert ( error_information["error_message"] diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 86d064050eb..bc31bcb2692 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -7326,6 +7326,46 @@ def test_failure_error_information_masks_ip_addresses_in_long_text_and_at_its_tr assert sanitized["error_code"] == "500" +def test_spend_row_sanitizes_the_session_failure_a_realtime_success_row_carries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.proxy_server import general_settings + + monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", False) + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "200") + provider_error: Final = 'stream ended {"messages": [{"role": "user", "content": "private-prompt"}]} ' + "x" * 2000 + now: Final = datetime.datetime.now(timezone.utc) + + row: Final = get_logging_payload( + kwargs={ + "model": "bedrock/amazon.nova-sonic-v1:0", + "call_type": "_arealtime", + "litellm_params": { + "metadata": { + "user_api_key": "hashed-key", + "status": "failure", + "error_information": { + "error_code": "503", + "error_class": "BedrockError", + "error_message": provider_error, + "llm_provider": "bedrock", + }, + } + }, + }, + response_obj=None, + start_time=now, + end_time=now, + ) + + stored: Final = json.loads(row["metadata"])["error_information"] + assert row["status"] == "failure" + assert "private-prompt" not in stored["error_message"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored["error_message"] + assert len(stored["error_message"]) < len(provider_error) + assert stored["error_class"] == "BedrockError" + + @pytest.mark.parametrize( "address", ["203.0.113.9", "2001:db8::9", "1:2:3:4:5:6:7:8", "1:2:3:4:5:6:7::", "::1:2:3:4:5:6:7", "fe80::1"] ) diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 3edb5f74454..17e954ccd9d 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -539,27 +539,29 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( @pytest.mark.asyncio +@pytest.mark.parametrize("realtime_model", ["bedrock/amazon.nova-sonic-v1:0", "openai/gpt-realtime-2.1"]) @pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)]) async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served( - clean_sessions: int, cooled_down: bool + realtime_model: str, clean_sessions: int, cooled_down: bool ) -> None: import time + provider, _, provider_model = realtime_model.partition("/") router: Final = Router( model_list=[ { - "model_name": "nova-sonic", - "litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0", "order": order}, + "model_name": "realtime-voice", + "litellm_params": {"model": realtime_model, "order": order}, "model_info": {"id": deployment_id}, } - for order, deployment_id in ((1, "sonic-a"), (2, "sonic-b")) + for order, deployment_id in ((1, "realtime-a"), (2, "realtime-b")) ], num_retries=0, ) drop: Final = litellm.ServiceUnavailableError( - message="Bedrock realtime stream ended before the session closed", - llm_provider="bedrock", - model="amazon.nova-sonic-v1:0", + message="realtime provider stream ended before the session closed", + llm_provider=provider, + model=provider_model, ) def realtime_session(failure: Exception | None): @@ -567,7 +569,7 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy await router.deployment_callback_on_success( kwargs={ "call_type": "_arealtime", - "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": kwargs["model_info"]}, + "litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": kwargs["model_info"]}, "standard_logging_object": create_standard_logging_payload(), }, completion_response=None, @@ -580,15 +582,15 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy return _arealtime for _ in range(clean_sessions): - await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(None)) + await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(None)) async def drop_session() -> bool: with pytest.raises(litellm.ServiceUnavailableError): - await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(drop)) + await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(drop)) return router.deployment_callback_on_failure( kwargs={ "exception": drop, - "litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": "sonic-a"}}, + "litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": {"id": "realtime-a"}}, }, completion_response=None, start_time=time.time(),