mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(bedrock): agree on a dropped realtime session's failure across loggers and the spend row
A realtime session's success event now reports status failure, with the same error_information the spend row stores, when the session recorded a provider failure in its metadata. Usage already streamed stays billed and the router still counts one cooldown failure The spend row builder now runs every row's error_information through sanitize_error_information_for_spend_logs (DB size cap plus prompt redaction), so the realtime failure rows get the same treatment as the proxy failure hook, which no longer sanitizes on its own to avoid truncating twice Drop the selection-failure tag on the committed-session refusal, since the shared logging object already keeps a retried refusal out of cooldown, and pin the router's realtime success counting with an OpenAI realtime deployment
This commit is contained in:
parent
8a3248de0d
commit
edb9aaeee4
9 changed files with 183 additions and 82 deletions
|
|
@ -7478,10 +7478,25 @@ def get_standard_logging_object_payload(
|
|||
router_model_id=logging_obj.get_router_model_id(),
|
||||
)
|
||||
|
||||
error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
|
||||
metadata=metadata,
|
||||
original_exception=original_exception,
|
||||
error_str=error_str,
|
||||
metadata_fields: Final = cast(Mapping[str, object], metadata) # cast-ok: merged request metadata is str-keyed
|
||||
recorded_error: Final = metadata_fields.get("error_information")
|
||||
recorded_failure: Final = (
|
||||
cast(StandardLoggingPayloadErrorInformation, recorded_error) # cast-ok: a reserved key only litellm writes
|
||||
if isinstance(recorded_error, dict)
|
||||
and status == "success"
|
||||
and call_type == CallTypes.arealtime.value
|
||||
and metadata_fields.get("status") == "failure"
|
||||
else None
|
||||
)
|
||||
payload_status: Final[StandardLoggingPayloadStatus] = "failure" if recorded_failure is not None else status
|
||||
error_information, error_str = (
|
||||
(recorded_failure, error_str or recorded_failure.get("error_message"))
|
||||
if recorded_failure is not None
|
||||
else StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
|
||||
metadata=metadata,
|
||||
original_exception=original_exception,
|
||||
error_str=error_str,
|
||||
)
|
||||
)
|
||||
|
||||
## get final response object ##
|
||||
|
|
@ -7572,9 +7587,9 @@ def get_standard_logging_object_payload(
|
|||
call_type=call_type or "",
|
||||
cache_hit=cache_hit,
|
||||
stream=stream,
|
||||
status=status,
|
||||
status=payload_status,
|
||||
status_fields=_get_status_fields(
|
||||
status=status,
|
||||
status=payload_status,
|
||||
guardrail_information=metadata.get("standard_logging_guardrail_information", None),
|
||||
error_str=error_str,
|
||||
),
|
||||
|
|
|
|||
|
|
@ -192,18 +192,14 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None:
|
|||
"""A fallback attempt on a websocket whose Bedrock stream already took input would silently start over."""
|
||||
if scope is None or not _is_session_committed(scope):
|
||||
return
|
||||
from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure
|
||||
|
||||
committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY)
|
||||
refusal: Final = BedrockError(
|
||||
raise BedrockError(
|
||||
status_code=400,
|
||||
message=(
|
||||
(f"{committed_failure}. " if committed_failure else "")
|
||||
+ "Bedrock realtime session already committed to a provider stream; it cannot be replayed"
|
||||
),
|
||||
)
|
||||
mark_deployment_selection_failure(refusal)
|
||||
raise refusal
|
||||
|
||||
|
||||
def _pending_opening_message(scope: Mapping[str, object]) -> str | None:
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 #
|
|||
_sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
deployment_params_provider,
|
||||
get_request_model_access_groups,
|
||||
sanitize_error_information_for_spend_logs,
|
||||
sanitize_error_information_for_spend_logs, # pyright: ignore[reportUnusedImport] # backwards-compatible package export
|
||||
should_store_prompts_and_responses_in_spend_logs,
|
||||
)
|
||||
from litellm.proxy.utils import (
|
||||
|
|
@ -72,7 +72,6 @@ from litellm.types.utils import (
|
|||
CallTypes,
|
||||
LiteLLMBatch,
|
||||
StandardLoggingPayload,
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
)
|
||||
from litellm.utils import get_end_user_id_for_cost_tracking
|
||||
|
||||
|
|
@ -429,13 +428,6 @@ class _ProxyDBLogger(CustomLogger):
|
|||
# ``.get("traceback")`` / truthy checks, and the TypedDict marks
|
||||
# the field as optional, so omitting is type-safe.
|
||||
_error_information.pop("traceback", None)
|
||||
# Strip echoed request input + apply DB-size cap before storing in
|
||||
# the spend-log metadata column (LIT-2992). Result is never None
|
||||
# here because the input above is constructed non-None.
|
||||
_error_information = cast(
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
sanitize_error_information_for_spend_logs(_error_information, original_exception=original_exception),
|
||||
)
|
||||
_metadata["error_information"] = _error_information
|
||||
|
||||
_metadata = await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
|
|
|
|||
|
|
@ -582,6 +582,7 @@ def get_logging_payload(
|
|||
kwargs = {}
|
||||
|
||||
rejected_as_unknown_model: Final = isinstance(response_obj, ProxyModelNotFoundError)
|
||||
failure_exception: Final = response_obj if isinstance(response_obj, BaseException) else None
|
||||
response_served_from_cache: Final = served_from_cache(response_obj)
|
||||
if response_obj is None:
|
||||
response_obj = {}
|
||||
|
|
@ -851,14 +852,11 @@ def get_logging_payload(
|
|||
additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens
|
||||
clean_metadata["additional_usage_values"] = additional_usage_values
|
||||
drop_requester_ip_address: Final = requester_ip_address_logging_disabled()
|
||||
error_information: Final = clean_metadata["error_information"]
|
||||
if drop_requester_ip_address:
|
||||
clean_metadata["requester_ip_address"] = None
|
||||
if drop_requester_ip_address and isinstance(error_information, Mapping):
|
||||
clean_metadata["error_information"] = cast( # cast-ok: same keys, only the str values are masked
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
_error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(error_information)),
|
||||
)
|
||||
clean_metadata["error_information"] = sanitize_error_information_for_spend_logs(
|
||||
clean_metadata["error_information"], original_exception=failure_exception
|
||||
)
|
||||
|
||||
if litellm.cache is None:
|
||||
cache_key = "Cache OFF"
|
||||
|
|
|
|||
|
|
@ -6511,6 +6511,73 @@ def test_get_error_information_for_logging_payload_client_disconnect():
|
|||
assert error_str is None
|
||||
|
||||
|
||||
_SESSION_FAILURE: Final = {
|
||||
"error_code": "503",
|
||||
"error_class": "BedrockError",
|
||||
"error_message": "Bedrock realtime stream ended before the session closed",
|
||||
"llm_provider": "bedrock",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metadata_bucket", ["metadata", "litellm_metadata"])
|
||||
def test_realtime_success_payload_reports_the_session_failure_its_metadata_recorded(
|
||||
logging_obj: LitellmLogging, metadata_bucket: str
|
||||
) -> None:
|
||||
from datetime import datetime
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
|
||||
|
||||
now: Final = datetime(2026, 1, 1)
|
||||
payload: Final = get_standard_logging_object_payload(
|
||||
kwargs={
|
||||
"model": "bedrock/amazon.nova-sonic-v1:0",
|
||||
"call_type": "_arealtime",
|
||||
"litellm_params": {metadata_bucket: {"status": "failure", "error_information": dict(_SESSION_FAILURE)}},
|
||||
},
|
||||
init_response_obj={},
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
logging_obj=logging_obj,
|
||||
status="success",
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert payload["status"] == "failure"
|
||||
assert payload["status_fields"]["llm_api_status"] == "failure"
|
||||
assert payload["error_information"] == _SESSION_FAILURE
|
||||
assert payload["error_str"] == _SESSION_FAILURE["error_message"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("call_type", "metadata"),
|
||||
[
|
||||
("_arealtime", {"status": "failure"}),
|
||||
("_arealtime", {"error_information": dict(_SESSION_FAILURE)}),
|
||||
("acompletion", {"status": "failure", "error_information": dict(_SESSION_FAILURE)}),
|
||||
],
|
||||
)
|
||||
def test_success_payload_stays_success_without_a_recorded_realtime_session_failure(
|
||||
logging_obj: LitellmLogging, call_type: str, metadata: dict[str, object]
|
||||
) -> None:
|
||||
from datetime import datetime
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
|
||||
|
||||
now: Final = datetime(2026, 1, 1)
|
||||
payload: Final = get_standard_logging_object_payload(
|
||||
kwargs={"model": "gpt-4o", "messages": [], "call_type": call_type, "litellm_params": {"metadata": metadata}},
|
||||
init_response_obj={},
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
logging_obj=logging_obj,
|
||||
status="success",
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert payload["status"] == "success"
|
||||
assert payload["error_information"] != _SESSION_FAILURE
|
||||
|
||||
|
||||
def test_get_error_information_proxy_exception_preserves_message():
|
||||
"""ProxyException keeps its text in ``.message`` (str() was empty pre-fix),
|
||||
so error_information must still surface the message and code."""
|
||||
|
|
|
|||
|
|
@ -11,17 +11,14 @@ from unittest.mock import MagicMock
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.constants import (
|
||||
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
|
||||
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
|
||||
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY,
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
WEBSOCKET_CLOSE_REASON_MAX_BYTES,
|
||||
)
|
||||
from litellm.litellm_core_utils.realtime_errors import websocket_close_reason
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals, raise_if_session_committed
|
||||
from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
|
||||
from litellm.realtime_api import main as realtime_main
|
||||
|
|
@ -1122,44 +1119,6 @@ class TestBedrockRealtimeProviderFailurePropagation:
|
|||
await spend_dispatch["coro"]
|
||||
assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replay_refusals_do_not_count_toward_deployment_cooldown(self):
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "nova-sonic",
|
||||
"litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0"},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for deployment_id in ("sonic-a", "sonic-b")
|
||||
]
|
||||
)
|
||||
with pytest.raises(BedrockError) as refusal:
|
||||
raise_if_session_committed(
|
||||
{
|
||||
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: True,
|
||||
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: "Bedrock realtime stream ended before the session closed",
|
||||
}
|
||||
)
|
||||
drop: Final = BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed")
|
||||
|
||||
def cooled_down_after(exception: Exception, deployment_id: str) -> bool:
|
||||
return router.deployment_callback_on_failure(
|
||||
kwargs={
|
||||
"exception": exception,
|
||||
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": deployment_id}},
|
||||
},
|
||||
completion_response=None,
|
||||
start_time=time.time(),
|
||||
end_time=time.time(),
|
||||
)
|
||||
|
||||
refusals_then_drop: Final = [cooled_down_after(error, "sonic-a") for error in (*[refusal.value] * 4, drop)]
|
||||
drops_only: Final = [cooled_down_after(drop, "sonic-b") for _ in range(5)]
|
||||
|
||||
assert not any(refusals_then_drop), "one drop retried four times is one deployment failure, not five"
|
||||
assert drops_only[-1], "five real drops in a minute still cool the deployment down"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish(
|
||||
self, stub_aws_sdk_client, spend_dispatch
|
||||
|
|
|
|||
|
|
@ -3213,6 +3213,40 @@ async def test_sidecar_ignores_an_undecodable_event(): # test-quality-ok: a dis
|
|||
mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited()
|
||||
|
||||
|
||||
def _spend_row_error_information(update_database_kwargs: Mapping[str, object]) -> dict[str, object]:
|
||||
row: Final = get_logging_payload(
|
||||
kwargs=cast(dict, update_database_kwargs["kwargs"]),
|
||||
response_obj=update_database_kwargs["completion_response"],
|
||||
start_time=cast(datetime, update_database_kwargs["start_time"]),
|
||||
end_time=cast(datetime, update_database_kwargs["end_time"]),
|
||||
)
|
||||
return json.loads(row["metadata"])["error_information"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_hook_spend_row_truncates_a_long_provider_error_exactly_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "100")
|
||||
writer: Final = MagicMock(spec=DBSpendUpdateWriter)
|
||||
writer.update_database = AsyncMock()
|
||||
logger: Final = ProxyDBLogger(spend_writer=lambda: writer)
|
||||
provider_error: Final = "upstream failure " + "x" * 1000
|
||||
|
||||
await logger.async_post_call_failure_hook(
|
||||
request_data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]},
|
||||
original_exception=litellm.InternalServerError(message=provider_error, llm_provider="openai", model="gpt-4o"),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"),
|
||||
)
|
||||
|
||||
stored_message: Final = _spend_row_error_information(writer.update_database.call_args.kwargs)["error_message"]
|
||||
assert isinstance(stored_message, str)
|
||||
assert stored_message.count("skipped") == 1
|
||||
kept_chars: Final = len(stored_message.split("... (")[0]) + len(stored_message.rsplit(") ...", 1)[1])
|
||||
skipped_chars: Final = int(stored_message.split("skipped ", 1)[1].split(" chars", 1)[0])
|
||||
assert kept_chars + skipped_chars > len(provider_error)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_model_rejection():
|
||||
raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes"
|
||||
|
|
@ -3226,9 +3260,7 @@ async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_
|
|||
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"),
|
||||
)
|
||||
|
||||
error_information: Final = writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][
|
||||
"error_information"
|
||||
]
|
||||
error_information: Final = _spend_row_error_information(writer.update_database.call_args.kwargs)
|
||||
assert "medical records" not in json.dumps(error_information)
|
||||
assert (
|
||||
error_information["error_message"]
|
||||
|
|
|
|||
|
|
@ -7326,6 +7326,46 @@ def test_failure_error_information_masks_ip_addresses_in_long_text_and_at_its_tr
|
|||
assert sanitized["error_code"] == "500"
|
||||
|
||||
|
||||
def test_spend_row_sanitizes_the_session_failure_a_realtime_success_row_carries(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", False)
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "200")
|
||||
provider_error: Final = 'stream ended {"messages": [{"role": "user", "content": "private-prompt"}]} ' + "x" * 2000
|
||||
now: Final = datetime.datetime.now(timezone.utc)
|
||||
|
||||
row: Final = get_logging_payload(
|
||||
kwargs={
|
||||
"model": "bedrock/amazon.nova-sonic-v1:0",
|
||||
"call_type": "_arealtime",
|
||||
"litellm_params": {
|
||||
"metadata": {
|
||||
"user_api_key": "hashed-key",
|
||||
"status": "failure",
|
||||
"error_information": {
|
||||
"error_code": "503",
|
||||
"error_class": "BedrockError",
|
||||
"error_message": provider_error,
|
||||
"llm_provider": "bedrock",
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
response_obj=None,
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
)
|
||||
|
||||
stored: Final = json.loads(row["metadata"])["error_information"]
|
||||
assert row["status"] == "failure"
|
||||
assert "private-prompt" not in stored["error_message"]
|
||||
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored["error_message"]
|
||||
assert len(stored["error_message"]) < len(provider_error)
|
||||
assert stored["error_class"] == "BedrockError"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"address", ["203.0.113.9", "2001:db8::9", "1:2:3:4:5:6:7:8", "1:2:3:4:5:6:7::", "::1:2:3:4:5:6:7", "fe80::1"]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -539,27 +539,29 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("realtime_model", ["bedrock/amazon.nova-sonic-v1:0", "openai/gpt-realtime-2.1"])
|
||||
@pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)])
|
||||
async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served(
|
||||
clean_sessions: int, cooled_down: bool
|
||||
realtime_model: str, clean_sessions: int, cooled_down: bool
|
||||
) -> None:
|
||||
import time
|
||||
|
||||
provider, _, provider_model = realtime_model.partition("/")
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "nova-sonic",
|
||||
"litellm_params": {"model": "bedrock/amazon.nova-sonic-v1:0", "order": order},
|
||||
"model_name": "realtime-voice",
|
||||
"litellm_params": {"model": realtime_model, "order": order},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for order, deployment_id in ((1, "sonic-a"), (2, "sonic-b"))
|
||||
for order, deployment_id in ((1, "realtime-a"), (2, "realtime-b"))
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
drop: Final = litellm.ServiceUnavailableError(
|
||||
message="Bedrock realtime stream ended before the session closed",
|
||||
llm_provider="bedrock",
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
message="realtime provider stream ended before the session closed",
|
||||
llm_provider=provider,
|
||||
model=provider_model,
|
||||
)
|
||||
|
||||
def realtime_session(failure: Exception | None):
|
||||
|
|
@ -567,7 +569,7 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy
|
|||
await router.deployment_callback_on_success(
|
||||
kwargs={
|
||||
"call_type": "_arealtime",
|
||||
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": kwargs["model_info"]},
|
||||
"litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": kwargs["model_info"]},
|
||||
"standard_logging_object": create_standard_logging_payload(),
|
||||
},
|
||||
completion_response=None,
|
||||
|
|
@ -580,15 +582,15 @@ async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deploy
|
|||
return _arealtime
|
||||
|
||||
for _ in range(clean_sessions):
|
||||
await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(None))
|
||||
await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(None))
|
||||
|
||||
async def drop_session() -> bool:
|
||||
with pytest.raises(litellm.ServiceUnavailableError):
|
||||
await router._ageneric_api_call_with_fallbacks(model="nova-sonic", original_function=realtime_session(drop))
|
||||
await router._ageneric_api_call_with_fallbacks(model="realtime-voice", original_function=realtime_session(drop))
|
||||
return router.deployment_callback_on_failure(
|
||||
kwargs={
|
||||
"exception": drop,
|
||||
"litellm_params": {"metadata": {"model_group": "nova-sonic"}, "model_info": {"id": "sonic-a"}},
|
||||
"litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": {"id": "realtime-a"}},
|
||||
},
|
||||
completion_response=None,
|
||||
start_time=time.time(),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue