diff --git a/litellm/anthropic_interface/exceptions/__init__.py b/litellm/anthropic_interface/exceptions/__init__.py index e1749757010..7c3cea0a28a 100644 --- a/litellm/anthropic_interface/exceptions/__init__.py +++ b/litellm/anthropic_interface/exceptions/__init__.py @@ -2,7 +2,6 @@ from .exception_mapping_utils import ( ANTHROPIC_ERROR_TYPE_MAP, - ANTHROPIC_STATUS_CODE_MAP, AnthropicErrorSseFrame, AnthropicExceptionMapping, anthropic_error_sse_frame, @@ -15,7 +14,6 @@ from .exceptions import ( __all__ = [ "ANTHROPIC_ERROR_TYPE_MAP", - "ANTHROPIC_STATUS_CODE_MAP", "AnthropicErrorDetail", "AnthropicErrorResponse", "AnthropicErrorSseFrame", diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index 47ddffed0f0..b4c95fafff3 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -5,12 +5,7 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop """ import json -from collections.abc import Mapping -from types import MappingProxyType -from typing import Final, Literal - -from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config -from typing_extensions import ReadOnly, TypedDict +from typing import Final from pydantic import TypeAdapter @@ -33,17 +28,6 @@ ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = { 529: "overloaded_error", } -ANTHROPIC_STATUS_CODE_MAP: Final = MappingProxyType({503: 529}) - - -@with_config(ConfigDict(extra="allow")) -class _AnthropicShapedEnvelope(TypedDict): - type: ReadOnly[Literal["error"]] - error: ReadOnly[Mapping[str, object]] - - -_ANTHROPIC_SHAPED_ENVELOPE: Final = TypeAdapter(_AnthropicShapedEnvelope) - class AnthropicExceptionMapping: """ @@ -52,16 +36,10 @@ class AnthropicExceptionMapping: Similar pattern to ExceptionCheckers in litellm_core_utils/exception_mapping_utils.py """ - @staticmethod - def get_status_code(status_code: int) -> int: - """Map an HTTP status code to the one Anthropic's contract answers with: it has no 503, so an upstream that - cannot take the request right now is its 529 overloaded_error.""" - return ANTHROPIC_STATUS_CODE_MAP.get(status_code, status_code) - @staticmethod def get_error_type(status_code: int) -> AnthropicErrorType: """Map HTTP status code to Anthropic error type.""" - return ANTHROPIC_ERROR_TYPE_MAP.get(AnthropicExceptionMapping.get_status_code(status_code), "api_error") + return ANTHROPIC_ERROR_TYPE_MAP.get(status_code, "api_error") @staticmethod def create_error_response( @@ -114,28 +92,22 @@ class AnthropicExceptionMapping: return raw_message @staticmethod - def _anthropic_error_envelope(parsed: dict | None) -> _AnthropicShapedEnvelope | None: - """The parsed body when it already is an Anthropic error envelope, `{"type": "error", "error": {"type": ..., - "message": ...}}`, with every other field it carries kept.""" - try: - envelope: Final = _ANTHROPIC_SHAPED_ENVELOPE.validate_python(parsed) - except ValidationError: - return None - return envelope if "type" in envelope["error"] and "message" in envelope["error"] else None + def _is_anthropic_error_dict(parsed: dict) -> bool: + """ + Check if a parsed dict is in Anthropic error format. - @staticmethod - def _passthrough( - envelope: _AnthropicShapedEnvelope, status_code: int, request_id: str | None - ) -> AnthropicErrorResponse: - """An already Anthropic-shaped body passes through with the request id it lacks and the error type a - remapped status names: a 503 envelope answers as 529, and a 529 answering `api_error` contradicts itself.""" - request_id_field: Final[Mapping[str, str]] = ( - {"request_id": request_id} if request_id and "request_id" not in envelope else {} + Anthropic error format: + { + "type": "error", + "error": {"type": "...", "message": "..."} + } + """ + return ( + parsed.get("type") == "error" + and isinstance(parsed.get("error"), dict) + and "type" in parsed["error"] + and "message" in parsed["error"] ) - error_type_field: Final[Mapping[str, AnthropicErrorType]] = ( - {"type": ANTHROPIC_ERROR_TYPE_MAP[status_code]} if status_code in ANTHROPIC_STATUS_CODE_MAP.values() else {} - ) - return {**envelope, **request_id_field, "error": {**envelope["error"], **error_type_field}} @staticmethod def _extract_message_from_dict(parsed: dict, raw_message: str) -> str: @@ -163,8 +135,7 @@ class AnthropicExceptionMapping: """ Transform an error message to Anthropic format. - - If already in Anthropic format: passthrough, with the error type a remapped status names (a 503 answers - as 529, and 529 is overloaded_error) + - If already in Anthropic format: passthrough unchanged - Otherwise: extract message and create Anthropic error Parses JSON only once for efficiency. @@ -183,9 +154,11 @@ class AnthropicExceptionMapping: parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None # If parsed and already in Anthropic format - passthrough - envelope: Final = AnthropicExceptionMapping._anthropic_error_envelope(parsed) - if envelope is not None: - return AnthropicExceptionMapping._passthrough(envelope, status_code, request_id) + if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed): + # Optionally add request_id if provided and not present + if request_id and "request_id" not in parsed: + parsed["request_id"] = request_id + return parsed # Extract message - use parsed dict if available, otherwise raw string if parsed is not None: @@ -228,8 +201,7 @@ class AnthropicErrorSseFrame(str): def anthropic_error_sse_frame(status_code: int, raw_message: str) -> AnthropicErrorSseFrame: - answered_status: Final = AnthropicExceptionMapping.get_status_code(status_code) return AnthropicErrorSseFrame( - answered_status, - AnthropicExceptionMapping.transform_to_anthropic_error(status_code=answered_status, raw_message=raw_message), + status_code, + AnthropicExceptionMapping.transform_to_anthropic_error(status_code=status_code, raw_message=raw_message), ) diff --git a/litellm/proxy/anthropic_endpoints/answered_status.py b/litellm/proxy/anthropic_endpoints/answered_status.py index 3eaa9d64324..8a3469f9119 100644 --- a/litellm/proxy/anthropic_endpoints/answered_status.py +++ b/litellm/proxy/anthropic_endpoints/answered_status.py @@ -1,17 +1,10 @@ -from typing import Final, Protocol, TypeVar, runtime_checkable +from typing import Final, TypeVar -from litellm.anthropic_interface.exceptions import AnthropicExceptionMapping from litellm.exceptions import InternalServerError, MidStreamFallbackError -from litellm.proxy._types import ProxyException _Raised = TypeVar("_Raised", bound=Exception) -@runtime_checkable -class _CarriesStatusCode(Protocol): - status_code: int - - def _provider_status_code(exc: Exception) -> int | None: unwrapped = exc # rebind-ok: the recursion gate requires iterative wrapper traversal while isinstance(unwrapped, MidStreamFallbackError) and unwrapped.original_exception is not None: @@ -33,17 +26,5 @@ def _copy_carrying(exc: _Raised, attribute: str, value: int | str) -> _Raised: def with_answered_status(exc: Exception) -> Exception: - """A copy of the exception, same class, carrying the status the Anthropic routes answer with (a 503 and an - upstream Anthropic overload answer as 529), so the proxy-level recorders the failure hook runs (the spend log - row, the OTel server span, the failed-request metric, alerting) agree with the client. The raised object keeps - its own status, because the litellm layer already handed it to the deployment-level recorders (the deployment - failure metric, cooldown, retry policy, custom loggers), which may still read it.""" - if isinstance(exc, ProxyException): - if not exc.code.isdecimal(): - return exc - return _copy_carrying(exc, "code", str(AnthropicExceptionMapping.get_status_code(int(exc.code)))) - if isinstance(exc, _CarriesStatusCode): - return _copy_carrying( - exc, "status_code", AnthropicExceptionMapping.get_status_code(_provider_status_code(exc) or exc.status_code) - ) - return exc + provider_status_code: Final = _provider_status_code(exc) + return exc if provider_status_code is None else _copy_carrying(exc, "status_code", provider_status_code) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 0d12dfac28e..7f096eebb0a 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2991,7 +2991,6 @@ class ProxyBaseLLMRequestProcessing: idle. See ``open_sse_before_first_byte``; unwrapped unless an operator sets ``litellm_settings.sse_keepalive_ping_interval_seconds``. """ - answers_as_anthropic: Final = route_type == "anthropic_messages" async def _audit_late_failure(exc: Exception) -> HTTPException | None: diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 5304bc2b91e..2aea32909ce 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -321,15 +321,12 @@ def request_with_retry[T: RetryableResponse]( """Bounded retry on statuses attributable to the PROVIDER, never the proxy. The system under test is the proxy, so the transport may only absorb - statuses that mean an overloaded provider; today that is exactly 529, the - Anthropic overloaded_error (their own SDK retries it too). The Anthropic - routes also answer the proxy's own 503s as 529, so a test asserting one of - those wraps its call in without_retries(), and a proxy that keeps failing - still fails once the bounded retries run out. 500/502/503/504 stay - first-class failures: at this layer a 5xx from the proxy is - indistinguishable from one it relayed, and retrying them could mask an - intermittently failing proxy. Widen the set only for a status that means - an overloaded provider, with an observed flake in hand. + statuses the proxy itself cannot emit; today that is exactly 529, the + Anthropic overloaded_error passed through verbatim (their own SDK retries + it too). 500/502/503/504 stay first-class failures: at this layer a 5xx + from the proxy is indistinguishable from one it relayed, and retrying them + could mask an intermittently failing proxy. Widen the set only for a + status litellm provably never originates, with an observed flake in hand. Also deliberately NOT retried: 429, because this suite asserts the proxy's own rate-limit and budget 429s; network errors and timeouts, because a diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py index 99a7d554cab..bae775164dd 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py @@ -126,6 +126,13 @@ def _assert_overloaded(response: httpx.Response, *, message: str) -> None: assert "content_block_delta" not in response.text, response.text +def _assert_api_error(response: httpx.Response, *, status: int, message: str) -> None: + assert response.status_code == status, response.text + error: Final = _anthropic_error(response) + assert error["type"] == "api_error", response.text + assert message in string_value(error["message"]), response.text + + def _assert_internal_server_error(response: httpx.Response) -> None: assert response.status_code == 500, response.text error: Final = _openai_error(response) @@ -193,17 +200,21 @@ async def test_messages_529_answers_overloaded_error_to_the_async_sdk(gateway: G @pytest.mark.parametrize( - ("upstream_status", "error_class"), ((529, "InternalServerError"), (503, "ServiceUnavailableError")) + ("upstream_status", "error_class", "error_type"), + ((529, "InternalServerError", "overloaded_error"), (503, "ServiceUnavailableError", "api_error")), ) -def test_messages_stream_rejected_before_its_first_byte_answers_529_json( - gateway: Gateway, upstream_status: int, error_class: str +def test_messages_stream_rejected_before_its_first_byte_answers_the_provider_status_as_json( + gateway: Gateway, upstream_status: int, error_class: str, error_type: str ) -> None: with wire_server(_always(status_reply(upstream_status))) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True)) - _assert_overloaded(response, message=f"scripted {upstream_status}") + assert response.status_code == upstream_status, response.text + error: Final = _anthropic_error(response) + assert error["type"] == error_type, response.text + assert f"scripted {upstream_status}" in string_value(error["message"]), response.text _one_post(wire, "/v1/messages") - _assert_logged_failure(model, rows=1, error_class=error_class) + _assert_logged_failure(model, rows=1, error_class=error_class, code=str(upstream_status)) def test_messages_stream_529_raises_529_in_the_sync_sdk(gateway: Gateway) -> None: @@ -266,7 +277,7 @@ def _one_bedrock_post(wire: Wire) -> None: assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request -def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_an_overloaded_error_event( +def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_an_api_error_event( gateway: Gateway, ) -> None: with ( @@ -281,10 +292,10 @@ def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_ (failure,) = (event for event in parse_sse(response.text) if event.data and event.event != "ping") assert failure.event == "error", response.text error: Final = object_value(failure.data["error"]) - assert error["type"] == "overloaded_error", response.text + assert error["type"] == "api_error", response.text assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text _one_bedrock_post(wire) - _assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError") + _assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503") def test_chat_completions_bedrock_503_after_the_keepalive_opened_the_response_keeps_the_openai_error_frame( @@ -403,7 +414,7 @@ def test_responses_529_answers_500(gateway: Gateway, shape: Shape) -> None: @pytest.mark.parametrize("shape", ("non-stream", "stream")) -def test_messages_bedrock_503_answers_529_overloaded_error(gateway: Gateway, shape: Shape) -> None: +def test_messages_bedrock_503_answers_503_api_error(gateway: Gateway, shape: Shape) -> None: with wire_server(_always(Reply(status=503, body=_BEDROCK_503))) as wire, gateway.scenario() as scenario: model: Final = scenario.model( model=f"bedrock/{_BEDROCK_MODEL}", api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url @@ -411,10 +422,10 @@ def test_messages_bedrock_503_answers_529_overloaded_error(gateway: Gateway, sha response: Final = gateway.request( "POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream") ) - _assert_overloaded(response, message="Bedrock is unable to process your request.") + _assert_api_error(response, status=503, message="Bedrock is unable to process your request.") (request,) = wire.drain() assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request - _assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError") + _assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503") def _converse_event(event: str, payload: Mapping[str, JsonValue]) -> bytes: @@ -445,7 +456,7 @@ def _converse_deployment(scenario: Scenario, wire: Wire) -> str: return scenario.model(model=_CONVERSE_MODEL, api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url) -def test_messages_bedrock_stream_503_after_text_ends_with_an_overloaded_error_event(gateway: Gateway) -> None: +def test_messages_bedrock_stream_503_after_text_ends_with_an_api_error_event(gateway: Gateway) -> None: with wire_server(_always(_converse_503_after_text())) as wire, gateway.scenario() as scenario: model: Final = _converse_deployment(scenario, wire) response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True)) @@ -457,10 +468,10 @@ def test_messages_bedrock_stream_503_after_text_ends_with_an_overloaded_error_ev assert _TEXT in response.text, response.text (failure,) = (event for event in events if event_type(event) == "error") error: Final = object_value(failure.data["error"]) - assert error["type"] == "overloaded_error", response.text + assert error["type"] == "api_error", response.text assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text _one_post(wire, _CONVERSE_STREAM_TARGET) - _assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError") + _assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503") def test_chat_completions_bedrock_stream_503_after_text_ends_with_a_503_error_frame(gateway: Gateway) -> None: @@ -481,7 +492,7 @@ def test_chat_completions_bedrock_stream_503_after_text_ends_with_a_503_error_fr @pytest.mark.parametrize("shape", ("non-stream", "stream")) -def test_chat_bridge_503_answers_529_overloaded_error_on_messages(gateway: Gateway, shape: Shape) -> None: +def test_chat_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None: with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario: model: Final = scenario.model( model=f"hosted_vllm/{_CHAT_ONLY_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-vllm-key" @@ -489,12 +500,12 @@ def test_chat_bridge_503_answers_529_overloaded_error_on_messages(gateway: Gatew response: Final = gateway.request( "POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream") ) - _assert_overloaded(response, message="scripted 503") + _assert_api_error(response, status=503, message="scripted 503") _one_post(wire, "/v1/chat/completions") @pytest.mark.parametrize("shape", ("non-stream", "stream")) -def test_responses_bridge_503_answers_529_overloaded_error_on_messages(gateway: Gateway, shape: Shape) -> None: +def test_responses_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None: with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario: model: Final = scenario.model( model=f"openai/{_CODEX_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-openai-key" @@ -502,7 +513,7 @@ def test_responses_bridge_503_answers_529_overloaded_error_on_messages(gateway: response: Final = gateway.request( "POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream") ) - _assert_overloaded(response, message="scripted 503") + _assert_api_error(response, status=503, message="scripted 503") _one_post(wire, "/v1/responses") @@ -561,14 +572,14 @@ def _assert_refused_without_an_upstream_call( assert reached.isdisjoint(prompt for prompt, _ in answers), reached -def test_messages_to_a_paused_deployment_named_by_its_id_answer_529_overloaded_error(gateway: Gateway) -> None: +def test_messages_to_a_paused_deployment_named_by_its_id_answer_503_api_error(gateway: Gateway) -> None: with _paused_deployment(gateway) as (wire, identity, name): _assert_refused_without_an_upstream_call( - gateway, wire, identity, status=529, error_type="overloaded_error", message="is currently paused" + gateway, wire, identity, status=503, error_type="api_error", message="is currently paused" ) rows: Final = _failure_rows_of((identity, name), _SETTLED_ANSWERS) assert {(row["code"], row["error_class"], row["normalized_error"]) for row in rows} == { - ("529", "ServiceUnavailableError", _OVERLOAD_CLASS) + ("503", "ServiceUnavailableError", _OVERLOAD_CLASS) }, rows diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py index dcc18d3f772..83a9ba3d14c 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py @@ -404,7 +404,7 @@ def test_retry_rejected_before_it_opens_counts_against_the_same_budget(gateway: seconds=70, ) rejected: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "request", budget=0)) - _assert_rejected_before_content(rejected, wire, status=500, attempts=1, error="error") + _assert_rejected_before_content(rejected, wire, status=529, attempts=1, error="overloaded_error") rejection_rows: Final = eventually( lambda: _request_rows(gateway, rejected.headers["x-litellm-call-id"]), lambda values: len(values) >= 1, diff --git a/tests/integration/observability/test_anthropic_overloaded_failure_signals.py b/tests/integration/observability/test_anthropic_overloaded_failure_signals.py index 81687ac82fb..044c58e44cc 100644 --- a/tests/integration/observability/test_anthropic_overloaded_failure_signals.py +++ b/tests/integration/observability/test_anthropic_overloaded_failure_signals.py @@ -214,7 +214,7 @@ def _until(rig: _Rig, name: str, model: str) -> tuple[Sample, ...]: ("route", "model", "answered", "deployment_status", "deployment_class"), ( pytest.param("/v1/messages", _MESSAGES_ANTHROPIC, 529, "500", "InternalServerError", id="messages-anthropic"), - pytest.param("/v1/messages", _MESSAGES_BEDROCK, 529, "503", "ServiceUnavailableError", id="messages-bedrock"), + pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, "503", "ServiceUnavailableError", id="messages-bedrock"), pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 500, "500", "InternalServerError", id="chat-anthropic"), ), ) @@ -312,7 +312,7 @@ def _datadog_status_codes(rig: _Rig, alias: str, expected: int) -> tuple[tuple[s ("route", "model", "answered", "deployment_status"), ( pytest.param("/v1/messages", _DATADOG_MESSAGES_ANTHROPIC, 529, "500", id="messages-anthropic"), - pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 529, "503", id="messages-bedrock"), + pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 503, "503", id="messages-bedrock"), pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 500, "500", id="chat-anthropic"), ), ) @@ -332,7 +332,7 @@ def test_the_datadog_proxy_failure_log_carries_the_answered_status_and_the_llm_f @pytest.mark.parametrize( ("route", "model", "answered"), ( - pytest.param("/v1/messages", _GUARDED_MESSAGES, 529, id="messages"), + pytest.param("/v1/messages", _GUARDED_MESSAGES, 503, id="messages"), pytest.param("/v1/chat/completions", _GUARDED_CHAT, 503, id="chat"), ), ) @@ -360,7 +360,7 @@ def test_a_fail_closed_guardrail_outage_is_answered_and_recorded_with_the_routes ("route", "model", "answered"), ( pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages-anthropic"), - pytest.param("/v1/messages", _MESSAGES_BEDROCK, 529, id="messages-bedrock"), + pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, id="messages-bedrock"), pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat-anthropic"), ), ) diff --git a/tests/integration/observability/test_bedrock_error_request_id.py b/tests/integration/observability/test_bedrock_error_request_id.py index 1e7d096a2b5..436dcc572d5 100644 --- a/tests/integration/observability/test_bedrock_error_request_id.py +++ b/tests/integration/observability/test_bedrock_error_request_id.py @@ -424,9 +424,7 @@ def _bedrock_logged_error(endpoint: BedrockEndpoint, failure: BedrockFailure) -> shape: Final = _BEDROCK_FAILURES[failure] if endpoint == "messages-invoke": return "BedrockError", str(shape.bedrock_status) - if failure != "exception-event": - return shape.converse_class, str(shape.bedrock_status) - return shape.converse_class, "529" if endpoint == "messages" else "503" + return shape.converse_class, "503" if failure == "exception-event" else str(shape.bedrock_status) def _bedrock_client_status(endpoint: BedrockEndpoint, failure: BedrockFailure, after_output: bool) -> int: @@ -553,7 +551,7 @@ def test_bedrock_non_stream_500_logs_amzn_request_id_on_every_endpoint( endpoint, _bedrock_model(scenario, endpoint, wire.url), amzn_request_id, stream=False ) response: Final = gateway.request("POST", path, body) - assert response.status_code == (503 if endpoint == "responses" else 529), response.text + assert response.status_code == 503, response.text if endpoint == "responses": assert response.headers.get("llm_provider-x-amzn-requestid") == amzn_request_id, dict(response.headers) assert len(wire.drain()) == 1 diff --git a/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py b/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py index 5feccdf972d..380881a6796 100644 --- a/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py +++ b/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py @@ -476,7 +476,7 @@ def test_messages_count_tokens_surfaces_the_peer_status_when_the_peer_rejects_th with _listening(deployment, _rejecting(status)) as peer: body: Final = count_request(deployment.model, STRING_CASE) response: Final = _count(gateway, body) - assert response.status_code == (529 if status == 503 else status), response.text + assert response.status_code == status, response.text assert "input_tokens" not in response.text, response.text assert _bodies(peer, deployment.provider) == (_STRING_BODY,) diff --git a/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py b/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py index bf393869623..bcbf6d0f6f3 100644 --- a/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py +++ b/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py @@ -235,7 +235,7 @@ async def test_upstream_aborts_then_an_outage_fail_each_call_once_and_the_native refused: Final = await _burst(proxy, gateway.key, model, during_outage) assert len(refused) == 4 for item in refused: - assert item.status == (529 if item.call.endpoint == "messages" else 503), item.text + assert item.status == 503, item.text assert "Cannot connect to host" in item.text, item.text assert await asyncio.to_thread( eventually, lambda: _unhealthy_count(gateway, model), lambda count: count == 1, 30 diff --git a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py index 7615629ed3a..c392e355708 100644 --- a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py @@ -1172,7 +1172,7 @@ def test_messages_count_tokens_surfaces_the_mantle_status_when_mantle_errors( ): model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, _full(model)) - assert response.status_code == (529 if status == 503 else status), response.text + assert response.status_code == status, response.text assert "input_tokens" not in response.text, response.text assert len(_runtime_count_targets(runtime)) == 1 assert _mantle_bodies(mantle) == (_MANTLE_FULL,) diff --git a/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py b/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py index 53cbcb85695..42950fb18c3 100644 --- a/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py +++ b/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py @@ -329,8 +329,10 @@ async def test_peer_killed_mid_burst_fails_only_the_held_calls_and_a_restarted_p assert (len(succeeded), len(failed)) == (6, 6), [(item.call.marker, item.status) for item in served] for item in succeeded: _assert_answered_with_its_own_marker(item) + assert {item.status for item in failed} == {503}, [ + (item.call.endpoint, item.call.stream, item.status, item.text) for item in failed + ] for item in failed: - assert item.status == (529 if item.call.endpoint == "messages" else 503), (item.call, item.status, item.text) assert "ServiceUnavailableError: BedrockException - Server disconnected" in item.text, item.text assert "marker-" not in item.text and item.call_id is not None, item.text with _child_peer(deployment.peer_port, answer_first=10**6) as revived: diff --git a/tests/integration/providers/test_databricks_chat_wire.py b/tests/integration/providers/test_databricks_chat_wire.py index f3eedd4e1d5..3d9d749eb04 100644 --- a/tests/integration/providers/test_databricks_chat_wire.py +++ b/tests/integration/providers/test_databricks_chat_wire.py @@ -1261,8 +1261,7 @@ def test_databricks_json_schema_burst_through_an_upstream_outage_keeps_every_ref for request in received: _assert_burst_outbound(request) expected: Final = tuple( - (529 if endpoint == "messages" else 503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON) - for endpoint, _stream, down in plan + (503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON) for _endpoint, _stream, down in plan ) assert tuple(_outcome(*result) for result in results) == expected assert tuple(_outcome(*result) for result in recovered) == ((200, _EXPECTED_JSON),) * len(recovery) @@ -1572,8 +1571,7 @@ def test_databricks_one_block_burst_through_an_upstream_outage_reaches_upstream_ json.dumps([{"role": "user", "content": prompt}]) for prompt in prompts ) expected: Final = tuple( - (529 if endpoint == "messages" else 503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON) - for endpoint, _stream, down in plan + (503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON) for _endpoint, _stream, down in plan ) assert tuple(_outcome(*result) for result in results) == expected diff --git a/tests/integration/providers/test_responses_bridge_tool_search_chaos.py b/tests/integration/providers/test_responses_bridge_tool_search_chaos.py index 2cbee8f9b7d..d4a5e6bf0db 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_chaos.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_chaos.py @@ -1215,8 +1215,7 @@ async def test_upstream_outage_mid_burst_fails_only_its_calls_and_every_replay_l assert len(served) == len(calls) for item in served: if item.call.marker in down: - assert item.status == (529 if item.call.endpoint == "messages" else 503), item.text - assert _BRIDGE_OUTAGE in item.text, item.text + assert item.status == 503 and _BRIDGE_OUTAGE in item.text, item.text assert rv.answer(item.call.marker) not in item.text, item.text else: _bridge_assert_answered(item) diff --git a/tests/integration/providers/test_sagemaker_chat_wire.py b/tests/integration/providers/test_sagemaker_chat_wire.py index 9fed22f01fb..784e43f85e2 100644 --- a/tests/integration/providers/test_sagemaker_chat_wire.py +++ b/tests/integration/providers/test_sagemaker_chat_wire.py @@ -659,7 +659,7 @@ async def _burst(gateway: Gateway, model: str, calls: tuple[_Call, ...]) -> tupl def _assert_answered(served: _Served, down: frozenset[str]) -> None: if served.call.marker in down: - assert served.status == (529 if served.call.endpoint == "messages" else 503), (served.call, served.text) + assert served.status == 503, (served.call, served.text) assert "synthetic container outage" in served.text, (served.call, served.text) return assert served.status == 200, (served.call, served.text) diff --git a/tests/integration/providers/test_vertex_gemini_count_tokens_wire.py b/tests/integration/providers/test_vertex_gemini_count_tokens_wire.py index b515c1c0868..1632e424c55 100644 --- a/tests/integration/providers/test_vertex_gemini_count_tokens_wire.py +++ b/tests/integration/providers/test_vertex_gemini_count_tokens_wire.py @@ -980,7 +980,7 @@ def test_concurrent_count_waves_recover_after_vertex_count_outage(gateway: Gatew body: Final = _messages(model, None, None) assert _count_wave(clients, body) == ((200, _COUNT), (200, _COUNT), (200, _COUNT)) peer.set_status(503) - assert _count_wave(clients, body) == ((529, None), (529, None), (529, None)) + assert _count_wave(clients, body) == ((503, None), (503, None), (503, None)) peer.set_status(200) assert _count_wave(clients, body) == ((200, _COUNT), (200, _COUNT), (200, _COUNT)) requests: Final = _count_requests(wire) diff --git a/tests/integration/providers/test_vertex_partner_count_tokens_wire.py b/tests/integration/providers/test_vertex_partner_count_tokens_wire.py index 7d310e8070c..c679311e4cc 100644 --- a/tests/integration/providers/test_vertex_partner_count_tokens_wire.py +++ b/tests/integration/providers/test_vertex_partner_count_tokens_wire.py @@ -622,7 +622,7 @@ def test_messages_count_tokens_surfaces_the_peer_error_when_peer_errors(gateway: with wire_server(_peer(_rejecting(status))) as wire, gateway.scenario() as scenario: model: Final = _deployment(gateway, scenario, wire.url) response: Final = _count(gateway, _full(model)) - assert response.status_code == (529 if status == 503 else status), response.text + assert response.status_code == status, response.text assert "input_tokens" not in response.text, response.text assert _counted_bodies(wire) == (_PEER_FULL,) diff --git a/tests/integration/providers/test_vertex_passthrough_stream_wire.py b/tests/integration/providers/test_vertex_passthrough_stream_wire.py index 8c9b9fa62cc..1ec2f740480 100644 --- a/tests/integration/providers/test_vertex_passthrough_stream_wire.py +++ b/tests/integration/providers/test_vertex_passthrough_stream_wire.py @@ -115,8 +115,7 @@ def test_vertex_pass_through_stream_relays_an_upstream_503_as_503(vertex: _Verte response: Final = vertex.stream(_UNAVAILABLE_MODEL, uuid.uuid4().hex) assert response.status_code == 503, response.text - assert _UNAVAILABLE_MESSAGE in response.text, response.text - assert "overloaded_error" not in response.text, response.text + assert response.content == _UNAVAILABLE_BODY, response.text assert [request.target for request in vertex.wire.drain()] == [ f"/v1/{_model_path(_UNAVAILABLE_MODEL)}:streamGenerateContent?alt=sse" ] diff --git a/tests/integration/routing/test_anthropic_overloaded_cooldown_owned_proxy.py b/tests/integration/routing/test_anthropic_overloaded_cooldown_owned_proxy.py index 6526895cea5..e6df49a73bd 100644 --- a/tests/integration/routing/test_anthropic_overloaded_cooldown_owned_proxy.py +++ b/tests/integration/routing/test_anthropic_overloaded_cooldown_owned_proxy.py @@ -3,7 +3,6 @@ import os import uuid from collections.abc import Iterator from dataclasses import dataclass -from itertools import chain from pathlib import Path from typing import Final from urllib.parse import unquote @@ -25,12 +24,10 @@ from integration._support.anthropic_sse import ( from integration._support.client import ( Gateway, JsonValue, - eventually, gateway_from_environment, object_value, string_value, ) -from integration._support.database import read_rows from integration._support.process import graceful_stop_seconds, owned_proxy from integration._support.wire import Reply, Request, Wire, wire_server @@ -53,8 +50,6 @@ _GATEWAY_GROUPS: Final = { "stream": "audit-gateway-overloaded-stream", } _COUNT_STATUSES: Final = {_COUNTED: 503, _COUNTED_OVERLOADED: 529, _COUNTED_RATE_LIMITED: 429} -_ADAPTIVE_UPSTREAMS: Final = {503: "claude-adaptive-unavailable"} -_ADAPTIVE_FLUSH_SECONDS: Final = 40 _COUNT_TOKENS_TARGET: Final = "/v1/messages/count_tokens" _UPSTREAM_URL_PLACEHOLDER: Final = "upstream-url" @@ -73,8 +68,6 @@ def _respond(request: Request) -> Reply: return Reply(status=status, body=error_body(status, f"scripted {status}")) if backend == _OVERLOADED: return status_reply(529) - if backend in _ADAPTIVE_UPSTREAMS.values(): - return status_reply(next(status for status, name in _ADAPTIVE_UPSTREAMS.items() if name == backend)) if backend == _HEALTHY: return stream_reply(message_stream(_served_id(user_prompt(body)), backend, _TEXT)) return Reply(status=404, body=b'{"error": "no scripted backend"}') @@ -84,36 +77,6 @@ def _deployment(name: str, model: str, api_base: str, api_key: str, **params: Js return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": api_key, **params}} -def _adaptive_router(status: int) -> str: - return f"audit-adaptive-router-{status}" - - -def _adaptive_backend(status: int) -> str: - return f"audit-adaptive-backend-{status}" - - -def _adaptive_deployments(status: int) -> tuple[dict[str, JsonValue], ...]: - return ( - { - "model_name": _adaptive_router(status), - "litellm_params": { - "model": "auto_router/adaptive_router", - "adaptive_router_config": {"available_models": [_adaptive_backend(status)]}, - }, - }, - { - **_deployment( - _adaptive_backend(status), - f"anthropic/{_ADAPTIVE_UPSTREAMS[status]}", - _UPSTREAM_URL_PLACEHOLDER, - _API_KEY, - input_cost_per_token=0.000001, - ), - "model_info": {"adaptive_router_preferences": {"quality_tier": 2, "strengths": []}}, - }, - ) - - def _config(wire: Wire, directory: Path) -> Path: config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) config["model_list"] = [ @@ -134,7 +97,6 @@ def _config(wire: Wire, directory: Path) -> Path: _deployment(group, f"anthropic/{_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY) for group in _GATEWAY_GROUPS.values() ), - *chain.from_iterable(_adaptive_deployments(status) for status in _ADAPTIVE_UPSTREAMS), ] config["router_settings"] = { "num_retries": 0, @@ -224,22 +186,20 @@ def test_an_overloaded_deployment_is_cooled_down_by_the_internal_server_error_al assert rig.backends_called() == (_HEALTHY,) -def test_count_tokens_disabled_without_a_provider_counter_answers_529_overloaded_error(rig: _Rig) -> None: +def test_count_tokens_disabled_without_a_provider_counter_answers_503_api_error(rig: _Rig) -> None: response: Final = rig.count_tokens(_COUNT_DISABLED_GROUP) - error: Final = _count_tokens_error(response, 529) - assert error["type"] == "overloaded_error", response.text + error: Final = _count_tokens_error(response, 503) + assert error["type"] == "api_error", response.text assert "Token counting is disabled" in string_value(error["message"]), response.text assert rig.targets_called() == () -def test_count_tokens_whose_upstream_answers_503_answers_529_overloaded_error(rig: _Rig) -> None: +def test_count_tokens_whose_upstream_answers_503_stays_503_api_error(rig: _Rig) -> None: response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP) - error: Final = _count_tokens_error(response, 529) - assert error["type"] == "overloaded_error", response.text + error: Final = _count_tokens_error(response, 503) + assert error["type"] == "api_error", response.text assert "scripted 503" in string_value(error["message"]), response.text assert rig.targets_called() == (_COUNT_TOKENS_TARGET,) - - @pytest.mark.parametrize( ("group", "status", "error_type"), ( @@ -279,46 +239,9 @@ def test_the_claude_code_gateway_messages_route_answers_an_upstream_529_as_overl assert rig.backends_called() == (_OVERLOADED,) -def test_the_claude_code_gateway_count_tokens_route_answers_an_upstream_503_as_529(rig: _Rig) -> None: +def test_the_claude_code_gateway_count_tokens_route_answers_an_upstream_503_as_503(rig: _Rig) -> None: response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP, prefix=_GATEWAY) - error: Final = _count_tokens_error(response, 529) - assert error["type"] == "overloaded_error", response.text + error: Final = _count_tokens_error(response, 503) + assert error["type"] == "api_error", response.text assert "scripted 503" in string_value(error["message"]), response.text assert rig.targets_called() == (_COUNT_TOKENS_TARGET,) - - -def _adaptive_session_rows(session: str) -> list[dict[str, JsonValue]]: - return read_rows( - 'SELECT model_name, exhaustion_count FROM "LiteLLM_AdaptiveRouterSession" WHERE session_id = %s', (session,) - ) - - -@pytest.mark.parametrize( - ("status", "answered"), - ( - pytest.param(503, 503, id="upstream-503"), - ), -) -def test_an_adaptive_router_session_counts_an_upstream_overload_as_exhaustion( - rig: _Rig, status: int, answered: int -) -> None: - session: Final = f"audit-adaptive-{uuid.uuid4().hex}" - response: Final = rig.proxy.request( - "POST", - "/v1/chat/completions", - { - "model": _adaptive_router(status), - "max_tokens": 16, - "messages": [{"role": "user", "content": session}], - }, - headers={"x-litellm-session-id": session}, - ) - assert response.status_code == answered, response.text - assert rig.backends_called() == (_ADAPTIVE_UPSTREAMS[status],) - rows: Final = eventually( - lambda: _adaptive_session_rows(session), - lambda rows: len(rows) > 0, - seconds=_ADAPTIVE_FLUSH_SECONDS, - return_last_on_timeout=True, - ) - assert rows == [{"model_name": _adaptive_backend(status), "exhaustion_count": 1}], response.text diff --git a/tests/integration/routing/test_messages_stream_retry_budget_sources_owned_proxy.py b/tests/integration/routing/test_messages_stream_retry_budget_sources_owned_proxy.py index 9b55e6bf51b..e0900bc0926 100644 --- a/tests/integration/routing/test_messages_stream_retry_budget_sources_owned_proxy.py +++ b/tests/integration/routing/test_messages_stream_retry_budget_sources_owned_proxy.py @@ -55,7 +55,7 @@ Behavior = Literal[ "hold-ping", "drop-then-too-long", "drop-then-401", - "overloaded-frames-fallback-503", + "overloaded-frames-fallback-529", "overloaded-frames", "overloaded-once", ] @@ -90,7 +90,7 @@ def _primary_reply( return Reply(status=400, body=error_body(400, "prompt is too long: 250000 tokens > 200000 maximum")) case "drop-then-401": return dropped if attempt == 1 else status_reply(401) - case "overloaded-frames-fallback-503" | "overloaded-frames": + case "overloaded-frames-fallback-529" | "overloaded-frames": return stream_reply((error_frame(529, "scripted overloaded"),)) case "overloaded-once": return status_reply(529) if attempt == 1 else stream_reply(full) @@ -98,7 +98,7 @@ def _primary_reply( def _fallback_reply(behavior: str, full: tuple[bytes, bytes, bytes, bytes]) -> Reply: - if behavior == "overloaded-frames-fallback-503": + if behavior == "overloaded-frames-fallback-529": return status_reply(503) return stream_reply(full) @@ -309,7 +309,7 @@ def test_a_retry_rejected_with_401_ends_the_retries_and_reaches_the_client_uncha def test_overloaded_frames_whose_fallback_fails_answer_529_overloaded_error(rig: _Rig) -> None: marker: Final = uuid.uuid4().hex - response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-503", marker) + response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-529", marker) assert response.status_code == 529, response.text assert "overloaded_error" in response.text, response.text assert "content_block_delta" not in response.text, response.text diff --git a/tests/integration/streaming/test_noma_v2_streaming_settings.py b/tests/integration/streaming/test_noma_v2_streaming_settings.py index 407400a9311..877978732f8 100644 --- a/tests/integration/streaming/test_noma_v2_streaming_settings.py +++ b/tests/integration/streaming/test_noma_v2_streaming_settings.py @@ -67,7 +67,7 @@ BLOCKED: Final = "Request blocked by Noma guardrail" SLOW_SECONDS: Final = 1.5 SAMPLED_SCANS: Final = range(2, 7) BUFFERED_BLOCK_STATUS: Final[Mapping[Endpoint, int]] = {"chat": 400, "messages": 400, "responses": 200} -BUFFERED_OUTAGE_STATUS: Final[Mapping[Endpoint, int]] = {"chat": 503, "messages": 529, "responses": 200} +BUFFERED_OUTAGE_STATUS: Final[Mapping[Endpoint, int]] = {"chat": 503, "messages": 503, "responses": 200} PIECES: Final = ( " Contact", " me", diff --git a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py index c23f4e95cad..210ab6717aa 100644 --- a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py +++ b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py @@ -56,17 +56,6 @@ class TestCreateErrorResponse: ) assert response["error"]["type"] == "api_error" - def test_503_answers_as_anthropics_529_overloaded_error(self): - """LIT-9450: Anthropic's contract has no 503, so an upstream that cannot take the request right now - (a Bedrock capacity 503) answers as 529 overloaded_error, the status Claude Code's overload path keys on.""" - assert AnthropicExceptionMapping.get_status_code(503) == 529 - response = AnthropicExceptionMapping.create_error_response(503, "Bedrock is unable to process your request.") - assert response["error"]["type"] == "overloaded_error" - - @pytest.mark.parametrize("status_code", [400, 429, 500, 529]) - def test_other_statuses_keep_their_code(self, status_code: int): - assert AnthropicExceptionMapping.get_status_code(status_code) == status_code - def test_with_request_id(self): """Test request_id is included when provided.""" response = AnthropicExceptionMapping.create_error_response( @@ -179,21 +168,6 @@ class TestTransformToAnthropicError: ) assert result["request_id"] == "req_123" - def test_an_anthropic_shaped_503_body_answers_529_with_the_overloaded_type(self): - """A 503 whose message is already an Anthropic envelope keeps its message but takes the error type the - 529 it answers as names, so the status and the body agree.""" - raw = json.dumps({"type": "error", "error": {"type": "api_error", "message": "Service unavailable"}}) - result = AnthropicExceptionMapping.transform_to_anthropic_error( - status_code=AnthropicExceptionMapping.get_status_code(503), - raw_message=raw, - request_id="req_529", - ) - assert result == { - "type": "error", - "error": {"type": "overloaded_error", "message": "Service unavailable"}, - "request_id": "req_529", - } - def test_translates_bedrock_error(self): """Test that Bedrock errors are translated to Anthropic format.""" bedrock_error = json.dumps({"detail": {"message": "Access denied"}}) @@ -250,7 +224,7 @@ class TestTransformToAnthropicError: class TestAnthropicErrorSseFrame: @pytest.mark.parametrize( ("status_code", "expected_error_type"), - [(429, "rate_limit_error"), (503, "overloaded_error"), (400, "invalid_request_error")], + [(429, "rate_limit_error"), (503, "api_error"), (400, "invalid_request_error")], ) def test_the_frame_is_one_error_event_carrying_the_anthropic_envelope( self, status_code: int, expected_error_type: str @@ -265,22 +239,11 @@ class TestAnthropicErrorSseFrame: "error": {"type": expected_error_type, "message": "upstream unavailable"}, } - def test_a_503_envelope_frame_takes_the_overloaded_type_of_the_529_it_answers(self) -> None: - envelope: Final = json.dumps({"type": "error", "error": {"type": "api_error", "message": "Service unavailable"}}) - - frame: Final = anthropic_error_sse_frame(status_code=503, raw_message=envelope) - - assert frame.status_code == 529 - assert frame.json_body(call_id=None) == { - "type": "error", - "error": {"type": "overloaded_error", "message": "Service unavailable"}, - } - - def test_the_frame_remembers_the_status_anthropic_answers_with_and_the_body(self) -> None: + def test_the_frame_remembers_the_status_and_body_it_was_built_from(self) -> None: frame: Final = anthropic_error_sse_frame(status_code=503, raw_message="upstream unavailable") assert isinstance(frame, AnthropicErrorSseFrame) - assert frame.status_code == 529 + assert frame.status_code == 503 data_line: Final = frame.split("\n")[1] assert data_line == f"data: {json.dumps(frame.json_body(call_id=None))}" @@ -289,9 +252,9 @@ class TestAnthropicErrorSseFrame: assert frame.json_body(call_id="call-1") == { "type": "error", - "error": {"type": "overloaded_error", "message": "upstream unavailable", "litellm_call_id": "call-1"}, + "error": {"type": "api_error", "message": "upstream unavailable", "litellm_call_id": "call-1"}, } assert frame.json_body(call_id=None) == { "type": "error", - "error": {"type": "overloaded_error", "message": "upstream unavailable"}, + "error": {"type": "api_error", "message": "upstream unavailable"}, } diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index ae8ce51ee81..32a99a37870 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1438,8 +1438,6 @@ def test_an_already_mapped_litellm_exception_passes_through_untouched(provider, ['{"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}', "upstream failure"], ) def test_an_anthropic_529_stays_an_internal_server_error_that_carries_the_529(message, quiet_exception_mapping): - """An Anthropic overload keeps the InternalServerError class and 500 status that retry and allowed-fails - policies and SDK callers key on, and carries the provider's 529 for the Anthropic-shaped routes to answer.""" with pytest.raises(litellm.InternalServerError) as raised: exception_type( model="claude-opus-5-5", diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py index a2e63aeeb73..4626f76c8fc 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -561,7 +561,7 @@ class TestUpstreamFailureEndsStreamWithErrorEvent: [ ( MidStreamFallbackError(message="boom", model="m", llm_provider="openai"), - {"type": "overloaded_error", "message": "litellm.MidStreamFallbackError: boom"}, + {"type": "api_error", "message": "litellm.MidStreamFallbackError: boom"}, ), ( type("StringStatusError", (Exception,), {"status_code": "429"})("throttled"), diff --git a/tests/unit/proxy/anthropic_endpoints/test_answered_status.py b/tests/unit/proxy/anthropic_endpoints/test_answered_status.py index f30d8b674d1..881a4f28a82 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_answered_status.py +++ b/tests/unit/proxy/anthropic_endpoints/test_answered_status.py @@ -16,27 +16,28 @@ def _anthropic_internal_server_error(provider_status_code: int | None) -> litell ) -@pytest.mark.parametrize(("code", "answered"), [(503, "529"), (400, "400"), (500, "500")]) -def test_a_proxy_exception_answers_its_503_as_529_and_keeps_every_other_code(code: int, answered: str) -> None: +@pytest.mark.parametrize("code", [400, 500, 503]) +def test_a_proxy_exception_passes_through_unchanged(code: int) -> None: exc = ProxyException(message="budget service unavailable", type="None", param=None, code=code) answered_exc = with_answered_status(exc) - assert (type(answered_exc), answered_exc.code, answered_exc.message) == (ProxyException, answered, exc.message) + assert answered_exc is exc assert exc.code == str(code) @pytest.mark.parametrize( - ("exc", "answered"), + ("exc", "answered", "copy"), [ pytest.param( litellm.ServiceUnavailableError(message="BedrockException", llm_provider="bedrock", model="m"), - 529, + 503, + False, id="service-unavailable-503", ), - pytest.param(_anthropic_internal_server_error(529), 529, id="anthropic-overload"), - pytest.param(_anthropic_internal_server_error(500), 500, id="anthropic-500"), - pytest.param(_anthropic_internal_server_error(None), 500, id="internal-server-error"), + pytest.param(_anthropic_internal_server_error(529), 529, True, id="anthropic-overload"), + pytest.param(_anthropic_internal_server_error(500), 500, True, id="anthropic-500"), + pytest.param(_anthropic_internal_server_error(None), 500, False, id="internal-server-error"), pytest.param( MidStreamFallbackError( message="fallbacks failed", @@ -45,6 +46,7 @@ def test_a_proxy_exception_answers_its_503_as_529_and_keeps_every_other_code(cod original_exception=_anthropic_internal_server_error(529), ), 529, + True, id="fallback-over-an-anthropic-overload", ), pytest.param( @@ -60,20 +62,25 @@ def test_a_proxy_exception_answers_its_503_as_529_and_keeps_every_other_code(cod ), ), 529, + True, id="nested-fallback-over-an-anthropic-overload", ), pytest.param( - litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="m"), 429, id="rate-limit" + litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="m"), + 429, + False, + id="rate-limit", ), ], ) def test_a_litellm_exception_answers_the_status_anthropic_gives_its_failure( - exc: openai.APIStatusError, answered: int + exc: openai.APIStatusError, answered: int, copy: bool ) -> None: raised_status = exc.status_code answered_exc = with_answered_status(exc) + assert (answered_exc is not exc) is copy assert (type(answered_exc), answered_exc.status_code, str(answered_exc)) == (type(exc), answered, str(exc)) assert exc.status_code == raised_status @@ -82,13 +89,16 @@ def _raise_a_bedrock_overload_caused_by_a_reset() -> None: try: raise ConnectionError("upstream reset") except ConnectionError as upstream: - raise litellm.ServiceUnavailableError( - message="BedrockException", llm_provider="bedrock", model="m" + raise litellm.InternalServerError( + message="AnthropicError - Overloaded", + llm_provider="anthropic", + model="m", + provider_status_code=529, ) from upstream def test_the_answered_copy_keeps_the_raised_exceptions_cause_and_traceback() -> None: - with pytest.raises(litellm.ServiceUnavailableError) as raised: + with pytest.raises(litellm.InternalServerError) as raised: _raise_a_bedrock_overload_caused_by_a_reset() answered_exc = with_answered_status(raised.value) diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 3cce01fe9d0..dccd1e877fc 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -303,7 +303,7 @@ class TestProxyExceptionAnthropicEnvelope: import litellm.proxy.anthropic_endpoints.endpoints as ep from litellm.proxy._types import ProxyException - request = MagicMock() + request: Final = MagicMock() request.headers = {} response = ep._anthropic_error_json_response( @@ -361,7 +361,7 @@ class TestProxyExceptionAnthropicEnvelope: from litellm.proxy._types import UserAPIKeyAuth raised_status: Final = unavailable.status_code - request = MagicMock() + request: Final = MagicMock() request.headers = {} with ( @@ -380,7 +380,7 @@ class TestProxyExceptionAnthropicEnvelope: user_api_key_dict=UserAPIKeyAuth(), ) - recorded = mock_logging.post_call_failure_hook.await_args.kwargs["original_exception"] + recorded: Final = mock_logging.post_call_failure_hook.await_args.kwargs["original_exception"] assert recorded is not None assert (recorded is not unavailable) is hook_receives_copy assert recorded.status_code == expected_status @@ -438,9 +438,9 @@ class TestProxyExceptionAnthropicEnvelope: proxy_server, "count_request_tokens", new=AsyncMock(side_effect=unavailable) ), ): - response = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) - body = json.loads(response.body) + body: Final = json.loads(response.body) assert response.status_code == expected_status assert body["type"] == "error" assert body["error"]["type"] == expected_error_type @@ -553,7 +553,7 @@ class TestHttpExceptionDictDetail: ) as mock_logging, # test-quality-ok: module global imported at call time; no injection seam ): mock_logging.post_call_failure_hook = AsyncMock() - response = await ep.anthropic_response( + response: Final = await ep.anthropic_response( fastapi_response=MagicMock(), request=request, user_api_key_dict=UserAPIKeyAuth(), diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 3316424ccd4..1fa60e71f11 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -2810,8 +2810,7 @@ class TestCommonRequestProcessingHelpers: async def test_a_stream_that_fails_before_its_first_byte_answers_as_an_anthropic_json_error(self): """A /v1/messages stream whose first chunk is already the error frame has nothing - streamed yet, so the failure answers as JSON with the status Anthropic answers that upstream failure with - (a 503 is its 529 overloaded_error, LIT-9450), + streamed yet, so the failure answers as JSON with the status the upstream gave, the shape Anthropic clients raise their status-specific errors on""" async def stream(): @@ -2822,12 +2821,12 @@ class TestCommonRequestProcessingHelpers: response = await create_response(generator, "text/event-stream", {"x-litellm-call-id": "call-8609"}) assert isinstance(response, JSONResponse) - assert response.status_code == 529 + assert response.status_code == 503 assert response.headers["content-type"] == "application/json" assert response.headers["x-litellm-call-id"] == "call-8609" assert json.loads(response.body) == { "type": "error", - "error": {"type": "overloaded_error", "message": "upstream unavailable"}, + "error": {"type": "api_error", "message": "upstream unavailable"}, } assert generator.ag_frame is None @@ -2863,7 +2862,7 @@ class TestCommonRequestProcessingHelpers: assert response.status_code == 200 assert "".join(await self.consume_stream(response)) == ( ANTHROPIC_PING_SSE_CHUNK - + 'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "upstream unavailable"}}\n\n' + + 'event: error\ndata: {"type": "error", "error": {"type": "api_error", "message": "upstream unavailable"}}\n\n' ) async def test_create_streaming_response_disables_proxy_buffering(self): @@ -10981,22 +10980,27 @@ async def test_base_process_llm_request_audits_a_failure_that_lands_after_its_ke @pytest.mark.asyncio -async def test_anthropic_messages_answers_a_late_overload_after_its_keepalive_as_an_overloaded_error_event(): - async def slow_overload(self, **kwargs): - await asyncio.sleep(0.25) - raise litellm.ServiceUnavailableError( - message='BedrockException - {"message": "Bedrock is unable to process your request."}', - llm_provider="bedrock", - model="global.anthropic.claude-opus-5-5", - ) +async def test_anthropic_messages_answers_a_late_503_after_its_keepalive_as_an_api_error_event() -> None: + unavailable: Final = litellm.ServiceUnavailableError( + message='BedrockException - {"message": "Bedrock is unable to process your request."}', + llm_provider="bedrock", + model="global.anthropic.claude-opus-5-5", + ) + release_failure: Final = asyncio.Event() - proxy_logging_obj = MagicMock(spec=ProxyLogging) + async def fail_after_keepalive(self: ProxyBaseLLMRequestProcessing, **kwargs: object) -> None: + await release_failure.wait() + raise unavailable + + proxy_logging_obj: Final = MagicMock(spec=ProxyLogging) proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) - processor = ProxyBaseLLMRequestProcessing(data={"model": "global.anthropic.claude-opus-5-5", "stream": True}) + processor: Final = ProxyBaseLLMRequestProcessing( + data={"model": "global.anthropic.claude-opus-5-5", "stream": True} + ) with patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05): - with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", slow_overload): - response = await processor.base_process_llm_request( + with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", fail_after_keepalive): + response: Final = await processor.base_process_llm_request( request=MagicMock(spec=Request), fastapi_response=Response(), user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), @@ -11005,15 +11009,20 @@ async def test_anthropic_messages_answers_a_late_overload_after_its_keepalive_as general_settings={}, proxy_config=MagicMock(spec=ProxyConfig), ) - collected = await _drain(response) + body_iterator: Final = response.body_iterator + first_chunk: Final = await body_iterator.__anext__() + release_failure.set() + remaining_chunks: Final = [ + chunk.encode("utf-8") if isinstance(chunk, str) else bytes(chunk) async for chunk in body_iterator + ] - assert proxy_logging_obj.post_call_failure_hook.await_args.kwargs["original_exception"].status_code == 529 - assert collected[0] == TTFT_PING - event_line, data_line = collected[-1].decode().strip().split("\n") + assert proxy_logging_obj.post_call_failure_hook.await_args.kwargs["original_exception"] is unavailable + assert first_chunk == TTFT_PING + event_line, data_line = remaining_chunks[-1].decode().strip().split("\n") assert event_line == "event: error" - error = json.loads(data_line.removeprefix("data: ")) + error: Final = json.loads(data_line.removeprefix("data: ")) assert error["type"] == "error" - assert error["error"]["type"] == "overloaded_error" + assert error["error"]["type"] == "api_error" assert "Bedrock is unable to process your request." in error["error"]["message"]