fix(anthropic): preserve 503s and answer upstream 529s on messages routes

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-11-03 21:32:47 +00:00
parent dabaa6d4ed
commit d89164512a
28 changed files with 166 additions and 309 deletions

View file

@ -2,7 +2,6 @@
from .exception_mapping_utils import (
ANTHROPIC_ERROR_TYPE_MAP,
ANTHROPIC_STATUS_CODE_MAP,
AnthropicErrorSseFrame,
AnthropicExceptionMapping,
anthropic_error_sse_frame,
@ -15,7 +14,6 @@ from .exceptions import (
__all__ = [
"ANTHROPIC_ERROR_TYPE_MAP",
"ANTHROPIC_STATUS_CODE_MAP",
"AnthropicErrorDetail",
"AnthropicErrorResponse",
"AnthropicErrorSseFrame",

View file

@ -5,12 +5,7 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
"""
import json
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final, Literal
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import ReadOnly, TypedDict
from typing import Final
from pydantic import TypeAdapter
@ -33,17 +28,6 @@ ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
529: "overloaded_error",
}
ANTHROPIC_STATUS_CODE_MAP: Final = MappingProxyType({503: 529})
@with_config(ConfigDict(extra="allow"))
class _AnthropicShapedEnvelope(TypedDict):
type: ReadOnly[Literal["error"]]
error: ReadOnly[Mapping[str, object]]
_ANTHROPIC_SHAPED_ENVELOPE: Final = TypeAdapter(_AnthropicShapedEnvelope)
class AnthropicExceptionMapping:
"""
@ -52,16 +36,10 @@ class AnthropicExceptionMapping:
Similar pattern to ExceptionCheckers in litellm_core_utils/exception_mapping_utils.py
"""
@staticmethod
def get_status_code(status_code: int) -> int:
"""Map an HTTP status code to the one Anthropic's contract answers with: it has no 503, so an upstream that
cannot take the request right now is its 529 overloaded_error."""
return ANTHROPIC_STATUS_CODE_MAP.get(status_code, status_code)
@staticmethod
def get_error_type(status_code: int) -> AnthropicErrorType:
"""Map HTTP status code to Anthropic error type."""
return ANTHROPIC_ERROR_TYPE_MAP.get(AnthropicExceptionMapping.get_status_code(status_code), "api_error")
return ANTHROPIC_ERROR_TYPE_MAP.get(status_code, "api_error")
@staticmethod
def create_error_response(
@ -114,28 +92,22 @@ class AnthropicExceptionMapping:
return raw_message
@staticmethod
def _anthropic_error_envelope(parsed: dict | None) -> _AnthropicShapedEnvelope | None:
"""The parsed body when it already is an Anthropic error envelope, `{"type": "error", "error": {"type": ...,
"message": ...}}`, with every other field it carries kept."""
try:
envelope: Final = _ANTHROPIC_SHAPED_ENVELOPE.validate_python(parsed)
except ValidationError:
return None
return envelope if "type" in envelope["error"] and "message" in envelope["error"] else None
def _is_anthropic_error_dict(parsed: dict) -> bool:
"""
Check if a parsed dict is in Anthropic error format.
@staticmethod
def _passthrough(
envelope: _AnthropicShapedEnvelope, status_code: int, request_id: str | None
) -> AnthropicErrorResponse:
"""An already Anthropic-shaped body passes through with the request id it lacks and the error type a
remapped status names: a 503 envelope answers as 529, and a 529 answering `api_error` contradicts itself."""
request_id_field: Final[Mapping[str, str]] = (
{"request_id": request_id} if request_id and "request_id" not in envelope else {}
Anthropic error format:
{
"type": "error",
"error": {"type": "...", "message": "..."}
}
"""
return (
parsed.get("type") == "error"
and isinstance(parsed.get("error"), dict)
and "type" in parsed["error"]
and "message" in parsed["error"]
)
error_type_field: Final[Mapping[str, AnthropicErrorType]] = (
{"type": ANTHROPIC_ERROR_TYPE_MAP[status_code]} if status_code in ANTHROPIC_STATUS_CODE_MAP.values() else {}
)
return {**envelope, **request_id_field, "error": {**envelope["error"], **error_type_field}}
@staticmethod
def _extract_message_from_dict(parsed: dict, raw_message: str) -> str:
@ -163,8 +135,7 @@ class AnthropicExceptionMapping:
"""
Transform an error message to Anthropic format.
- If already in Anthropic format: passthrough, with the error type a remapped status names (a 503 answers
as 529, and 529 is overloaded_error)
- If already in Anthropic format: passthrough unchanged
- Otherwise: extract message and create Anthropic error
Parses JSON only once for efficiency.
@ -183,9 +154,11 @@ class AnthropicExceptionMapping:
parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None
# If parsed and already in Anthropic format - passthrough
envelope: Final = AnthropicExceptionMapping._anthropic_error_envelope(parsed)
if envelope is not None:
return AnthropicExceptionMapping._passthrough(envelope, status_code, request_id)
if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed):
# Optionally add request_id if provided and not present
if request_id and "request_id" not in parsed:
parsed["request_id"] = request_id
return parsed
# Extract message - use parsed dict if available, otherwise raw string
if parsed is not None:
@ -228,8 +201,7 @@ class AnthropicErrorSseFrame(str):
def anthropic_error_sse_frame(status_code: int, raw_message: str) -> AnthropicErrorSseFrame:
answered_status: Final = AnthropicExceptionMapping.get_status_code(status_code)
return AnthropicErrorSseFrame(
answered_status,
AnthropicExceptionMapping.transform_to_anthropic_error(status_code=answered_status, raw_message=raw_message),
status_code,
AnthropicExceptionMapping.transform_to_anthropic_error(status_code=status_code, raw_message=raw_message),
)

View file

@ -1,17 +1,10 @@
from typing import Final, Protocol, TypeVar, runtime_checkable
from typing import Final, TypeVar
from litellm.anthropic_interface.exceptions import AnthropicExceptionMapping
from litellm.exceptions import InternalServerError, MidStreamFallbackError
from litellm.proxy._types import ProxyException
_Raised = TypeVar("_Raised", bound=Exception)
@runtime_checkable
class _CarriesStatusCode(Protocol):
status_code: int
def _provider_status_code(exc: Exception) -> int | None:
unwrapped = exc # rebind-ok: the recursion gate requires iterative wrapper traversal
while isinstance(unwrapped, MidStreamFallbackError) and unwrapped.original_exception is not None:
@ -33,17 +26,5 @@ def _copy_carrying(exc: _Raised, attribute: str, value: int | str) -> _Raised:
def with_answered_status(exc: Exception) -> Exception:
"""A copy of the exception, same class, carrying the status the Anthropic routes answer with (a 503 and an
upstream Anthropic overload answer as 529), so the proxy-level recorders the failure hook runs (the spend log
row, the OTel server span, the failed-request metric, alerting) agree with the client. The raised object keeps
its own status, because the litellm layer already handed it to the deployment-level recorders (the deployment
failure metric, cooldown, retry policy, custom loggers), which may still read it."""
if isinstance(exc, ProxyException):
if not exc.code.isdecimal():
return exc
return _copy_carrying(exc, "code", str(AnthropicExceptionMapping.get_status_code(int(exc.code))))
if isinstance(exc, _CarriesStatusCode):
return _copy_carrying(
exc, "status_code", AnthropicExceptionMapping.get_status_code(_provider_status_code(exc) or exc.status_code)
)
return exc
provider_status_code: Final = _provider_status_code(exc)
return exc if provider_status_code is None else _copy_carrying(exc, "status_code", provider_status_code)

View file

@ -2991,7 +2991,6 @@ class ProxyBaseLLMRequestProcessing:
idle. See ``open_sse_before_first_byte``; unwrapped unless an operator sets
``litellm_settings.sse_keepalive_ping_interval_seconds``.
"""
answers_as_anthropic: Final = route_type == "anthropic_messages"
async def _audit_late_failure(exc: Exception) -> HTTPException | None:

View file

@ -321,15 +321,12 @@ def request_with_retry[T: RetryableResponse](
"""Bounded retry on statuses attributable to the PROVIDER, never the proxy.
The system under test is the proxy, so the transport may only absorb
statuses that mean an overloaded provider; today that is exactly 529, the
Anthropic overloaded_error (their own SDK retries it too). The Anthropic
routes also answer the proxy's own 503s as 529, so a test asserting one of
those wraps its call in without_retries(), and a proxy that keeps failing
still fails once the bounded retries run out. 500/502/503/504 stay
first-class failures: at this layer a 5xx from the proxy is
indistinguishable from one it relayed, and retrying them could mask an
intermittently failing proxy. Widen the set only for a status that means
an overloaded provider, with an observed flake in hand.
statuses the proxy itself cannot emit; today that is exactly 529, the
Anthropic overloaded_error passed through verbatim (their own SDK retries
it too). 500/502/503/504 stay first-class failures: at this layer a 5xx
from the proxy is indistinguishable from one it relayed, and retrying them
could mask an intermittently failing proxy. Widen the set only for a
status litellm provably never originates, with an observed flake in hand.
Also deliberately NOT retried: 429, because this suite asserts the proxy's
own rate-limit and budget 429s; network errors and timeouts, because a

View file

@ -126,6 +126,13 @@ def _assert_overloaded(response: httpx.Response, *, message: str) -> None:
assert "content_block_delta" not in response.text, response.text
def _assert_api_error(response: httpx.Response, *, status: int, message: str) -> None:
assert response.status_code == status, response.text
error: Final = _anthropic_error(response)
assert error["type"] == "api_error", response.text
assert message in string_value(error["message"]), response.text
def _assert_internal_server_error(response: httpx.Response) -> None:
assert response.status_code == 500, response.text
error: Final = _openai_error(response)
@ -193,17 +200,21 @@ async def test_messages_529_answers_overloaded_error_to_the_async_sdk(gateway: G
@pytest.mark.parametrize(
("upstream_status", "error_class"), ((529, "InternalServerError"), (503, "ServiceUnavailableError"))
("upstream_status", "error_class", "error_type"),
((529, "InternalServerError", "overloaded_error"), (503, "ServiceUnavailableError", "api_error")),
)
def test_messages_stream_rejected_before_its_first_byte_answers_529_json(
gateway: Gateway, upstream_status: int, error_class: str
def test_messages_stream_rejected_before_its_first_byte_answers_the_provider_status_as_json(
gateway: Gateway, upstream_status: int, error_class: str, error_type: str
) -> None:
with wire_server(_always(status_reply(upstream_status))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
_assert_overloaded(response, message=f"scripted {upstream_status}")
assert response.status_code == upstream_status, response.text
error: Final = _anthropic_error(response)
assert error["type"] == error_type, response.text
assert f"scripted {upstream_status}" in string_value(error["message"]), response.text
_one_post(wire, "/v1/messages")
_assert_logged_failure(model, rows=1, error_class=error_class)
_assert_logged_failure(model, rows=1, error_class=error_class, code=str(upstream_status))
def test_messages_stream_529_raises_529_in_the_sync_sdk(gateway: Gateway) -> None:
@ -266,7 +277,7 @@ def _one_bedrock_post(wire: Wire) -> None:
assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request
def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_an_overloaded_error_event(
def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_an_api_error_event(
gateway: Gateway,
) -> None:
with (
@ -281,10 +292,10 @@ def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_
(failure,) = (event for event in parse_sse(response.text) if event.data and event.event != "ping")
assert failure.event == "error", response.text
error: Final = object_value(failure.data["error"])
assert error["type"] == "overloaded_error", response.text
assert error["type"] == "api_error", response.text
assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text
_one_bedrock_post(wire)
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError")
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
def test_chat_completions_bedrock_503_after_the_keepalive_opened_the_response_keeps_the_openai_error_frame(
@ -403,7 +414,7 @@ def test_responses_529_answers_500(gateway: Gateway, shape: Shape) -> None:
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
def test_messages_bedrock_503_answers_529_overloaded_error(gateway: Gateway, shape: Shape) -> None:
def test_messages_bedrock_503_answers_503_api_error(gateway: Gateway, shape: Shape) -> None:
with wire_server(_always(Reply(status=503, body=_BEDROCK_503))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/{_BEDROCK_MODEL}", api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url
@ -411,10 +422,10 @@ def test_messages_bedrock_503_answers_529_overloaded_error(gateway: Gateway, sha
response: Final = gateway.request(
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
)
_assert_overloaded(response, message="Bedrock is unable to process your request.")
_assert_api_error(response, status=503, message="Bedrock is unable to process your request.")
(request,) = wire.drain()
assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError")
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
def _converse_event(event: str, payload: Mapping[str, JsonValue]) -> bytes:
@ -445,7 +456,7 @@ def _converse_deployment(scenario: Scenario, wire: Wire) -> str:
return scenario.model(model=_CONVERSE_MODEL, api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url)
def test_messages_bedrock_stream_503_after_text_ends_with_an_overloaded_error_event(gateway: Gateway) -> None:
def test_messages_bedrock_stream_503_after_text_ends_with_an_api_error_event(gateway: Gateway) -> None:
with wire_server(_always(_converse_503_after_text())) as wire, gateway.scenario() as scenario:
model: Final = _converse_deployment(scenario, wire)
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
@ -457,10 +468,10 @@ def test_messages_bedrock_stream_503_after_text_ends_with_an_overloaded_error_ev
assert _TEXT in response.text, response.text
(failure,) = (event for event in events if event_type(event) == "error")
error: Final = object_value(failure.data["error"])
assert error["type"] == "overloaded_error", response.text
assert error["type"] == "api_error", response.text
assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text
_one_post(wire, _CONVERSE_STREAM_TARGET)
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError")
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
def test_chat_completions_bedrock_stream_503_after_text_ends_with_a_503_error_frame(gateway: Gateway) -> None:
@ -481,7 +492,7 @@ def test_chat_completions_bedrock_stream_503_after_text_ends_with_a_503_error_fr
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
def test_chat_bridge_503_answers_529_overloaded_error_on_messages(gateway: Gateway, shape: Shape) -> None:
def test_chat_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None:
with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"hosted_vllm/{_CHAT_ONLY_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-vllm-key"
@ -489,12 +500,12 @@ def test_chat_bridge_503_answers_529_overloaded_error_on_messages(gateway: Gatew
response: Final = gateway.request(
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
)
_assert_overloaded(response, message="scripted 503")
_assert_api_error(response, status=503, message="scripted 503")
_one_post(wire, "/v1/chat/completions")
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
def test_responses_bridge_503_answers_529_overloaded_error_on_messages(gateway: Gateway, shape: Shape) -> None:
def test_responses_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None:
with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"openai/{_CODEX_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-openai-key"
@ -502,7 +513,7 @@ def test_responses_bridge_503_answers_529_overloaded_error_on_messages(gateway:
response: Final = gateway.request(
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
)
_assert_overloaded(response, message="scripted 503")
_assert_api_error(response, status=503, message="scripted 503")
_one_post(wire, "/v1/responses")
@ -561,14 +572,14 @@ def _assert_refused_without_an_upstream_call(
assert reached.isdisjoint(prompt for prompt, _ in answers), reached
def test_messages_to_a_paused_deployment_named_by_its_id_answer_529_overloaded_error(gateway: Gateway) -> None:
def test_messages_to_a_paused_deployment_named_by_its_id_answer_503_api_error(gateway: Gateway) -> None:
with _paused_deployment(gateway) as (wire, identity, name):
_assert_refused_without_an_upstream_call(
gateway, wire, identity, status=529, error_type="overloaded_error", message="is currently paused"
gateway, wire, identity, status=503, error_type="api_error", message="is currently paused"
)
rows: Final = _failure_rows_of((identity, name), _SETTLED_ANSWERS)
assert {(row["code"], row["error_class"], row["normalized_error"]) for row in rows} == {
("529", "ServiceUnavailableError", _OVERLOAD_CLASS)
("503", "ServiceUnavailableError", _OVERLOAD_CLASS)
}, rows

View file

@ -404,7 +404,7 @@ def test_retry_rejected_before_it_opens_counts_against_the_same_budget(gateway:
seconds=70,
)
rejected: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "request", budget=0))
_assert_rejected_before_content(rejected, wire, status=500, attempts=1, error="error")
_assert_rejected_before_content(rejected, wire, status=529, attempts=1, error="overloaded_error")
rejection_rows: Final = eventually(
lambda: _request_rows(gateway, rejected.headers["x-litellm-call-id"]),
lambda values: len(values) >= 1,

View file

@ -214,7 +214,7 @@ def _until(rig: _Rig, name: str, model: str) -> tuple[Sample, ...]:
("route", "model", "answered", "deployment_status", "deployment_class"),
(
pytest.param("/v1/messages", _MESSAGES_ANTHROPIC, 529, "500", "InternalServerError", id="messages-anthropic"),
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 529, "503", "ServiceUnavailableError", id="messages-bedrock"),
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, "503", "ServiceUnavailableError", id="messages-bedrock"),
pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 500, "500", "InternalServerError", id="chat-anthropic"),
),
)
@ -312,7 +312,7 @@ def _datadog_status_codes(rig: _Rig, alias: str, expected: int) -> tuple[tuple[s
("route", "model", "answered", "deployment_status"),
(
pytest.param("/v1/messages", _DATADOG_MESSAGES_ANTHROPIC, 529, "500", id="messages-anthropic"),
pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 529, "503", id="messages-bedrock"),
pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 503, "503", id="messages-bedrock"),
pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 500, "500", id="chat-anthropic"),
),
)
@ -332,7 +332,7 @@ def test_the_datadog_proxy_failure_log_carries_the_answered_status_and_the_llm_f
@pytest.mark.parametrize(
("route", "model", "answered"),
(
pytest.param("/v1/messages", _GUARDED_MESSAGES, 529, id="messages"),
pytest.param("/v1/messages", _GUARDED_MESSAGES, 503, id="messages"),
pytest.param("/v1/chat/completions", _GUARDED_CHAT, 503, id="chat"),
),
)
@ -360,7 +360,7 @@ def test_a_fail_closed_guardrail_outage_is_answered_and_recorded_with_the_routes
("route", "model", "answered"),
(
pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages-anthropic"),
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 529, id="messages-bedrock"),
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, id="messages-bedrock"),
pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat-anthropic"),
),
)

View file

@ -424,9 +424,7 @@ def _bedrock_logged_error(endpoint: BedrockEndpoint, failure: BedrockFailure) ->
shape: Final = _BEDROCK_FAILURES[failure]
if endpoint == "messages-invoke":
return "BedrockError", str(shape.bedrock_status)
if failure != "exception-event":
return shape.converse_class, str(shape.bedrock_status)
return shape.converse_class, "529" if endpoint == "messages" else "503"
return shape.converse_class, "503" if failure == "exception-event" else str(shape.bedrock_status)
def _bedrock_client_status(endpoint: BedrockEndpoint, failure: BedrockFailure, after_output: bool) -> int:
@ -553,7 +551,7 @@ def test_bedrock_non_stream_500_logs_amzn_request_id_on_every_endpoint(
endpoint, _bedrock_model(scenario, endpoint, wire.url), amzn_request_id, stream=False
)
response: Final = gateway.request("POST", path, body)
assert response.status_code == (503 if endpoint == "responses" else 529), response.text
assert response.status_code == 503, response.text
if endpoint == "responses":
assert response.headers.get("llm_provider-x-amzn-requestid") == amzn_request_id, dict(response.headers)
assert len(wire.drain()) == 1

View file

@ -476,7 +476,7 @@ def test_messages_count_tokens_surfaces_the_peer_status_when_the_peer_rejects_th
with _listening(deployment, _rejecting(status)) as peer:
body: Final = count_request(deployment.model, STRING_CASE)
response: Final = _count(gateway, body)
assert response.status_code == (529 if status == 503 else status), response.text
assert response.status_code == status, response.text
assert "input_tokens" not in response.text, response.text
assert _bodies(peer, deployment.provider) == (_STRING_BODY,)

View file

@ -235,7 +235,7 @@ async def test_upstream_aborts_then_an_outage_fail_each_call_once_and_the_native
refused: Final = await _burst(proxy, gateway.key, model, during_outage)
assert len(refused) == 4
for item in refused:
assert item.status == (529 if item.call.endpoint == "messages" else 503), item.text
assert item.status == 503, item.text
assert "Cannot connect to host" in item.text, item.text
assert await asyncio.to_thread(
eventually, lambda: _unhealthy_count(gateway, model), lambda count: count == 1, 30

View file

@ -1172,7 +1172,7 @@ def test_messages_count_tokens_surfaces_the_mantle_status_when_mantle_errors(
):
model: Final = _fallback_deployment(scenario, runtime.url)
response: Final = _count(gateway, _full(model))
assert response.status_code == (529 if status == 503 else status), response.text
assert response.status_code == status, response.text
assert "input_tokens" not in response.text, response.text
assert len(_runtime_count_targets(runtime)) == 1
assert _mantle_bodies(mantle) == (_MANTLE_FULL,)

View file

@ -329,8 +329,10 @@ async def test_peer_killed_mid_burst_fails_only_the_held_calls_and_a_restarted_p
assert (len(succeeded), len(failed)) == (6, 6), [(item.call.marker, item.status) for item in served]
for item in succeeded:
_assert_answered_with_its_own_marker(item)
assert {item.status for item in failed} == {503}, [
(item.call.endpoint, item.call.stream, item.status, item.text) for item in failed
]
for item in failed:
assert item.status == (529 if item.call.endpoint == "messages" else 503), (item.call, item.status, item.text)
assert "ServiceUnavailableError: BedrockException - Server disconnected" in item.text, item.text
assert "marker-" not in item.text and item.call_id is not None, item.text
with _child_peer(deployment.peer_port, answer_first=10**6) as revived:

View file

@ -1261,8 +1261,7 @@ def test_databricks_json_schema_burst_through_an_upstream_outage_keeps_every_ref
for request in received:
_assert_burst_outbound(request)
expected: Final = tuple(
(529 if endpoint == "messages" else 503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON)
for endpoint, _stream, down in plan
(503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON) for _endpoint, _stream, down in plan
)
assert tuple(_outcome(*result) for result in results) == expected
assert tuple(_outcome(*result) for result in recovered) == ((200, _EXPECTED_JSON),) * len(recovery)
@ -1572,8 +1571,7 @@ def test_databricks_one_block_burst_through_an_upstream_outage_reaches_upstream_
json.dumps([{"role": "user", "content": prompt}]) for prompt in prompts
)
expected: Final = tuple(
(529 if endpoint == "messages" else 503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON)
for endpoint, _stream, down in plan
(503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON) for _endpoint, _stream, down in plan
)
assert tuple(_outcome(*result) for result in results) == expected

View file

@ -1215,8 +1215,7 @@ async def test_upstream_outage_mid_burst_fails_only_its_calls_and_every_replay_l
assert len(served) == len(calls)
for item in served:
if item.call.marker in down:
assert item.status == (529 if item.call.endpoint == "messages" else 503), item.text
assert _BRIDGE_OUTAGE in item.text, item.text
assert item.status == 503 and _BRIDGE_OUTAGE in item.text, item.text
assert rv.answer(item.call.marker) not in item.text, item.text
else:
_bridge_assert_answered(item)

View file

@ -659,7 +659,7 @@ async def _burst(gateway: Gateway, model: str, calls: tuple[_Call, ...]) -> tupl
def _assert_answered(served: _Served, down: frozenset[str]) -> None:
if served.call.marker in down:
assert served.status == (529 if served.call.endpoint == "messages" else 503), (served.call, served.text)
assert served.status == 503, (served.call, served.text)
assert "synthetic container outage" in served.text, (served.call, served.text)
return
assert served.status == 200, (served.call, served.text)

View file

@ -980,7 +980,7 @@ def test_concurrent_count_waves_recover_after_vertex_count_outage(gateway: Gatew
body: Final = _messages(model, None, None)
assert _count_wave(clients, body) == ((200, _COUNT), (200, _COUNT), (200, _COUNT))
peer.set_status(503)
assert _count_wave(clients, body) == ((529, None), (529, None), (529, None))
assert _count_wave(clients, body) == ((503, None), (503, None), (503, None))
peer.set_status(200)
assert _count_wave(clients, body) == ((200, _COUNT), (200, _COUNT), (200, _COUNT))
requests: Final = _count_requests(wire)

View file

@ -622,7 +622,7 @@ def test_messages_count_tokens_surfaces_the_peer_error_when_peer_errors(gateway:
with wire_server(_peer(_rejecting(status))) as wire, gateway.scenario() as scenario:
model: Final = _deployment(gateway, scenario, wire.url)
response: Final = _count(gateway, _full(model))
assert response.status_code == (529 if status == 503 else status), response.text
assert response.status_code == status, response.text
assert "input_tokens" not in response.text, response.text
assert _counted_bodies(wire) == (_PEER_FULL,)

View file

@ -115,8 +115,7 @@ def test_vertex_pass_through_stream_relays_an_upstream_503_as_503(vertex: _Verte
response: Final = vertex.stream(_UNAVAILABLE_MODEL, uuid.uuid4().hex)
assert response.status_code == 503, response.text
assert _UNAVAILABLE_MESSAGE in response.text, response.text
assert "overloaded_error" not in response.text, response.text
assert response.content == _UNAVAILABLE_BODY, response.text
assert [request.target for request in vertex.wire.drain()] == [
f"/v1/{_model_path(_UNAVAILABLE_MODEL)}:streamGenerateContent?alt=sse"
]

View file

@ -3,7 +3,6 @@ import os
import uuid
from collections.abc import Iterator
from dataclasses import dataclass
from itertools import chain
from pathlib import Path
from typing import Final
from urllib.parse import unquote
@ -25,12 +24,10 @@ from integration._support.anthropic_sse import (
from integration._support.client import (
Gateway,
JsonValue,
eventually,
gateway_from_environment,
object_value,
string_value,
)
from integration._support.database import read_rows
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.wire import Reply, Request, Wire, wire_server
@ -53,8 +50,6 @@ _GATEWAY_GROUPS: Final = {
"stream": "audit-gateway-overloaded-stream",
}
_COUNT_STATUSES: Final = {_COUNTED: 503, _COUNTED_OVERLOADED: 529, _COUNTED_RATE_LIMITED: 429}
_ADAPTIVE_UPSTREAMS: Final = {503: "claude-adaptive-unavailable"}
_ADAPTIVE_FLUSH_SECONDS: Final = 40
_COUNT_TOKENS_TARGET: Final = "/v1/messages/count_tokens"
_UPSTREAM_URL_PLACEHOLDER: Final = "upstream-url"
@ -73,8 +68,6 @@ def _respond(request: Request) -> Reply:
return Reply(status=status, body=error_body(status, f"scripted {status}"))
if backend == _OVERLOADED:
return status_reply(529)
if backend in _ADAPTIVE_UPSTREAMS.values():
return status_reply(next(status for status, name in _ADAPTIVE_UPSTREAMS.items() if name == backend))
if backend == _HEALTHY:
return stream_reply(message_stream(_served_id(user_prompt(body)), backend, _TEXT))
return Reply(status=404, body=b'{"error": "no scripted backend"}')
@ -84,36 +77,6 @@ def _deployment(name: str, model: str, api_base: str, api_key: str, **params: Js
return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": api_key, **params}}
def _adaptive_router(status: int) -> str:
return f"audit-adaptive-router-{status}"
def _adaptive_backend(status: int) -> str:
return f"audit-adaptive-backend-{status}"
def _adaptive_deployments(status: int) -> tuple[dict[str, JsonValue], ...]:
return (
{
"model_name": _adaptive_router(status),
"litellm_params": {
"model": "auto_router/adaptive_router",
"adaptive_router_config": {"available_models": [_adaptive_backend(status)]},
},
},
{
**_deployment(
_adaptive_backend(status),
f"anthropic/{_ADAPTIVE_UPSTREAMS[status]}",
_UPSTREAM_URL_PLACEHOLDER,
_API_KEY,
input_cost_per_token=0.000001,
),
"model_info": {"adaptive_router_preferences": {"quality_tier": 2, "strengths": []}},
},
)
def _config(wire: Wire, directory: Path) -> Path:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
config["model_list"] = [
@ -134,7 +97,6 @@ def _config(wire: Wire, directory: Path) -> Path:
_deployment(group, f"anthropic/{_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY)
for group in _GATEWAY_GROUPS.values()
),
*chain.from_iterable(_adaptive_deployments(status) for status in _ADAPTIVE_UPSTREAMS),
]
config["router_settings"] = {
"num_retries": 0,
@ -224,22 +186,20 @@ def test_an_overloaded_deployment_is_cooled_down_by_the_internal_server_error_al
assert rig.backends_called() == (_HEALTHY,)
def test_count_tokens_disabled_without_a_provider_counter_answers_529_overloaded_error(rig: _Rig) -> None:
def test_count_tokens_disabled_without_a_provider_counter_answers_503_api_error(rig: _Rig) -> None:
response: Final = rig.count_tokens(_COUNT_DISABLED_GROUP)
error: Final = _count_tokens_error(response, 529)
assert error["type"] == "overloaded_error", response.text
error: Final = _count_tokens_error(response, 503)
assert error["type"] == "api_error", response.text
assert "Token counting is disabled" in string_value(error["message"]), response.text
assert rig.targets_called() == ()
def test_count_tokens_whose_upstream_answers_503_answers_529_overloaded_error(rig: _Rig) -> None:
def test_count_tokens_whose_upstream_answers_503_stays_503_api_error(rig: _Rig) -> None:
response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP)
error: Final = _count_tokens_error(response, 529)
assert error["type"] == "overloaded_error", response.text
error: Final = _count_tokens_error(response, 503)
assert error["type"] == "api_error", response.text
assert "scripted 503" in string_value(error["message"]), response.text
assert rig.targets_called() == (_COUNT_TOKENS_TARGET,)
@pytest.mark.parametrize(
("group", "status", "error_type"),
(
@ -279,46 +239,9 @@ def test_the_claude_code_gateway_messages_route_answers_an_upstream_529_as_overl
assert rig.backends_called() == (_OVERLOADED,)
def test_the_claude_code_gateway_count_tokens_route_answers_an_upstream_503_as_529(rig: _Rig) -> None:
def test_the_claude_code_gateway_count_tokens_route_answers_an_upstream_503_as_503(rig: _Rig) -> None:
response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP, prefix=_GATEWAY)
error: Final = _count_tokens_error(response, 529)
assert error["type"] == "overloaded_error", response.text
error: Final = _count_tokens_error(response, 503)
assert error["type"] == "api_error", response.text
assert "scripted 503" in string_value(error["message"]), response.text
assert rig.targets_called() == (_COUNT_TOKENS_TARGET,)
def _adaptive_session_rows(session: str) -> list[dict[str, JsonValue]]:
return read_rows(
'SELECT model_name, exhaustion_count FROM "LiteLLM_AdaptiveRouterSession" WHERE session_id = %s', (session,)
)
@pytest.mark.parametrize(
("status", "answered"),
(
pytest.param(503, 503, id="upstream-503"),
),
)
def test_an_adaptive_router_session_counts_an_upstream_overload_as_exhaustion(
rig: _Rig, status: int, answered: int
) -> None:
session: Final = f"audit-adaptive-{uuid.uuid4().hex}"
response: Final = rig.proxy.request(
"POST",
"/v1/chat/completions",
{
"model": _adaptive_router(status),
"max_tokens": 16,
"messages": [{"role": "user", "content": session}],
},
headers={"x-litellm-session-id": session},
)
assert response.status_code == answered, response.text
assert rig.backends_called() == (_ADAPTIVE_UPSTREAMS[status],)
rows: Final = eventually(
lambda: _adaptive_session_rows(session),
lambda rows: len(rows) > 0,
seconds=_ADAPTIVE_FLUSH_SECONDS,
return_last_on_timeout=True,
)
assert rows == [{"model_name": _adaptive_backend(status), "exhaustion_count": 1}], response.text

View file

@ -55,7 +55,7 @@ Behavior = Literal[
"hold-ping",
"drop-then-too-long",
"drop-then-401",
"overloaded-frames-fallback-503",
"overloaded-frames-fallback-529",
"overloaded-frames",
"overloaded-once",
]
@ -90,7 +90,7 @@ def _primary_reply(
return Reply(status=400, body=error_body(400, "prompt is too long: 250000 tokens > 200000 maximum"))
case "drop-then-401":
return dropped if attempt == 1 else status_reply(401)
case "overloaded-frames-fallback-503" | "overloaded-frames":
case "overloaded-frames-fallback-529" | "overloaded-frames":
return stream_reply((error_frame(529, "scripted overloaded"),))
case "overloaded-once":
return status_reply(529) if attempt == 1 else stream_reply(full)
@ -98,7 +98,7 @@ def _primary_reply(
def _fallback_reply(behavior: str, full: tuple[bytes, bytes, bytes, bytes]) -> Reply:
if behavior == "overloaded-frames-fallback-503":
if behavior == "overloaded-frames-fallback-529":
return status_reply(503)
return stream_reply(full)
@ -309,7 +309,7 @@ def test_a_retry_rejected_with_401_ends_the_retries_and_reaches_the_client_uncha
def test_overloaded_frames_whose_fallback_fails_answer_529_overloaded_error(rig: _Rig) -> None:
marker: Final = uuid.uuid4().hex
response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-503", marker)
response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-529", marker)
assert response.status_code == 529, response.text
assert "overloaded_error" in response.text, response.text
assert "content_block_delta" not in response.text, response.text

View file

@ -67,7 +67,7 @@ BLOCKED: Final = "Request blocked by Noma guardrail"
SLOW_SECONDS: Final = 1.5
SAMPLED_SCANS: Final = range(2, 7)
BUFFERED_BLOCK_STATUS: Final[Mapping[Endpoint, int]] = {"chat": 400, "messages": 400, "responses": 200}
BUFFERED_OUTAGE_STATUS: Final[Mapping[Endpoint, int]] = {"chat": 503, "messages": 529, "responses": 200}
BUFFERED_OUTAGE_STATUS: Final[Mapping[Endpoint, int]] = {"chat": 503, "messages": 503, "responses": 200}
PIECES: Final = (
" Contact",
" me",

View file

@ -56,17 +56,6 @@ class TestCreateErrorResponse:
)
assert response["error"]["type"] == "api_error"
def test_503_answers_as_anthropics_529_overloaded_error(self):
"""LIT-9450: Anthropic's contract has no 503, so an upstream that cannot take the request right now
(a Bedrock capacity 503) answers as 529 overloaded_error, the status Claude Code's overload path keys on."""
assert AnthropicExceptionMapping.get_status_code(503) == 529
response = AnthropicExceptionMapping.create_error_response(503, "Bedrock is unable to process your request.")
assert response["error"]["type"] == "overloaded_error"
@pytest.mark.parametrize("status_code", [400, 429, 500, 529])
def test_other_statuses_keep_their_code(self, status_code: int):
assert AnthropicExceptionMapping.get_status_code(status_code) == status_code
def test_with_request_id(self):
"""Test request_id is included when provided."""
response = AnthropicExceptionMapping.create_error_response(
@ -179,21 +168,6 @@ class TestTransformToAnthropicError:
)
assert result["request_id"] == "req_123"
def test_an_anthropic_shaped_503_body_answers_529_with_the_overloaded_type(self):
"""A 503 whose message is already an Anthropic envelope keeps its message but takes the error type the
529 it answers as names, so the status and the body agree."""
raw = json.dumps({"type": "error", "error": {"type": "api_error", "message": "Service unavailable"}})
result = AnthropicExceptionMapping.transform_to_anthropic_error(
status_code=AnthropicExceptionMapping.get_status_code(503),
raw_message=raw,
request_id="req_529",
)
assert result == {
"type": "error",
"error": {"type": "overloaded_error", "message": "Service unavailable"},
"request_id": "req_529",
}
def test_translates_bedrock_error(self):
"""Test that Bedrock errors are translated to Anthropic format."""
bedrock_error = json.dumps({"detail": {"message": "Access denied"}})
@ -250,7 +224,7 @@ class TestTransformToAnthropicError:
class TestAnthropicErrorSseFrame:
@pytest.mark.parametrize(
("status_code", "expected_error_type"),
[(429, "rate_limit_error"), (503, "overloaded_error"), (400, "invalid_request_error")],
[(429, "rate_limit_error"), (503, "api_error"), (400, "invalid_request_error")],
)
def test_the_frame_is_one_error_event_carrying_the_anthropic_envelope(
self, status_code: int, expected_error_type: str
@ -265,22 +239,11 @@ class TestAnthropicErrorSseFrame:
"error": {"type": expected_error_type, "message": "upstream unavailable"},
}
def test_a_503_envelope_frame_takes_the_overloaded_type_of_the_529_it_answers(self) -> None:
envelope: Final = json.dumps({"type": "error", "error": {"type": "api_error", "message": "Service unavailable"}})
frame: Final = anthropic_error_sse_frame(status_code=503, raw_message=envelope)
assert frame.status_code == 529
assert frame.json_body(call_id=None) == {
"type": "error",
"error": {"type": "overloaded_error", "message": "Service unavailable"},
}
def test_the_frame_remembers_the_status_anthropic_answers_with_and_the_body(self) -> None:
def test_the_frame_remembers_the_status_and_body_it_was_built_from(self) -> None:
frame: Final = anthropic_error_sse_frame(status_code=503, raw_message="upstream unavailable")
assert isinstance(frame, AnthropicErrorSseFrame)
assert frame.status_code == 529
assert frame.status_code == 503
data_line: Final = frame.split("\n")[1]
assert data_line == f"data: {json.dumps(frame.json_body(call_id=None))}"
@ -289,9 +252,9 @@ class TestAnthropicErrorSseFrame:
assert frame.json_body(call_id="call-1") == {
"type": "error",
"error": {"type": "overloaded_error", "message": "upstream unavailable", "litellm_call_id": "call-1"},
"error": {"type": "api_error", "message": "upstream unavailable", "litellm_call_id": "call-1"},
}
assert frame.json_body(call_id=None) == {
"type": "error",
"error": {"type": "overloaded_error", "message": "upstream unavailable"},
"error": {"type": "api_error", "message": "upstream unavailable"},
}

View file

@ -1438,8 +1438,6 @@ def test_an_already_mapped_litellm_exception_passes_through_untouched(provider,
['{"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}', "upstream failure"],
)
def test_an_anthropic_529_stays_an_internal_server_error_that_carries_the_529(message, quiet_exception_mapping):
"""An Anthropic overload keeps the InternalServerError class and 500 status that retry and allowed-fails
policies and SDK callers key on, and carries the provider's 529 for the Anthropic-shaped routes to answer."""
with pytest.raises(litellm.InternalServerError) as raised:
exception_type(
model="claude-opus-5-5",

View file

@ -561,7 +561,7 @@ class TestUpstreamFailureEndsStreamWithErrorEvent:
[
(
MidStreamFallbackError(message="boom", model="m", llm_provider="openai"),
{"type": "overloaded_error", "message": "litellm.MidStreamFallbackError: boom"},
{"type": "api_error", "message": "litellm.MidStreamFallbackError: boom"},
),
(
type("StringStatusError", (Exception,), {"status_code": "429"})("throttled"),

View file

@ -16,27 +16,28 @@ def _anthropic_internal_server_error(provider_status_code: int | None) -> litell
)
@pytest.mark.parametrize(("code", "answered"), [(503, "529"), (400, "400"), (500, "500")])
def test_a_proxy_exception_answers_its_503_as_529_and_keeps_every_other_code(code: int, answered: str) -> None:
@pytest.mark.parametrize("code", [400, 500, 503])
def test_a_proxy_exception_passes_through_unchanged(code: int) -> None:
exc = ProxyException(message="budget service unavailable", type="None", param=None, code=code)
answered_exc = with_answered_status(exc)
assert (type(answered_exc), answered_exc.code, answered_exc.message) == (ProxyException, answered, exc.message)
assert answered_exc is exc
assert exc.code == str(code)
@pytest.mark.parametrize(
("exc", "answered"),
("exc", "answered", "copy"),
[
pytest.param(
litellm.ServiceUnavailableError(message="BedrockException", llm_provider="bedrock", model="m"),
529,
503,
False,
id="service-unavailable-503",
),
pytest.param(_anthropic_internal_server_error(529), 529, id="anthropic-overload"),
pytest.param(_anthropic_internal_server_error(500), 500, id="anthropic-500"),
pytest.param(_anthropic_internal_server_error(None), 500, id="internal-server-error"),
pytest.param(_anthropic_internal_server_error(529), 529, True, id="anthropic-overload"),
pytest.param(_anthropic_internal_server_error(500), 500, True, id="anthropic-500"),
pytest.param(_anthropic_internal_server_error(None), 500, False, id="internal-server-error"),
pytest.param(
MidStreamFallbackError(
message="fallbacks failed",
@ -45,6 +46,7 @@ def test_a_proxy_exception_answers_its_503_as_529_and_keeps_every_other_code(cod
original_exception=_anthropic_internal_server_error(529),
),
529,
True,
id="fallback-over-an-anthropic-overload",
),
pytest.param(
@ -60,20 +62,25 @@ def test_a_proxy_exception_answers_its_503_as_529_and_keeps_every_other_code(cod
),
),
529,
True,
id="nested-fallback-over-an-anthropic-overload",
),
pytest.param(
litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="m"), 429, id="rate-limit"
litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="m"),
429,
False,
id="rate-limit",
),
],
)
def test_a_litellm_exception_answers_the_status_anthropic_gives_its_failure(
exc: openai.APIStatusError, answered: int
exc: openai.APIStatusError, answered: int, copy: bool
) -> None:
raised_status = exc.status_code
answered_exc = with_answered_status(exc)
assert (answered_exc is not exc) is copy
assert (type(answered_exc), answered_exc.status_code, str(answered_exc)) == (type(exc), answered, str(exc))
assert exc.status_code == raised_status
@ -82,13 +89,16 @@ def _raise_a_bedrock_overload_caused_by_a_reset() -> None:
try:
raise ConnectionError("upstream reset")
except ConnectionError as upstream:
raise litellm.ServiceUnavailableError(
message="BedrockException", llm_provider="bedrock", model="m"
raise litellm.InternalServerError(
message="AnthropicError - Overloaded",
llm_provider="anthropic",
model="m",
provider_status_code=529,
) from upstream
def test_the_answered_copy_keeps_the_raised_exceptions_cause_and_traceback() -> None:
with pytest.raises(litellm.ServiceUnavailableError) as raised:
with pytest.raises(litellm.InternalServerError) as raised:
_raise_a_bedrock_overload_caused_by_a_reset()
answered_exc = with_answered_status(raised.value)

View file

@ -303,7 +303,7 @@ class TestProxyExceptionAnthropicEnvelope:
import litellm.proxy.anthropic_endpoints.endpoints as ep
from litellm.proxy._types import ProxyException
request = MagicMock()
request: Final = MagicMock()
request.headers = {}
response = ep._anthropic_error_json_response(
@ -361,7 +361,7 @@ class TestProxyExceptionAnthropicEnvelope:
from litellm.proxy._types import UserAPIKeyAuth
raised_status: Final = unavailable.status_code
request = MagicMock()
request: Final = MagicMock()
request.headers = {}
with (
@ -380,7 +380,7 @@ class TestProxyExceptionAnthropicEnvelope:
user_api_key_dict=UserAPIKeyAuth(),
)
recorded = mock_logging.post_call_failure_hook.await_args.kwargs["original_exception"]
recorded: Final = mock_logging.post_call_failure_hook.await_args.kwargs["original_exception"]
assert recorded is not None
assert (recorded is not unavailable) is hook_receives_copy
assert recorded.status_code == expected_status
@ -438,9 +438,9 @@ class TestProxyExceptionAnthropicEnvelope:
proxy_server, "count_request_tokens", new=AsyncMock(side_effect=unavailable)
),
):
response = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
body = json.loads(response.body)
body: Final = json.loads(response.body)
assert response.status_code == expected_status
assert body["type"] == "error"
assert body["error"]["type"] == expected_error_type
@ -553,7 +553,7 @@ class TestHttpExceptionDictDetail:
) as mock_logging, # test-quality-ok: module global imported at call time; no injection seam
):
mock_logging.post_call_failure_hook = AsyncMock()
response = await ep.anthropic_response(
response: Final = await ep.anthropic_response(
fastapi_response=MagicMock(),
request=request,
user_api_key_dict=UserAPIKeyAuth(),

View file

@ -2810,8 +2810,7 @@ class TestCommonRequestProcessingHelpers:
async def test_a_stream_that_fails_before_its_first_byte_answers_as_an_anthropic_json_error(self):
"""A /v1/messages stream whose first chunk is already the error frame has nothing
streamed yet, so the failure answers as JSON with the status Anthropic answers that upstream failure with
(a 503 is its 529 overloaded_error, LIT-9450),
streamed yet, so the failure answers as JSON with the status the upstream gave,
the shape Anthropic clients raise their status-specific errors on"""
async def stream():
@ -2822,12 +2821,12 @@ class TestCommonRequestProcessingHelpers:
response = await create_response(generator, "text/event-stream", {"x-litellm-call-id": "call-8609"})
assert isinstance(response, JSONResponse)
assert response.status_code == 529
assert response.status_code == 503
assert response.headers["content-type"] == "application/json"
assert response.headers["x-litellm-call-id"] == "call-8609"
assert json.loads(response.body) == {
"type": "error",
"error": {"type": "overloaded_error", "message": "upstream unavailable"},
"error": {"type": "api_error", "message": "upstream unavailable"},
}
assert generator.ag_frame is None
@ -2863,7 +2862,7 @@ class TestCommonRequestProcessingHelpers:
assert response.status_code == 200
assert "".join(await self.consume_stream(response)) == (
ANTHROPIC_PING_SSE_CHUNK
+ 'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "upstream unavailable"}}\n\n'
+ 'event: error\ndata: {"type": "error", "error": {"type": "api_error", "message": "upstream unavailable"}}\n\n'
)
async def test_create_streaming_response_disables_proxy_buffering(self):
@ -10981,22 +10980,27 @@ async def test_base_process_llm_request_audits_a_failure_that_lands_after_its_ke
@pytest.mark.asyncio
async def test_anthropic_messages_answers_a_late_overload_after_its_keepalive_as_an_overloaded_error_event():
async def slow_overload(self, **kwargs):
await asyncio.sleep(0.25)
raise litellm.ServiceUnavailableError(
message='BedrockException - {"message": "Bedrock is unable to process your request."}',
llm_provider="bedrock",
model="global.anthropic.claude-opus-5-5",
)
async def test_anthropic_messages_answers_a_late_503_after_its_keepalive_as_an_api_error_event() -> None:
unavailable: Final = litellm.ServiceUnavailableError(
message='BedrockException - {"message": "Bedrock is unable to process your request."}',
llm_provider="bedrock",
model="global.anthropic.claude-opus-5-5",
)
release_failure: Final = asyncio.Event()
proxy_logging_obj = MagicMock(spec=ProxyLogging)
async def fail_after_keepalive(self: ProxyBaseLLMRequestProcessing, **kwargs: object) -> None:
await release_failure.wait()
raise unavailable
proxy_logging_obj: Final = MagicMock(spec=ProxyLogging)
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
processor = ProxyBaseLLMRequestProcessing(data={"model": "global.anthropic.claude-opus-5-5", "stream": True})
processor: Final = ProxyBaseLLMRequestProcessing(
data={"model": "global.anthropic.claude-opus-5-5", "stream": True}
)
with patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05):
with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", slow_overload):
response = await processor.base_process_llm_request(
with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", fail_after_keepalive):
response: Final = await processor.base_process_llm_request(
request=MagicMock(spec=Request),
fastapi_response=Response(),
user_api_key_dict=MagicMock(spec=UserAPIKeyAuth),
@ -11005,15 +11009,20 @@ async def test_anthropic_messages_answers_a_late_overload_after_its_keepalive_as
general_settings={},
proxy_config=MagicMock(spec=ProxyConfig),
)
collected = await _drain(response)
body_iterator: Final = response.body_iterator
first_chunk: Final = await body_iterator.__anext__()
release_failure.set()
remaining_chunks: Final = [
chunk.encode("utf-8") if isinstance(chunk, str) else bytes(chunk) async for chunk in body_iterator
]
assert proxy_logging_obj.post_call_failure_hook.await_args.kwargs["original_exception"].status_code == 529
assert collected[0] == TTFT_PING
event_line, data_line = collected[-1].decode().strip().split("\n")
assert proxy_logging_obj.post_call_failure_hook.await_args.kwargs["original_exception"] is unavailable
assert first_chunk == TTFT_PING
event_line, data_line = remaining_chunks[-1].decode().strip().split("\n")
assert event_line == "event: error"
error = json.loads(data_line.removeprefix("data: "))
error: Final = json.loads(data_line.removeprefix("data: "))
assert error["type"] == "error"
assert error["error"]["type"] == "overloaded_error"
assert error["error"]["type"] == "api_error"
assert "Bedrock is unable to process your request." in error["error"]["message"]