mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(anthropic): preserve 503s and answer upstream 529s on messages routes
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
dabaa6d4ed
commit
d89164512a
28 changed files with 166 additions and 309 deletions
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
from .exception_mapping_utils import (
|
||||
ANTHROPIC_ERROR_TYPE_MAP,
|
||||
ANTHROPIC_STATUS_CODE_MAP,
|
||||
AnthropicErrorSseFrame,
|
||||
AnthropicExceptionMapping,
|
||||
anthropic_error_sse_frame,
|
||||
|
|
@ -15,7 +14,6 @@ from .exceptions import (
|
|||
|
||||
__all__ = [
|
||||
"ANTHROPIC_ERROR_TYPE_MAP",
|
||||
"ANTHROPIC_STATUS_CODE_MAP",
|
||||
"AnthropicErrorDetail",
|
||||
"AnthropicErrorResponse",
|
||||
"AnthropicErrorSseFrame",
|
||||
|
|
|
|||
|
|
@ -5,12 +5,7 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
|
|
@ -33,17 +28,6 @@ ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
|
|||
529: "overloaded_error",
|
||||
}
|
||||
|
||||
ANTHROPIC_STATUS_CODE_MAP: Final = MappingProxyType({503: 529})
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow"))
|
||||
class _AnthropicShapedEnvelope(TypedDict):
|
||||
type: ReadOnly[Literal["error"]]
|
||||
error: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
_ANTHROPIC_SHAPED_ENVELOPE: Final = TypeAdapter(_AnthropicShapedEnvelope)
|
||||
|
||||
|
||||
class AnthropicExceptionMapping:
|
||||
"""
|
||||
|
|
@ -52,16 +36,10 @@ class AnthropicExceptionMapping:
|
|||
Similar pattern to ExceptionCheckers in litellm_core_utils/exception_mapping_utils.py
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_status_code(status_code: int) -> int:
|
||||
"""Map an HTTP status code to the one Anthropic's contract answers with: it has no 503, so an upstream that
|
||||
cannot take the request right now is its 529 overloaded_error."""
|
||||
return ANTHROPIC_STATUS_CODE_MAP.get(status_code, status_code)
|
||||
|
||||
@staticmethod
|
||||
def get_error_type(status_code: int) -> AnthropicErrorType:
|
||||
"""Map HTTP status code to Anthropic error type."""
|
||||
return ANTHROPIC_ERROR_TYPE_MAP.get(AnthropicExceptionMapping.get_status_code(status_code), "api_error")
|
||||
return ANTHROPIC_ERROR_TYPE_MAP.get(status_code, "api_error")
|
||||
|
||||
@staticmethod
|
||||
def create_error_response(
|
||||
|
|
@ -114,28 +92,22 @@ class AnthropicExceptionMapping:
|
|||
return raw_message
|
||||
|
||||
@staticmethod
|
||||
def _anthropic_error_envelope(parsed: dict | None) -> _AnthropicShapedEnvelope | None:
|
||||
"""The parsed body when it already is an Anthropic error envelope, `{"type": "error", "error": {"type": ...,
|
||||
"message": ...}}`, with every other field it carries kept."""
|
||||
try:
|
||||
envelope: Final = _ANTHROPIC_SHAPED_ENVELOPE.validate_python(parsed)
|
||||
except ValidationError:
|
||||
return None
|
||||
return envelope if "type" in envelope["error"] and "message" in envelope["error"] else None
|
||||
def _is_anthropic_error_dict(parsed: dict) -> bool:
|
||||
"""
|
||||
Check if a parsed dict is in Anthropic error format.
|
||||
|
||||
@staticmethod
|
||||
def _passthrough(
|
||||
envelope: _AnthropicShapedEnvelope, status_code: int, request_id: str | None
|
||||
) -> AnthropicErrorResponse:
|
||||
"""An already Anthropic-shaped body passes through with the request id it lacks and the error type a
|
||||
remapped status names: a 503 envelope answers as 529, and a 529 answering `api_error` contradicts itself."""
|
||||
request_id_field: Final[Mapping[str, str]] = (
|
||||
{"request_id": request_id} if request_id and "request_id" not in envelope else {}
|
||||
Anthropic error format:
|
||||
{
|
||||
"type": "error",
|
||||
"error": {"type": "...", "message": "..."}
|
||||
}
|
||||
"""
|
||||
return (
|
||||
parsed.get("type") == "error"
|
||||
and isinstance(parsed.get("error"), dict)
|
||||
and "type" in parsed["error"]
|
||||
and "message" in parsed["error"]
|
||||
)
|
||||
error_type_field: Final[Mapping[str, AnthropicErrorType]] = (
|
||||
{"type": ANTHROPIC_ERROR_TYPE_MAP[status_code]} if status_code in ANTHROPIC_STATUS_CODE_MAP.values() else {}
|
||||
)
|
||||
return {**envelope, **request_id_field, "error": {**envelope["error"], **error_type_field}}
|
||||
|
||||
@staticmethod
|
||||
def _extract_message_from_dict(parsed: dict, raw_message: str) -> str:
|
||||
|
|
@ -163,8 +135,7 @@ class AnthropicExceptionMapping:
|
|||
"""
|
||||
Transform an error message to Anthropic format.
|
||||
|
||||
- If already in Anthropic format: passthrough, with the error type a remapped status names (a 503 answers
|
||||
as 529, and 529 is overloaded_error)
|
||||
- If already in Anthropic format: passthrough unchanged
|
||||
- Otherwise: extract message and create Anthropic error
|
||||
|
||||
Parses JSON only once for efficiency.
|
||||
|
|
@ -183,9 +154,11 @@ class AnthropicExceptionMapping:
|
|||
parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None
|
||||
|
||||
# If parsed and already in Anthropic format - passthrough
|
||||
envelope: Final = AnthropicExceptionMapping._anthropic_error_envelope(parsed)
|
||||
if envelope is not None:
|
||||
return AnthropicExceptionMapping._passthrough(envelope, status_code, request_id)
|
||||
if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed):
|
||||
# Optionally add request_id if provided and not present
|
||||
if request_id and "request_id" not in parsed:
|
||||
parsed["request_id"] = request_id
|
||||
return parsed
|
||||
|
||||
# Extract message - use parsed dict if available, otherwise raw string
|
||||
if parsed is not None:
|
||||
|
|
@ -228,8 +201,7 @@ class AnthropicErrorSseFrame(str):
|
|||
|
||||
|
||||
def anthropic_error_sse_frame(status_code: int, raw_message: str) -> AnthropicErrorSseFrame:
|
||||
answered_status: Final = AnthropicExceptionMapping.get_status_code(status_code)
|
||||
return AnthropicErrorSseFrame(
|
||||
answered_status,
|
||||
AnthropicExceptionMapping.transform_to_anthropic_error(status_code=answered_status, raw_message=raw_message),
|
||||
status_code,
|
||||
AnthropicExceptionMapping.transform_to_anthropic_error(status_code=status_code, raw_message=raw_message),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,17 +1,10 @@
|
|||
from typing import Final, Protocol, TypeVar, runtime_checkable
|
||||
from typing import Final, TypeVar
|
||||
|
||||
from litellm.anthropic_interface.exceptions import AnthropicExceptionMapping
|
||||
from litellm.exceptions import InternalServerError, MidStreamFallbackError
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
_Raised = TypeVar("_Raised", bound=Exception)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _CarriesStatusCode(Protocol):
|
||||
status_code: int
|
||||
|
||||
|
||||
def _provider_status_code(exc: Exception) -> int | None:
|
||||
unwrapped = exc # rebind-ok: the recursion gate requires iterative wrapper traversal
|
||||
while isinstance(unwrapped, MidStreamFallbackError) and unwrapped.original_exception is not None:
|
||||
|
|
@ -33,17 +26,5 @@ def _copy_carrying(exc: _Raised, attribute: str, value: int | str) -> _Raised:
|
|||
|
||||
|
||||
def with_answered_status(exc: Exception) -> Exception:
|
||||
"""A copy of the exception, same class, carrying the status the Anthropic routes answer with (a 503 and an
|
||||
upstream Anthropic overload answer as 529), so the proxy-level recorders the failure hook runs (the spend log
|
||||
row, the OTel server span, the failed-request metric, alerting) agree with the client. The raised object keeps
|
||||
its own status, because the litellm layer already handed it to the deployment-level recorders (the deployment
|
||||
failure metric, cooldown, retry policy, custom loggers), which may still read it."""
|
||||
if isinstance(exc, ProxyException):
|
||||
if not exc.code.isdecimal():
|
||||
return exc
|
||||
return _copy_carrying(exc, "code", str(AnthropicExceptionMapping.get_status_code(int(exc.code))))
|
||||
if isinstance(exc, _CarriesStatusCode):
|
||||
return _copy_carrying(
|
||||
exc, "status_code", AnthropicExceptionMapping.get_status_code(_provider_status_code(exc) or exc.status_code)
|
||||
)
|
||||
return exc
|
||||
provider_status_code: Final = _provider_status_code(exc)
|
||||
return exc if provider_status_code is None else _copy_carrying(exc, "status_code", provider_status_code)
|
||||
|
|
|
|||
|
|
@ -2991,7 +2991,6 @@ class ProxyBaseLLMRequestProcessing:
|
|||
idle. See ``open_sse_before_first_byte``; unwrapped unless an operator sets
|
||||
``litellm_settings.sse_keepalive_ping_interval_seconds``.
|
||||
"""
|
||||
|
||||
answers_as_anthropic: Final = route_type == "anthropic_messages"
|
||||
|
||||
async def _audit_late_failure(exc: Exception) -> HTTPException | None:
|
||||
|
|
|
|||
|
|
@ -321,15 +321,12 @@ def request_with_retry[T: RetryableResponse](
|
|||
"""Bounded retry on statuses attributable to the PROVIDER, never the proxy.
|
||||
|
||||
The system under test is the proxy, so the transport may only absorb
|
||||
statuses that mean an overloaded provider; today that is exactly 529, the
|
||||
Anthropic overloaded_error (their own SDK retries it too). The Anthropic
|
||||
routes also answer the proxy's own 503s as 529, so a test asserting one of
|
||||
those wraps its call in without_retries(), and a proxy that keeps failing
|
||||
still fails once the bounded retries run out. 500/502/503/504 stay
|
||||
first-class failures: at this layer a 5xx from the proxy is
|
||||
indistinguishable from one it relayed, and retrying them could mask an
|
||||
intermittently failing proxy. Widen the set only for a status that means
|
||||
an overloaded provider, with an observed flake in hand.
|
||||
statuses the proxy itself cannot emit; today that is exactly 529, the
|
||||
Anthropic overloaded_error passed through verbatim (their own SDK retries
|
||||
it too). 500/502/503/504 stay first-class failures: at this layer a 5xx
|
||||
from the proxy is indistinguishable from one it relayed, and retrying them
|
||||
could mask an intermittently failing proxy. Widen the set only for a
|
||||
status litellm provably never originates, with an observed flake in hand.
|
||||
|
||||
Also deliberately NOT retried: 429, because this suite asserts the proxy's
|
||||
own rate-limit and budget 429s; network errors and timeouts, because a
|
||||
|
|
|
|||
|
|
@ -126,6 +126,13 @@ def _assert_overloaded(response: httpx.Response, *, message: str) -> None:
|
|||
assert "content_block_delta" not in response.text, response.text
|
||||
|
||||
|
||||
def _assert_api_error(response: httpx.Response, *, status: int, message: str) -> None:
|
||||
assert response.status_code == status, response.text
|
||||
error: Final = _anthropic_error(response)
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert message in string_value(error["message"]), response.text
|
||||
|
||||
|
||||
def _assert_internal_server_error(response: httpx.Response) -> None:
|
||||
assert response.status_code == 500, response.text
|
||||
error: Final = _openai_error(response)
|
||||
|
|
@ -193,17 +200,21 @@ async def test_messages_529_answers_overloaded_error_to_the_async_sdk(gateway: G
|
|||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("upstream_status", "error_class"), ((529, "InternalServerError"), (503, "ServiceUnavailableError"))
|
||||
("upstream_status", "error_class", "error_type"),
|
||||
((529, "InternalServerError", "overloaded_error"), (503, "ServiceUnavailableError", "api_error")),
|
||||
)
|
||||
def test_messages_stream_rejected_before_its_first_byte_answers_529_json(
|
||||
gateway: Gateway, upstream_status: int, error_class: str
|
||||
def test_messages_stream_rejected_before_its_first_byte_answers_the_provider_status_as_json(
|
||||
gateway: Gateway, upstream_status: int, error_class: str, error_type: str
|
||||
) -> None:
|
||||
with wire_server(_always(status_reply(upstream_status))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
|
||||
_assert_overloaded(response, message=f"scripted {upstream_status}")
|
||||
assert response.status_code == upstream_status, response.text
|
||||
error: Final = _anthropic_error(response)
|
||||
assert error["type"] == error_type, response.text
|
||||
assert f"scripted {upstream_status}" in string_value(error["message"]), response.text
|
||||
_one_post(wire, "/v1/messages")
|
||||
_assert_logged_failure(model, rows=1, error_class=error_class)
|
||||
_assert_logged_failure(model, rows=1, error_class=error_class, code=str(upstream_status))
|
||||
|
||||
|
||||
def test_messages_stream_529_raises_529_in_the_sync_sdk(gateway: Gateway) -> None:
|
||||
|
|
@ -266,7 +277,7 @@ def _one_bedrock_post(wire: Wire) -> None:
|
|||
assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request
|
||||
|
||||
|
||||
def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_an_overloaded_error_event(
|
||||
def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_an_api_error_event(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with (
|
||||
|
|
@ -281,10 +292,10 @@ def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_
|
|||
(failure,) = (event for event in parse_sse(response.text) if event.data and event.event != "ping")
|
||||
assert failure.event == "error", response.text
|
||||
error: Final = object_value(failure.data["error"])
|
||||
assert error["type"] == "overloaded_error", response.text
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text
|
||||
_one_bedrock_post(wire)
|
||||
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError")
|
||||
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
|
||||
|
||||
|
||||
def test_chat_completions_bedrock_503_after_the_keepalive_opened_the_response_keeps_the_openai_error_frame(
|
||||
|
|
@ -403,7 +414,7 @@ def test_responses_529_answers_500(gateway: Gateway, shape: Shape) -> None:
|
|||
|
||||
|
||||
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
|
||||
def test_messages_bedrock_503_answers_529_overloaded_error(gateway: Gateway, shape: Shape) -> None:
|
||||
def test_messages_bedrock_503_answers_503_api_error(gateway: Gateway, shape: Shape) -> None:
|
||||
with wire_server(_always(Reply(status=503, body=_BEDROCK_503))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"bedrock/{_BEDROCK_MODEL}", api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url
|
||||
|
|
@ -411,10 +422,10 @@ def test_messages_bedrock_503_answers_529_overloaded_error(gateway: Gateway, sha
|
|||
response: Final = gateway.request(
|
||||
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
|
||||
)
|
||||
_assert_overloaded(response, message="Bedrock is unable to process your request.")
|
||||
_assert_api_error(response, status=503, message="Bedrock is unable to process your request.")
|
||||
(request,) = wire.drain()
|
||||
assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request
|
||||
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError")
|
||||
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
|
||||
|
||||
|
||||
def _converse_event(event: str, payload: Mapping[str, JsonValue]) -> bytes:
|
||||
|
|
@ -445,7 +456,7 @@ def _converse_deployment(scenario: Scenario, wire: Wire) -> str:
|
|||
return scenario.model(model=_CONVERSE_MODEL, api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url)
|
||||
|
||||
|
||||
def test_messages_bedrock_stream_503_after_text_ends_with_an_overloaded_error_event(gateway: Gateway) -> None:
|
||||
def test_messages_bedrock_stream_503_after_text_ends_with_an_api_error_event(gateway: Gateway) -> None:
|
||||
with wire_server(_always(_converse_503_after_text())) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _converse_deployment(scenario, wire)
|
||||
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
|
||||
|
|
@ -457,10 +468,10 @@ def test_messages_bedrock_stream_503_after_text_ends_with_an_overloaded_error_ev
|
|||
assert _TEXT in response.text, response.text
|
||||
(failure,) = (event for event in events if event_type(event) == "error")
|
||||
error: Final = object_value(failure.data["error"])
|
||||
assert error["type"] == "overloaded_error", response.text
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text
|
||||
_one_post(wire, _CONVERSE_STREAM_TARGET)
|
||||
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError")
|
||||
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
|
||||
|
||||
|
||||
def test_chat_completions_bedrock_stream_503_after_text_ends_with_a_503_error_frame(gateway: Gateway) -> None:
|
||||
|
|
@ -481,7 +492,7 @@ def test_chat_completions_bedrock_stream_503_after_text_ends_with_a_503_error_fr
|
|||
|
||||
|
||||
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
|
||||
def test_chat_bridge_503_answers_529_overloaded_error_on_messages(gateway: Gateway, shape: Shape) -> None:
|
||||
def test_chat_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None:
|
||||
with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"hosted_vllm/{_CHAT_ONLY_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-vllm-key"
|
||||
|
|
@ -489,12 +500,12 @@ def test_chat_bridge_503_answers_529_overloaded_error_on_messages(gateway: Gatew
|
|||
response: Final = gateway.request(
|
||||
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
|
||||
)
|
||||
_assert_overloaded(response, message="scripted 503")
|
||||
_assert_api_error(response, status=503, message="scripted 503")
|
||||
_one_post(wire, "/v1/chat/completions")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
|
||||
def test_responses_bridge_503_answers_529_overloaded_error_on_messages(gateway: Gateway, shape: Shape) -> None:
|
||||
def test_responses_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None:
|
||||
with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"openai/{_CODEX_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-openai-key"
|
||||
|
|
@ -502,7 +513,7 @@ def test_responses_bridge_503_answers_529_overloaded_error_on_messages(gateway:
|
|||
response: Final = gateway.request(
|
||||
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
|
||||
)
|
||||
_assert_overloaded(response, message="scripted 503")
|
||||
_assert_api_error(response, status=503, message="scripted 503")
|
||||
_one_post(wire, "/v1/responses")
|
||||
|
||||
|
||||
|
|
@ -561,14 +572,14 @@ def _assert_refused_without_an_upstream_call(
|
|||
assert reached.isdisjoint(prompt for prompt, _ in answers), reached
|
||||
|
||||
|
||||
def test_messages_to_a_paused_deployment_named_by_its_id_answer_529_overloaded_error(gateway: Gateway) -> None:
|
||||
def test_messages_to_a_paused_deployment_named_by_its_id_answer_503_api_error(gateway: Gateway) -> None:
|
||||
with _paused_deployment(gateway) as (wire, identity, name):
|
||||
_assert_refused_without_an_upstream_call(
|
||||
gateway, wire, identity, status=529, error_type="overloaded_error", message="is currently paused"
|
||||
gateway, wire, identity, status=503, error_type="api_error", message="is currently paused"
|
||||
)
|
||||
rows: Final = _failure_rows_of((identity, name), _SETTLED_ANSWERS)
|
||||
assert {(row["code"], row["error_class"], row["normalized_error"]) for row in rows} == {
|
||||
("529", "ServiceUnavailableError", _OVERLOAD_CLASS)
|
||||
("503", "ServiceUnavailableError", _OVERLOAD_CLASS)
|
||||
}, rows
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -404,7 +404,7 @@ def test_retry_rejected_before_it_opens_counts_against_the_same_budget(gateway:
|
|||
seconds=70,
|
||||
)
|
||||
rejected: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "request", budget=0))
|
||||
_assert_rejected_before_content(rejected, wire, status=500, attempts=1, error="error")
|
||||
_assert_rejected_before_content(rejected, wire, status=529, attempts=1, error="overloaded_error")
|
||||
rejection_rows: Final = eventually(
|
||||
lambda: _request_rows(gateway, rejected.headers["x-litellm-call-id"]),
|
||||
lambda values: len(values) >= 1,
|
||||
|
|
|
|||
|
|
@ -214,7 +214,7 @@ def _until(rig: _Rig, name: str, model: str) -> tuple[Sample, ...]:
|
|||
("route", "model", "answered", "deployment_status", "deployment_class"),
|
||||
(
|
||||
pytest.param("/v1/messages", _MESSAGES_ANTHROPIC, 529, "500", "InternalServerError", id="messages-anthropic"),
|
||||
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 529, "503", "ServiceUnavailableError", id="messages-bedrock"),
|
||||
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, "503", "ServiceUnavailableError", id="messages-bedrock"),
|
||||
pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 500, "500", "InternalServerError", id="chat-anthropic"),
|
||||
),
|
||||
)
|
||||
|
|
@ -312,7 +312,7 @@ def _datadog_status_codes(rig: _Rig, alias: str, expected: int) -> tuple[tuple[s
|
|||
("route", "model", "answered", "deployment_status"),
|
||||
(
|
||||
pytest.param("/v1/messages", _DATADOG_MESSAGES_ANTHROPIC, 529, "500", id="messages-anthropic"),
|
||||
pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 529, "503", id="messages-bedrock"),
|
||||
pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 503, "503", id="messages-bedrock"),
|
||||
pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 500, "500", id="chat-anthropic"),
|
||||
),
|
||||
)
|
||||
|
|
@ -332,7 +332,7 @@ def test_the_datadog_proxy_failure_log_carries_the_answered_status_and_the_llm_f
|
|||
@pytest.mark.parametrize(
|
||||
("route", "model", "answered"),
|
||||
(
|
||||
pytest.param("/v1/messages", _GUARDED_MESSAGES, 529, id="messages"),
|
||||
pytest.param("/v1/messages", _GUARDED_MESSAGES, 503, id="messages"),
|
||||
pytest.param("/v1/chat/completions", _GUARDED_CHAT, 503, id="chat"),
|
||||
),
|
||||
)
|
||||
|
|
@ -360,7 +360,7 @@ def test_a_fail_closed_guardrail_outage_is_answered_and_recorded_with_the_routes
|
|||
("route", "model", "answered"),
|
||||
(
|
||||
pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages-anthropic"),
|
||||
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 529, id="messages-bedrock"),
|
||||
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, id="messages-bedrock"),
|
||||
pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat-anthropic"),
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -424,9 +424,7 @@ def _bedrock_logged_error(endpoint: BedrockEndpoint, failure: BedrockFailure) ->
|
|||
shape: Final = _BEDROCK_FAILURES[failure]
|
||||
if endpoint == "messages-invoke":
|
||||
return "BedrockError", str(shape.bedrock_status)
|
||||
if failure != "exception-event":
|
||||
return shape.converse_class, str(shape.bedrock_status)
|
||||
return shape.converse_class, "529" if endpoint == "messages" else "503"
|
||||
return shape.converse_class, "503" if failure == "exception-event" else str(shape.bedrock_status)
|
||||
|
||||
|
||||
def _bedrock_client_status(endpoint: BedrockEndpoint, failure: BedrockFailure, after_output: bool) -> int:
|
||||
|
|
@ -553,7 +551,7 @@ def test_bedrock_non_stream_500_logs_amzn_request_id_on_every_endpoint(
|
|||
endpoint, _bedrock_model(scenario, endpoint, wire.url), amzn_request_id, stream=False
|
||||
)
|
||||
response: Final = gateway.request("POST", path, body)
|
||||
assert response.status_code == (503 if endpoint == "responses" else 529), response.text
|
||||
assert response.status_code == 503, response.text
|
||||
if endpoint == "responses":
|
||||
assert response.headers.get("llm_provider-x-amzn-requestid") == amzn_request_id, dict(response.headers)
|
||||
assert len(wire.drain()) == 1
|
||||
|
|
|
|||
|
|
@ -476,7 +476,7 @@ def test_messages_count_tokens_surfaces_the_peer_status_when_the_peer_rejects_th
|
|||
with _listening(deployment, _rejecting(status)) as peer:
|
||||
body: Final = count_request(deployment.model, STRING_CASE)
|
||||
response: Final = _count(gateway, body)
|
||||
assert response.status_code == (529 if status == 503 else status), response.text
|
||||
assert response.status_code == status, response.text
|
||||
assert "input_tokens" not in response.text, response.text
|
||||
assert _bodies(peer, deployment.provider) == (_STRING_BODY,)
|
||||
|
||||
|
|
|
|||
|
|
@ -235,7 +235,7 @@ async def test_upstream_aborts_then_an_outage_fail_each_call_once_and_the_native
|
|||
refused: Final = await _burst(proxy, gateway.key, model, during_outage)
|
||||
assert len(refused) == 4
|
||||
for item in refused:
|
||||
assert item.status == (529 if item.call.endpoint == "messages" else 503), item.text
|
||||
assert item.status == 503, item.text
|
||||
assert "Cannot connect to host" in item.text, item.text
|
||||
assert await asyncio.to_thread(
|
||||
eventually, lambda: _unhealthy_count(gateway, model), lambda count: count == 1, 30
|
||||
|
|
|
|||
|
|
@ -1172,7 +1172,7 @@ def test_messages_count_tokens_surfaces_the_mantle_status_when_mantle_errors(
|
|||
):
|
||||
model: Final = _fallback_deployment(scenario, runtime.url)
|
||||
response: Final = _count(gateway, _full(model))
|
||||
assert response.status_code == (529 if status == 503 else status), response.text
|
||||
assert response.status_code == status, response.text
|
||||
assert "input_tokens" not in response.text, response.text
|
||||
assert len(_runtime_count_targets(runtime)) == 1
|
||||
assert _mantle_bodies(mantle) == (_MANTLE_FULL,)
|
||||
|
|
|
|||
|
|
@ -329,8 +329,10 @@ async def test_peer_killed_mid_burst_fails_only_the_held_calls_and_a_restarted_p
|
|||
assert (len(succeeded), len(failed)) == (6, 6), [(item.call.marker, item.status) for item in served]
|
||||
for item in succeeded:
|
||||
_assert_answered_with_its_own_marker(item)
|
||||
assert {item.status for item in failed} == {503}, [
|
||||
(item.call.endpoint, item.call.stream, item.status, item.text) for item in failed
|
||||
]
|
||||
for item in failed:
|
||||
assert item.status == (529 if item.call.endpoint == "messages" else 503), (item.call, item.status, item.text)
|
||||
assert "ServiceUnavailableError: BedrockException - Server disconnected" in item.text, item.text
|
||||
assert "marker-" not in item.text and item.call_id is not None, item.text
|
||||
with _child_peer(deployment.peer_port, answer_first=10**6) as revived:
|
||||
|
|
|
|||
|
|
@ -1261,8 +1261,7 @@ def test_databricks_json_schema_burst_through_an_upstream_outage_keeps_every_ref
|
|||
for request in received:
|
||||
_assert_burst_outbound(request)
|
||||
expected: Final = tuple(
|
||||
(529 if endpoint == "messages" else 503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON)
|
||||
for endpoint, _stream, down in plan
|
||||
(503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON) for _endpoint, _stream, down in plan
|
||||
)
|
||||
assert tuple(_outcome(*result) for result in results) == expected
|
||||
assert tuple(_outcome(*result) for result in recovered) == ((200, _EXPECTED_JSON),) * len(recovery)
|
||||
|
|
@ -1572,8 +1571,7 @@ def test_databricks_one_block_burst_through_an_upstream_outage_reaches_upstream_
|
|||
json.dumps([{"role": "user", "content": prompt}]) for prompt in prompts
|
||||
)
|
||||
expected: Final = tuple(
|
||||
(529 if endpoint == "messages" else 503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON)
|
||||
for endpoint, _stream, down in plan
|
||||
(503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON) for _endpoint, _stream, down in plan
|
||||
)
|
||||
assert tuple(_outcome(*result) for result in results) == expected
|
||||
|
||||
|
|
|
|||
|
|
@ -1215,8 +1215,7 @@ async def test_upstream_outage_mid_burst_fails_only_its_calls_and_every_replay_l
|
|||
assert len(served) == len(calls)
|
||||
for item in served:
|
||||
if item.call.marker in down:
|
||||
assert item.status == (529 if item.call.endpoint == "messages" else 503), item.text
|
||||
assert _BRIDGE_OUTAGE in item.text, item.text
|
||||
assert item.status == 503 and _BRIDGE_OUTAGE in item.text, item.text
|
||||
assert rv.answer(item.call.marker) not in item.text, item.text
|
||||
else:
|
||||
_bridge_assert_answered(item)
|
||||
|
|
|
|||
|
|
@ -659,7 +659,7 @@ async def _burst(gateway: Gateway, model: str, calls: tuple[_Call, ...]) -> tupl
|
|||
|
||||
def _assert_answered(served: _Served, down: frozenset[str]) -> None:
|
||||
if served.call.marker in down:
|
||||
assert served.status == (529 if served.call.endpoint == "messages" else 503), (served.call, served.text)
|
||||
assert served.status == 503, (served.call, served.text)
|
||||
assert "synthetic container outage" in served.text, (served.call, served.text)
|
||||
return
|
||||
assert served.status == 200, (served.call, served.text)
|
||||
|
|
|
|||
|
|
@ -980,7 +980,7 @@ def test_concurrent_count_waves_recover_after_vertex_count_outage(gateway: Gatew
|
|||
body: Final = _messages(model, None, None)
|
||||
assert _count_wave(clients, body) == ((200, _COUNT), (200, _COUNT), (200, _COUNT))
|
||||
peer.set_status(503)
|
||||
assert _count_wave(clients, body) == ((529, None), (529, None), (529, None))
|
||||
assert _count_wave(clients, body) == ((503, None), (503, None), (503, None))
|
||||
peer.set_status(200)
|
||||
assert _count_wave(clients, body) == ((200, _COUNT), (200, _COUNT), (200, _COUNT))
|
||||
requests: Final = _count_requests(wire)
|
||||
|
|
|
|||
|
|
@ -622,7 +622,7 @@ def test_messages_count_tokens_surfaces_the_peer_error_when_peer_errors(gateway:
|
|||
with wire_server(_peer(_rejecting(status))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(gateway, scenario, wire.url)
|
||||
response: Final = _count(gateway, _full(model))
|
||||
assert response.status_code == (529 if status == 503 else status), response.text
|
||||
assert response.status_code == status, response.text
|
||||
assert "input_tokens" not in response.text, response.text
|
||||
assert _counted_bodies(wire) == (_PEER_FULL,)
|
||||
|
||||
|
|
|
|||
|
|
@ -115,8 +115,7 @@ def test_vertex_pass_through_stream_relays_an_upstream_503_as_503(vertex: _Verte
|
|||
response: Final = vertex.stream(_UNAVAILABLE_MODEL, uuid.uuid4().hex)
|
||||
|
||||
assert response.status_code == 503, response.text
|
||||
assert _UNAVAILABLE_MESSAGE in response.text, response.text
|
||||
assert "overloaded_error" not in response.text, response.text
|
||||
assert response.content == _UNAVAILABLE_BODY, response.text
|
||||
assert [request.target for request in vertex.wire.drain()] == [
|
||||
f"/v1/{_model_path(_UNAVAILABLE_MODEL)}:streamGenerateContent?alt=sse"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import os
|
|||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import unquote
|
||||
|
|
@ -25,12 +24,10 @@ from integration._support.anthropic_sse import (
|
|||
from integration._support.client import (
|
||||
Gateway,
|
||||
JsonValue,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
|
|
@ -53,8 +50,6 @@ _GATEWAY_GROUPS: Final = {
|
|||
"stream": "audit-gateway-overloaded-stream",
|
||||
}
|
||||
_COUNT_STATUSES: Final = {_COUNTED: 503, _COUNTED_OVERLOADED: 529, _COUNTED_RATE_LIMITED: 429}
|
||||
_ADAPTIVE_UPSTREAMS: Final = {503: "claude-adaptive-unavailable"}
|
||||
_ADAPTIVE_FLUSH_SECONDS: Final = 40
|
||||
_COUNT_TOKENS_TARGET: Final = "/v1/messages/count_tokens"
|
||||
_UPSTREAM_URL_PLACEHOLDER: Final = "upstream-url"
|
||||
|
||||
|
|
@ -73,8 +68,6 @@ def _respond(request: Request) -> Reply:
|
|||
return Reply(status=status, body=error_body(status, f"scripted {status}"))
|
||||
if backend == _OVERLOADED:
|
||||
return status_reply(529)
|
||||
if backend in _ADAPTIVE_UPSTREAMS.values():
|
||||
return status_reply(next(status for status, name in _ADAPTIVE_UPSTREAMS.items() if name == backend))
|
||||
if backend == _HEALTHY:
|
||||
return stream_reply(message_stream(_served_id(user_prompt(body)), backend, _TEXT))
|
||||
return Reply(status=404, body=b'{"error": "no scripted backend"}')
|
||||
|
|
@ -84,36 +77,6 @@ def _deployment(name: str, model: str, api_base: str, api_key: str, **params: Js
|
|||
return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": api_key, **params}}
|
||||
|
||||
|
||||
def _adaptive_router(status: int) -> str:
|
||||
return f"audit-adaptive-router-{status}"
|
||||
|
||||
|
||||
def _adaptive_backend(status: int) -> str:
|
||||
return f"audit-adaptive-backend-{status}"
|
||||
|
||||
|
||||
def _adaptive_deployments(status: int) -> tuple[dict[str, JsonValue], ...]:
|
||||
return (
|
||||
{
|
||||
"model_name": _adaptive_router(status),
|
||||
"litellm_params": {
|
||||
"model": "auto_router/adaptive_router",
|
||||
"adaptive_router_config": {"available_models": [_adaptive_backend(status)]},
|
||||
},
|
||||
},
|
||||
{
|
||||
**_deployment(
|
||||
_adaptive_backend(status),
|
||||
f"anthropic/{_ADAPTIVE_UPSTREAMS[status]}",
|
||||
_UPSTREAM_URL_PLACEHOLDER,
|
||||
_API_KEY,
|
||||
input_cost_per_token=0.000001,
|
||||
),
|
||||
"model_info": {"adaptive_router_preferences": {"quality_tier": 2, "strengths": []}},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _config(wire: Wire, directory: Path) -> Path:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["model_list"] = [
|
||||
|
|
@ -134,7 +97,6 @@ def _config(wire: Wire, directory: Path) -> Path:
|
|||
_deployment(group, f"anthropic/{_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY)
|
||||
for group in _GATEWAY_GROUPS.values()
|
||||
),
|
||||
*chain.from_iterable(_adaptive_deployments(status) for status in _ADAPTIVE_UPSTREAMS),
|
||||
]
|
||||
config["router_settings"] = {
|
||||
"num_retries": 0,
|
||||
|
|
@ -224,22 +186,20 @@ def test_an_overloaded_deployment_is_cooled_down_by_the_internal_server_error_al
|
|||
assert rig.backends_called() == (_HEALTHY,)
|
||||
|
||||
|
||||
def test_count_tokens_disabled_without_a_provider_counter_answers_529_overloaded_error(rig: _Rig) -> None:
|
||||
def test_count_tokens_disabled_without_a_provider_counter_answers_503_api_error(rig: _Rig) -> None:
|
||||
response: Final = rig.count_tokens(_COUNT_DISABLED_GROUP)
|
||||
error: Final = _count_tokens_error(response, 529)
|
||||
assert error["type"] == "overloaded_error", response.text
|
||||
error: Final = _count_tokens_error(response, 503)
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert "Token counting is disabled" in string_value(error["message"]), response.text
|
||||
assert rig.targets_called() == ()
|
||||
|
||||
|
||||
def test_count_tokens_whose_upstream_answers_503_answers_529_overloaded_error(rig: _Rig) -> None:
|
||||
def test_count_tokens_whose_upstream_answers_503_stays_503_api_error(rig: _Rig) -> None:
|
||||
response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP)
|
||||
error: Final = _count_tokens_error(response, 529)
|
||||
assert error["type"] == "overloaded_error", response.text
|
||||
error: Final = _count_tokens_error(response, 503)
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert "scripted 503" in string_value(error["message"]), response.text
|
||||
assert rig.targets_called() == (_COUNT_TOKENS_TARGET,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("group", "status", "error_type"),
|
||||
(
|
||||
|
|
@ -279,46 +239,9 @@ def test_the_claude_code_gateway_messages_route_answers_an_upstream_529_as_overl
|
|||
assert rig.backends_called() == (_OVERLOADED,)
|
||||
|
||||
|
||||
def test_the_claude_code_gateway_count_tokens_route_answers_an_upstream_503_as_529(rig: _Rig) -> None:
|
||||
def test_the_claude_code_gateway_count_tokens_route_answers_an_upstream_503_as_503(rig: _Rig) -> None:
|
||||
response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP, prefix=_GATEWAY)
|
||||
error: Final = _count_tokens_error(response, 529)
|
||||
assert error["type"] == "overloaded_error", response.text
|
||||
error: Final = _count_tokens_error(response, 503)
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert "scripted 503" in string_value(error["message"]), response.text
|
||||
assert rig.targets_called() == (_COUNT_TOKENS_TARGET,)
|
||||
|
||||
|
||||
def _adaptive_session_rows(session: str) -> list[dict[str, JsonValue]]:
|
||||
return read_rows(
|
||||
'SELECT model_name, exhaustion_count FROM "LiteLLM_AdaptiveRouterSession" WHERE session_id = %s', (session,)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status", "answered"),
|
||||
(
|
||||
pytest.param(503, 503, id="upstream-503"),
|
||||
),
|
||||
)
|
||||
def test_an_adaptive_router_session_counts_an_upstream_overload_as_exhaustion(
|
||||
rig: _Rig, status: int, answered: int
|
||||
) -> None:
|
||||
session: Final = f"audit-adaptive-{uuid.uuid4().hex}"
|
||||
response: Final = rig.proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": _adaptive_router(status),
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": session}],
|
||||
},
|
||||
headers={"x-litellm-session-id": session},
|
||||
)
|
||||
assert response.status_code == answered, response.text
|
||||
assert rig.backends_called() == (_ADAPTIVE_UPSTREAMS[status],)
|
||||
rows: Final = eventually(
|
||||
lambda: _adaptive_session_rows(session),
|
||||
lambda rows: len(rows) > 0,
|
||||
seconds=_ADAPTIVE_FLUSH_SECONDS,
|
||||
return_last_on_timeout=True,
|
||||
)
|
||||
assert rows == [{"model_name": _adaptive_backend(status), "exhaustion_count": 1}], response.text
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ Behavior = Literal[
|
|||
"hold-ping",
|
||||
"drop-then-too-long",
|
||||
"drop-then-401",
|
||||
"overloaded-frames-fallback-503",
|
||||
"overloaded-frames-fallback-529",
|
||||
"overloaded-frames",
|
||||
"overloaded-once",
|
||||
]
|
||||
|
|
@ -90,7 +90,7 @@ def _primary_reply(
|
|||
return Reply(status=400, body=error_body(400, "prompt is too long: 250000 tokens > 200000 maximum"))
|
||||
case "drop-then-401":
|
||||
return dropped if attempt == 1 else status_reply(401)
|
||||
case "overloaded-frames-fallback-503" | "overloaded-frames":
|
||||
case "overloaded-frames-fallback-529" | "overloaded-frames":
|
||||
return stream_reply((error_frame(529, "scripted overloaded"),))
|
||||
case "overloaded-once":
|
||||
return status_reply(529) if attempt == 1 else stream_reply(full)
|
||||
|
|
@ -98,7 +98,7 @@ def _primary_reply(
|
|||
|
||||
|
||||
def _fallback_reply(behavior: str, full: tuple[bytes, bytes, bytes, bytes]) -> Reply:
|
||||
if behavior == "overloaded-frames-fallback-503":
|
||||
if behavior == "overloaded-frames-fallback-529":
|
||||
return status_reply(503)
|
||||
return stream_reply(full)
|
||||
|
||||
|
|
@ -309,7 +309,7 @@ def test_a_retry_rejected_with_401_ends_the_retries_and_reaches_the_client_uncha
|
|||
|
||||
def test_overloaded_frames_whose_fallback_fails_answer_529_overloaded_error(rig: _Rig) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-503", marker)
|
||||
response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-529", marker)
|
||||
assert response.status_code == 529, response.text
|
||||
assert "overloaded_error" in response.text, response.text
|
||||
assert "content_block_delta" not in response.text, response.text
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ BLOCKED: Final = "Request blocked by Noma guardrail"
|
|||
SLOW_SECONDS: Final = 1.5
|
||||
SAMPLED_SCANS: Final = range(2, 7)
|
||||
BUFFERED_BLOCK_STATUS: Final[Mapping[Endpoint, int]] = {"chat": 400, "messages": 400, "responses": 200}
|
||||
BUFFERED_OUTAGE_STATUS: Final[Mapping[Endpoint, int]] = {"chat": 503, "messages": 529, "responses": 200}
|
||||
BUFFERED_OUTAGE_STATUS: Final[Mapping[Endpoint, int]] = {"chat": 503, "messages": 503, "responses": 200}
|
||||
PIECES: Final = (
|
||||
" Contact",
|
||||
" me",
|
||||
|
|
|
|||
|
|
@ -56,17 +56,6 @@ class TestCreateErrorResponse:
|
|||
)
|
||||
assert response["error"]["type"] == "api_error"
|
||||
|
||||
def test_503_answers_as_anthropics_529_overloaded_error(self):
|
||||
"""LIT-9450: Anthropic's contract has no 503, so an upstream that cannot take the request right now
|
||||
(a Bedrock capacity 503) answers as 529 overloaded_error, the status Claude Code's overload path keys on."""
|
||||
assert AnthropicExceptionMapping.get_status_code(503) == 529
|
||||
response = AnthropicExceptionMapping.create_error_response(503, "Bedrock is unable to process your request.")
|
||||
assert response["error"]["type"] == "overloaded_error"
|
||||
|
||||
@pytest.mark.parametrize("status_code", [400, 429, 500, 529])
|
||||
def test_other_statuses_keep_their_code(self, status_code: int):
|
||||
assert AnthropicExceptionMapping.get_status_code(status_code) == status_code
|
||||
|
||||
def test_with_request_id(self):
|
||||
"""Test request_id is included when provided."""
|
||||
response = AnthropicExceptionMapping.create_error_response(
|
||||
|
|
@ -179,21 +168,6 @@ class TestTransformToAnthropicError:
|
|||
)
|
||||
assert result["request_id"] == "req_123"
|
||||
|
||||
def test_an_anthropic_shaped_503_body_answers_529_with_the_overloaded_type(self):
|
||||
"""A 503 whose message is already an Anthropic envelope keeps its message but takes the error type the
|
||||
529 it answers as names, so the status and the body agree."""
|
||||
raw = json.dumps({"type": "error", "error": {"type": "api_error", "message": "Service unavailable"}})
|
||||
result = AnthropicExceptionMapping.transform_to_anthropic_error(
|
||||
status_code=AnthropicExceptionMapping.get_status_code(503),
|
||||
raw_message=raw,
|
||||
request_id="req_529",
|
||||
)
|
||||
assert result == {
|
||||
"type": "error",
|
||||
"error": {"type": "overloaded_error", "message": "Service unavailable"},
|
||||
"request_id": "req_529",
|
||||
}
|
||||
|
||||
def test_translates_bedrock_error(self):
|
||||
"""Test that Bedrock errors are translated to Anthropic format."""
|
||||
bedrock_error = json.dumps({"detail": {"message": "Access denied"}})
|
||||
|
|
@ -250,7 +224,7 @@ class TestTransformToAnthropicError:
|
|||
class TestAnthropicErrorSseFrame:
|
||||
@pytest.mark.parametrize(
|
||||
("status_code", "expected_error_type"),
|
||||
[(429, "rate_limit_error"), (503, "overloaded_error"), (400, "invalid_request_error")],
|
||||
[(429, "rate_limit_error"), (503, "api_error"), (400, "invalid_request_error")],
|
||||
)
|
||||
def test_the_frame_is_one_error_event_carrying_the_anthropic_envelope(
|
||||
self, status_code: int, expected_error_type: str
|
||||
|
|
@ -265,22 +239,11 @@ class TestAnthropicErrorSseFrame:
|
|||
"error": {"type": expected_error_type, "message": "upstream unavailable"},
|
||||
}
|
||||
|
||||
def test_a_503_envelope_frame_takes_the_overloaded_type_of_the_529_it_answers(self) -> None:
|
||||
envelope: Final = json.dumps({"type": "error", "error": {"type": "api_error", "message": "Service unavailable"}})
|
||||
|
||||
frame: Final = anthropic_error_sse_frame(status_code=503, raw_message=envelope)
|
||||
|
||||
assert frame.status_code == 529
|
||||
assert frame.json_body(call_id=None) == {
|
||||
"type": "error",
|
||||
"error": {"type": "overloaded_error", "message": "Service unavailable"},
|
||||
}
|
||||
|
||||
def test_the_frame_remembers_the_status_anthropic_answers_with_and_the_body(self) -> None:
|
||||
def test_the_frame_remembers_the_status_and_body_it_was_built_from(self) -> None:
|
||||
frame: Final = anthropic_error_sse_frame(status_code=503, raw_message="upstream unavailable")
|
||||
|
||||
assert isinstance(frame, AnthropicErrorSseFrame)
|
||||
assert frame.status_code == 529
|
||||
assert frame.status_code == 503
|
||||
data_line: Final = frame.split("\n")[1]
|
||||
assert data_line == f"data: {json.dumps(frame.json_body(call_id=None))}"
|
||||
|
||||
|
|
@ -289,9 +252,9 @@ class TestAnthropicErrorSseFrame:
|
|||
|
||||
assert frame.json_body(call_id="call-1") == {
|
||||
"type": "error",
|
||||
"error": {"type": "overloaded_error", "message": "upstream unavailable", "litellm_call_id": "call-1"},
|
||||
"error": {"type": "api_error", "message": "upstream unavailable", "litellm_call_id": "call-1"},
|
||||
}
|
||||
assert frame.json_body(call_id=None) == {
|
||||
"type": "error",
|
||||
"error": {"type": "overloaded_error", "message": "upstream unavailable"},
|
||||
"error": {"type": "api_error", "message": "upstream unavailable"},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1438,8 +1438,6 @@ def test_an_already_mapped_litellm_exception_passes_through_untouched(provider,
|
|||
['{"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}', "upstream failure"],
|
||||
)
|
||||
def test_an_anthropic_529_stays_an_internal_server_error_that_carries_the_529(message, quiet_exception_mapping):
|
||||
"""An Anthropic overload keeps the InternalServerError class and 500 status that retry and allowed-fails
|
||||
policies and SDK callers key on, and carries the provider's 529 for the Anthropic-shaped routes to answer."""
|
||||
with pytest.raises(litellm.InternalServerError) as raised:
|
||||
exception_type(
|
||||
model="claude-opus-5-5",
|
||||
|
|
|
|||
|
|
@ -561,7 +561,7 @@ class TestUpstreamFailureEndsStreamWithErrorEvent:
|
|||
[
|
||||
(
|
||||
MidStreamFallbackError(message="boom", model="m", llm_provider="openai"),
|
||||
{"type": "overloaded_error", "message": "litellm.MidStreamFallbackError: boom"},
|
||||
{"type": "api_error", "message": "litellm.MidStreamFallbackError: boom"},
|
||||
),
|
||||
(
|
||||
type("StringStatusError", (Exception,), {"status_code": "429"})("throttled"),
|
||||
|
|
|
|||
|
|
@ -16,27 +16,28 @@ def _anthropic_internal_server_error(provider_status_code: int | None) -> litell
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("code", "answered"), [(503, "529"), (400, "400"), (500, "500")])
|
||||
def test_a_proxy_exception_answers_its_503_as_529_and_keeps_every_other_code(code: int, answered: str) -> None:
|
||||
@pytest.mark.parametrize("code", [400, 500, 503])
|
||||
def test_a_proxy_exception_passes_through_unchanged(code: int) -> None:
|
||||
exc = ProxyException(message="budget service unavailable", type="None", param=None, code=code)
|
||||
|
||||
answered_exc = with_answered_status(exc)
|
||||
|
||||
assert (type(answered_exc), answered_exc.code, answered_exc.message) == (ProxyException, answered, exc.message)
|
||||
assert answered_exc is exc
|
||||
assert exc.code == str(code)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("exc", "answered"),
|
||||
("exc", "answered", "copy"),
|
||||
[
|
||||
pytest.param(
|
||||
litellm.ServiceUnavailableError(message="BedrockException", llm_provider="bedrock", model="m"),
|
||||
529,
|
||||
503,
|
||||
False,
|
||||
id="service-unavailable-503",
|
||||
),
|
||||
pytest.param(_anthropic_internal_server_error(529), 529, id="anthropic-overload"),
|
||||
pytest.param(_anthropic_internal_server_error(500), 500, id="anthropic-500"),
|
||||
pytest.param(_anthropic_internal_server_error(None), 500, id="internal-server-error"),
|
||||
pytest.param(_anthropic_internal_server_error(529), 529, True, id="anthropic-overload"),
|
||||
pytest.param(_anthropic_internal_server_error(500), 500, True, id="anthropic-500"),
|
||||
pytest.param(_anthropic_internal_server_error(None), 500, False, id="internal-server-error"),
|
||||
pytest.param(
|
||||
MidStreamFallbackError(
|
||||
message="fallbacks failed",
|
||||
|
|
@ -45,6 +46,7 @@ def test_a_proxy_exception_answers_its_503_as_529_and_keeps_every_other_code(cod
|
|||
original_exception=_anthropic_internal_server_error(529),
|
||||
),
|
||||
529,
|
||||
True,
|
||||
id="fallback-over-an-anthropic-overload",
|
||||
),
|
||||
pytest.param(
|
||||
|
|
@ -60,20 +62,25 @@ def test_a_proxy_exception_answers_its_503_as_529_and_keeps_every_other_code(cod
|
|||
),
|
||||
),
|
||||
529,
|
||||
True,
|
||||
id="nested-fallback-over-an-anthropic-overload",
|
||||
),
|
||||
pytest.param(
|
||||
litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="m"), 429, id="rate-limit"
|
||||
litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="m"),
|
||||
429,
|
||||
False,
|
||||
id="rate-limit",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_litellm_exception_answers_the_status_anthropic_gives_its_failure(
|
||||
exc: openai.APIStatusError, answered: int
|
||||
exc: openai.APIStatusError, answered: int, copy: bool
|
||||
) -> None:
|
||||
raised_status = exc.status_code
|
||||
|
||||
answered_exc = with_answered_status(exc)
|
||||
|
||||
assert (answered_exc is not exc) is copy
|
||||
assert (type(answered_exc), answered_exc.status_code, str(answered_exc)) == (type(exc), answered, str(exc))
|
||||
assert exc.status_code == raised_status
|
||||
|
||||
|
|
@ -82,13 +89,16 @@ def _raise_a_bedrock_overload_caused_by_a_reset() -> None:
|
|||
try:
|
||||
raise ConnectionError("upstream reset")
|
||||
except ConnectionError as upstream:
|
||||
raise litellm.ServiceUnavailableError(
|
||||
message="BedrockException", llm_provider="bedrock", model="m"
|
||||
raise litellm.InternalServerError(
|
||||
message="AnthropicError - Overloaded",
|
||||
llm_provider="anthropic",
|
||||
model="m",
|
||||
provider_status_code=529,
|
||||
) from upstream
|
||||
|
||||
|
||||
def test_the_answered_copy_keeps_the_raised_exceptions_cause_and_traceback() -> None:
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as raised:
|
||||
with pytest.raises(litellm.InternalServerError) as raised:
|
||||
_raise_a_bedrock_overload_caused_by_a_reset()
|
||||
|
||||
answered_exc = with_answered_status(raised.value)
|
||||
|
|
|
|||
|
|
@ -303,7 +303,7 @@ class TestProxyExceptionAnthropicEnvelope:
|
|||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
request = MagicMock()
|
||||
request: Final = MagicMock()
|
||||
request.headers = {}
|
||||
|
||||
response = ep._anthropic_error_json_response(
|
||||
|
|
@ -361,7 +361,7 @@ class TestProxyExceptionAnthropicEnvelope:
|
|||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
raised_status: Final = unavailable.status_code
|
||||
request = MagicMock()
|
||||
request: Final = MagicMock()
|
||||
request.headers = {}
|
||||
|
||||
with (
|
||||
|
|
@ -380,7 +380,7 @@ class TestProxyExceptionAnthropicEnvelope:
|
|||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
)
|
||||
|
||||
recorded = mock_logging.post_call_failure_hook.await_args.kwargs["original_exception"]
|
||||
recorded: Final = mock_logging.post_call_failure_hook.await_args.kwargs["original_exception"]
|
||||
assert recorded is not None
|
||||
assert (recorded is not unavailable) is hook_receives_copy
|
||||
assert recorded.status_code == expected_status
|
||||
|
|
@ -438,9 +438,9 @@ class TestProxyExceptionAnthropicEnvelope:
|
|||
proxy_server, "count_request_tokens", new=AsyncMock(side_effect=unavailable)
|
||||
),
|
||||
):
|
||||
response = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
body = json.loads(response.body)
|
||||
body: Final = json.loads(response.body)
|
||||
assert response.status_code == expected_status
|
||||
assert body["type"] == "error"
|
||||
assert body["error"]["type"] == expected_error_type
|
||||
|
|
@ -553,7 +553,7 @@ class TestHttpExceptionDictDetail:
|
|||
) as mock_logging, # test-quality-ok: module global imported at call time; no injection seam
|
||||
):
|
||||
mock_logging.post_call_failure_hook = AsyncMock()
|
||||
response = await ep.anthropic_response(
|
||||
response: Final = await ep.anthropic_response(
|
||||
fastapi_response=MagicMock(),
|
||||
request=request,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
|
|
|
|||
|
|
@ -2810,8 +2810,7 @@ class TestCommonRequestProcessingHelpers:
|
|||
|
||||
async def test_a_stream_that_fails_before_its_first_byte_answers_as_an_anthropic_json_error(self):
|
||||
"""A /v1/messages stream whose first chunk is already the error frame has nothing
|
||||
streamed yet, so the failure answers as JSON with the status Anthropic answers that upstream failure with
|
||||
(a 503 is its 529 overloaded_error, LIT-9450),
|
||||
streamed yet, so the failure answers as JSON with the status the upstream gave,
|
||||
the shape Anthropic clients raise their status-specific errors on"""
|
||||
|
||||
async def stream():
|
||||
|
|
@ -2822,12 +2821,12 @@ class TestCommonRequestProcessingHelpers:
|
|||
response = await create_response(generator, "text/event-stream", {"x-litellm-call-id": "call-8609"})
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 529
|
||||
assert response.status_code == 503
|
||||
assert response.headers["content-type"] == "application/json"
|
||||
assert response.headers["x-litellm-call-id"] == "call-8609"
|
||||
assert json.loads(response.body) == {
|
||||
"type": "error",
|
||||
"error": {"type": "overloaded_error", "message": "upstream unavailable"},
|
||||
"error": {"type": "api_error", "message": "upstream unavailable"},
|
||||
}
|
||||
assert generator.ag_frame is None
|
||||
|
||||
|
|
@ -2863,7 +2862,7 @@ class TestCommonRequestProcessingHelpers:
|
|||
assert response.status_code == 200
|
||||
assert "".join(await self.consume_stream(response)) == (
|
||||
ANTHROPIC_PING_SSE_CHUNK
|
||||
+ 'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "upstream unavailable"}}\n\n'
|
||||
+ 'event: error\ndata: {"type": "error", "error": {"type": "api_error", "message": "upstream unavailable"}}\n\n'
|
||||
)
|
||||
|
||||
async def test_create_streaming_response_disables_proxy_buffering(self):
|
||||
|
|
@ -10981,22 +10980,27 @@ async def test_base_process_llm_request_audits_a_failure_that_lands_after_its_ke
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_answers_a_late_overload_after_its_keepalive_as_an_overloaded_error_event():
|
||||
async def slow_overload(self, **kwargs):
|
||||
await asyncio.sleep(0.25)
|
||||
raise litellm.ServiceUnavailableError(
|
||||
message='BedrockException - {"message": "Bedrock is unable to process your request."}',
|
||||
llm_provider="bedrock",
|
||||
model="global.anthropic.claude-opus-5-5",
|
||||
)
|
||||
async def test_anthropic_messages_answers_a_late_503_after_its_keepalive_as_an_api_error_event() -> None:
|
||||
unavailable: Final = litellm.ServiceUnavailableError(
|
||||
message='BedrockException - {"message": "Bedrock is unable to process your request."}',
|
||||
llm_provider="bedrock",
|
||||
model="global.anthropic.claude-opus-5-5",
|
||||
)
|
||||
release_failure: Final = asyncio.Event()
|
||||
|
||||
proxy_logging_obj = MagicMock(spec=ProxyLogging)
|
||||
async def fail_after_keepalive(self: ProxyBaseLLMRequestProcessing, **kwargs: object) -> None:
|
||||
await release_failure.wait()
|
||||
raise unavailable
|
||||
|
||||
proxy_logging_obj: Final = MagicMock(spec=ProxyLogging)
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
|
||||
processor = ProxyBaseLLMRequestProcessing(data={"model": "global.anthropic.claude-opus-5-5", "stream": True})
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(
|
||||
data={"model": "global.anthropic.claude-opus-5-5", "stream": True}
|
||||
)
|
||||
|
||||
with patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05):
|
||||
with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", slow_overload):
|
||||
response = await processor.base_process_llm_request(
|
||||
with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", fail_after_keepalive):
|
||||
response: Final = await processor.base_process_llm_request(
|
||||
request=MagicMock(spec=Request),
|
||||
fastapi_response=Response(),
|
||||
user_api_key_dict=MagicMock(spec=UserAPIKeyAuth),
|
||||
|
|
@ -11005,15 +11009,20 @@ async def test_anthropic_messages_answers_a_late_overload_after_its_keepalive_as
|
|||
general_settings={},
|
||||
proxy_config=MagicMock(spec=ProxyConfig),
|
||||
)
|
||||
collected = await _drain(response)
|
||||
body_iterator: Final = response.body_iterator
|
||||
first_chunk: Final = await body_iterator.__anext__()
|
||||
release_failure.set()
|
||||
remaining_chunks: Final = [
|
||||
chunk.encode("utf-8") if isinstance(chunk, str) else bytes(chunk) async for chunk in body_iterator
|
||||
]
|
||||
|
||||
assert proxy_logging_obj.post_call_failure_hook.await_args.kwargs["original_exception"].status_code == 529
|
||||
assert collected[0] == TTFT_PING
|
||||
event_line, data_line = collected[-1].decode().strip().split("\n")
|
||||
assert proxy_logging_obj.post_call_failure_hook.await_args.kwargs["original_exception"] is unavailable
|
||||
assert first_chunk == TTFT_PING
|
||||
event_line, data_line = remaining_chunks[-1].decode().strip().split("\n")
|
||||
assert event_line == "event: error"
|
||||
error = json.loads(data_line.removeprefix("data: "))
|
||||
error: Final = json.loads(data_line.removeprefix("data: "))
|
||||
assert error["type"] == "error"
|
||||
assert error["error"]["type"] == "overloaded_error"
|
||||
assert error["error"]["type"] == "api_error"
|
||||
assert "Bedrock is unable to process your request." in error["error"]["message"]
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue