mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-10 08:28:25 +00:00
fix(proxy): answer an upstream overload with 503 server_is_overloaded on OpenAI routes (#49993)
* fix(anthropic): answer a 503 as 529 overloaded_error on /v1/messages
Anthropic's error contract has no 503, so a client built against it (Claude Code) treats a 503 from /v1/messages as a plain 5xx and retries it ten times instead of taking its overload path. A Bedrock 503 now leaves /v1/messages and /v1/messages/count_tokens as 529 overloaded_error, for both a JSON answer and a stream that fails before its first byte.
An Anthropic 529 overloaded_error now maps to litellm.ServiceUnavailableError (503) instead of InternalServerError (500), so OpenAI-shaped routes answer it as 503, the status OpenAI documents for an overloaded engine, and router retry and allowed-fails policies key it as ServiceUnavailableError.
* test(proxy): expect overloaded_error on a 503 stream error after a keepalive ping
* fix(proxy): record the 529 the Anthropic routes answer and type the chat stream error frame
The messages route stamps the answered status on the exception before post_call_failure_hook runs, so the spend log row, the OTel server span, the failed-request metric, and alerting record the 529 the client saw instead of the deployment's 503, while the deployment failure metric and cooldown keep the deployment's own status. A chat completions stream error frame names the type its status stands for instead of the null .type ServiceUnavailableError carries. An Anthropic-shaped 503 envelope passes through as 529 overloaded_error through a validated TypedDict, and count_tokens answers every ProxyException as the Anthropic envelope. Five integration tests that pinned 503 on the messages route now expect 529
* test(proxy): expect the top-level Anthropic envelope from count_tokens errors
count_tokens now answers a ProxyException as a JSONResponse carrying the
Anthropic error envelope instead of raising HTTPException, which FastAPI
wrapped in {"detail": ...}. The three legacy error-format tests now read
the response body and check that no detail wrap is left.
* test(proxy): cover the 529 overloaded_error answer across routes, cooldown, and retry budgets
* fix(anthropic): keep Anthropic overloads as InternalServerError and answer 529 from the provider status
An upstream Anthropic 529 stays litellm.InternalServerError (status 500) so
InternalServerErrorRetries, InternalServerErrorAllowedFails, SDK handlers,
and the OpenAI-shaped routes behave as on main. The error now carries
provider_status_code, and the /v1/messages routes answer 529
overloaded_error from it.
* refactor(proxy): unwrap a mid-stream fallback error once instead of recursing for the provider status
* fix(anthropic): keep count_tokens' detail body, end keepalive streams with an Anthropic error event, and log any 529 as an overload
* test(integration): cover the audit round 3 cells and expect 529 on the count_tokens 503 cases
* fix(proxy): re-raise an unchanged stream HTTPException as itself and walk nested mid-stream fallbacks for the provider status
* fix(proxy): walk nested mid-stream fallbacks without recursion
* fix(anthropic): preserve 503s and answer upstream 529s on messages routes
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* fix(anthropic): answer an upstream Anthropic 529 as 529 on count_tokens
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* test(anthropic): use integration coverage for keepalive errors
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* fix(proxy): answer an upstream overload with 503 server_is_overloaded on OpenAI routes
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* fix(proxy): pass answered overload status to failure hooks
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* fix(proxy): preserve Anthropic route exception handling
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* refactor(proxy): bind the transformed failure without rebinding the parameter
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* test(proxy): observe the overload failure hook through a real ProxyLogging
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---------
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Mateo Wang
GitHub
parent
733b776403
commit
7a641e5172
7 changed files with 291 additions and 56 deletions
|
|
@ -1,10 +1,29 @@
|
|||
from typing import Final, TypeVar
|
||||
|
||||
from litellm.exceptions import InternalServerError, MidStreamFallbackError
|
||||
from litellm.exceptions import APIError, InternalServerError, MidStreamFallbackError
|
||||
|
||||
_PROVIDER_OVERLOAD_STATUS_CODE: Final = 529
|
||||
_OPENAI_OVERLOAD_ANSWERED_STATUS_CODE: Final = 503
|
||||
|
||||
_Raised = TypeVar("_Raised", bound=Exception)
|
||||
|
||||
|
||||
def is_provider_overload(exc: Exception) -> bool:
|
||||
return _provider_status_code(exc) == _PROVIDER_OVERLOAD_STATUS_CODE
|
||||
|
||||
|
||||
def with_openai_overload_status(exc: _Raised) -> _Raised:
|
||||
return (
|
||||
_copy_carrying(
|
||||
_copy_carrying(exc, "provider_status_code", _PROVIDER_OVERLOAD_STATUS_CODE),
|
||||
"status_code",
|
||||
_OPENAI_OVERLOAD_ANSWERED_STATUS_CODE,
|
||||
)
|
||||
if is_provider_overload(exc)
|
||||
else exc
|
||||
)
|
||||
|
||||
|
||||
def _provider_status_code(exc: Exception) -> int | None:
|
||||
unwrapped = exc # rebind-ok: the recursion gate requires iterative wrapper traversal
|
||||
while isinstance(unwrapped, MidStreamFallbackError) and unwrapped.original_exception is not None:
|
||||
|
|
@ -12,6 +31,10 @@ def _provider_status_code(exc: Exception) -> int | None:
|
|||
match unwrapped:
|
||||
case InternalServerError(provider_status_code=int() as provider_status):
|
||||
return provider_status
|
||||
case APIError(provider_status_code=int() as provider_status):
|
||||
return provider_status
|
||||
case APIError(status_code=int() as provider_status):
|
||||
return provider_status
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ from litellm.litellm_core_utils.streaming_handler import (
|
|||
)
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields
|
||||
from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.anthropic_endpoints.answered_status import is_provider_overload, with_openai_overload_status
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
can_key_call_resolved_model,
|
||||
is_model_cost_zero,
|
||||
|
|
@ -115,6 +116,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
|
|||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import (
|
||||
LITELLM_CALL_ID_HEADER,
|
||||
OverloadedProxyException,
|
||||
attribute_of,
|
||||
error_status_code,
|
||||
openai_error_param,
|
||||
|
|
@ -164,6 +166,7 @@ _ChunkT: Final = TypeVar("_ChunkT")
|
|||
KNOWN_PROXY_ROUTES: Final = frozenset(
|
||||
route for member in LiteLLMRoutes for route in member.value if route.startswith("/")
|
||||
)
|
||||
_OPENAI_OVERLOAD_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion", "aresponses"})
|
||||
|
||||
ProxyRouteType: TypeAlias = Literal[
|
||||
"acompletion",
|
||||
|
|
@ -4398,28 +4401,28 @@ class ProxyBaseLLMRequestProcessing:
|
|||
version: str | None = None,
|
||||
) -> Never:
|
||||
"""Raises ProxyException (OpenAI API compatible) if an exception is raised"""
|
||||
log_llm_api_exception(e, self.litellm_call_id)
|
||||
answered: Final = with_openai_overload_status(e) if self.route_type in _OPENAI_OVERLOAD_ROUTE_TYPES else e
|
||||
log_llm_api_exception(answered, self.litellm_call_id)
|
||||
billing_logging_obj: Final = _REQUEST_FIELDS.validate_python(self.data).get("litellm_logging_obj")
|
||||
failed_in_agentic_follow_up: Final = is_agentic_followup_failure(e)
|
||||
# Allow callbacks to transform the error response
|
||||
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
original_exception=answered,
|
||||
request_data=self.data,
|
||||
dispatched_call_type=self.route_type,
|
||||
)
|
||||
# Use transformed exception if callback returned one, otherwise use original
|
||||
if transformed_exception is not None:
|
||||
e = transformed_exception
|
||||
litellm_debug_info: Final = getattr(e, "litellm_debug_info", "")
|
||||
failure: Final = transformed_exception if transformed_exception is not None else answered
|
||||
litellm_debug_info: Final = getattr(failure, "litellm_debug_info", "")
|
||||
verbose_proxy_logger.debug(
|
||||
"An error occurred: %s %s",
|
||||
e,
|
||||
failure,
|
||||
litellm_debug_info,
|
||||
)
|
||||
|
||||
timeout: Final = getattr(
|
||||
e, "timeout", None
|
||||
failure, "timeout", None
|
||||
) # returns the timeout set by the wrapper. Used for testing if model-specific timeout are set correctly
|
||||
_litellm_logging_obj: Final = (
|
||||
billing_logging_obj if isinstance(billing_logging_obj, LiteLLMLoggingObj) else None
|
||||
|
|
@ -4451,11 +4454,11 @@ class ProxyBaseLLMRequestProcessing:
|
|||
# a failed request reports no timing, matching /v1/chat/completions
|
||||
read_timing_from_logging_obj=False,
|
||||
)
|
||||
headers = getattr(e, "headers", None) or {}
|
||||
headers = getattr(failure, "headers", None) or {}
|
||||
if not headers:
|
||||
_response: Final = attribute_of(e, "response")
|
||||
_response: Final = attribute_of(failure, "response")
|
||||
_response_headers: Final = getattr(_response, "headers", None) if _response is not None else None
|
||||
_provider_headers: Final = _response_headers or getattr(e, "litellm_response_headers", None)
|
||||
_provider_headers: Final = _response_headers or getattr(failure, "litellm_response_headers", None)
|
||||
if _provider_headers:
|
||||
headers = get_response_headers(dict(_provider_headers))
|
||||
headers.update(custom_headers)
|
||||
|
|
@ -4475,21 +4478,21 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
safe_headers: Final = {k: v for k, v in headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS}
|
||||
|
||||
self._apply_router_cooldown_retry_after(safe_headers, e)
|
||||
self._apply_router_cooldown_retry_after(safe_headers, failure)
|
||||
|
||||
if isinstance(e, ProxyException):
|
||||
e.headers = {
|
||||
**{k: v for k, v in e.headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS},
|
||||
if isinstance(failure, ProxyException):
|
||||
failure.headers = {
|
||||
**{k: v for k, v in failure.headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS},
|
||||
**{k: v if isinstance(v, str) else str(v) for k, v in safe_headers.items()},
|
||||
}
|
||||
raise e
|
||||
raise failure
|
||||
|
||||
if isinstance(e, HTTPException):
|
||||
raise proxy_exception_from_http_exception(e, safe_headers)
|
||||
elif isinstance(e, httpx.HTTPStatusError):
|
||||
if isinstance(failure, HTTPException):
|
||||
raise proxy_exception_from_http_exception(failure, safe_headers)
|
||||
elif isinstance(failure, httpx.HTTPStatusError):
|
||||
# Handle httpx.HTTPStatusError - extract actual error from response
|
||||
# This matches the original behavior before the refactor in commit 511d435f6f
|
||||
http_status_error: Final[httpx.HTTPStatusError] = e
|
||||
http_status_error: Final[httpx.HTTPStatusError] = failure
|
||||
error_body: Final = await http_status_error.response.aread()
|
||||
error_text: Final = error_body.decode("utf-8")
|
||||
|
||||
|
|
@ -4499,12 +4502,12 @@ class ProxyBaseLLMRequestProcessing:
|
|||
detail={"error": error_text},
|
||||
headers=error_headers,
|
||||
)
|
||||
error_msg: Final = f"{e}"
|
||||
error_msg: Final = f"{failure}"
|
||||
# Check for AttributeError in the exception chain.
|
||||
# The AttributeError may be wrapped in multiple layers
|
||||
# (e.g. AttributeError -> OpenAIException -> APIConnectionError),
|
||||
# so walk __cause__, __context__, and original_exception recursively.
|
||||
has_attribute_error: Final = _has_attribute_error_in_chain(e)
|
||||
has_attribute_error: Final = _has_attribute_error_in_chain(failure)
|
||||
|
||||
if has_attribute_error:
|
||||
raise ProxyException(
|
||||
|
|
@ -4519,12 +4522,12 @@ class ProxyBaseLLMRequestProcessing:
|
|||
# VertexAIError, etc.) all have a status_code attribute reflecting
|
||||
# the upstream API response. Use it to return the correct HTTP code
|
||||
# instead of defaulting to 500.
|
||||
_exc_status_code: Final = getattr(e, "status_code", None)
|
||||
_exc_status_code: Final = getattr(failure, "status_code", None)
|
||||
if _exc_status_code is not None and isinstance(_exc_status_code, int) and 400 <= _exc_status_code <= 599:
|
||||
_code = _exc_status_code
|
||||
else:
|
||||
_code = status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
if should_report_bug(e):
|
||||
if should_report_bug(failure):
|
||||
proxy_server_request: Final = self.data.get("proxy_server_request")
|
||||
request_url: Final = (
|
||||
proxy_server_request.get("url") if isinstance(proxy_server_request, Mapping) else None
|
||||
|
|
@ -4533,24 +4536,33 @@ class ProxyBaseLLMRequestProcessing:
|
|||
verbose_proxy_logger.error(
|
||||
bug_report_notice(
|
||||
build_proxy_bug_report(
|
||||
e,
|
||||
failure,
|
||||
call_type=allowlisted(request_path, KNOWN_PROXY_ROUTES),
|
||||
custom_llm_provider=self.data.get("custom_llm_provider"),
|
||||
stream=self.data.get("stream"),
|
||||
)
|
||||
)
|
||||
)
|
||||
client_message: Final = getattr(e, "message", error_msg)
|
||||
client_message: Final = getattr(failure, "message", error_msg)
|
||||
redacted_client_message: Final = redact_internal_details_from_client_message(
|
||||
strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg
|
||||
)
|
||||
provider_specific_fields: Final = getattr(
|
||||
failure, "provider_specific_fields", None
|
||||
) or relayed_provider_specific_fields(failure)
|
||||
if self.route_type in _OPENAI_OVERLOAD_ROUTE_TYPES and is_provider_overload(failure):
|
||||
raise OverloadedProxyException(
|
||||
message=redacted_client_message,
|
||||
headers=cast(dict[str, str], safe_headers), # cast-ok: ProxyException coerces header values to strings
|
||||
provider_specific_fields=provider_specific_fields,
|
||||
)
|
||||
raise ProxyException(
|
||||
message=redact_internal_details_from_client_message(
|
||||
strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg
|
||||
),
|
||||
type=openai_error_type(e, _code),
|
||||
param=openai_error_param(e),
|
||||
openai_code=getattr(e, "code", None),
|
||||
message=redacted_client_message,
|
||||
type=openai_error_type(failure, _code),
|
||||
param=openai_error_param(failure),
|
||||
openai_code=getattr(failure, "code", None),
|
||||
code=_code,
|
||||
provider_specific_fields=getattr(e, "provider_specific_fields", None)
|
||||
or relayed_provider_specific_fields(e),
|
||||
provider_specific_fields=provider_specific_fields,
|
||||
headers=safe_headers,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
|
|||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
|
||||
|
||||
LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id"
|
||||
_SERVER_IS_OVERLOADED_CODE: Final = "server_is_overloaded"
|
||||
|
||||
_OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
|
||||
{
|
||||
|
|
@ -44,6 +45,31 @@ _FAILOVER_REFUSALS: Final = (
|
|||
)
|
||||
|
||||
|
||||
class OverloadedProxyException(ProxyException):
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
headers: dict[str, str] | None, # mutable-ok: matches ProxyException's constructor
|
||||
provider_specific_fields: dict[str, object] | None, # mutable-ok: matches ProxyException's constructor
|
||||
) -> None:
|
||||
super().__init__( # pyright: ignore[reportUnknownMemberType] # ProxyException omits generic arguments
|
||||
message=message,
|
||||
type="service_unavailable_error",
|
||||
param=None,
|
||||
code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
headers=None if headers is None else dict(headers),
|
||||
provider_specific_fields=provider_specific_fields,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, object]: # mutable-ok: matches ProxyException.to_dict
|
||||
return {
|
||||
**_ERROR_BODY.validate_python(
|
||||
super().to_dict() # pyright: ignore[reportUnknownMemberType] # ProxyException omits generic arguments
|
||||
),
|
||||
"code": _SERVER_IS_OVERLOADED_CODE,
|
||||
}
|
||||
|
||||
|
||||
def attribute_of(value: object, name: str, default: object = None) -> object:
|
||||
return getattr(value, name, default)
|
||||
|
||||
|
|
|
|||
|
|
@ -426,6 +426,7 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolv
|
|||
from litellm.proxy.analytics_endpoints.analytics_endpoints import (
|
||||
router as analytics_router,
|
||||
)
|
||||
from litellm.proxy.anthropic_endpoints.answered_status import is_provider_overload
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
ROLE_BASED_PERMISSIONS_ADAPTER,
|
||||
ExperimentalUIJWTToken,
|
||||
|
|
@ -13375,6 +13376,13 @@ async def completion(
|
|||
_response.choices[0].text = e.message
|
||||
return _response
|
||||
except Exception as e:
|
||||
if is_provider_overload(e):
|
||||
raise await base_llm_response_processor.handle_llm_api_exception(
|
||||
e=e,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
version=version,
|
||||
)
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
|
|
|
|||
|
|
@ -140,6 +140,16 @@ def _assert_internal_server_error(response: httpx.Response) -> None:
|
|||
assert "InternalServerError" in string_value(error["message"]), response.text
|
||||
|
||||
|
||||
def _assert_openai_overloaded(response: httpx.Response) -> None:
|
||||
assert response.status_code == 503, response.text
|
||||
error: Final = _openai_error(response)
|
||||
assert set(error) == {"message", "type", "param", "code"}, response.text
|
||||
assert string_value(error["message"]), response.text
|
||||
assert error["type"] == "service_unavailable_error", response.text
|
||||
assert error["param"] is None, response.text
|
||||
assert error["code"] == "server_is_overloaded", response.text
|
||||
|
||||
|
||||
def _one_post(wire: Wire, target: str) -> None:
|
||||
assert [(request.method, unquote(request.target)) for request in wire.drain()] == [("POST", target)]
|
||||
|
||||
|
|
@ -361,7 +371,7 @@ def test_messages_overloaded_frame_before_content_without_a_retry_budget_is_rela
|
|||
]
|
||||
|
||||
|
||||
def test_chat_completions_overloaded_frame_before_content_answers_500(gateway: Gateway) -> None:
|
||||
def test_chat_completions_overloaded_frame_before_content_answers_503(gateway: Gateway) -> None:
|
||||
prompt: Final = _prompt()
|
||||
with wire_server(_always(_overloaded_before_content(prompt))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
|
|
@ -370,24 +380,24 @@ def test_chat_completions_overloaded_frame_before_content_answers_500(gateway: G
|
|||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True, "num_retries": 0},
|
||||
)
|
||||
_assert_internal_server_error(response)
|
||||
_assert_openai_overloaded(response)
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
def test_chat_completions_529_answers_500_to_the_sync_sdk(gateway: Gateway) -> None:
|
||||
def test_chat_completions_529_answers_503_to_the_sync_sdk(gateway: Gateway) -> None:
|
||||
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
with pytest.raises(openai.APIStatusError) as caught:
|
||||
_openai_sdk(gateway).chat.completions.create(
|
||||
model=model, messages=[{"role": "user", "content": _prompt()}], extra_body={"num_retries": 0}
|
||||
)
|
||||
assert caught.value.status_code == 500, caught.value.response.text
|
||||
_assert_internal_server_error(caught.value.response)
|
||||
assert caught.value.status_code == 503, caught.value.response.text
|
||||
_assert_openai_overloaded(caught.value.response)
|
||||
_one_post(wire, "/v1/messages")
|
||||
_assert_logged_failure(model, rows=1, error_class="InternalServerError", code="500")
|
||||
_assert_logged_failure(model, rows=1, error_class="InternalServerError", code="503")
|
||||
|
||||
|
||||
async def test_chat_completions_stream_529_answers_500_to_the_async_sdk(gateway: Gateway) -> None:
|
||||
async def test_chat_completions_stream_529_answers_503_to_the_async_sdk(gateway: Gateway) -> None:
|
||||
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
with pytest.raises(openai.APIStatusError) as caught:
|
||||
|
|
@ -397,19 +407,31 @@ async def test_chat_completions_stream_529_answers_500_to_the_async_sdk(gateway:
|
|||
stream=True,
|
||||
extra_body={"num_retries": 0},
|
||||
)
|
||||
assert caught.value.status_code == 500, caught.value.response.text
|
||||
_assert_internal_server_error(caught.value.response)
|
||||
assert caught.value.status_code == 503, caught.value.response.text
|
||||
_assert_openai_overloaded(caught.value.response)
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
def test_completions_529_answers_503(gateway: Gateway) -> None:
|
||||
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/completions",
|
||||
{"model": model, "prompt": _prompt(), "max_tokens": 16, "num_retries": 0},
|
||||
)
|
||||
_assert_openai_overloaded(response)
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
|
||||
def test_responses_529_answers_500(gateway: Gateway, shape: Shape) -> None:
|
||||
def test_responses_529_answers_503(gateway: Gateway, shape: Shape) -> None:
|
||||
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": _prompt(), "stream": shape == "stream", "num_retries": 0}
|
||||
)
|
||||
_assert_internal_server_error(response)
|
||||
_assert_openai_overloaded(response)
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ def _until(rig: _Rig, name: str, model: str) -> tuple[Sample, ...]:
|
|||
(
|
||||
pytest.param("/v1/messages", _MESSAGES_ANTHROPIC, 529, "500", "InternalServerError", id="messages-anthropic"),
|
||||
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, "503", "ServiceUnavailableError", id="messages-bedrock"),
|
||||
pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 500, "500", "InternalServerError", id="chat-anthropic"),
|
||||
pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 503, "500", "InternalServerError", id="chat-anthropic"),
|
||||
),
|
||||
)
|
||||
def test_the_failed_request_metric_counts_the_answered_status_and_the_deployment_metric_the_deployment_status(
|
||||
|
|
@ -245,7 +245,7 @@ def _server_spans(rig: _Rig, since: int, trace_id: str) -> tuple[Span, ...]:
|
|||
("route", "model", "answered"),
|
||||
(
|
||||
pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages"),
|
||||
pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat"),
|
||||
pytest.param("/v1/chat/completions", _TRACED_CHAT, 503, id="chat"),
|
||||
),
|
||||
)
|
||||
def test_the_server_span_carries_the_status_the_route_answered(
|
||||
|
|
@ -313,7 +313,7 @@ def _datadog_status_codes(rig: _Rig, alias: str, expected: int) -> tuple[tuple[s
|
|||
(
|
||||
pytest.param("/v1/messages", _DATADOG_MESSAGES_ANTHROPIC, 529, "500", id="messages-anthropic"),
|
||||
pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 503, "503", id="messages-bedrock"),
|
||||
pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 500, "500", id="chat-anthropic"),
|
||||
pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 503, "500", id="chat-anthropic"),
|
||||
),
|
||||
)
|
||||
def test_the_datadog_proxy_failure_log_carries_the_answered_status_and_the_llm_failure_log_the_deployment_status(
|
||||
|
|
@ -361,7 +361,7 @@ def test_a_fail_closed_guardrail_outage_is_answered_and_recorded_with_the_routes
|
|||
(
|
||||
pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages-anthropic"),
|
||||
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, id="messages-bedrock"),
|
||||
pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat-anthropic"),
|
||||
pytest.param("/v1/chat/completions", _TRACED_CHAT, 503, id="chat-anthropic"),
|
||||
),
|
||||
)
|
||||
def test_the_otel_v2_server_span_carries_the_status_the_route_answered(
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
|
|||
|
||||
import litellm
|
||||
from litellm._uuid import uuid
|
||||
from litellm.exceptions import APIError, InternalServerError, MidStreamFallbackError
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame
|
||||
from litellm.caching.caching import DualCache
|
||||
|
|
@ -5127,6 +5128,138 @@ class TestHandleLLMApiExceptionRetryAfter:
|
|||
assert "All deployments for selected model are in cooldown" not in body["message"]
|
||||
|
||||
|
||||
class TestHandleLLMApiExceptionProviderOverload:
|
||||
class _RecordingFailureHook(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.original_exception: Exception | None = None
|
||||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict,
|
||||
original_exception: Exception,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
traceback_str: str | None = None,
|
||||
) -> None:
|
||||
self.original_exception = original_exception
|
||||
|
||||
async def _invoke(
|
||||
self,
|
||||
exc: Exception,
|
||||
route_type: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
recording_hook: _RecordingFailureHook | None = None,
|
||||
) -> ProxyException:
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={})
|
||||
processor.route_type = route_type
|
||||
callback: Final = recording_hook if recording_hook is not None else self._RecordingFailureHook()
|
||||
monkeypatch.setattr(litellm, "callbacks", [callback])
|
||||
proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache())
|
||||
|
||||
with pytest.raises(ProxyException) as raised:
|
||||
await processor.handle_llm_api_exception(
|
||||
e=exc,
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
return raised.value
|
||||
|
||||
@pytest.mark.parametrize("route_type", ("acompletion", "atext_completion", "aresponses"))
|
||||
async def test_openai_routes_answer_provider_overload_as_503(
|
||||
self, route_type: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
exc: Final = InternalServerError(
|
||||
message="Overloaded",
|
||||
llm_provider="anthropic",
|
||||
model="claude",
|
||||
provider_status_code=529,
|
||||
)
|
||||
|
||||
proxy_exc: Final = await self._invoke(exc, route_type, monkeypatch)
|
||||
|
||||
assert proxy_exc.code == "503"
|
||||
assert proxy_exc.type == "service_unavailable_error"
|
||||
assert proxy_exc.to_dict()["code"] == "server_is_overloaded"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("route_type", "provider_status_code"),
|
||||
(("acompletion", 500), ("anthropic_messages", 529)),
|
||||
)
|
||||
async def test_non_overload_and_non_openai_routes_keep_their_status(
|
||||
self, route_type: str, provider_status_code: int, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
exc: Final = InternalServerError(
|
||||
message="Internal server error",
|
||||
llm_provider="anthropic",
|
||||
model="claude",
|
||||
provider_status_code=provider_status_code,
|
||||
)
|
||||
|
||||
proxy_exc: Final = await self._invoke(exc, route_type, monkeypatch)
|
||||
|
||||
assert proxy_exc.code == "500"
|
||||
assert proxy_exc.type == "internal_server_error"
|
||||
assert proxy_exc.to_dict()["code"] == "500"
|
||||
|
||||
async def test_openai_route_answers_wrapped_provider_overload_as_503(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
original: Final = InternalServerError(
|
||||
message="Overloaded",
|
||||
llm_provider="anthropic",
|
||||
model="claude",
|
||||
provider_status_code=529,
|
||||
)
|
||||
exc: Final = MidStreamFallbackError(
|
||||
message="Fallback failed",
|
||||
llm_provider="anthropic",
|
||||
model="claude",
|
||||
original_exception=original,
|
||||
)
|
||||
|
||||
proxy_exc: Final = await self._invoke(exc, "acompletion", monkeypatch)
|
||||
|
||||
assert proxy_exc.code == "503"
|
||||
assert proxy_exc.type == "service_unavailable_error"
|
||||
assert proxy_exc.to_dict()["code"] == "server_is_overloaded"
|
||||
|
||||
async def test_anthropic_text_api_overload_answers_503(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
exc: Final = APIError(
|
||||
status_code=529,
|
||||
message="Overloaded",
|
||||
llm_provider="anthropic_text",
|
||||
model="claude",
|
||||
)
|
||||
|
||||
proxy_exc: Final = await self._invoke(exc, "atext_completion", monkeypatch)
|
||||
|
||||
assert proxy_exc.code == "503"
|
||||
assert proxy_exc.type == "service_unavailable_error"
|
||||
assert proxy_exc.to_dict()["code"] == "server_is_overloaded"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("route_type", "expected_status"),
|
||||
(("acompletion", 503), ("anthropic_messages", 500)),
|
||||
)
|
||||
async def test_failure_hook_receives_the_answered_overload_status(
|
||||
self, route_type: str, expected_status: int, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
exc: Final = InternalServerError(
|
||||
message="Overloaded",
|
||||
llm_provider="anthropic",
|
||||
model="claude",
|
||||
provider_status_code=529,
|
||||
)
|
||||
recording_hook: Final = self._RecordingFailureHook()
|
||||
|
||||
await self._invoke(exc, route_type, monkeypatch, recording_hook)
|
||||
|
||||
original_exception: Final = recording_hook.original_exception
|
||||
assert original_exception is not None
|
||||
assert original_exception.status_code == expected_status
|
||||
|
||||
|
||||
class TestHandleLLMApiExceptionFramingHeaders:
|
||||
"""HTTP-framing headers on the provider exception must be stripped before the
|
||||
proxy builds its own response, or they conflict with the framing the proxy
|
||||
|
|
@ -7755,7 +7888,10 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
|
|||
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "sk-synthetic"},
|
||||
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
|
||||
},
|
||||
{"model_name": "chat-model", "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-synthetic"}},
|
||||
{
|
||||
"model_name": "chat-model",
|
||||
"litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-synthetic"},
|
||||
},
|
||||
],
|
||||
fallbacks=[{"rate-limited": ["embed-alias", "chat-model"]}],
|
||||
)
|
||||
|
|
@ -9829,10 +9965,13 @@ def _hook_data(**kwargs: object) -> object:
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("prompt, proxy_field, client_field", [
|
||||
({"input": "hi"}, "litellm_metadata", "metadata"),
|
||||
({"messages": [{"role": "user", "content": "hi"}]}, "metadata", "litellm_metadata"),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"prompt, proxy_field, client_field",
|
||||
[
|
||||
({"input": "hi"}, "litellm_metadata", "metadata"),
|
||||
({"messages": [{"role": "user", "content": "hi"}]}, "metadata", "litellm_metadata"),
|
||||
],
|
||||
)
|
||||
async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape_picks(
|
||||
prompt: dict[str, str | list[dict[str, str]]],
|
||||
proxy_field: str,
|
||||
|
|
@ -9842,7 +9981,9 @@ async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape
|
|||
|
||||
logging: Final = MagicMock(spec=ProxyLogging)
|
||||
logging.pre_call_hook = AsyncMock(side_effect=_hook_data)
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={"model": "gpt-x", "metadata": {"client": "cursor"}, **prompt})
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(
|
||||
data={"model": "gpt-x", "metadata": {"client": "cursor"}, **prompt}
|
||||
)
|
||||
data, _ = await processor.common_processing_pre_call_logic(
|
||||
request=Request({"type": "http", "method": "POST", "path": "/cursor/chat/completions", "headers": []}),
|
||||
general_settings={},
|
||||
|
|
@ -9858,6 +9999,7 @@ async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape
|
|||
assert "queue_time_seconds" not in data.get(client_field, {})
|
||||
assert data["metadata"]["client"] == "cursor"
|
||||
|
||||
|
||||
class TestPerRequestModelGroupAlias:
|
||||
"""``router_settings.model_group_alias`` on a key or team has to be resolved
|
||||
by the proxy: the Router resolves aliases from its own shared instance
|
||||
|
|
@ -12693,7 +12835,9 @@ class TestResponseCostFromLoggingObjForCacheHits:
|
|||
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
|
||||
)
|
||||
|
||||
cost_headers: Final = {name: value for name, value in headers.items() if name.startswith("x-litellm-response-cost")}
|
||||
cost_headers: Final = {
|
||||
name: value for name, value in headers.items() if name.startswith("x-litellm-response-cost")
|
||||
}
|
||||
assert cost_headers == {"x-litellm-response-cost-output": "0.0012"}
|
||||
|
||||
def test_a_fresh_zero_cost_v1_messages_call_still_sends_no_cost_header(self):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue