This commit is contained in:
Mateo Wang 2026-10-29 02:58:34 +00:00 • committed by GitHub
commit 8f00ea20f9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 224 additions and 36 deletions

View file

@ -3816,6 +3816,16 @@ class ManagementEndpointLoggingPayload(LiteLLMPydanticObjectBase):
end_time: datetime | None = None
def router_error_status_code(message: str) -> int | None:
"""The status a router failure answers with when its message names it: no deployment left maps to 429
(https://github.com/BerriAI/litellm/issues/2487) and no deployment matching the request's tags to 401."""
if "No healthy deployment available" in message or "No deployments available" in message:
return 429
if RouterErrors.no_deployments_with_tag_routing.value in message:
return 401
return None
class ProxyException(Exception):
# NOTE: DO NOT MODIFY THIS
# This is used to map exactly to OPENAI Exceptions
@ -3837,20 +3847,14 @@ class ProxyException(Exception):
# If we look on official python OpenAI lib, the code should be a string:
# https://github.com/openai/openai-python/blob/195c05a64d39c87b2dfdf1eca2d339597f1fce03/src/openai/types/shared/error_object.py#L11
# Related LiteLLM issue: https://github.com/BerriAI/litellm/discussions/4834
self.code = str(code)
router_status_code: Final = router_error_status_code(self.message)
self.code = str(code if router_status_code is None else router_status_code)
if headers is not None:
for k, v in headers.items():
if not isinstance(v, str):
headers[k] = str(v)
self.headers = headers or {}
self.provider_specific_fields = provider_specific_fields
# rules for proxyExceptions
# Litellm router.py returns "No healthy deployment available" when there are no deployments available
# Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487
if "No healthy deployment available" in self.message or "No deployments available" in self.message:
self.code = "429"
elif RouterErrors.no_deployments_with_tag_routing.value in self.message:
self.code = "401"
def to_dict(self) -> dict:
"""Converts the ProxyException instance to a dictionary."""

View file

@ -9,7 +9,7 @@ from typing import Final
from fastapi import status
from litellm.constants import STRINGIFIED_NONE
from litellm.proxy._types import ProxyException
from litellm.proxy._types import ProxyException, router_error_status_code
LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id"
@ -45,6 +45,16 @@ def openai_error_type(exc: object, status_code: int) -> str:
return openai_error_type_for_status(status_code)
def proxy_error_status_code(exc: object, message: str) -> int:
"""The status ``ProxyException`` answers ``exc`` with once its router-message rules apply."""
routed: Final = router_error_status_code(message)
return routed if routed is not None else error_status_code(exc, status.HTTP_500_INTERNAL_SERVER_ERROR)
def is_gateway_failure(status_code: int) -> bool:
return status_code == status.HTTP_408_REQUEST_TIMEOUT or status_code >= status.HTTP_500_INTERNAL_SERVER_ERROR
def openai_error_type_for_status(status_code: int) -> str:
mapped: Final = _OPENAI_ERROR_TYPE_BY_STATUS.get(status_code)
if mapped is not None:

View file

@ -7243,15 +7243,10 @@ async def list_keys(
return response
except Exception as e:
verbose_proxy_logger.exception("Error in list_keys: %s", e)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"error({e})"),
type=ProxyErrorTypes.internal_server_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
raise handle_exception_on_proxy(e)
verbose_proxy_logger.exception("Error in list_keys: %s", e)
if isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),

View file

@ -85,8 +85,11 @@ from litellm.proxy._types import (
from litellm.proxy.bug_report_config import build_proxy_bug_report
from litellm.proxy.common_utils.openai_error_payload import (
attribute_of,
is_gateway_failure,
litellm_call_id_headers,
openai_error_param,
openai_error_type,
proxy_error_status_code,
with_litellm_call_id,
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
@ -133,7 +136,7 @@ from litellm import (
Router,
)
from litellm._internal_context import service_target
from litellm._logging import redact_string, verbose_proxy_logger
from litellm._logging import redact_internal_details_from_client_message, redact_string, verbose_proxy_logger
from litellm._service_logger import ServiceLogging, ServiceTypes
from litellm.caching.caching import DualCache, RedisCache
from litellm.caching.dual_cache import LimitedSizeOrderedDict
@ -9177,12 +9180,14 @@ def _recreate_writer_on_read_only_transaction(prisma_client: "PrismaClient | Non
asyncio.create_task(prisma_client.recreate_read_only_writer(reason="postgres_read_only_transaction"))
def _client_error_message(message: str, status_code: int) -> str:
return redact_internal_details_from_client_message(message) if is_gateway_failure(status_code) else message
def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) -> ProxyException:
"""
Returns an Exception as ProxyException, this ensures all exceptions are OpenAI API compatible
"""
from fastapi import status
verbose_proxy_logger.exception("Exception: %s", e)
if PrismaDBExceptionHandler.is_read_only_transaction_error(e):
from litellm.proxy.proxy_server import prisma_client
@ -9191,21 +9196,24 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None)
headers: Final = litellm_call_id_headers(litellm_call_id)
if isinstance(e, HTTPException):
detail: Final = str(attribute_of(e, "detail", f"error({e})"))
http_status_code: Final = proxy_error_status_code(e, detail)
return ProxyException(
message=getattr(e, "detail", f"error({e})"),
type=ProxyErrorTypes.internal_server_error,
message=_client_error_message(detail, http_status_code),
type=openai_error_type(e, http_status_code),
param=openai_error_param(e),
headers=headers,
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
code=http_status_code,
)
elif isinstance(e, ProxyException):
return with_litellm_call_id(e, litellm_call_id)
_status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
message: Final = strip_bug_report_notice(str(e))
_status_code: Final = proxy_error_status_code(e, message)
if should_report_bug(e):
verbose_proxy_logger.error(bug_report_notice(build_proxy_bug_report(e)))
return ProxyException(
message=strip_bug_report_notice(str(e)),
type=ProxyErrorTypes.internal_server_error,
message=_client_error_message(message, _status_code),
type=openai_error_type(e, _status_code),
param=openai_error_param(e),
headers=headers,
code=_status_code,

View file

@ -432,7 +432,7 @@ def test_team_guardrail_submissions_require_admin_approval(
f"Guardrail '{first_name}' not found. "
"Please ensure the guardrail is configured in your LiteLLM proxy."
),
"type": "internal_server_error",
"type": "invalid_request_error",
"param": None,
"code": "404",
}
@ -481,7 +481,7 @@ def test_team_guardrail_submissions_require_admin_approval(
f"Guardrail '{second_name}' not found. "
"Please ensure the guardrail is configured in your LiteLLM proxy."
),
"type": "internal_server_error",
"type": "invalid_request_error",
"param": None,
"code": "404",
}

View file

@ -279,7 +279,7 @@ def test_new_user_with_admin_role_requires_proxy_admin(gateway: Gateway, role: s
assert refused.json() == {
"error": {
"message": f"Only proxy admins can create administrative users (proxy_admin, proxy_admin_viewer). Attempted to create user with role: {role}. Your role: internal_user",
"type": "internal_server_error",
"type": "permission_error",
"param": None,
"code": "403",
}

View file

@ -170,7 +170,7 @@ def test_user_delete_revokes_keys_and_memberships(gateway: Gateway, peer: Gatewa
assert info.json() == {
"error": {
"message": f"User {user_id} not found",
"type": "internal_server_error",
"type": "invalid_request_error",
"param": None,
"code": "404",
}
@ -337,7 +337,7 @@ def test_scim_user_delete_revokes_keys(gateway: Gateway, peer: Gateway, monkeypa
expected_info: Final = {
"error": {
"message": f"User {user_id} not found",
"type": "internal_server_error",
"type": "invalid_request_error",
"param": None,
"code": "404",
}

View file

@ -156,7 +156,7 @@ def test_apply_guardrail_returns_a_block_error_for_generic_guardrails(gateway: G
assert error.model_dump() == {
"error": {
"message": _BLOCK_REASON,
"type": "internal_server_error",
"type": "invalid_request_error",
"param": None,
"code": 400,
}

View file

@ -3014,3 +3014,20 @@ class TestCredentialDisplayName:
assert (by_name["display_name"], by_name["source"]) == ("Prod OpenAI", "db")
assert config_by_name["source"] == "config"
assert listed["openai-prod"]["credential_values"]["api_key"] != "sk-old"
def test_looking_up_a_missing_credential_is_labelled_a_client_error_not_a_gateway_failure(
restore_credential_list: None,
):
"""Naming a credential that does not exist is the caller's own mistake, and the shared
handler used to label every one of these internal_server_error, so an SDK branching on
the type read a 404 it can never retry past as a gateway outage and backed off forever."""
response: Final = _call_as("GET", "/credentials/by_name/definitely-not-there")
assert response.status_code == 404, f"lookup answered {response.status_code}: {response.text}"
assert response.json()["error"] == {
"message": "Credential not found. Got credential name: definitely-not-there",
"type": "invalid_request_error",
"param": None,
"code": "404",
}

View file

@ -6538,7 +6538,7 @@ async def test_list_keys_with_invalid_status():
)
# Verify ProxyException properties
assert exc_info.value.code == "400"
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
assert "Invalid status value" in str(exc_info.value.message)
assert "deleted" in str(exc_info.value.message)
@ -18736,7 +18736,7 @@ async def test_list_keys_rejects_invalid_expires():
expires="expred",
)
assert exc_info.value.code == "400"
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
assert "Invalid expires value" in str(exc_info.value.message)

View file

@ -1,13 +1,19 @@
import asyncio
import json
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import httpx
import openai
import pytest
from fastapi import HTTPException
from prisma.errors import DataError
import litellm
from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.proxy.common_request_processing import proxy_exception_from_route_error
from litellm.proxy.utils import get_error_message_str, handle_exception_on_proxy
from litellm.types.router import RouterErrors
def normalize(value):
@ -96,7 +102,7 @@ def test_handle_exception_on_proxy_happy_path_http_exception():
assert snapshot == {
"is_proxy_exception": True,
"message": "forbidden",
"type": ProxyErrorTypes.internal_server_error.value,
"type": "permission_error",
"code": "403",
}
@ -156,7 +162,7 @@ def test_handle_exception_on_proxy_uses_attached_status_code_when_present():
assert snapshot == {
"code": "418",
"message": "teapot",
"type": ProxyErrorTypes.internal_server_error.value,
"type": "invalid_request_error",
}
@ -176,6 +182,154 @@ def test_handle_exception_on_proxy_error_path_none_input_wraps_as_500():
}
@pytest.mark.parametrize(
"status_code, expected_type",
[
(400, "invalid_request_error"),
(401, "authentication_error"),
(403, "permission_error"),
(404, "invalid_request_error"),
(408, "invalid_request_error"),
(409, "invalid_request_error"),
(422, "invalid_request_error"),
(429, "rate_limit_error"),
(500, ProxyErrorTypes.internal_server_error.value),
(503, ProxyErrorTypes.internal_server_error.value),
],
)
def test_handle_exception_on_proxy_labels_an_http_exception_by_the_status_it_answers(
status_code: int, expected_type: str
):
"""A management route that rejects a caller's own argument answers 400, and calling that
internal_server_error tells the caller's retry loop the gateway is down when nothing but
their request is wrong."""
result: Final = handle_exception_on_proxy(HTTPException(status_code=status_code, detail="boom"))
assert (result.code, result.type) == (str(status_code), expected_type)
def _exception_carrying_status(carried: int | None) -> Exception:
class _CarriedStatus(Exception):
status_code = carried
return _CarriedStatus("boom")
@pytest.mark.parametrize(
"status_code, expected_type",
[
(400, "invalid_request_error"),
(404, "invalid_request_error"),
(408, "invalid_request_error"),
(429, "rate_limit_error"),
(500, ProxyErrorTypes.internal_server_error.value),
],
)
def test_handle_exception_on_proxy_labels_a_carried_status_the_same_way(status_code: int, expected_type: str):
"""Routes also reach this handler with plain exceptions carrying their own status, and
those must be named by that status rather than all landing on the 500 label."""
result: Final = handle_exception_on_proxy(_exception_carrying_status(status_code))
assert (result.code, result.type) == (str(status_code), expected_type)
def test_handle_exception_on_proxy_keeps_the_type_the_exception_already_names():
"""A rate limit litellm raised already names itself, and overwriting that with the status
map's guess would drop what the client branches on."""
result: Final = handle_exception_on_proxy(
litellm.RateLimitError(message="slow down", llm_provider="openai", model="gpt-4o")
)
assert (result.code, result.type) == ("429", "throttling_error")
@pytest.mark.parametrize(
"exc",
[
openai.InternalServerError(
message="upstream down",
response=httpx.Response(500, request=httpx.Request("GET", "https://api.openai.com/v1/batches/b")),
body={"type": "server_error"},
),
litellm.Timeout(message="took too long", model="gpt-6.1-sol", llm_provider="openai"),
litellm.RateLimitError(message="slow down", llm_provider="openai", model="gpt-6.1-sol"),
litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-6.1-sol"),
litellm.BadRequestError(message="bad input", model="gpt-6.1-sol", llm_provider="openai"),
HTTPException(status_code=404, detail="no such batch"),
],
ids=["provider_5xx_naming_its_own_type", "timeout", "rate_limit", "bad_key", "bad_request", "http_404"],
)
def test_handle_exception_on_proxy_labels_a_failure_the_way_the_chat_route_does(exc: Exception):
"""Batch, file, and management routes answer through this handler while chat answers through the route
error path, and one client calling both must read the same type for the same failure."""
shared: Final = handle_exception_on_proxy(exc)
chat: Final = proxy_exception_from_route_error(exc)
assert (shared.code, shared.type) == (chat.code, chat.type)
@pytest.mark.parametrize(
"message, expected",
[
(f"{RouterErrors.no_deployments_with_provider_budget_routing.value}, try later", ("429", "rate_limit_error")),
(f"{RouterErrors.no_deployments_with_tag_routing.value}=['paid']", ("401", "authentication_error")),
],
ids=["no_deployment_left", "no_deployment_for_tags"],
)
def test_handle_exception_on_proxy_names_a_router_failure_by_the_status_it_answers(
message: str, expected: tuple[str, str]
):
"""A router failure that carries no status answers the 429 or 401 its message maps to, and a 429 labelled
internal_server_error reads as an outage to a client that branches on the type."""
result: Final = handle_exception_on_proxy(ValueError(message))
assert (result.code, result.type) == expected
def test_handle_exception_on_proxy_keeps_the_path_a_rejected_request_names():
"""A provider rejecting a batch input names the caller's own bucket path, and redacting the /tmp part of it
leaves the caller nothing to fix their request with."""
result: Final = handle_exception_on_proxy(
litellm.BadRequestError(
message="Invalid input_file_id gs://my-bucket/tmp/batch_input.jsonl",
model="gemini-3.1-pro-preview",
llm_provider="vertex_ai",
)
)
assert result.code == "400"
assert "gs://my-bucket/tmp/batch_input.jsonl" in result.message
def test_handle_exception_on_proxy_redacts_the_internal_host_an_unreachable_upstream_names():
"""A guardrail or provider the gateway cannot reach names its private address in the
error, and any virtual key can call those routes, so the host must not reach the caller."""
result: Final = handle_exception_on_proxy(ConnectionError("Cannot connect to host 10.12.4.7:8443"))
assert result.code == "500"
assert "Cannot connect to host" in result.message
assert "10.12.4.7" not in result.message
def test_handle_exception_on_proxy_redacts_the_internal_host_a_route_puts_in_a_gateway_failure_detail():
"""Routes wrap an upstream failure as an HTTPException whose detail is the raw error, so a 5xx detail carries
the same private address an unwrapped connection error does."""
result: Final = handle_exception_on_proxy(HTTPException(status_code=503, detail="Cannot connect to host 10.0.0.9"))
assert result.code == "503"
assert "Cannot connect to host" in result.message
assert "10.0.0.9" not in result.message
def test_handle_exception_on_proxy_answers_500_for_an_exception_whose_status_is_none():
"""A guardrail whose upstream refuses the connection raises with status_code=None, and
passing that through as the code crashed the proxy's error handler, which then blamed the
auth database with a 503 instead of naming the unreachable guardrail."""
result: Final = handle_exception_on_proxy(_exception_carrying_status(None))
assert (result.code, result.type, result.message) == ("500", ProxyErrorTypes.internal_server_error.value, "boom")
@pytest.mark.parametrize(
"exc",
[