diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5ce6cd93a22..e49487afc72 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3816,6 +3816,16 @@ class ManagementEndpointLoggingPayload(LiteLLMPydanticObjectBase): end_time: datetime | None = None +def router_error_status_code(message: str) -> int | None: + """The status a router failure answers with when its message names it: no deployment left maps to 429 + (https://github.com/BerriAI/litellm/issues/2487) and no deployment matching the request's tags to 401.""" + if "No healthy deployment available" in message or "No deployments available" in message: + return 429 + if RouterErrors.no_deployments_with_tag_routing.value in message: + return 401 + return None + + class ProxyException(Exception): # NOTE: DO NOT MODIFY THIS # This is used to map exactly to OPENAI Exceptions @@ -3837,20 +3847,14 @@ class ProxyException(Exception): # If we look on official python OpenAI lib, the code should be a string: # https://github.com/openai/openai-python/blob/195c05a64d39c87b2dfdf1eca2d339597f1fce03/src/openai/types/shared/error_object.py#L11 # Related LiteLLM issue: https://github.com/BerriAI/litellm/discussions/4834 - self.code = str(code) + router_status_code: Final = router_error_status_code(self.message) + self.code = str(code if router_status_code is None else router_status_code) if headers is not None: for k, v in headers.items(): if not isinstance(v, str): headers[k] = str(v) self.headers = headers or {} self.provider_specific_fields = provider_specific_fields - # rules for proxyExceptions - # Litellm router.py returns "No healthy deployment available" when there are no deployments available - # Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487 - if "No healthy deployment available" in self.message or "No deployments available" in self.message: - self.code = "429" - elif RouterErrors.no_deployments_with_tag_routing.value in self.message: - self.code = "401" def to_dict(self) -> dict: """Converts the ProxyException instance to a dictionary.""" diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index 2f01dbe11f7..aea60d5217f 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -9,7 +9,7 @@ from typing import Final from fastapi import status from litellm.constants import STRINGIFIED_NONE -from litellm.proxy._types import ProxyException +from litellm.proxy._types import ProxyException, router_error_status_code LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id" @@ -45,6 +45,16 @@ def openai_error_type(exc: object, status_code: int) -> str: return openai_error_type_for_status(status_code) +def proxy_error_status_code(exc: object, message: str) -> int: + """The status ``ProxyException`` answers ``exc`` with once its router-message rules apply.""" + routed: Final = router_error_status_code(message) + return routed if routed is not None else error_status_code(exc, status.HTTP_500_INTERNAL_SERVER_ERROR) + + +def is_gateway_failure(status_code: int) -> bool: + return status_code == status.HTTP_408_REQUEST_TIMEOUT or status_code >= status.HTTP_500_INTERNAL_SERVER_ERROR + + def openai_error_type_for_status(status_code: int) -> str: mapped: Final = _OPENAI_ERROR_TYPE_BY_STATUS.get(status_code) if mapped is not None: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a19ecb8a9e9..5b2c0fb1bd6 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -7243,15 +7243,10 @@ async def list_keys( return response except Exception as e: - verbose_proxy_logger.exception("Error in list_keys: %s", e) if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"error({e})"), - type=ProxyErrorTypes.internal_server_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR), - ) - elif isinstance(e, ProxyException): + raise handle_exception_on_proxy(e) + verbose_proxy_logger.exception("Error in list_keys: %s", e) + if isinstance(e, ProxyException): raise e raise ProxyException( message="Authentication Error, " + str(e), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..1487774cd64 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -85,8 +85,11 @@ from litellm.proxy._types import ( from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.openai_error_payload import ( attribute_of, + is_gateway_failure, litellm_call_id_headers, openai_error_param, + openai_error_type, + proxy_error_status_code, with_litellm_call_id, ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error @@ -133,7 +136,7 @@ from litellm import ( Router, ) from litellm._internal_context import service_target -from litellm._logging import redact_string, verbose_proxy_logger +from litellm._logging import redact_internal_details_from_client_message, redact_string, verbose_proxy_logger from litellm._service_logger import ServiceLogging, ServiceTypes from litellm.caching.caching import DualCache, RedisCache from litellm.caching.dual_cache import LimitedSizeOrderedDict @@ -9177,12 +9180,14 @@ def _recreate_writer_on_read_only_transaction(prisma_client: "PrismaClient | Non asyncio.create_task(prisma_client.recreate_read_only_writer(reason="postgres_read_only_transaction")) +def _client_error_message(message: str, status_code: int) -> str: + return redact_internal_details_from_client_message(message) if is_gateway_failure(status_code) else message + + def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) -> ProxyException: """ Returns an Exception as ProxyException, this ensures all exceptions are OpenAI API compatible """ - from fastapi import status - verbose_proxy_logger.exception("Exception: %s", e) if PrismaDBExceptionHandler.is_read_only_transaction_error(e): from litellm.proxy.proxy_server import prisma_client @@ -9191,21 +9196,24 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) headers: Final = litellm_call_id_headers(litellm_call_id) if isinstance(e, HTTPException): + detail: Final = str(attribute_of(e, "detail", f"error({e})")) + http_status_code: Final = proxy_error_status_code(e, detail) return ProxyException( - message=getattr(e, "detail", f"error({e})"), - type=ProxyErrorTypes.internal_server_error, + message=_client_error_message(detail, http_status_code), + type=openai_error_type(e, http_status_code), param=openai_error_param(e), headers=headers, - code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + code=http_status_code, ) elif isinstance(e, ProxyException): return with_litellm_call_id(e, litellm_call_id) - _status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) + message: Final = strip_bug_report_notice(str(e)) + _status_code: Final = proxy_error_status_code(e, message) if should_report_bug(e): verbose_proxy_logger.error(bug_report_notice(build_proxy_bug_report(e))) return ProxyException( - message=strip_bug_report_notice(str(e)), - type=ProxyErrorTypes.internal_server_error, + message=_client_error_message(message, _status_code), + type=openai_error_type(e, _status_code), param=openai_error_param(e), headers=headers, code=_status_code, diff --git a/tests/integration/authorization/test_guardrail_submissions.py b/tests/integration/authorization/test_guardrail_submissions.py index f120028f46d..07d9a96c979 100644 --- a/tests/integration/authorization/test_guardrail_submissions.py +++ b/tests/integration/authorization/test_guardrail_submissions.py @@ -432,7 +432,7 @@ def test_team_guardrail_submissions_require_admin_approval( f"Guardrail '{first_name}' not found. " "Please ensure the guardrail is configured in your LiteLLM proxy." ), - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } @@ -481,7 +481,7 @@ def test_team_guardrail_submissions_require_admin_approval( f"Guardrail '{second_name}' not found. " "Please ensure the guardrail is configured in your LiteLLM proxy." ), - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } diff --git a/tests/integration/authorization/test_user_self_update_guard.py b/tests/integration/authorization/test_user_self_update_guard.py index d83960e1e85..722046ca6bd 100644 --- a/tests/integration/authorization/test_user_self_update_guard.py +++ b/tests/integration/authorization/test_user_self_update_guard.py @@ -279,7 +279,7 @@ def test_new_user_with_admin_role_requires_proxy_admin(gateway: Gateway, role: s assert refused.json() == { "error": { "message": f"Only proxy admins can create administrative users (proxy_admin, proxy_admin_viewer). Attempted to create user with role: {role}. Your role: internal_user", - "type": "internal_server_error", + "type": "permission_error", "param": None, "code": "403", } diff --git a/tests/integration/management/test_user_delete_revokes_keys.py b/tests/integration/management/test_user_delete_revokes_keys.py index 324963a0da9..edbbf550871 100644 --- a/tests/integration/management/test_user_delete_revokes_keys.py +++ b/tests/integration/management/test_user_delete_revokes_keys.py @@ -170,7 +170,7 @@ def test_user_delete_revokes_keys_and_memberships(gateway: Gateway, peer: Gatewa assert info.json() == { "error": { "message": f"User {user_id} not found", - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } @@ -337,7 +337,7 @@ def test_scim_user_delete_revokes_keys(gateway: Gateway, peer: Gateway, monkeypa expected_info: Final = { "error": { "message": f"User {user_id} not found", - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } diff --git a/tests/integration/observability/test_apply_guardrail_endpoint.py b/tests/integration/observability/test_apply_guardrail_endpoint.py index 9fa7d1a74db..f50679bd54e 100644 --- a/tests/integration/observability/test_apply_guardrail_endpoint.py +++ b/tests/integration/observability/test_apply_guardrail_endpoint.py @@ -156,7 +156,7 @@ def test_apply_guardrail_returns_a_block_error_for_generic_guardrails(gateway: G assert error.model_dump() == { "error": { "message": _BLOCK_REASON, - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": 400, } diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index c905d38dd41..3c5343eb7e7 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -3014,3 +3014,20 @@ class TestCredentialDisplayName: assert (by_name["display_name"], by_name["source"]) == ("Prod OpenAI", "db") assert config_by_name["source"] == "config" assert listed["openai-prod"]["credential_values"]["api_key"] != "sk-old" + + +def test_looking_up_a_missing_credential_is_labelled_a_client_error_not_a_gateway_failure( + restore_credential_list: None, +): + """Naming a credential that does not exist is the caller's own mistake, and the shared + handler used to label every one of these internal_server_error, so an SDK branching on + the type read a 404 it can never retry past as a gateway outage and backed off forever.""" + response: Final = _call_as("GET", "/credentials/by_name/definitely-not-there") + + assert response.status_code == 404, f"lookup answered {response.status_code}: {response.text}" + assert response.json()["error"] == { + "message": "Credential not found. Got credential name: definitely-not-there", + "type": "invalid_request_error", + "param": None, + "code": "404", + } diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index b13a7bfc33e..ba5d0f6b8f4 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -6538,7 +6538,7 @@ async def test_list_keys_with_invalid_status(): ) # Verify ProxyException properties - assert exc_info.value.code == "400" + assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error") assert "Invalid status value" in str(exc_info.value.message) assert "deleted" in str(exc_info.value.message) @@ -18736,7 +18736,7 @@ async def test_list_keys_rejects_invalid_expires(): expires="expred", ) - assert exc_info.value.code == "400" + assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error") assert "Invalid expires value" in str(exc_info.value.message) diff --git a/tests/unit/proxy/utils/helpers/test_error_helpers.py b/tests/unit/proxy/utils/helpers/test_error_helpers.py index df399c8b1d2..6f6e2dcc469 100644 --- a/tests/unit/proxy/utils/helpers/test_error_helpers.py +++ b/tests/unit/proxy/utils/helpers/test_error_helpers.py @@ -1,13 +1,19 @@ import asyncio import json +from typing import Final from unittest.mock import AsyncMock, MagicMock +import httpx +import openai import pytest from fastapi import HTTPException from prisma.errors import DataError +import litellm from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.proxy.common_request_processing import proxy_exception_from_route_error from litellm.proxy.utils import get_error_message_str, handle_exception_on_proxy +from litellm.types.router import RouterErrors def normalize(value): @@ -96,7 +102,7 @@ def test_handle_exception_on_proxy_happy_path_http_exception(): assert snapshot == { "is_proxy_exception": True, "message": "forbidden", - "type": ProxyErrorTypes.internal_server_error.value, + "type": "permission_error", "code": "403", } @@ -156,7 +162,7 @@ def test_handle_exception_on_proxy_uses_attached_status_code_when_present(): assert snapshot == { "code": "418", "message": "teapot", - "type": ProxyErrorTypes.internal_server_error.value, + "type": "invalid_request_error", } @@ -176,6 +182,154 @@ def test_handle_exception_on_proxy_error_path_none_input_wraps_as_500(): } +@pytest.mark.parametrize( + "status_code, expected_type", + [ + (400, "invalid_request_error"), + (401, "authentication_error"), + (403, "permission_error"), + (404, "invalid_request_error"), + (408, "invalid_request_error"), + (409, "invalid_request_error"), + (422, "invalid_request_error"), + (429, "rate_limit_error"), + (500, ProxyErrorTypes.internal_server_error.value), + (503, ProxyErrorTypes.internal_server_error.value), + ], +) +def test_handle_exception_on_proxy_labels_an_http_exception_by_the_status_it_answers( + status_code: int, expected_type: str +): + """A management route that rejects a caller's own argument answers 400, and calling that + internal_server_error tells the caller's retry loop the gateway is down when nothing but + their request is wrong.""" + result: Final = handle_exception_on_proxy(HTTPException(status_code=status_code, detail="boom")) + + assert (result.code, result.type) == (str(status_code), expected_type) + + +def _exception_carrying_status(carried: int | None) -> Exception: + class _CarriedStatus(Exception): + status_code = carried + + return _CarriedStatus("boom") + + +@pytest.mark.parametrize( + "status_code, expected_type", + [ + (400, "invalid_request_error"), + (404, "invalid_request_error"), + (408, "invalid_request_error"), + (429, "rate_limit_error"), + (500, ProxyErrorTypes.internal_server_error.value), + ], +) +def test_handle_exception_on_proxy_labels_a_carried_status_the_same_way(status_code: int, expected_type: str): + """Routes also reach this handler with plain exceptions carrying their own status, and + those must be named by that status rather than all landing on the 500 label.""" + result: Final = handle_exception_on_proxy(_exception_carrying_status(status_code)) + + assert (result.code, result.type) == (str(status_code), expected_type) + + +def test_handle_exception_on_proxy_keeps_the_type_the_exception_already_names(): + """A rate limit litellm raised already names itself, and overwriting that with the status + map's guess would drop what the client branches on.""" + result: Final = handle_exception_on_proxy( + litellm.RateLimitError(message="slow down", llm_provider="openai", model="gpt-4o") + ) + + assert (result.code, result.type) == ("429", "throttling_error") + + +@pytest.mark.parametrize( + "exc", + [ + openai.InternalServerError( + message="upstream down", + response=httpx.Response(500, request=httpx.Request("GET", "https://api.openai.com/v1/batches/b")), + body={"type": "server_error"}, + ), + litellm.Timeout(message="took too long", model="gpt-6.1-sol", llm_provider="openai"), + litellm.RateLimitError(message="slow down", llm_provider="openai", model="gpt-6.1-sol"), + litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-6.1-sol"), + litellm.BadRequestError(message="bad input", model="gpt-6.1-sol", llm_provider="openai"), + HTTPException(status_code=404, detail="no such batch"), + ], + ids=["provider_5xx_naming_its_own_type", "timeout", "rate_limit", "bad_key", "bad_request", "http_404"], +) +def test_handle_exception_on_proxy_labels_a_failure_the_way_the_chat_route_does(exc: Exception): + """Batch, file, and management routes answer through this handler while chat answers through the route + error path, and one client calling both must read the same type for the same failure.""" + shared: Final = handle_exception_on_proxy(exc) + chat: Final = proxy_exception_from_route_error(exc) + + assert (shared.code, shared.type) == (chat.code, chat.type) + + +@pytest.mark.parametrize( + "message, expected", + [ + (f"{RouterErrors.no_deployments_with_provider_budget_routing.value}, try later", ("429", "rate_limit_error")), + (f"{RouterErrors.no_deployments_with_tag_routing.value}=['paid']", ("401", "authentication_error")), + ], + ids=["no_deployment_left", "no_deployment_for_tags"], +) +def test_handle_exception_on_proxy_names_a_router_failure_by_the_status_it_answers( + message: str, expected: tuple[str, str] +): + """A router failure that carries no status answers the 429 or 401 its message maps to, and a 429 labelled + internal_server_error reads as an outage to a client that branches on the type.""" + result: Final = handle_exception_on_proxy(ValueError(message)) + + assert (result.code, result.type) == expected + + +def test_handle_exception_on_proxy_keeps_the_path_a_rejected_request_names(): + """A provider rejecting a batch input names the caller's own bucket path, and redacting the /tmp part of it + leaves the caller nothing to fix their request with.""" + result: Final = handle_exception_on_proxy( + litellm.BadRequestError( + message="Invalid input_file_id gs://my-bucket/tmp/batch_input.jsonl", + model="gemini-3.1-pro-preview", + llm_provider="vertex_ai", + ) + ) + + assert result.code == "400" + assert "gs://my-bucket/tmp/batch_input.jsonl" in result.message + + +def test_handle_exception_on_proxy_redacts_the_internal_host_an_unreachable_upstream_names(): + """A guardrail or provider the gateway cannot reach names its private address in the + error, and any virtual key can call those routes, so the host must not reach the caller.""" + result: Final = handle_exception_on_proxy(ConnectionError("Cannot connect to host 10.12.4.7:8443")) + + assert result.code == "500" + assert "Cannot connect to host" in result.message + assert "10.12.4.7" not in result.message + + +def test_handle_exception_on_proxy_redacts_the_internal_host_a_route_puts_in_a_gateway_failure_detail(): + """Routes wrap an upstream failure as an HTTPException whose detail is the raw error, so a 5xx detail carries + the same private address an unwrapped connection error does.""" + result: Final = handle_exception_on_proxy(HTTPException(status_code=503, detail="Cannot connect to host 10.0.0.9")) + + assert result.code == "503" + assert "Cannot connect to host" in result.message + assert "10.0.0.9" not in result.message + + +def test_handle_exception_on_proxy_answers_500_for_an_exception_whose_status_is_none(): + """A guardrail whose upstream refuses the connection raises with status_code=None, and + passing that through as the code crashed the proxy's error handler, which then blamed the + auth database with a 503 instead of naming the unreachable guardrail.""" + result: Final = handle_exception_on_proxy(_exception_carrying_status(None)) + + assert (result.code, result.type, result.message) == ("500", ProxyErrorTypes.internal_server_error.value, "boom") + + @pytest.mark.parametrize( "exc", [