From 6a9cb48cf3d1a2438db3de4199343cae89828b8c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 04:31:30 -0700 Subject: [PATCH 1/3] fix(proxy): name shared-handler errors by the status they answer handle_exception_on_proxy pinned type=internal_server_error on every exception it wrapped, so a management route rejecting a caller's own argument answered {"type":"internal_server_error","code":"400"} and an SDK branching on the type read its own bad request as a gateway outage. Derive the type from the status the response actually carries, keeping a type the exception already names. --- litellm/proxy/utils.py | 15 +++-- .../credential_endpoints/test_endpoints.py | 28 ++++++++ .../proxy/utils/helpers/test_error_helpers.py | 64 ++++++++++++++++++- 3 files changed, 100 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 5c0fbd64744..ff5ce75a8c8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -37,7 +37,11 @@ from litellm.proxy._types import ( SpendLogsMetadata, SpendLogsPayload, ) -from litellm.proxy.common_utils.openai_error_payload import openai_error_param +from litellm.proxy.common_utils.openai_error_payload import ( + error_status_code, + openai_error_param, + openai_error_type, +) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse @@ -7096,18 +7100,19 @@ def handle_exception_on_proxy(e: Exception) -> ProxyException: verbose_proxy_logger.exception("Exception: %s", e) if isinstance(e, HTTPException): + http_status_code: Final = error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR) return ProxyException( message=getattr(e, "detail", f"error({e})"), - type=ProxyErrorTypes.internal_server_error, + type=openai_error_type(e, http_status_code), param=openai_error_param(e), - code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + code=http_status_code, ) elif isinstance(e, ProxyException): return e - _status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) + _status_code: Final = error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR) return ProxyException( message=str(e), - type=ProxyErrorTypes.internal_server_error, + type=openai_error_type(e, _status_code), param=openai_error_param(e), code=_status_code, ) diff --git a/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py b/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py index dcd8e6881bd..fd38c22398a 100644 --- a/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py @@ -86,3 +86,31 @@ def test_update_credential_still_answers_200_on_a_successful_write(): assert response.status_code == 200, response.text assert response.json()["success"] is True + + +def _get_credential_by_name(name: str): + missing = object() + previous_override = app.dependency_overrides.get(user_api_key_auth, missing) + app.dependency_overrides[user_api_key_auth] = _as_admin + try: + return client.get(f"/credentials/by_name/{name}", headers={"Authorization": "Bearer test-key"}) + finally: + if previous_override is missing: + app.dependency_overrides.pop(user_api_key_auth, None) + else: + app.dependency_overrides[user_api_key_auth] = previous_override + + +def test_looking_up_a_missing_credential_is_labelled_a_client_error_not_a_gateway_failure(): + """Naming a credential that does not exist is the caller's own mistake, and the shared + handler used to label every one of these internal_server_error, so an SDK branching on + the type read a 404 it can never retry past as a gateway outage and backed off forever.""" + response = _get_credential_by_name("definitely-not-there") + + assert response.status_code == 404, f"lookup answered {response.status_code}: {response.text}" + assert response.json()["error"] == { + "message": "Credential not found. Got credential name: definitely-not-there", + "type": "invalid_request_error", + "param": None, + "code": "404", + } diff --git a/tests/test_litellm/proxy/utils/helpers/test_error_helpers.py b/tests/test_litellm/proxy/utils/helpers/test_error_helpers.py index dc30798df55..3333bcc80e5 100644 --- a/tests/test_litellm/proxy/utils/helpers/test_error_helpers.py +++ b/tests/test_litellm/proxy/utils/helpers/test_error_helpers.py @@ -3,6 +3,7 @@ import json import pytest from fastapi import HTTPException +import litellm from litellm.proxy._types import ProxyErrorTypes, ProxyException from litellm.proxy.utils import get_error_message_str, handle_exception_on_proxy @@ -93,7 +94,7 @@ def test_handle_exception_on_proxy_happy_path_http_exception(): assert snapshot == { "is_proxy_exception": True, "message": "forbidden", - "type": ProxyErrorTypes.internal_server_error.value, + "type": "permission_error", "code": "403", } @@ -153,7 +154,7 @@ def test_handle_exception_on_proxy_uses_attached_status_code_when_present(): assert snapshot == { "code": "418", "message": "teapot", - "type": ProxyErrorTypes.internal_server_error.value, + "type": "invalid_request_error", } @@ -171,3 +172,62 @@ def test_handle_exception_on_proxy_error_path_none_input_wraps_as_500(): "code": "500", "type": ProxyErrorTypes.internal_server_error.value, } + + +@pytest.mark.parametrize( + "status_code, expected_type", + [ + (400, "invalid_request_error"), + (401, "authentication_error"), + (403, "permission_error"), + (404, "invalid_request_error"), + (409, "invalid_request_error"), + (422, "invalid_request_error"), + (429, "rate_limit_error"), + (500, ProxyErrorTypes.internal_server_error.value), + (503, ProxyErrorTypes.internal_server_error.value), + ], +) +def test_handle_exception_on_proxy_labels_an_http_exception_by_the_status_it_answers( + status_code: int, expected_type: str +): + """A management route that rejects a caller's own argument answers 400, and calling that + internal_server_error tells the caller's retry loop the gateway is down when nothing but + their request is wrong.""" + result = handle_exception_on_proxy(HTTPException(status_code=status_code, detail="boom")) + + assert (result.code, result.type) == (str(status_code), expected_type) + + +def _exception_carrying_status(carried: int) -> Exception: + class _CarriedStatus(Exception): + status_code = carried + + return _CarriedStatus("boom") + + +@pytest.mark.parametrize( + "status_code, expected_type", + [ + (400, "invalid_request_error"), + (404, "invalid_request_error"), + (429, "rate_limit_error"), + (500, ProxyErrorTypes.internal_server_error.value), + ], +) +def test_handle_exception_on_proxy_labels_a_carried_status_the_same_way(status_code: int, expected_type: str): + """Routes also reach this handler with plain exceptions carrying their own status, and + those must be named by that status rather than all landing on the 500 label.""" + result = handle_exception_on_proxy(_exception_carrying_status(status_code)) + + assert (result.code, result.type) == (str(status_code), expected_type) + + +def test_handle_exception_on_proxy_keeps_the_type_the_exception_already_names(): + """A rate limit litellm raised already names itself, and overwriting that with the status + map's guess would drop what the client branches on.""" + result = handle_exception_on_proxy( + litellm.RateLimitError(message="slow down", llm_provider="openai", model="gpt-4o") + ) + + assert (result.code, result.type) == ("429", "throttling_error") From 07df4ff09c0481b788c4e64dd0e5f1bbe34ce58f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:04:05 -0700 Subject: [PATCH 2/3] test(proxy): pin a 500 for exceptions whose status is None --- tests/unit/proxy/utils/helpers/test_error_helpers.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/utils/helpers/test_error_helpers.py b/tests/unit/proxy/utils/helpers/test_error_helpers.py index 5d530e6d89f..9d7fb9cbadc 100644 --- a/tests/unit/proxy/utils/helpers/test_error_helpers.py +++ b/tests/unit/proxy/utils/helpers/test_error_helpers.py @@ -202,7 +202,7 @@ def test_handle_exception_on_proxy_labels_an_http_exception_by_the_status_it_ans assert (result.code, result.type) == (str(status_code), expected_type) -def _exception_carrying_status(carried: int) -> Exception: +def _exception_carrying_status(carried: int | None) -> Exception: class _CarriedStatus(Exception): status_code = carried @@ -236,6 +236,15 @@ def test_handle_exception_on_proxy_keeps_the_type_the_exception_already_names(): assert (result.code, result.type) == ("429", "throttling_error") +def test_handle_exception_on_proxy_answers_500_for_an_exception_whose_status_is_none(): + """A guardrail whose upstream refuses the connection raises with status_code=None, and + passing that through as the code crashed the proxy's error handler, which then blamed the + auth database with a 503 instead of naming the unreachable guardrail.""" + result = handle_exception_on_proxy(_exception_carrying_status(None)) + + assert (result.code, result.type, result.message) == ("500", ProxyErrorTypes.internal_server_error.value, "boom") + + @pytest.mark.parametrize( "exc", [ From 1d416418e484999fa8933e9f60fd0473e27b132f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:30:26 -0700 Subject: [PATCH 3/3] fix(proxy): keep internal_server_error for timeouts and 5xx, redact hosts in shared handler errors Provider 5xx and timeouts keep internal_server_error so retry loops still retry them, the shared handler redacts internal hosts from raw exception messages, and /key/list routes its HTTPException through the shared handler instead of its own copy of the old label. --- .../common_utils/openai_error_payload.py | 6 +++ .../key_management_endpoints.py | 7 +-- litellm/proxy/utils.py | 10 ++-- .../credential_endpoints/test_endpoints.py | 6 ++- .../test_key_management_endpoints.py | 4 +- .../proxy/utils/helpers/test_error_helpers.py | 49 +++++++++++++++++-- 6 files changed, 63 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index 2f01dbe11f7..3f852ca7fec 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -45,6 +45,12 @@ def openai_error_type(exc: object, status_code: int) -> str: return openai_error_type_for_status(status_code) +def caller_error_type(exc: object, status_code: int) -> str: + if status_code == status.HTTP_408_REQUEST_TIMEOUT or status_code >= status.HTTP_500_INTERNAL_SERVER_ERROR: + return "internal_server_error" + return openai_error_type(exc, status_code) + + def openai_error_type_for_status(status_code: int) -> str: mapped: Final = _OPENAI_ERROR_TYPE_BY_STATUS.get(status_code) if mapped is not None: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 824b310a1dd..1f3a882469f 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -7225,12 +7225,7 @@ async def list_keys( except Exception as e: verbose_proxy_logger.exception("Error in list_keys: %s", e) if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"error({e})"), - type=ProxyErrorTypes.internal_server_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR), - ) + raise handle_exception_on_proxy(e) elif isinstance(e, ProxyException): raise e raise ProxyException( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a10f9e9ee9d..00b1de408c5 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -84,10 +84,10 @@ from litellm.proxy._types import ( from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.openai_error_payload import ( attribute_of, + caller_error_type, error_status_code, litellm_call_id_headers, openai_error_param, - openai_error_type, with_litellm_call_id, ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error @@ -133,7 +133,7 @@ from litellm import ( Router, ) from litellm._internal_context import service_target -from litellm._logging import redact_string, verbose_proxy_logger +from litellm._logging import redact_internal_details_from_client_message, redact_string, verbose_proxy_logger from litellm._service_logger import ServiceLogging, ServiceTypes from litellm.caching.caching import DualCache, RedisCache from litellm.caching.dual_cache import LimitedSizeOrderedDict @@ -9017,7 +9017,7 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) http_status_code: Final = error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR) return ProxyException( message=getattr(e, "detail", f"error({e})"), - type=openai_error_type(e, http_status_code), + type=caller_error_type(e, http_status_code), param=openai_error_param(e), headers=headers, code=http_status_code, @@ -9028,8 +9028,8 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) if should_report_bug(e): verbose_proxy_logger.error(bug_report_notice(build_proxy_bug_report(e))) return ProxyException( - message=strip_bug_report_notice(str(e)), - type=openai_error_type(e, _status_code), + message=redact_internal_details_from_client_message(strip_bug_report_notice(str(e))), + type=caller_error_type(e, _status_code), param=openai_error_param(e), headers=headers, code=_status_code, diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index d1993909ece..3c5343eb7e7 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -3016,11 +3016,13 @@ class TestCredentialDisplayName: assert listed["openai-prod"]["credential_values"]["api_key"] != "sk-old" -def test_looking_up_a_missing_credential_is_labelled_a_client_error_not_a_gateway_failure(restore_credential_list): +def test_looking_up_a_missing_credential_is_labelled_a_client_error_not_a_gateway_failure( + restore_credential_list: None, +): """Naming a credential that does not exist is the caller's own mistake, and the shared handler used to label every one of these internal_server_error, so an SDK branching on the type read a 404 it can never retry past as a gateway outage and backed off forever.""" - response = _call_as("GET", "/credentials/by_name/definitely-not-there") + response: Final = _call_as("GET", "/credentials/by_name/definitely-not-there") assert response.status_code == 404, f"lookup answered {response.status_code}: {response.text}" assert response.json()["error"] == { diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index d6ee9b1d29e..cafb96735c5 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -6540,7 +6540,7 @@ async def test_list_keys_with_invalid_status(): ) # Verify ProxyException properties - assert exc_info.value.code == "400" + assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error") assert "Invalid status value" in str(exc_info.value.message) assert "deleted" in str(exc_info.value.message) @@ -18733,7 +18733,7 @@ async def test_list_keys_rejects_invalid_expires(): expires="expred", ) - assert exc_info.value.code == "400" + assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error") assert "Invalid expires value" in str(exc_info.value.message) diff --git a/tests/unit/proxy/utils/helpers/test_error_helpers.py b/tests/unit/proxy/utils/helpers/test_error_helpers.py index 9d7fb9cbadc..a6667f7a46d 100644 --- a/tests/unit/proxy/utils/helpers/test_error_helpers.py +++ b/tests/unit/proxy/utils/helpers/test_error_helpers.py @@ -1,7 +1,10 @@ import asyncio import json +from typing import Final from unittest.mock import AsyncMock, MagicMock +import httpx +import openai import pytest from fastapi import HTTPException from prisma.errors import DataError @@ -184,6 +187,7 @@ def test_handle_exception_on_proxy_error_path_none_input_wraps_as_500(): (401, "authentication_error"), (403, "permission_error"), (404, "invalid_request_error"), + (408, ProxyErrorTypes.internal_server_error.value), (409, "invalid_request_error"), (422, "invalid_request_error"), (429, "rate_limit_error"), @@ -197,7 +201,7 @@ def test_handle_exception_on_proxy_labels_an_http_exception_by_the_status_it_ans """A management route that rejects a caller's own argument answers 400, and calling that internal_server_error tells the caller's retry loop the gateway is down when nothing but their request is wrong.""" - result = handle_exception_on_proxy(HTTPException(status_code=status_code, detail="boom")) + result: Final = handle_exception_on_proxy(HTTPException(status_code=status_code, detail="boom")) assert (result.code, result.type) == (str(status_code), expected_type) @@ -214,6 +218,7 @@ def _exception_carrying_status(carried: int | None) -> Exception: [ (400, "invalid_request_error"), (404, "invalid_request_error"), + (408, ProxyErrorTypes.internal_server_error.value), (429, "rate_limit_error"), (500, ProxyErrorTypes.internal_server_error.value), ], @@ -221,7 +226,7 @@ def _exception_carrying_status(carried: int | None) -> Exception: def test_handle_exception_on_proxy_labels_a_carried_status_the_same_way(status_code: int, expected_type: str): """Routes also reach this handler with plain exceptions carrying their own status, and those must be named by that status rather than all landing on the 500 label.""" - result = handle_exception_on_proxy(_exception_carrying_status(status_code)) + result: Final = handle_exception_on_proxy(_exception_carrying_status(status_code)) assert (result.code, result.type) == (str(status_code), expected_type) @@ -229,18 +234,54 @@ def test_handle_exception_on_proxy_labels_a_carried_status_the_same_way(status_c def test_handle_exception_on_proxy_keeps_the_type_the_exception_already_names(): """A rate limit litellm raised already names itself, and overwriting that with the status map's guess would drop what the client branches on.""" - result = handle_exception_on_proxy( + result: Final = handle_exception_on_proxy( litellm.RateLimitError(message="slow down", llm_provider="openai", model="gpt-4o") ) assert (result.code, result.type) == ("429", "throttling_error") +@pytest.mark.parametrize( + "exc, expected_code", + [ + ( + openai.InternalServerError( + message="upstream down", + response=httpx.Response(500, request=httpx.Request("GET", "https://api.openai.com/v1/batches/b")), + body={"type": "server_error"}, + ), + "500", + ), + (litellm.Timeout(message="took too long", model="gpt-6.1-sol", llm_provider="openai"), "408"), + ], + ids=["provider_5xx_naming_its_own_type", "provider_timeout"], +) +def test_handle_exception_on_proxy_keeps_internal_server_error_for_failures_the_caller_cannot_fix( + exc: Exception, expected_code: str +): + """Batch and fine-tuning routes pass provider errors straight through, and a retry loop + that treats internal_server_error as transient must keep retrying a provider outage or a + timeout instead of reading the provider's own type or a fix-your-request label.""" + result: Final = handle_exception_on_proxy(exc) + + assert (result.code, result.type) == (expected_code, ProxyErrorTypes.internal_server_error.value) + + +def test_handle_exception_on_proxy_redacts_the_internal_host_an_unreachable_upstream_names(): + """A guardrail or provider the gateway cannot reach names its private address in the + error, and any virtual key can call those routes, so the host must not reach the caller.""" + result: Final = handle_exception_on_proxy(ConnectionError("Cannot connect to host 10.12.4.7:8443")) + + assert result.code == "500" + assert "Cannot connect to host" in result.message + assert "10.12.4.7" not in result.message + + def test_handle_exception_on_proxy_answers_500_for_an_exception_whose_status_is_none(): """A guardrail whose upstream refuses the connection raises with status_code=None, and passing that through as the code crashed the proxy's error handler, which then blamed the auth database with a 503 instead of naming the unreachable guardrail.""" - result = handle_exception_on_proxy(_exception_carrying_status(None)) + result: Final = handle_exception_on_proxy(_exception_carrying_status(None)) assert (result.code, result.type, result.message) == ("500", ProxyErrorTypes.internal_server_error.value, "boom")