fix(proxy): type a route failure by the status it answers with after the tag-routing and no-deployment remap

This commit is contained in:
mateo-berri 2026-10-29 03:42:29 -07:00
parent 6dc45a6eb5
commit 8db3643411
6 changed files with 192 additions and 54 deletions

View file

@ -131,8 +131,30 @@ def _is_failover_refusal(exc: Exception) -> bool:
return any(refusal in message for refusal in _FAILOVER_REFUSALS)
def answered_status_code(message: str, status_code: int) -> int:
return int(proxy_exception_status_code(message, str(status_code)))
def route_failure_proxy_exception(
exc: Exception,
message: str,
default_status_code: int,
headers: dict[str, str] | None = None, # mutable-ok: ProxyException.headers
openai_code: str | None = None,
) -> ProxyException:
answered_status: Final = answered_status_code(message, error_status_code(exc, default_status_code))
return ProxyException(
message=message,
type=openai_error_type(exc, answered_status),
param=openai_error_param(exc),
headers=headers,
openai_code=openai_code,
code=answered_status,
)
def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
answered_status: Final = int(proxy_exception_status_code(message, str(status_code)))
answered_status: Final = answered_status_code(message, status_code)
return ProxyException(
message=message,
type=_catch_all_error_type(exc, answered_status),

View file

@ -522,12 +522,12 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
)
from litellm.proxy.common_utils.openai_error_payload import (
LITELLM_CALL_ID_HEADER,
error_status_code,
attribute_of,
headers_with_litellm_call_id,
litellm_call_id_headers,
openai_error_param,
openai_error_type,
proxy_exception_for,
route_failure_proxy_exception,
with_litellm_call_id,
)
from litellm.proxy.common_utils.periodic_reload_schedule import (
@ -9840,11 +9840,8 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe
# Including it in the SSE response leaks internal details to clients.
error_msg: Final = str(e)
proxy_exception: Final = ProxyException(
message=getattr(e, "message", error_msg),
type=openai_error_type(e, error_status_code(e, 500)),
param=openai_error_param(e),
code=error_status_code(e, 500),
proxy_exception: Final = route_failure_proxy_exception(
e, str(attribute_of(e, "message", error_msg)), status.HTTP_500_INTERNAL_SERVER_ERROR
)
error_returned: Final = json.dumps({"error": proxy_exception.to_dict()})
yield f"data: {error_returned}\n\n"
@ -10677,11 +10674,10 @@ async def async_data_generator(
{
"error": sse_error_payload(client_error)[1]
if isinstance(client_error, HTTPException)
else ProxyException(
message=getattr(client_error, "message", str(client_error)),
type=openai_error_type(client_error, error_status_code(client_error, 500)),
param=openai_error_param(client_error),
code=error_status_code(client_error, 500),
else route_failure_proxy_exception(
client_error,
str(attribute_of(client_error, "message", client_error)),
status.HTTP_500_INTERNAL_SERVER_ERROR,
).to_dict()
},
default=str,
@ -13045,14 +13041,12 @@ async def completion(
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, ProxyException):
raise with_litellm_call_id(e, litellm_call_id)
error_msg: Final = f"{e}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=openai_error_type(e, error_status_code(e, 500)),
param=openai_error_param(e),
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e)),
status.HTTP_500_INTERNAL_SERVER_ERROR,
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
code=error_status_code(e, 500),
)
@ -13273,22 +13267,18 @@ async def moderations(
if isinstance(e, ProxyException):
raise with_litellm_call_id(e, litellm_call_id)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e)),
type=openai_error_type(e, error_status_code(e, status.HTTP_400_BAD_REQUEST)),
param=openai_error_param(e),
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e)),
status.HTTP_400_BAD_REQUEST,
headers=litellm_call_id_headers(litellm_call_id),
code=error_status_code(e, status.HTTP_400_BAD_REQUEST),
)
else:
error_msg: Final = f"{e}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=openai_error_type(e, error_status_code(e, 500)),
param=openai_error_param(e),
headers=litellm_call_id_headers(litellm_call_id),
code=error_status_code(e, 500),
)
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e)),
status.HTTP_500_INTERNAL_SERVER_ERROR,
headers=litellm_call_id_headers(litellm_call_id),
)
async def _audio_speech_chunk_generator(
@ -13426,13 +13416,12 @@ async def audio_speech(
detail=e.detail,
headers=headers_with_litellm_call_id(e.headers, litellm_call_id),
)
raise ProxyException(
message=getattr(e, "message", f"{e}"),
type=openai_error_type(e, error_status_code(e, 500)),
param=openai_error_param(e),
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e)),
status.HTTP_500_INTERNAL_SERVER_ERROR,
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
code=error_status_code(e, 500),
)
@ -13572,23 +13561,19 @@ async def audio_transcriptions(
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
type=openai_error_type(e, error_status_code(e, status.HTTP_400_BAD_REQUEST)),
param=openai_error_param(e),
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e.detail)),
status.HTTP_400_BAD_REQUEST,
headers=litellm_call_id_headers(litellm_call_id),
code=error_status_code(e, status.HTTP_400_BAD_REQUEST),
)
else:
error_msg: Final = f"{e}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=openai_error_type(e, error_status_code(e, 500)),
param=openai_error_param(e),
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
code=error_status_code(e, 500),
)
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e)),
status.HTTP_500_INTERNAL_SERVER_ERROR,
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
)
######################################################################

View file

@ -1310,6 +1310,33 @@ async def test_auth_rejects_wrong_or_empty_present_custom_header(
assert exc_info.value.code == "401"
@pytest.mark.asyncio
@pytest.mark.parametrize("api_key", [None, ""], ids=["no_key", "no_bearer_prefix"])
async def test_auth_rejection_without_a_usable_key_has_a_json_null_param(
api_key: str | None, monkeypatch: pytest.MonkeyPatch
):
from fastapi import Request
from litellm.proxy import proxy_server
from litellm.proxy._types import ProxyErrorTypes, ProxyException
monkeypatch.setattr(proxy_server, "master_key", "sk-test-master")
monkeypatch.setattr(proxy_server, "general_settings", {})
request = Request(
scope={"type": "http", "method": "POST", "path": "/chat/completions", "headers": []},
receive=AsyncMock(return_value={"type": "http.request", "body": b"", "more_body": False}),
)
request._url = URL(url="/chat/completions")
with pytest.raises(ProxyException) as exc_info:
await user_api_key_auth(request=request, api_key=api_key)
body = json.loads(json.dumps({"error": exc_info.value.to_dict()}))
assert body["error"]["code"] == "401"
assert body["error"]["type"] == ProxyErrorTypes.auth_error.value
assert body["error"]["param"] is None
@pytest.mark.asyncio
async def test_user_api_key_from_query_param():
"""Ensure user_api_key_auth reads API key from `key` query parameter."""

View file

@ -16,7 +16,11 @@ import httpx
import pytest
from litellm.proxy import proxy_server
from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.types.llms.openai import HttpxBinaryResponseContent
from litellm.types.router import RouterErrors
_TAG_ROUTING_DENIAL = f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=tts-1 and tags=['x']"
@pytest.fixture
@ -208,7 +212,8 @@ def test_audio_speech_error(client, auth_as, patched_speech_error, path):
with auth_as():
response = client.post(path, json=payload)
assert response.status_code == 500
assert len(response.content) > 0
error = response.json()["error"]
assert (error["type"], error["param"]) == ("internal_server_error", None)
def test_audio_speech_bad_request_maps_to_400(client, auth_as, patched_speech_provider_rejection):
@ -265,6 +270,69 @@ def test_audio_transcription_over_the_file_size_cap_is_rejected_as_a_bad_request
assert error["param"] == "file"
def test_audio_speech_tag_routing_denial_is_typed_as_an_authentication_error(
client, auth_as, patched_speech_error, monkeypatch
):
async def _deny(*args, **kwargs):
raise ValueError(_TAG_ROUTING_DENIAL)
monkeypatch.setattr(proxy_server, "route_request", _deny)
with auth_as():
response = client.post("/v1/audio/speech", json={"model": "tts-1", "input": "Hi", "voice": "alloy"})
assert response.status_code == 401
error = response.json()["error"]
assert (error["type"], error["param"]) == ("authentication_error", None)
def test_audio_transcription_tag_routing_denial_is_typed_as_an_authentication_error(
client, auth_as, patched_transcription, monkeypatch
):
async def _deny(*args, **kwargs):
raise ValueError(_TAG_ROUTING_DENIAL)
monkeypatch.setattr(proxy_server, "route_request", _deny)
files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")}
with auth_as():
response = client.post("/v1/audio/transcriptions", files=files, data={"model": "whisper-1"})
assert response.status_code == 401
error = response.json()["error"]
assert (error["type"], error["param"]) == ("authentication_error", None)
@pytest.mark.parametrize(
"rejection",
[
pytest.param(
ProxyException(
message="Invalid type for 'metadata': expected an object, but got an integer instead.",
type=ProxyErrorTypes.bad_request_error,
param="metadata",
code=400,
),
id="bad_metadata",
),
pytest.param(
ProxyException(
message="tag budgets are unavailable", type=ProxyErrorTypes.no_db_connection, param=None, code=503
),
id="tag_budget_store_down",
),
],
)
def test_audio_transcription_keeps_the_status_of_a_proxy_rejection(
client, auth_as, patched_transcription, monkeypatch, rejection
):
async def _reject(data, **kwargs):
raise rejection
monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _reject)
files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")}
with auth_as():
response = client.post("/v1/audio/transcriptions", files=files, data={"model": "whisper-1"})
assert response.status_code == int(rejection.code)
assert response.json()["error"] == rejection.to_dict()
@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"])
def test_audio_transcription_error(client, auth_as, patched_transcription_error, path):
"""Pins ``POST /v1/audio/transcriptions`` / ``POST /audio/transcriptions`` (error)."""

View file

@ -14,6 +14,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from litellm.proxy import common_request_processing, proxy_server
from litellm.types.router import RouterErrors
from .conftest import normalize # type: ignore[import-not-found]
@ -124,3 +125,32 @@ def test_completion_pipeline_error(client, auth_as, completion_pipeline_raises,
response = client.post(path, json=payload)
assert response.status_code == 500
assert response.headers.get("content-type", "").startswith("application/json")
@pytest.mark.parametrize(
("failure", "expected_status", "expected_type"),
[
pytest.param(ValueError("boom"), 500, "internal_server_error", id="unexpected_failure"),
pytest.param(
ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=gpt-4o and tags=['x']"),
401,
"authentication_error",
id="tag_routing_denial",
),
],
)
def test_completion_failure_body_types_the_status_it_answers_with(
client, auth_as, monkeypatch, failure, expected_status, expected_type
):
monkeypatch.setattr(proxy_server, "llm_router", MagicMock())
monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock(post_call_failure_hook=AsyncMock()))
async def _raise(self, *args, **kwargs):
raise failure
monkeypatch.setattr(common_request_processing.ProxyBaseLLMRequestProcessing, "base_process_llm_request", _raise)
with auth_as():
response = client.post("/v1/completions", json={"model": "gpt-4o", "prompt": "hi"})
assert response.status_code == expected_status
error = response.json()["error"]
assert (error["type"], error["param"], error["code"]) == (expected_type, None, str(expected_status))

View file

@ -14260,6 +14260,12 @@ async def test_moderations_reraises_proxy_exception_unwrapped():
pytest.param(HTTPException(status_code=400, detail="bad input"), "invalid_request_error", "400", id="http_400"),
pytest.param(HTTPException(status_code=429, detail="slow down"), "rate_limit_error", "429", id="http_429"),
pytest.param(Exception("provider down"), "internal_server_error", "500", id="bare_exception"),
pytest.param(
ValueError("Not allowed to access model due to tags configuration. Passed model=m and tags=['x']"),
"authentication_error",
"401",
id="tag_routing_denial",
),
],
)
async def test_moderations_failure_body_is_openai_shaped(failure, expected_type, expected_code):