mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): type a route failure by the status it answers with after the tag-routing and no-deployment remap
This commit is contained in:
parent
6dc45a6eb5
commit
8db3643411
6 changed files with 192 additions and 54 deletions
|
|
@ -131,8 +131,30 @@ def _is_failover_refusal(exc: Exception) -> bool:
|
|||
return any(refusal in message for refusal in _FAILOVER_REFUSALS)
|
||||
|
||||
|
||||
def answered_status_code(message: str, status_code: int) -> int:
|
||||
return int(proxy_exception_status_code(message, str(status_code)))
|
||||
|
||||
|
||||
def route_failure_proxy_exception(
|
||||
exc: Exception,
|
||||
message: str,
|
||||
default_status_code: int,
|
||||
headers: dict[str, str] | None = None, # mutable-ok: ProxyException.headers
|
||||
openai_code: str | None = None,
|
||||
) -> ProxyException:
|
||||
answered_status: Final = answered_status_code(message, error_status_code(exc, default_status_code))
|
||||
return ProxyException(
|
||||
message=message,
|
||||
type=openai_error_type(exc, answered_status),
|
||||
param=openai_error_param(exc),
|
||||
headers=headers,
|
||||
openai_code=openai_code,
|
||||
code=answered_status,
|
||||
)
|
||||
|
||||
|
||||
def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
|
||||
answered_status: Final = int(proxy_exception_status_code(message, str(status_code)))
|
||||
answered_status: Final = answered_status_code(message, status_code)
|
||||
return ProxyException(
|
||||
message=message,
|
||||
type=_catch_all_error_type(exc, answered_status),
|
||||
|
|
|
|||
|
|
@ -522,12 +522,12 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
|
|||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import (
|
||||
LITELLM_CALL_ID_HEADER,
|
||||
error_status_code,
|
||||
attribute_of,
|
||||
headers_with_litellm_call_id,
|
||||
litellm_call_id_headers,
|
||||
openai_error_param,
|
||||
openai_error_type,
|
||||
proxy_exception_for,
|
||||
route_failure_proxy_exception,
|
||||
with_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.periodic_reload_schedule import (
|
||||
|
|
@ -9840,11 +9840,8 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe
|
|||
# Including it in the SSE response leaks internal details to clients.
|
||||
error_msg: Final = str(e)
|
||||
|
||||
proxy_exception: Final = ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=openai_error_type(e, error_status_code(e, 500)),
|
||||
param=openai_error_param(e),
|
||||
code=error_status_code(e, 500),
|
||||
proxy_exception: Final = route_failure_proxy_exception(
|
||||
e, str(attribute_of(e, "message", error_msg)), status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
)
|
||||
error_returned: Final = json.dumps({"error": proxy_exception.to_dict()})
|
||||
yield f"data: {error_returned}\n\n"
|
||||
|
|
@ -10677,11 +10674,10 @@ async def async_data_generator(
|
|||
{
|
||||
"error": sse_error_payload(client_error)[1]
|
||||
if isinstance(client_error, HTTPException)
|
||||
else ProxyException(
|
||||
message=getattr(client_error, "message", str(client_error)),
|
||||
type=openai_error_type(client_error, error_status_code(client_error, 500)),
|
||||
param=openai_error_param(client_error),
|
||||
code=error_status_code(client_error, 500),
|
||||
else route_failure_proxy_exception(
|
||||
client_error,
|
||||
str(attribute_of(client_error, "message", client_error)),
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
).to_dict()
|
||||
},
|
||||
default=str,
|
||||
|
|
@ -13045,14 +13041,12 @@ async def completion(
|
|||
log_llm_api_exception(e, litellm_call_id)
|
||||
if isinstance(e, ProxyException):
|
||||
raise with_litellm_call_id(e, litellm_call_id)
|
||||
error_msg: Final = f"{e}"
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=openai_error_type(e, error_status_code(e, 500)),
|
||||
param=openai_error_param(e),
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e)),
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
openai_code=getattr(e, "code", None),
|
||||
code=error_status_code(e, 500),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -13273,22 +13267,18 @@ async def moderations(
|
|||
if isinstance(e, ProxyException):
|
||||
raise with_litellm_call_id(e, litellm_call_id)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", str(e)),
|
||||
type=openai_error_type(e, error_status_code(e, status.HTTP_400_BAD_REQUEST)),
|
||||
param=openai_error_param(e),
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e)),
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
code=error_status_code(e, status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
else:
|
||||
error_msg: Final = f"{e}"
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=openai_error_type(e, error_status_code(e, 500)),
|
||||
param=openai_error_param(e),
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
code=error_status_code(e, 500),
|
||||
)
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e)),
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
)
|
||||
|
||||
|
||||
async def _audio_speech_chunk_generator(
|
||||
|
|
@ -13426,13 +13416,12 @@ async def audio_speech(
|
|||
detail=e.detail,
|
||||
headers=headers_with_litellm_call_id(e.headers, litellm_call_id),
|
||||
)
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", f"{e}"),
|
||||
type=openai_error_type(e, error_status_code(e, 500)),
|
||||
param=openai_error_param(e),
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e)),
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
openai_code=getattr(e, "code", None),
|
||||
code=error_status_code(e, 500),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -13572,23 +13561,19 @@ async def audio_transcriptions(
|
|||
)
|
||||
log_llm_api_exception(e, litellm_call_id)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", str(e.detail)),
|
||||
type=openai_error_type(e, error_status_code(e, status.HTTP_400_BAD_REQUEST)),
|
||||
param=openai_error_param(e),
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e.detail)),
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
code=error_status_code(e, status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
else:
|
||||
error_msg: Final = f"{e}"
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=openai_error_type(e, error_status_code(e, 500)),
|
||||
param=openai_error_param(e),
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
openai_code=getattr(e, "code", None),
|
||||
code=error_status_code(e, 500),
|
||||
)
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e)),
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
openai_code=getattr(e, "code", None),
|
||||
)
|
||||
|
||||
|
||||
######################################################################
|
||||
|
|
|
|||
|
|
@ -1310,6 +1310,33 @@ async def test_auth_rejects_wrong_or_empty_present_custom_header(
|
|||
assert exc_info.value.code == "401"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("api_key", [None, ""], ids=["no_key", "no_bearer_prefix"])
|
||||
async def test_auth_rejection_without_a_usable_key_has_a_json_null_param(
|
||||
api_key: str | None, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException
|
||||
|
||||
monkeypatch.setattr(proxy_server, "master_key", "sk-test-master")
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
request = Request(
|
||||
scope={"type": "http", "method": "POST", "path": "/chat/completions", "headers": []},
|
||||
receive=AsyncMock(return_value={"type": "http.request", "body": b"", "more_body": False}),
|
||||
)
|
||||
request._url = URL(url="/chat/completions")
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await user_api_key_auth(request=request, api_key=api_key)
|
||||
|
||||
body = json.loads(json.dumps({"error": exc_info.value.to_dict()}))
|
||||
assert body["error"]["code"] == "401"
|
||||
assert body["error"]["type"] == ProxyErrorTypes.auth_error.value
|
||||
assert body["error"]["param"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_api_key_from_query_param():
|
||||
"""Ensure user_api_key_auth reads API key from `key` query parameter."""
|
||||
|
|
|
|||
|
|
@ -16,7 +16,11 @@ import httpx
|
|||
import pytest
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
from litellm.types.router import RouterErrors
|
||||
|
||||
_TAG_ROUTING_DENIAL = f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=tts-1 and tags=['x']"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -208,7 +212,8 @@ def test_audio_speech_error(client, auth_as, patched_speech_error, path):
|
|||
with auth_as():
|
||||
response = client.post(path, json=payload)
|
||||
assert response.status_code == 500
|
||||
assert len(response.content) > 0
|
||||
error = response.json()["error"]
|
||||
assert (error["type"], error["param"]) == ("internal_server_error", None)
|
||||
|
||||
|
||||
def test_audio_speech_bad_request_maps_to_400(client, auth_as, patched_speech_provider_rejection):
|
||||
|
|
@ -265,6 +270,69 @@ def test_audio_transcription_over_the_file_size_cap_is_rejected_as_a_bad_request
|
|||
assert error["param"] == "file"
|
||||
|
||||
|
||||
def test_audio_speech_tag_routing_denial_is_typed_as_an_authentication_error(
|
||||
client, auth_as, patched_speech_error, monkeypatch
|
||||
):
|
||||
async def _deny(*args, **kwargs):
|
||||
raise ValueError(_TAG_ROUTING_DENIAL)
|
||||
|
||||
monkeypatch.setattr(proxy_server, "route_request", _deny)
|
||||
with auth_as():
|
||||
response = client.post("/v1/audio/speech", json={"model": "tts-1", "input": "Hi", "voice": "alloy"})
|
||||
assert response.status_code == 401
|
||||
error = response.json()["error"]
|
||||
assert (error["type"], error["param"]) == ("authentication_error", None)
|
||||
|
||||
|
||||
def test_audio_transcription_tag_routing_denial_is_typed_as_an_authentication_error(
|
||||
client, auth_as, patched_transcription, monkeypatch
|
||||
):
|
||||
async def _deny(*args, **kwargs):
|
||||
raise ValueError(_TAG_ROUTING_DENIAL)
|
||||
|
||||
monkeypatch.setattr(proxy_server, "route_request", _deny)
|
||||
files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")}
|
||||
with auth_as():
|
||||
response = client.post("/v1/audio/transcriptions", files=files, data={"model": "whisper-1"})
|
||||
assert response.status_code == 401
|
||||
error = response.json()["error"]
|
||||
assert (error["type"], error["param"]) == ("authentication_error", None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rejection",
|
||||
[
|
||||
pytest.param(
|
||||
ProxyException(
|
||||
message="Invalid type for 'metadata': expected an object, but got an integer instead.",
|
||||
type=ProxyErrorTypes.bad_request_error,
|
||||
param="metadata",
|
||||
code=400,
|
||||
),
|
||||
id="bad_metadata",
|
||||
),
|
||||
pytest.param(
|
||||
ProxyException(
|
||||
message="tag budgets are unavailable", type=ProxyErrorTypes.no_db_connection, param=None, code=503
|
||||
),
|
||||
id="tag_budget_store_down",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_audio_transcription_keeps_the_status_of_a_proxy_rejection(
|
||||
client, auth_as, patched_transcription, monkeypatch, rejection
|
||||
):
|
||||
async def _reject(data, **kwargs):
|
||||
raise rejection
|
||||
|
||||
monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _reject)
|
||||
files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")}
|
||||
with auth_as():
|
||||
response = client.post("/v1/audio/transcriptions", files=files, data={"model": "whisper-1"})
|
||||
assert response.status_code == int(rejection.code)
|
||||
assert response.json()["error"] == rejection.to_dict()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"])
|
||||
def test_audio_transcription_error(client, auth_as, patched_transcription_error, path):
|
||||
"""Pins ``POST /v1/audio/transcriptions`` / ``POST /audio/transcriptions`` (error)."""
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from unittest.mock import AsyncMock, MagicMock
|
|||
import pytest
|
||||
|
||||
from litellm.proxy import common_request_processing, proxy_server
|
||||
from litellm.types.router import RouterErrors
|
||||
|
||||
from .conftest import normalize # type: ignore[import-not-found]
|
||||
|
||||
|
|
@ -124,3 +125,32 @@ def test_completion_pipeline_error(client, auth_as, completion_pipeline_raises,
|
|||
response = client.post(path, json=payload)
|
||||
assert response.status_code == 500
|
||||
assert response.headers.get("content-type", "").startswith("application/json")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("failure", "expected_status", "expected_type"),
|
||||
[
|
||||
pytest.param(ValueError("boom"), 500, "internal_server_error", id="unexpected_failure"),
|
||||
pytest.param(
|
||||
ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=gpt-4o and tags=['x']"),
|
||||
401,
|
||||
"authentication_error",
|
||||
id="tag_routing_denial",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_completion_failure_body_types_the_status_it_answers_with(
|
||||
client, auth_as, monkeypatch, failure, expected_status, expected_type
|
||||
):
|
||||
monkeypatch.setattr(proxy_server, "llm_router", MagicMock())
|
||||
monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock(post_call_failure_hook=AsyncMock()))
|
||||
|
||||
async def _raise(self, *args, **kwargs):
|
||||
raise failure
|
||||
|
||||
monkeypatch.setattr(common_request_processing.ProxyBaseLLMRequestProcessing, "base_process_llm_request", _raise)
|
||||
with auth_as():
|
||||
response = client.post("/v1/completions", json={"model": "gpt-4o", "prompt": "hi"})
|
||||
assert response.status_code == expected_status
|
||||
error = response.json()["error"]
|
||||
assert (error["type"], error["param"], error["code"]) == (expected_type, None, str(expected_status))
|
||||
|
|
|
|||
|
|
@ -14260,6 +14260,12 @@ async def test_moderations_reraises_proxy_exception_unwrapped():
|
|||
pytest.param(HTTPException(status_code=400, detail="bad input"), "invalid_request_error", "400", id="http_400"),
|
||||
pytest.param(HTTPException(status_code=429, detail="slow down"), "rate_limit_error", "429", id="http_429"),
|
||||
pytest.param(Exception("provider down"), "internal_server_error", "500", id="bare_exception"),
|
||||
pytest.param(
|
||||
ValueError("Not allowed to access model due to tags configuration. Passed model=m and tags=['x']"),
|
||||
"authentication_error",
|
||||
"401",
|
||||
id="tag_routing_denial",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_moderations_failure_body_is_openai_shaped(failure, expected_type, expected_code):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue