mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(messages): answer 400 for a message without a valid role on bridged models
This commit is contained in:
parent
4c8b90945f
commit
a50aefef00
3 changed files with 164 additions and 3 deletions
|
|
@ -55,6 +55,7 @@ from ..safeguards import (
|
|||
proxy_serves_safeguards,
|
||||
with_safeguard_results,
|
||||
)
|
||||
from ..utils import messages_with_valid_roles
|
||||
from .interceptors import get_messages_interceptors
|
||||
from .utils import AnthropicMessagesRequestUtils, mock_response, prepare_native_messages
|
||||
|
||||
|
|
@ -691,13 +692,16 @@ def anthropic_messages_handler(
|
|||
model, custom_llm_provider, kwargs.get("model_info"), api_base
|
||||
)
|
||||
if anthropic_messages_provider_config is None:
|
||||
bridged_messages: Final[list[dict[str, object]]] = list( # mutable-ok: both bridge handlers take a list
|
||||
messages_with_valid_roles(messages, original_model, custom_llm_provider, litellm_params.drop_params)
|
||||
)
|
||||
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
|
||||
if kwargs.get("compaction") is None and _should_route_to_responses_api(
|
||||
custom_llm_provider, original_model, model, kwargs.get("model_info"), stop_sequences=stop_sequences
|
||||
):
|
||||
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
messages=bridged_messages,
|
||||
model=original_model,
|
||||
metadata=metadata,
|
||||
stop_sequences=stop_sequences,
|
||||
|
|
@ -727,7 +731,7 @@ def anthropic_messages_handler(
|
|||
# is passed through as a regular kwarg.
|
||||
return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
messages=bridged_messages,
|
||||
model=original_model,
|
||||
metadata=metadata,
|
||||
stop_sequences=stop_sequences,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import json
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import starmap
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, TypeVar
|
||||
|
||||
|
|
@ -9,6 +11,7 @@ from typing_extensions import ReadOnly
|
|||
|
||||
import litellm
|
||||
from litellm.constants import X_LITELLM_DISABLE_CALLBACKS
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.internal_call_metadata import sanitize_user_api_key_auth
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, ModelInfo, StandardCallbackDynamicParams
|
||||
|
|
@ -234,3 +237,45 @@ def normalize_reasoning_effort_value(
|
|||
|
||||
accepted_tiers: Final = tuple(level for level in supported if level != _THINKING_OFF)
|
||||
return next((level for level in (*chain, *accepted_tiers) if level in supported), chain[-1])
|
||||
|
||||
|
||||
_MESSAGE_ROLES: Final = frozenset({"user", "assistant", "system"})
|
||||
_Message: Final = TypeVar("_Message")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _MalformedMessage:
|
||||
param: str
|
||||
message: str
|
||||
|
||||
|
||||
def _malformed_message(index: int, message: object) -> _MalformedMessage | None:
|
||||
match as_str_mapping(message):
|
||||
case None:
|
||||
return _MalformedMessage(f"messages.{index}", f"messages.{index}: Input does not match the expected shape.")
|
||||
case {"role": str() as role} if role in _MESSAGE_ROLES:
|
||||
return None
|
||||
case {"role": _}:
|
||||
return _MalformedMessage(
|
||||
f"messages.{index}.role", f"messages.{index}.role: Input should be 'user', 'assistant', 'system'"
|
||||
)
|
||||
case _:
|
||||
return _MalformedMessage(f"messages.{index}.role", f"messages.{index}.role: Field required")
|
||||
|
||||
|
||||
def messages_with_valid_roles(
|
||||
messages: Sequence[_Message], model: str, custom_llm_provider: str, drop_params: object
|
||||
) -> Sequence[_Message]:
|
||||
if drops_unsupported_params(drop_params):
|
||||
return tuple(message for index, message in enumerate(messages) if _malformed_message(index, message) is None)
|
||||
malformed: Final = next(
|
||||
(malformed for malformed in starmap(_malformed_message, enumerate(messages)) if malformed is not None), None
|
||||
)
|
||||
if malformed is None:
|
||||
return messages
|
||||
raise litellm.MalformedRequestFieldError(
|
||||
message=malformed.message,
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
body={"type": "invalid_request_error", "param": malformed.param},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2455,3 +2455,115 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s
|
|||
assert first["tools"] == [tool]
|
||||
assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]}
|
||||
assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry"
|
||||
|
||||
|
||||
_BRIDGED_MODELS: Final = pytest.mark.parametrize(
|
||||
"model",
|
||||
["fireworks_ai/accounts/fireworks/models/kimi-k3", "openai/gpt-4o-mini"],
|
||||
ids=["chat_bridge", "responses_bridge"],
|
||||
)
|
||||
|
||||
|
||||
async def _send_bridged_messages(model: str, messages: list[object], *, is_async: bool, stream: bool) -> object:
|
||||
if is_async:
|
||||
return await litellm.anthropic.messages.acreate(
|
||||
model=model, max_tokens=8, messages=messages, stream=stream, api_key="test-key"
|
||||
)
|
||||
return litellm.anthropic.messages.create(
|
||||
model=model, max_tokens=8, messages=messages, stream=stream, api_key="test-key"
|
||||
)
|
||||
|
||||
|
||||
async def _bridged_messages_rejection(
|
||||
model: str, messages: list[object], *, is_async: bool, stream: bool
|
||||
) -> tuple[litellm.BadRequestError, int]:
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
provider: Final = upstream.route().mock(return_value=httpx.Response(500))
|
||||
with pytest.raises(litellm.BadRequestError) as rejected:
|
||||
await _send_bridged_messages(model, messages, is_async=is_async, stream=stream)
|
||||
return rejected.value, provider.call_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_BRIDGED_MODELS
|
||||
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["complete", "stream"])
|
||||
async def test_bridged_message_without_role_answers_400_before_any_provider_call(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, is_async: bool, stream: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
rejected, provider_calls = await _bridged_messages_rejection(
|
||||
model, [{"role": "user", "content": "hi"}, {"content": "no role"}], is_async=is_async, stream=stream
|
||||
)
|
||||
|
||||
assert rejected.status_code == 400
|
||||
assert rejected.message == "litellm.BadRequestError: messages.1.role: Field required"
|
||||
assert rejected.body == {"type": "invalid_request_error", "param": "messages.1.role"}
|
||||
assert rejected.llm_provider == model.split("/")[0]
|
||||
assert provider_calls == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_BRIDGED_MODELS
|
||||
@pytest.mark.parametrize(
|
||||
("message", "param", "error"),
|
||||
[
|
||||
({"role": "bogus", "content": "hi"}, "messages.1.role", "Input should be 'user', 'assistant', 'system'"),
|
||||
({"role": ["user"], "content": "hi"}, "messages.1.role", "Input should be 'user', 'assistant', 'system'"),
|
||||
({"role": None, "content": "hi"}, "messages.1.role", "Input should be 'user', 'assistant', 'system'"),
|
||||
("hi", "messages.1", "Input does not match the expected shape."),
|
||||
],
|
||||
ids=["unknown_role", "unhashable_role", "null_role", "not_an_object"],
|
||||
)
|
||||
async def test_bridged_message_with_an_invalid_role_answers_400_instead_of_being_dropped(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, message: object, param: str, error: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
rejected, provider_calls = await _bridged_messages_rejection(
|
||||
model, [{"role": "user", "content": "hi"}, message], is_async=True, stream=False
|
||||
)
|
||||
|
||||
assert rejected.status_code == 400
|
||||
assert rejected.message == f"litellm.BadRequestError: {param}: {error}"
|
||||
assert rejected.body == {"type": "invalid_request_error", "param": param}
|
||||
assert provider_calls == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
|
||||
async def test_bridged_messages_with_drop_params_drop_the_malformed_messages_and_keep_the_rest(
|
||||
monkeypatch: pytest.MonkeyPatch, is_async: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
completion: Final = {
|
||||
"id": "chatcmpl-drop-params",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "kimi-k3",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
provider: Final = upstream.route().mock(return_value=httpx.Response(200, json=completion))
|
||||
create: Final = litellm.anthropic.messages.acreate if is_async else litellm.anthropic.messages.create
|
||||
pending: Final = create(
|
||||
model="fireworks_ai/accounts/fireworks/models/kimi-k3",
|
||||
max_tokens=8,
|
||||
messages=[
|
||||
{"content": "no role"},
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "bogus", "content": "dropped"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
],
|
||||
drop_params=True,
|
||||
api_key="test-key",
|
||||
)
|
||||
result: Final = await pending if is_async else pending
|
||||
|
||||
assert result["content"] == [{"type": "text", "text": "ok"}]
|
||||
assert json.loads(provider.calls.last.request.content)["messages"] == [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue