mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(rust-bridge): apply the malformed user text rule before sync Rust messages calls
Sync anthropic messages calls have no Python path, so routing them off Rust for a malformed user text block raised "not implemented for sync calls". The sync call now stays on Rust and drops the block under drop_params or answers the 400 first, while async calls keep running on the Python path
This commit is contained in:
parent
a24a385e78
commit
eb16928919
4 changed files with 86 additions and 14 deletions
|
|
@ -371,9 +371,7 @@ def deployment_drops_params(deployment: Mapping[str, object] | None) -> bool:
|
|||
return drops_unsupported_params(None)
|
||||
|
||||
|
||||
def reject_malformed_text(
|
||||
messages: Iterable[Mapping[str, object]], model: str, forwarded_kwargs: Mapping[str, object]
|
||||
) -> None:
|
||||
def reject_malformed_text(messages: Iterable[object], model: str, forwarded_kwargs: Mapping[str, object]) -> None:
|
||||
if drops_unsupported_params(forwarded_kwargs.get("drop_params")):
|
||||
return
|
||||
error: Final = malformed_text_error(messages)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,19 @@
|
|||
import inspect
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping
|
||||
from dataclasses import replace
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import (
|
||||
drop_non_string_texts,
|
||||
malformed_text_error,
|
||||
reject_emptied_messages,
|
||||
reject_malformed_text,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native import handler as main
|
||||
from litellm.rust_bridge import detectors
|
||||
from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target
|
||||
|
|
@ -13,6 +22,7 @@ from litellm.rust_bridge.messages import detectors as route_detectors
|
|||
from litellm.rust_bridge.messages.entrypoints import (
|
||||
NATIVE_AMESSAGES,
|
||||
NATIVE_MESSAGES,
|
||||
NativeMessages,
|
||||
)
|
||||
from litellm.rust_bridge.public_call import (
|
||||
AsyncFlag,
|
||||
|
|
@ -52,6 +62,7 @@ _MESSAGES: Final = signature(_PYTHON_MESSAGES)
|
|||
_FLAG: Final = AsyncFlag("is_async", sync=_MESSAGES, is_set=bool)
|
||||
_PYTHON_AMESSAGES: Final = _python_amessages()
|
||||
_AMESSAGES: Final = signature(_PYTHON_AMESSAGES)
|
||||
_MESSAGE_LIST: Final = TypeAdapter(list[dict[str, object]])
|
||||
|
||||
|
||||
def _native_kwargs(fields: Mapping[str, object], kwargs: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
@ -112,7 +123,6 @@ _FEATURES: Final[detectors.Detectors] = MappingProxyType(
|
|||
Feature.STREAM_COST_INJECTION: route_detectors.stream_cost_injection,
|
||||
Feature.THINKING_SIGNATURE_RETRY: route_detectors.thinking_signature_retry,
|
||||
Feature.TOOL_CUSTOM_FIELD: route_detectors.tool_custom_field,
|
||||
Feature.MALFORMED_TEXT: route_detectors.malformed_text,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -122,6 +132,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType(
|
|||
Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook,
|
||||
Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception,
|
||||
Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted,
|
||||
Feature.MALFORMED_TEXT: route_detectors.malformed_text,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -151,6 +162,25 @@ _ADISPATCH: Final = PublicDispatch(
|
|||
)
|
||||
|
||||
|
||||
def _without_malformed_text(call: NativeCall) -> NativeCall:
|
||||
request: Final = call.resolved
|
||||
messages: Final = optional_sequence(request.get("messages")) or ()
|
||||
if malformed_text_error(messages) is None:
|
||||
return call
|
||||
model: Final = str(request["model"])
|
||||
reject_malformed_text(messages, model, request)
|
||||
sent: Final = _MESSAGE_LIST.validate_python(messages)
|
||||
kept: Final = drop_non_string_texts(sent, request.get("drop_params"))
|
||||
reject_emptied_messages(sent, kept, model, optional_str(request.get("custom_llm_provider")))
|
||||
return replace(call, kwargs={**call.kwargs, "messages": kept})
|
||||
|
||||
|
||||
def _native_messages(
|
||||
hook: NativeMessages, call: NativeCall, _args: tuple[object, ...], _kwargs: Mapping[str, object]
|
||||
) -> AnthropicMessagesResponse | Iterator[bytes]:
|
||||
return hook(_without_malformed_text(call))
|
||||
|
||||
|
||||
def anthropic_messages_handler(
|
||||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: preserve the public Anthropic Messages call shape
|
||||
|
|
@ -163,7 +193,7 @@ def anthropic_messages_handler(
|
|||
_FLAG.dropped(kwargs),
|
||||
python=python,
|
||||
binding=NATIVE_MESSAGES,
|
||||
native=native_call_hook,
|
||||
native=_native_messages,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -485,12 +485,6 @@ ADVISOR_TOOL: Final = {"type": ANTHROPIC_ADVISOR_TOOL_TYPE, "name": "advisor", "
|
|||
{"stream": True},
|
||||
{"include_cost_in_streaming_usage": True},
|
||||
),
|
||||
(
|
||||
Feature.MALFORMED_TEXT,
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
{"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": None}]}]},
|
||||
{},
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_each_detected_feature_reaches_its_rule(
|
||||
|
|
@ -544,6 +538,12 @@ ASYNC_ONLY_FEATURES: Final = (
|
|||
{},
|
||||
id="messages-interceptor",
|
||||
),
|
||||
pytest.param(
|
||||
Feature.MALFORMED_TEXT,
|
||||
{"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": None}]}]},
|
||||
{},
|
||||
id="malformed-text",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -559,7 +559,8 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_
|
|||
python_result: Final = response()
|
||||
native_result: Final = response()
|
||||
model: Final = str(kwargs.get("model", "anthropic/claude-sonnet-4-5"))
|
||||
call_kwargs: Final = {name: value for name, value in kwargs.items() if name != "model"}
|
||||
messages: Final = kwargs.get("messages", MESSAGES)
|
||||
call_kwargs: Final = {name: value for name, value in kwargs.items() if name not in {"model", "messages"}}
|
||||
|
||||
async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub
|
||||
return python_result
|
||||
|
|
@ -571,7 +572,7 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_
|
|||
for name, value in settings.items():
|
||||
monkeypatch.setattr(litellm, name, value)
|
||||
served_async: Final = await _ADISPATCH.arun(
|
||||
(16, MESSAGES, model),
|
||||
(16, messages, model),
|
||||
call_kwargs,
|
||||
python=python,
|
||||
binding=amessages_binding(native),
|
||||
|
|
@ -579,7 +580,7 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_
|
|||
rules=rules,
|
||||
)
|
||||
served_sync: Final = _DISPATCH.run(
|
||||
(16, MESSAGES, model),
|
||||
(16, messages, model),
|
||||
call_kwargs,
|
||||
python=lambda *call_args, **python_kwargs: python_result,
|
||||
binding=messages_binding(lambda request: native_result),
|
||||
|
|
|
|||
|
|
@ -146,3 +146,46 @@ async def test_optional_positional_body_parameters_reach_the_wire(server: Record
|
|||
assert response["id"] == MESSAGES_RESPONSE["id"]
|
||||
body, _ = sent(server)
|
||||
assert (body["metadata"], body["system"], body["temperature"]) == (args[3], args[6], args[7])
|
||||
|
||||
|
||||
def _user_text(*texts: object) -> tuple[dict[str, object], ...]:
|
||||
return ({"role": "user", "content": [{"type": "text", "text": text} for text in texts]},)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("messages", "error"),
|
||||
(
|
||||
(_user_text("hi", None), r"messages\.0\.content\.1\.text: Input should be a valid string"),
|
||||
(_user_text("hi", 42), r"messages\.0\.content\.1\.text: Input should be a valid string"),
|
||||
),
|
||||
)
|
||||
async def test_sync_call_rejects_non_string_user_text_before_io_without_drop_params(
|
||||
server: RecordingServer, monkeypatch: pytest.MonkeyPatch, messages: tuple[dict[str, object], ...], error: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
server.expected_requests = 0
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=error):
|
||||
await call(server, asynchronous=False, messages=messages)
|
||||
|
||||
assert server.requests == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("malformed", (None, 42))
|
||||
async def test_sync_call_drops_non_string_user_text_with_drop_params(server: RecordingServer, malformed: object) -> None:
|
||||
await call(server, asynchronous=False, messages=_user_text("hi", malformed), drop_params=True)
|
||||
|
||||
body, _ = sent(server)
|
||||
assert body["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_call_rejects_messages_emptied_by_drop_params(server: RecordingServer) -> None:
|
||||
server.expected_requests = 0
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="no message has content left"):
|
||||
await call(server, asynchronous=False, messages=_user_text(None), drop_params=True)
|
||||
|
||||
assert server.requests == []
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue