From eb16928919cbf09d570defb5ada6816018c18205 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:43:54 -0700 Subject: [PATCH] fix(rust-bridge): apply the malformed user text rule before sync Rust messages calls Sync anthropic messages calls have no Python path, so routing them off Rust for a malformed user text block raised "not implemented for sync calls". The sync call now stays on Rust and drops the block under drop_params or answers the 400 first, while async calls keep running on the Python path --- .../anthropic/messages/convertible_blocks.py | 4 +- litellm/messages/dispatch.py | 34 ++++++++++++++- .../messages/test_dispatch.py | 19 ++++---- .../messages/test_requests.py | 43 +++++++++++++++++++ 4 files changed, 86 insertions(+), 14 deletions(-) diff --git a/litellm/llms/anthropic/messages/convertible_blocks.py b/litellm/llms/anthropic/messages/convertible_blocks.py index 8a304e1d82e..117a58ca4dc 100644 --- a/litellm/llms/anthropic/messages/convertible_blocks.py +++ b/litellm/llms/anthropic/messages/convertible_blocks.py @@ -371,9 +371,7 @@ def deployment_drops_params(deployment: Mapping[str, object] | None) -> bool: return drops_unsupported_params(None) -def reject_malformed_text( - messages: Iterable[Mapping[str, object]], model: str, forwarded_kwargs: Mapping[str, object] -) -> None: +def reject_malformed_text(messages: Iterable[object], model: str, forwarded_kwargs: Mapping[str, object]) -> None: if drops_unsupported_params(forwarded_kwargs.get("drop_params")): return error: Final = malformed_text_error(messages) diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 376ec3b7cd1..acc67fb7bf2 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -1,10 +1,19 @@ import inspect from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping +from dataclasses import replace from types import MappingProxyType from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable +from pydantic import TypeAdapter + from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.anthropic.messages.convertible_blocks import ( + drop_non_string_texts, + malformed_text_error, + reject_emptied_messages, + reject_malformed_text, +) from litellm.llms.anthropic.messages.native import handler as main from litellm.rust_bridge import detectors from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target @@ -13,6 +22,7 @@ from litellm.rust_bridge.messages import detectors as route_detectors from litellm.rust_bridge.messages.entrypoints import ( NATIVE_AMESSAGES, NATIVE_MESSAGES, + NativeMessages, ) from litellm.rust_bridge.public_call import ( AsyncFlag, @@ -52,6 +62,7 @@ _MESSAGES: Final = signature(_PYTHON_MESSAGES) _FLAG: Final = AsyncFlag("is_async", sync=_MESSAGES, is_set=bool) _PYTHON_AMESSAGES: Final = _python_amessages() _AMESSAGES: Final = signature(_PYTHON_AMESSAGES) +_MESSAGE_LIST: Final = TypeAdapter(list[dict[str, object]]) def _native_kwargs(fields: Mapping[str, object], kwargs: Mapping[str, object]) -> Mapping[str, object]: @@ -112,7 +123,6 @@ _FEATURES: Final[detectors.Detectors] = MappingProxyType( Feature.STREAM_COST_INJECTION: route_detectors.stream_cost_injection, Feature.THINKING_SIGNATURE_RETRY: route_detectors.thinking_signature_retry, Feature.TOOL_CUSTOM_FIELD: route_detectors.tool_custom_field, - Feature.MALFORMED_TEXT: route_detectors.malformed_text, } ) @@ -122,6 +132,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType( Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook, Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception, Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted, + Feature.MALFORMED_TEXT: route_detectors.malformed_text, } ) @@ -151,6 +162,25 @@ _ADISPATCH: Final = PublicDispatch( ) +def _without_malformed_text(call: NativeCall) -> NativeCall: + request: Final = call.resolved + messages: Final = optional_sequence(request.get("messages")) or () + if malformed_text_error(messages) is None: + return call + model: Final = str(request["model"]) + reject_malformed_text(messages, model, request) + sent: Final = _MESSAGE_LIST.validate_python(messages) + kept: Final = drop_non_string_texts(sent, request.get("drop_params")) + reject_emptied_messages(sent, kept, model, optional_str(request.get("custom_llm_provider"))) + return replace(call, kwargs={**call.kwargs, "messages": kept}) + + +def _native_messages( + hook: NativeMessages, call: NativeCall, _args: tuple[object, ...], _kwargs: Mapping[str, object] +) -> AnthropicMessagesResponse | Iterator[bytes]: + return hook(_without_malformed_text(call)) + + def anthropic_messages_handler( *args: object, **kwargs: object, # kwargs-ok: preserve the public Anthropic Messages call shape @@ -163,7 +193,7 @@ def anthropic_messages_handler( _FLAG.dropped(kwargs), python=python, binding=NATIVE_MESSAGES, - native=native_call_hook, + native=_native_messages, ) diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 1e8d2f87abc..a9a3a623fc4 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -485,12 +485,6 @@ ADVISOR_TOOL: Final = {"type": ANTHROPIC_ADVISOR_TOOL_TYPE, "name": "advisor", " {"stream": True}, {"include_cost_in_streaming_usage": True}, ), - ( - Feature.MALFORMED_TEXT, - "anthropic/claude-sonnet-4-5", - {"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": None}]}]}, - {}, - ), ), ) def test_each_detected_feature_reaches_its_rule( @@ -544,6 +538,12 @@ ASYNC_ONLY_FEATURES: Final = ( {}, id="messages-interceptor", ), + pytest.param( + Feature.MALFORMED_TEXT, + {"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": None}]}]}, + {}, + id="malformed-text", + ), ) @@ -559,7 +559,8 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_ python_result: Final = response() native_result: Final = response() model: Final = str(kwargs.get("model", "anthropic/claude-sonnet-4-5")) - call_kwargs: Final = {name: value for name, value in kwargs.items() if name != "model"} + messages: Final = kwargs.get("messages", MESSAGES) + call_kwargs: Final = {name: value for name, value in kwargs.items() if name not in {"model", "messages"}} async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub return python_result @@ -571,7 +572,7 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_ for name, value in settings.items(): monkeypatch.setattr(litellm, name, value) served_async: Final = await _ADISPATCH.arun( - (16, MESSAGES, model), + (16, messages, model), call_kwargs, python=python, binding=amessages_binding(native), @@ -579,7 +580,7 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_ rules=rules, ) served_sync: Final = _DISPATCH.run( - (16, MESSAGES, model), + (16, messages, model), call_kwargs, python=lambda *call_args, **python_kwargs: python_result, binding=messages_binding(lambda request: native_result), diff --git a/tests/test_litellm_rust/messages/test_requests.py b/tests/test_litellm_rust/messages/test_requests.py index 92031e72aee..08f1e2b8363 100644 --- a/tests/test_litellm_rust/messages/test_requests.py +++ b/tests/test_litellm_rust/messages/test_requests.py @@ -146,3 +146,46 @@ async def test_optional_positional_body_parameters_reach_the_wire(server: Record assert response["id"] == MESSAGES_RESPONSE["id"] body, _ = sent(server) assert (body["metadata"], body["system"], body["temperature"]) == (args[3], args[6], args[7]) + + +def _user_text(*texts: object) -> tuple[dict[str, object], ...]: + return ({"role": "user", "content": [{"type": "text", "text": text} for text in texts]},) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("messages", "error"), + ( + (_user_text("hi", None), r"messages\.0\.content\.1\.text: Input should be a valid string"), + (_user_text("hi", 42), r"messages\.0\.content\.1\.text: Input should be a valid string"), + ), +) +async def test_sync_call_rejects_non_string_user_text_before_io_without_drop_params( + server: RecordingServer, monkeypatch: pytest.MonkeyPatch, messages: tuple[dict[str, object], ...], error: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + server.expected_requests = 0 + + with pytest.raises(litellm.BadRequestError, match=error): + await call(server, asynchronous=False, messages=messages) + + assert server.requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("malformed", (None, 42)) +async def test_sync_call_drops_non_string_user_text_with_drop_params(server: RecordingServer, malformed: object) -> None: + await call(server, asynchronous=False, messages=_user_text("hi", malformed), drop_params=True) + + body, _ = sent(server) + assert body["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + + +@pytest.mark.asyncio +async def test_sync_call_rejects_messages_emptied_by_drop_params(server: RecordingServer) -> None: + server.expected_requests = 0 + + with pytest.raises(litellm.BadRequestError, match="no message has content left"): + await call(server, asynchronous=False, messages=_user_text(None), drop_params=True) + + assert server.requests == []