fix(rust-bridge): apply the malformed user text rule where the native messages call reads its request

The sync pre-step ran before the native logging setup, so its 400 fired no failure callback, a stray non-object message raised a raw ValidationError under drop_params, and the error carried an empty llm_provider. The Python host now owns the rule at projection time for sync and async native calls, which replaces the async-only routing feature.
This commit is contained in:
mateo-berri 2026-10-29 13:38:33 -07:00
parent eb16928919
commit db6801bc74
11 changed files with 144 additions and 95 deletions

View file

@ -2,7 +2,7 @@ use crate::cache::{CacheCall, Cached, PythonCache, Selection};
use litellm_host_python::{PythonHostCalls, PythonOwned};
use bytes::Bytes;
use litellm_host_python::{InvokeError, PythonBinding, from_py, to_py};
use litellm_host_python::{InvokeError, PythonBinding, effective_py_args, from_py, to_py};
use litellm_inference_messages::{
Error, MessagesCall, MessagesSettings, MessagesShaping, litellm_params, messages_body,
route::{Messages, MessagesStreamHead},
@ -94,11 +94,12 @@ impl MessagesPythonHost {
argument(name)?.map(|value| value.extract()).transpose()
};
let model = string("model")?.ok_or(ArgumentError::Required("model"))?;
let messages = argument("messages")?.ok_or(ArgumentError::Required("messages"))?;
argument("messages")?.ok_or(ArgumentError::Required("messages"))?;
let messages = self.messages(py, arguments)?;
let fields = self.view.body_params(py, arguments, ROUTE_HOST_MODULE)?;
let body = [
("model".to_string(), Value::String(model.clone())),
("messages".to_string(), from_py(&messages)?),
("messages".to_string(), messages),
]
.into_iter()
.chain(fields)
@ -129,6 +130,15 @@ impl MessagesPythonHost {
}))
}
fn messages(&self, py: Python<'_>, arguments: &Bound<'_, PyDict>) -> PyResult<Value> {
let request = effective_py_args(self.view.request(py)?, arguments)?;
from_py(
&py.import(ROUTE_HOST_MODULE)?
.getattr("messages")?
.call1((request,))?,
)
}
fn merged_headers(
&self,
py: Python<'_>,

View file

@ -371,7 +371,9 @@ def deployment_drops_params(deployment: Mapping[str, object] | None) -> bool:
return drops_unsupported_params(None)
def reject_malformed_text(messages: Iterable[object], model: str, forwarded_kwargs: Mapping[str, object]) -> None:
def reject_malformed_text(
messages: Iterable[Mapping[str, object]], model: str, forwarded_kwargs: Mapping[str, object]
) -> None:
if drops_unsupported_params(forwarded_kwargs.get("drop_params")):
return
error: Final = malformed_text_error(messages)

View file

@ -1,19 +1,10 @@
import inspect
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping
from dataclasses import replace
from types import MappingProxyType
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
from pydantic import TypeAdapter
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.anthropic.messages.convertible_blocks import (
drop_non_string_texts,
malformed_text_error,
reject_emptied_messages,
reject_malformed_text,
)
from litellm.llms.anthropic.messages.native import handler as main
from litellm.rust_bridge import detectors
from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target
@ -22,7 +13,6 @@ from litellm.rust_bridge.messages import detectors as route_detectors
from litellm.rust_bridge.messages.entrypoints import (
NATIVE_AMESSAGES,
NATIVE_MESSAGES,
NativeMessages,
)
from litellm.rust_bridge.public_call import (
AsyncFlag,
@ -62,7 +52,6 @@ _MESSAGES: Final = signature(_PYTHON_MESSAGES)
_FLAG: Final = AsyncFlag("is_async", sync=_MESSAGES, is_set=bool)
_PYTHON_AMESSAGES: Final = _python_amessages()
_AMESSAGES: Final = signature(_PYTHON_AMESSAGES)
_MESSAGE_LIST: Final = TypeAdapter(list[dict[str, object]])
def _native_kwargs(fields: Mapping[str, object], kwargs: Mapping[str, object]) -> Mapping[str, object]:
@ -132,7 +121,6 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType(
Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook,
Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception,
Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted,
Feature.MALFORMED_TEXT: route_detectors.malformed_text,
}
)
@ -162,25 +150,6 @@ _ADISPATCH: Final = PublicDispatch(
)
def _without_malformed_text(call: NativeCall) -> NativeCall:
request: Final = call.resolved
messages: Final = optional_sequence(request.get("messages")) or ()
if malformed_text_error(messages) is None:
return call
model: Final = str(request["model"])
reject_malformed_text(messages, model, request)
sent: Final = _MESSAGE_LIST.validate_python(messages)
kept: Final = drop_non_string_texts(sent, request.get("drop_params"))
reject_emptied_messages(sent, kept, model, optional_str(request.get("custom_llm_provider")))
return replace(call, kwargs={**call.kwargs, "messages": kept})
def _native_messages(
hook: NativeMessages, call: NativeCall, _args: tuple[object, ...], _kwargs: Mapping[str, object]
) -> AnthropicMessagesResponse | Iterator[bytes]:
return hook(_without_malformed_text(call))
def anthropic_messages_handler(
*args: object,
**kwargs: object, # kwargs-ok: preserve the public Anthropic Messages call shape
@ -193,7 +162,7 @@ def anthropic_messages_handler(
_FLAG.dropped(kwargs),
python=python,
binding=NATIVE_MESSAGES,
native=_native_messages,
native=native_call_hook,
)

View file

@ -98,8 +98,6 @@ class Feature(str, Enum):
"""Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature."""
TOOL_CUSTOM_FIELD = "tool_custom_field"
"""Tools carrying a non-spec custom object, which Python drops or hoists for the provider."""
MALFORMED_TEXT = "malformed_text"
"""User text that is not a string, which Python rejects with a 400 unless drop_params drops it."""
CLIENT_HEADERS = "client_headers"
"""Proxy-forwarded headers that Python merges into extra_headers."""
DEPLOYMENT_HOOK = "deployment_hook"
@ -181,7 +179,6 @@ MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
Feature.STREAM_COST_INJECTION,
Feature.THINKING_SIGNATURE_RETRY,
Feature.TOOL_CUSTOM_FIELD,
Feature.MALFORMED_TEXT,
}
RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {

View file

@ -8,14 +8,20 @@ import httpx
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
from litellm.llms.anthropic.messages.convertible_blocks import (
drop_non_string_texts,
emptied_messages_error,
malformed_text_error,
)
from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled
from litellm.rust_bridge import failures
from litellm.rust_bridge.public_call import known_params, optional_str
from litellm.rust_bridge.public_call import known_params, optional_sequence, optional_str
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
_DROP_PATHS: Final = TypeAdapter(list[object])
_MESSAGE_LIST: Final = TypeAdapter(list[dict[str, object]])
_BODY_FIELDS: Final = frozenset(
{
"max_tokens",
@ -44,6 +50,10 @@ _BODY_FIELDS: Final = frozenset(
)
class _RejectedMessages(ValueError):
messages_request_error: Final = True
@dataclass(frozen=True, slots=True)
class MessagesSettings:
drop_params: bool
@ -63,6 +73,24 @@ def body_params(request: Mapping[str, object]) -> Mapping[str, object]:
return known_params(_BODY_FIELDS, request)
def messages(request: Mapping[str, object]) -> object:
sent: Final = request.get("messages")
error: Final = malformed_text_error(optional_sequence(sent) or ())
if error is None:
return sent
if not drops_unsupported_params(request.get("drop_params")):
raise _RejectedMessages(error)
try:
listed: Final = _MESSAGE_LIST.validate_python(sent)
except ValidationError:
return sent
kept: Final = drop_non_string_texts(listed, request.get("drop_params"))
emptied: Final = emptied_messages_error(listed, kept)
if emptied is not None:
raise _RejectedMessages(emptied)
return kept
def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]:
from litellm.llms.anthropic.messages.native.streaming_iterator import (
anthropic_messages_stream_hidden_params,

View file

@ -48,12 +48,6 @@ def tool_custom_field(request: NativeCall) -> bool:
return any("custom" in (optional_mapping(tool) or {}) for tool in tools)
def malformed_text(request: NativeCall) -> bool:
from litellm.llms.anthropic.messages.convertible_blocks import malformed_text_error
return malformed_text_error(optional_sequence(request.resolved.get("messages")) or ()) is not None
def stream_cost_injection(request: NativeCall) -> bool:
"""Python adds response cost to streaming usage when the global setting is truthy."""
return streaming(request) and bool(litellm.include_cost_in_streaming_usage)

View file

@ -538,12 +538,6 @@ ASYNC_ONLY_FEATURES: Final = (
{},
id="messages-interceptor",
),
pytest.param(
Feature.MALFORMED_TEXT,
{"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": None}]}]},
{},
id="malformed-text",
),
)
@ -559,8 +553,7 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_
python_result: Final = response()
native_result: Final = response()
model: Final = str(kwargs.get("model", "anthropic/claude-sonnet-4-5"))
messages: Final = kwargs.get("messages", MESSAGES)
call_kwargs: Final = {name: value for name, value in kwargs.items() if name not in {"model", "messages"}}
call_kwargs: Final = {name: value for name, value in kwargs.items() if name != "model"}
async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub
return python_result
@ -572,7 +565,7 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_
for name, value in settings.items():
monkeypatch.setattr(litellm, name, value)
served_async: Final = await _ADISPATCH.arun(
(16, messages, model),
(16, MESSAGES, model),
call_kwargs,
python=python,
binding=amessages_binding(native),
@ -580,7 +573,7 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_
rules=rules,
)
served_sync: Final = _DISPATCH.run(
(16, messages, model),
(16, MESSAGES, model),
call_kwargs,
python=lambda *call_args, **python_kwargs: python_result,
binding=messages_binding(lambda request: native_result),

View file

@ -4,6 +4,7 @@ import pytest
import litellm
from tests.test_litellm_rust.messages.data import MESSAGES, MESSAGES_RESPONSE, call
from tests.test_litellm_rust.support.callback_recorder import RecordingLogger
from tests.test_litellm_rust.support.recording_server import RecordingServer
from tests.test_litellm_rust.support.routing import assert_served_natively
@ -152,40 +153,61 @@ def _user_text(*texts: object) -> tuple[dict[str, object], ...]:
return ({"role": "user", "content": [{"type": "text", "text": text} for text in texts]},)
FAILURE_EVENTS: Final = {True: "async_log_failure_event", False: "log_failure_event"}
BOTH_MODES: Final = pytest.mark.parametrize("asynchronous", (True, False), ids=("async", "sync"))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("messages", "error"),
(
(_user_text("hi", None), r"messages\.0\.content\.1\.text: Input should be a valid string"),
(_user_text("hi", 42), r"messages\.0\.content\.1\.text: Input should be a valid string"),
),
)
async def test_sync_call_rejects_non_string_user_text_before_io_without_drop_params(
server: RecordingServer, monkeypatch: pytest.MonkeyPatch, messages: tuple[dict[str, object], ...], error: str
@BOTH_MODES
@pytest.mark.parametrize("malformed", (None, 42))
async def test_non_string_user_text_answers_400_before_io_and_reaches_the_failure_callback(
server: RecordingServer, monkeypatch: pytest.MonkeyPatch, asynchronous: bool, malformed: object
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
server.expected_requests = 0
recorder: Final = RecordingLogger()
with pytest.raises(litellm.BadRequestError, match=error):
await call(server, asynchronous=False, messages=messages)
with pytest.raises(litellm.BadRequestError, match=r"messages\.0\.content\.1\.text: Input should be a valid string") as raised:
await call(server, asynchronous=asynchronous, messages=_user_text("hi", malformed), callbacks=[recorder])
events: Final = await recorder.wait_for_async(FAILURE_EVENTS[asynchronous])
assert server.requests == []
assert raised.value.llm_provider == "anthropic"
assert isinstance(events[0].kwargs, dict)
assert events[0].kwargs["exception"] is raised.value
@pytest.mark.asyncio
@BOTH_MODES
@pytest.mark.parametrize("malformed", (None, 42))
async def test_sync_call_drops_non_string_user_text_with_drop_params(server: RecordingServer, malformed: object) -> None:
await call(server, asynchronous=False, messages=_user_text("hi", malformed), drop_params=True)
async def test_drop_params_drops_non_string_user_text(server: RecordingServer, asynchronous: bool, malformed: object) -> None:
await call(server, asynchronous=asynchronous, messages=_user_text("hi", malformed), drop_params=True)
body, _ = sent(server)
assert body["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
@pytest.mark.asyncio
async def test_sync_call_rejects_messages_emptied_by_drop_params(server: RecordingServer) -> None:
@BOTH_MODES
async def test_messages_emptied_by_drop_params_answer_400_before_io(server: RecordingServer, asynchronous: bool) -> None:
server.expected_requests = 0
recorder: Final = RecordingLogger()
with pytest.raises(litellm.BadRequestError, match="no message has content left"):
await call(server, asynchronous=False, messages=_user_text(None), drop_params=True)
await call(server, asynchronous=asynchronous, messages=_user_text(None), drop_params=True, callbacks=[recorder])
await recorder.wait_for_async(FAILURE_EVENTS[asynchronous])
assert server.requests == []
@pytest.mark.asyncio
@BOTH_MODES
async def test_drop_params_leaves_a_malformed_message_list_for_the_request_validation(
server: RecordingServer, asynchronous: bool
) -> None:
server.expected_requests = 0
with pytest.raises(litellm.BadRequestError, match="invalid Anthropic messages request"):
await call(server, asynchronous=asynchronous, messages=[*_user_text("hi", 42), "stray"], drop_params=True)
assert server.requests == []

View file

@ -138,3 +138,60 @@ def test_stream_hidden_params_projects_upstream_headers_the_way_the_python_handl
assert additional["llm_provider-request-id"] == "req_upstream_123"
assert additional["x-ratelimit-remaining-requests"] == "41"
assert additional["request-id"] == additional["llm_provider-request-id"]
def _user_text(*texts: object) -> list[dict[str, object]]:
return [{"role": "user", "content": [{"type": "text", "text": text} for text in texts]}]
def _rejection(request: dict[str, object], error: str) -> litellm.BadRequestError:
with pytest.raises(ValueError, match=error) as raised:
messages_host.messages(request)
mapped: Final = messages_host.map_failure(raised.value, request, "anthropic")
assert isinstance(mapped, litellm.BadRequestError)
return mapped
def test_messages_with_string_text_are_sent_as_the_caller_passed_them() -> None:
sent: Final = _user_text("hi")
assert messages_host.messages({"model": "anthropic/claude-haiku-5-5", "messages": sent}) is sent
@pytest.mark.parametrize("malformed", (None, 42))
def test_non_string_user_text_without_drop_params_maps_to_a_400_naming_the_field(
monkeypatch: pytest.MonkeyPatch, malformed: object
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
rejected: Final = _rejection(
{"model": "anthropic/claude-haiku-5-5", "messages": _user_text("hi", malformed)},
r"messages\.0\.content\.1\.text: Input should be a valid string",
)
assert "messages.0.content.1.text: Input should be a valid string" in rejected.message
assert (rejected.model, rejected.llm_provider) == ("claude-haiku-5-5", "anthropic")
@pytest.mark.parametrize("malformed", (None, 42))
def test_drop_params_drops_non_string_user_text(malformed: object) -> None:
kept: Final = messages_host.messages(
{"model": "anthropic/claude-haiku-5-5", "messages": _user_text("hi", malformed), "drop_params": True}
)
assert kept == _user_text("hi")
def test_messages_emptied_by_drop_params_map_to_a_400() -> None:
rejected: Final = _rejection(
{"model": "anthropic/claude-haiku-5-5", "messages": _user_text(None), "drop_params": True},
"no message has content left",
)
assert "no message has content left" in rejected.message
def test_drop_params_leaves_a_message_list_with_a_non_object_entry_for_the_native_validation() -> None:
sent: Final = [*_user_text("hi", 42), "stray"]
assert messages_host.messages({"model": "anthropic/claude-haiku-5-5", "messages": sent, "drop_params": True}) is sent

View file

@ -1,7 +1,6 @@
from __future__ import annotations
from collections.abc import Generator
from dataclasses import replace
from typing import Final
import pytest
@ -129,12 +128,3 @@ def test_empty_constraints_match_nothing() -> None:
assert catalog.decision(
RouteContext(Route.OCR, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), rules
) is Decision.PYTHON
def test_malformed_user_text_keeps_opted_in_anthropic_messages_on_python() -> None:
configuration.rust(True)
native: Final = RouteContext(Route.MESSAGES, provider="anthropic", target=Target.MESSAGES)
malformed: Final = replace(native, features=frozenset({Feature.MALFORMED_TEXT}))
assert catalog.decision(native) is Decision.RUST_WITH_FALLBACK
assert catalog.decision(malformed) is Decision.PYTHON

View file

@ -192,19 +192,6 @@ def _wrap_nested_level(value: object, depth: int) -> object:
(messages_detectors.tool_custom_field, {"tools": [{"name": "a"}, {"name": "b", "custom": {}}]}, True),
(messages_detectors.tool_custom_field, {"tools": [{"name": "a", "defer_loading": True}]}, False),
(messages_detectors.tool_custom_field, {}, False),
(
messages_detectors.malformed_text,
{"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": None}]}]},
True,
),
(messages_detectors.malformed_text, {"messages": [{"role": "user", "content": [{"type": "text", "text": 42}]}]}, True),
(messages_detectors.malformed_text, {"messages": [{"role": "user", "content": [{"type": "text", "text": ""}]}]}, False),
(
messages_detectors.malformed_text,
{"messages": [{"role": "assistant", "content": [{"type": "text", "text": None}]}]},
False,
),
(messages_detectors.malformed_text, {"messages": [{"role": "user", "content": "hi"}]}, False),
(detectors.drop_params, {"drop_params": True}, True),
(detectors.drop_params, {"drop_params": False}, False),
(detectors.drop_params, {"drop_params": " True "}, True),