diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 27ce8c943f0..befd56a56f7 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -2,7 +2,7 @@ use crate::cache::{CacheCall, Cached, PythonCache, Selection}; use litellm_host_python::{PythonHostCalls, PythonOwned}; use bytes::Bytes; -use litellm_host_python::{InvokeError, PythonBinding, from_py, to_py}; +use litellm_host_python::{InvokeError, PythonBinding, effective_py_args, from_py, to_py}; use litellm_inference_messages::{ Error, MessagesCall, MessagesSettings, MessagesShaping, litellm_params, messages_body, route::{Messages, MessagesStreamHead}, @@ -94,11 +94,12 @@ impl MessagesPythonHost { argument(name)?.map(|value| value.extract()).transpose() }; let model = string("model")?.ok_or(ArgumentError::Required("model"))?; - let messages = argument("messages")?.ok_or(ArgumentError::Required("messages"))?; + argument("messages")?.ok_or(ArgumentError::Required("messages"))?; + let messages = self.messages(py, arguments)?; let fields = self.view.body_params(py, arguments, ROUTE_HOST_MODULE)?; let body = [ ("model".to_string(), Value::String(model.clone())), - ("messages".to_string(), from_py(&messages)?), + ("messages".to_string(), messages), ] .into_iter() .chain(fields) @@ -129,6 +130,15 @@ impl MessagesPythonHost { })) } + fn messages(&self, py: Python<'_>, arguments: &Bound<'_, PyDict>) -> PyResult { + let request = effective_py_args(self.view.request(py)?, arguments)?; + from_py( + &py.import(ROUTE_HOST_MODULE)? + .getattr("messages")? + .call1((request,))?, + ) + } + fn merged_headers( &self, py: Python<'_>, diff --git a/litellm/llms/anthropic/messages/convertible_blocks.py b/litellm/llms/anthropic/messages/convertible_blocks.py index 117a58ca4dc..8a304e1d82e 100644 --- a/litellm/llms/anthropic/messages/convertible_blocks.py +++ b/litellm/llms/anthropic/messages/convertible_blocks.py @@ -371,7 +371,9 @@ def deployment_drops_params(deployment: Mapping[str, object] | None) -> bool: return drops_unsupported_params(None) -def reject_malformed_text(messages: Iterable[object], model: str, forwarded_kwargs: Mapping[str, object]) -> None: +def reject_malformed_text( + messages: Iterable[Mapping[str, object]], model: str, forwarded_kwargs: Mapping[str, object] +) -> None: if drops_unsupported_params(forwarded_kwargs.get("drop_params")): return error: Final = malformed_text_error(messages) diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index acc67fb7bf2..8e6becc60b4 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -1,19 +1,10 @@ import inspect from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping -from dataclasses import replace from types import MappingProxyType from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable -from pydantic import TypeAdapter - from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider -from litellm.llms.anthropic.messages.convertible_blocks import ( - drop_non_string_texts, - malformed_text_error, - reject_emptied_messages, - reject_malformed_text, -) from litellm.llms.anthropic.messages.native import handler as main from litellm.rust_bridge import detectors from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target @@ -22,7 +13,6 @@ from litellm.rust_bridge.messages import detectors as route_detectors from litellm.rust_bridge.messages.entrypoints import ( NATIVE_AMESSAGES, NATIVE_MESSAGES, - NativeMessages, ) from litellm.rust_bridge.public_call import ( AsyncFlag, @@ -62,7 +52,6 @@ _MESSAGES: Final = signature(_PYTHON_MESSAGES) _FLAG: Final = AsyncFlag("is_async", sync=_MESSAGES, is_set=bool) _PYTHON_AMESSAGES: Final = _python_amessages() _AMESSAGES: Final = signature(_PYTHON_AMESSAGES) -_MESSAGE_LIST: Final = TypeAdapter(list[dict[str, object]]) def _native_kwargs(fields: Mapping[str, object], kwargs: Mapping[str, object]) -> Mapping[str, object]: @@ -132,7 +121,6 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType( Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook, Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception, Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted, - Feature.MALFORMED_TEXT: route_detectors.malformed_text, } ) @@ -162,25 +150,6 @@ _ADISPATCH: Final = PublicDispatch( ) -def _without_malformed_text(call: NativeCall) -> NativeCall: - request: Final = call.resolved - messages: Final = optional_sequence(request.get("messages")) or () - if malformed_text_error(messages) is None: - return call - model: Final = str(request["model"]) - reject_malformed_text(messages, model, request) - sent: Final = _MESSAGE_LIST.validate_python(messages) - kept: Final = drop_non_string_texts(sent, request.get("drop_params")) - reject_emptied_messages(sent, kept, model, optional_str(request.get("custom_llm_provider"))) - return replace(call, kwargs={**call.kwargs, "messages": kept}) - - -def _native_messages( - hook: NativeMessages, call: NativeCall, _args: tuple[object, ...], _kwargs: Mapping[str, object] -) -> AnthropicMessagesResponse | Iterator[bytes]: - return hook(_without_malformed_text(call)) - - def anthropic_messages_handler( *args: object, **kwargs: object, # kwargs-ok: preserve the public Anthropic Messages call shape @@ -193,7 +162,7 @@ def anthropic_messages_handler( _FLAG.dropped(kwargs), python=python, binding=NATIVE_MESSAGES, - native=_native_messages, + native=native_call_hook, ) diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index f4d80bc80f1..fe52f8416d0 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -98,8 +98,6 @@ class Feature(str, Enum): """Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature.""" TOOL_CUSTOM_FIELD = "tool_custom_field" """Tools carrying a non-spec custom object, which Python drops or hoists for the provider.""" - MALFORMED_TEXT = "malformed_text" - """User text that is not a string, which Python rejects with a 400 unless drop_params drops it.""" CLIENT_HEADERS = "client_headers" """Proxy-forwarded headers that Python merges into extra_headers.""" DEPLOYMENT_HOOK = "deployment_hook" @@ -181,7 +179,6 @@ MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { Feature.STREAM_COST_INJECTION, Feature.THINKING_SIGNATURE_RETRY, Feature.TOOL_CUSTOM_FIELD, - Feature.MALFORMED_TEXT, } RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index 017ff65e2a7..d75db1aaced 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -8,14 +8,20 @@ import httpx from pydantic import TypeAdapter, ValidationError import litellm -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout +from litellm.llms.anthropic.messages.convertible_blocks import ( + drop_non_string_texts, + emptied_messages_error, + malformed_text_error, +) from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled from litellm.rust_bridge import failures -from litellm.rust_bridge.public_call import known_params, optional_str +from litellm.rust_bridge.public_call import known_params, optional_sequence, optional_str from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse _DROP_PATHS: Final = TypeAdapter(list[object]) +_MESSAGE_LIST: Final = TypeAdapter(list[dict[str, object]]) _BODY_FIELDS: Final = frozenset( { "max_tokens", @@ -44,6 +50,10 @@ _BODY_FIELDS: Final = frozenset( ) +class _RejectedMessages(ValueError): + messages_request_error: Final = True + + @dataclass(frozen=True, slots=True) class MessagesSettings: drop_params: bool @@ -63,6 +73,24 @@ def body_params(request: Mapping[str, object]) -> Mapping[str, object]: return known_params(_BODY_FIELDS, request) +def messages(request: Mapping[str, object]) -> object: + sent: Final = request.get("messages") + error: Final = malformed_text_error(optional_sequence(sent) or ()) + if error is None: + return sent + if not drops_unsupported_params(request.get("drop_params")): + raise _RejectedMessages(error) + try: + listed: Final = _MESSAGE_LIST.validate_python(sent) + except ValidationError: + return sent + kept: Final = drop_non_string_texts(listed, request.get("drop_params")) + emptied: Final = emptied_messages_error(listed, kept) + if emptied is not None: + raise _RejectedMessages(emptied) + return kept + + def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]: from litellm.llms.anthropic.messages.native.streaming_iterator import ( anthropic_messages_stream_hidden_params, diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 569b784dc1e..52917126b3b 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -48,12 +48,6 @@ def tool_custom_field(request: NativeCall) -> bool: return any("custom" in (optional_mapping(tool) or {}) for tool in tools) -def malformed_text(request: NativeCall) -> bool: - from litellm.llms.anthropic.messages.convertible_blocks import malformed_text_error - - return malformed_text_error(optional_sequence(request.resolved.get("messages")) or ()) is not None - - def stream_cost_injection(request: NativeCall) -> bool: """Python adds response cost to streaming usage when the global setting is truthy.""" return streaming(request) and bool(litellm.include_cost_in_streaming_usage) diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index a9a3a623fc4..15f58afb4e5 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -538,12 +538,6 @@ ASYNC_ONLY_FEATURES: Final = ( {}, id="messages-interceptor", ), - pytest.param( - Feature.MALFORMED_TEXT, - {"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": None}]}]}, - {}, - id="malformed-text", - ), ) @@ -559,8 +553,7 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_ python_result: Final = response() native_result: Final = response() model: Final = str(kwargs.get("model", "anthropic/claude-sonnet-4-5")) - messages: Final = kwargs.get("messages", MESSAGES) - call_kwargs: Final = {name: value for name, value in kwargs.items() if name not in {"model", "messages"}} + call_kwargs: Final = {name: value for name, value in kwargs.items() if name != "model"} async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub return python_result @@ -572,7 +565,7 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_ for name, value in settings.items(): monkeypatch.setattr(litellm, name, value) served_async: Final = await _ADISPATCH.arun( - (16, messages, model), + (16, MESSAGES, model), call_kwargs, python=python, binding=amessages_binding(native), @@ -580,7 +573,7 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_ rules=rules, ) served_sync: Final = _DISPATCH.run( - (16, messages, model), + (16, MESSAGES, model), call_kwargs, python=lambda *call_args, **python_kwargs: python_result, binding=messages_binding(lambda request: native_result), diff --git a/tests/test_litellm_rust/messages/test_requests.py b/tests/test_litellm_rust/messages/test_requests.py index 08f1e2b8363..7be0d11757c 100644 --- a/tests/test_litellm_rust/messages/test_requests.py +++ b/tests/test_litellm_rust/messages/test_requests.py @@ -4,6 +4,7 @@ import pytest import litellm from tests.test_litellm_rust.messages.data import MESSAGES, MESSAGES_RESPONSE, call +from tests.test_litellm_rust.support.callback_recorder import RecordingLogger from tests.test_litellm_rust.support.recording_server import RecordingServer from tests.test_litellm_rust.support.routing import assert_served_natively @@ -152,40 +153,61 @@ def _user_text(*texts: object) -> tuple[dict[str, object], ...]: return ({"role": "user", "content": [{"type": "text", "text": text} for text in texts]},) +FAILURE_EVENTS: Final = {True: "async_log_failure_event", False: "log_failure_event"} +BOTH_MODES: Final = pytest.mark.parametrize("asynchronous", (True, False), ids=("async", "sync")) + + @pytest.mark.asyncio -@pytest.mark.parametrize( - ("messages", "error"), - ( - (_user_text("hi", None), r"messages\.0\.content\.1\.text: Input should be a valid string"), - (_user_text("hi", 42), r"messages\.0\.content\.1\.text: Input should be a valid string"), - ), -) -async def test_sync_call_rejects_non_string_user_text_before_io_without_drop_params( - server: RecordingServer, monkeypatch: pytest.MonkeyPatch, messages: tuple[dict[str, object], ...], error: str +@BOTH_MODES +@pytest.mark.parametrize("malformed", (None, 42)) +async def test_non_string_user_text_answers_400_before_io_and_reaches_the_failure_callback( + server: RecordingServer, monkeypatch: pytest.MonkeyPatch, asynchronous: bool, malformed: object ) -> None: monkeypatch.setattr(litellm, "drop_params", False) server.expected_requests = 0 + recorder: Final = RecordingLogger() - with pytest.raises(litellm.BadRequestError, match=error): - await call(server, asynchronous=False, messages=messages) + with pytest.raises(litellm.BadRequestError, match=r"messages\.0\.content\.1\.text: Input should be a valid string") as raised: + await call(server, asynchronous=asynchronous, messages=_user_text("hi", malformed), callbacks=[recorder]) + events: Final = await recorder.wait_for_async(FAILURE_EVENTS[asynchronous]) assert server.requests == [] + assert raised.value.llm_provider == "anthropic" + assert isinstance(events[0].kwargs, dict) + assert events[0].kwargs["exception"] is raised.value @pytest.mark.asyncio +@BOTH_MODES @pytest.mark.parametrize("malformed", (None, 42)) -async def test_sync_call_drops_non_string_user_text_with_drop_params(server: RecordingServer, malformed: object) -> None: - await call(server, asynchronous=False, messages=_user_text("hi", malformed), drop_params=True) +async def test_drop_params_drops_non_string_user_text(server: RecordingServer, asynchronous: bool, malformed: object) -> None: + await call(server, asynchronous=asynchronous, messages=_user_text("hi", malformed), drop_params=True) body, _ = sent(server) assert body["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] @pytest.mark.asyncio -async def test_sync_call_rejects_messages_emptied_by_drop_params(server: RecordingServer) -> None: +@BOTH_MODES +async def test_messages_emptied_by_drop_params_answer_400_before_io(server: RecordingServer, asynchronous: bool) -> None: server.expected_requests = 0 + recorder: Final = RecordingLogger() with pytest.raises(litellm.BadRequestError, match="no message has content left"): - await call(server, asynchronous=False, messages=_user_text(None), drop_params=True) + await call(server, asynchronous=asynchronous, messages=_user_text(None), drop_params=True, callbacks=[recorder]) + await recorder.wait_for_async(FAILURE_EVENTS[asynchronous]) + + assert server.requests == [] + + +@pytest.mark.asyncio +@BOTH_MODES +async def test_drop_params_leaves_a_malformed_message_list_for_the_request_validation( + server: RecordingServer, asynchronous: bool +) -> None: + server.expected_requests = 0 + + with pytest.raises(litellm.BadRequestError, match="invalid Anthropic messages request"): + await call(server, asynchronous=asynchronous, messages=[*_user_text("hi", 42), "stray"], drop_params=True) assert server.requests == [] diff --git a/tests/unit/rust_bridge/host/routes/test_messages.py b/tests/unit/rust_bridge/host/routes/test_messages.py index abf0e055e73..aaddbfb22e0 100644 --- a/tests/unit/rust_bridge/host/routes/test_messages.py +++ b/tests/unit/rust_bridge/host/routes/test_messages.py @@ -138,3 +138,60 @@ def test_stream_hidden_params_projects_upstream_headers_the_way_the_python_handl assert additional["llm_provider-request-id"] == "req_upstream_123" assert additional["x-ratelimit-remaining-requests"] == "41" assert additional["request-id"] == additional["llm_provider-request-id"] + + +def _user_text(*texts: object) -> list[dict[str, object]]: + return [{"role": "user", "content": [{"type": "text", "text": text} for text in texts]}] + + +def _rejection(request: dict[str, object], error: str) -> litellm.BadRequestError: + with pytest.raises(ValueError, match=error) as raised: + messages_host.messages(request) + mapped: Final = messages_host.map_failure(raised.value, request, "anthropic") + assert isinstance(mapped, litellm.BadRequestError) + return mapped + + +def test_messages_with_string_text_are_sent_as_the_caller_passed_them() -> None: + sent: Final = _user_text("hi") + + assert messages_host.messages({"model": "anthropic/claude-haiku-5-5", "messages": sent}) is sent + + +@pytest.mark.parametrize("malformed", (None, 42)) +def test_non_string_user_text_without_drop_params_maps_to_a_400_naming_the_field( + monkeypatch: pytest.MonkeyPatch, malformed: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + + rejected: Final = _rejection( + {"model": "anthropic/claude-haiku-5-5", "messages": _user_text("hi", malformed)}, + r"messages\.0\.content\.1\.text: Input should be a valid string", + ) + + assert "messages.0.content.1.text: Input should be a valid string" in rejected.message + assert (rejected.model, rejected.llm_provider) == ("claude-haiku-5-5", "anthropic") + + +@pytest.mark.parametrize("malformed", (None, 42)) +def test_drop_params_drops_non_string_user_text(malformed: object) -> None: + kept: Final = messages_host.messages( + {"model": "anthropic/claude-haiku-5-5", "messages": _user_text("hi", malformed), "drop_params": True} + ) + + assert kept == _user_text("hi") + + +def test_messages_emptied_by_drop_params_map_to_a_400() -> None: + rejected: Final = _rejection( + {"model": "anthropic/claude-haiku-5-5", "messages": _user_text(None), "drop_params": True}, + "no message has content left", + ) + + assert "no message has content left" in rejected.message + + +def test_drop_params_leaves_a_message_list_with_a_non_object_entry_for_the_native_validation() -> None: + sent: Final = [*_user_text("hi", 42), "stray"] + + assert messages_host.messages({"model": "anthropic/claude-haiku-5-5", "messages": sent, "drop_params": True}) is sent diff --git a/tests/unit/rust_bridge/test_catalog.py b/tests/unit/rust_bridge/test_catalog.py index d8ced79c4eb..704a4f73fe8 100644 --- a/tests/unit/rust_bridge/test_catalog.py +++ b/tests/unit/rust_bridge/test_catalog.py @@ -1,7 +1,6 @@ from __future__ import annotations from collections.abc import Generator -from dataclasses import replace from typing import Final import pytest @@ -129,12 +128,3 @@ def test_empty_constraints_match_nothing() -> None: assert catalog.decision( RouteContext(Route.OCR, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), rules ) is Decision.PYTHON - - -def test_malformed_user_text_keeps_opted_in_anthropic_messages_on_python() -> None: - configuration.rust(True) - native: Final = RouteContext(Route.MESSAGES, provider="anthropic", target=Target.MESSAGES) - malformed: Final = replace(native, features=frozenset({Feature.MALFORMED_TEXT})) - - assert catalog.decision(native) is Decision.RUST_WITH_FALLBACK - assert catalog.decision(malformed) is Decision.PYTHON diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index 3e11100fa40..c02d2c12dbf 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -192,19 +192,6 @@ def _wrap_nested_level(value: object, depth: int) -> object: (messages_detectors.tool_custom_field, {"tools": [{"name": "a"}, {"name": "b", "custom": {}}]}, True), (messages_detectors.tool_custom_field, {"tools": [{"name": "a", "defer_loading": True}]}, False), (messages_detectors.tool_custom_field, {}, False), - ( - messages_detectors.malformed_text, - {"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": None}]}]}, - True, - ), - (messages_detectors.malformed_text, {"messages": [{"role": "user", "content": [{"type": "text", "text": 42}]}]}, True), - (messages_detectors.malformed_text, {"messages": [{"role": "user", "content": [{"type": "text", "text": ""}]}]}, False), - ( - messages_detectors.malformed_text, - {"messages": [{"role": "assistant", "content": [{"type": "text", "text": None}]}]}, - False, - ), - (messages_detectors.malformed_text, {"messages": [{"role": "user", "content": "hi"}]}, False), (detectors.drop_params, {"drop_params": True}, True), (detectors.drop_params, {"drop_params": False}, False), (detectors.drop_params, {"drop_params": " True "}, True),