From b51b9b7d43dc7da20b4baa49d431736c50c10600 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:13:46 +0000 Subject: [PATCH] fix(anthropic)!: answer 400 for a malformed tool_result text on /v1/messages and count_tokens (#47347) * fix(anthropic): answer 400 for a malformed tool_result text on the Responses bridge and count_tokens A tool_result text block whose text is null, a number, or missing made /v1/messages and /v1/messages/count_tokens answer 500 on OpenAI and Azure models. Both routes now answer 400 naming the block, and drop the block when drop_params is on. * fix(anthropic): keep a missing tool_result text as empty text, bound the error, honor body drop_params on count_tokens * fix(anthropic): apply the tool_result text check on the chat/completions bridge and every count route The chat/completions bridge used for Azure, Gemini and other providers now answers the same 400 for a null or numeric tool_result text and drops the block under drop_params, so a model group that mixes OpenAI with another provider answers the same way on every deployment. Gemini and Vertex count_tokens apply the same check, count_tokens turns on drop_params for Claude Code and Codex the way /v1/messages does, and a tool_result without a tool_use_id is checked too * fix(proxy): answer 400 for a malformed tool_result text on count_tokens models without a count API Providers with no token counting API fall back to the local tokenizer, which counted a null or numeric tool_result text as fine while /v1/messages on the same model answers 400. The fallback now runs the same check, unless drop_params applies to the deployment or the request * fix(proxy): honor request drop_params on the local count fallback when no deployment resolves * test(integration): cover non-string tool_result text on both /v1/messages bridges and count routes * fix(anthropic): keep a dropped tool_result on Mistral and reject it before a guardrail rewrite Under drop_params a tool_result whose only text block was null or a number became an empty tool message, and Mistral's empty-message filter removed it, so Mistral answered 400 for a missing tool reply. The filter now only removes empty assistant messages, which is what Mistral rejects. A guardrail that rewrites structured_messages wrote the dropped text back as "", so a null or numeric text was silently blanked even without drop_params. The rewrite path now answers the same 400 as the bridges unless the request or global drop_params is set. * test(integration): cover the Mistral empty tool reply and the guardrail rewrite tool_result text check * fix(anthropic): keep a malformed tool_result text through a guardrail rewrite and keep Mistral's null-message drop A guardrail that rewrites structured_messages now writes the caller's null or numeric tool_result text back in place, so the route's bridge and the deployment's drop_params decide after routing, the same as with no guardrail. The rewrite path no longer answers its own 400 before routing. Mistral drops a null-content system or user message again, as main does, and only a tool reply is exempt from that filter so a dropped tool_result keeps its tool call. * test(mistral): expect empty system and user turns dropped and the empty tool reply kept * perf(anthropic): restore kept tool_result text parts in one linear pass * test(integration): cover the tool_result text restore across tool_results, the chat bridge, and a batch upload behind a rewriting guardrail * fix(anthropic): let guardrails scan malformed tool_result text and honor drop_params on native Anthropic A rewriting guardrail now sees a non-string tool_result text as its JSON text and the write-back restores the caller's value by the sanitized tool_use_id when the guardrail left it unchanged. Native Anthropic drops the bad block under drop_params on /v1/messages and count_tokens, and Mistral keeps an empty tool reply only when it answers a tool call, sent as empty text. * fix(anthropic): compare a kept tool_result text to its scanned form once and align the wire tests with drop_params * test(integration): cover native Anthropic drop_params resolution, guardrail scanning of numeric tool_result text, and the Mistral orphan tool message * fix(count_tokens): honor the router's default drop_params when counting a malformed tool_result text * test(count_tokens): send the tool_result count tests through a real request body instead of a mocked parser * test(count_tokens): send the native Claude count test through a real request body * fix(anthropic): keep each non-string tool_result text when tool_use_ids sanitize to the same id * fix(anthropic): keep non-ASCII characters in the guardrail's view of a non-string tool_result text * test(gemini): mock the count_tokens upstream with respx instead of patching the client factory * fix(mistral): keep an empty tool reply only for an open call, and forward non-object message items on the native drop An empty tool reply re-answering a call the conversation already moved past made Mistral reject the request, and a tool call id that is not a string crashed the filter. Native Anthropic under drop_params assumed every message item was an object. * fix(mistral): find answering tool replies in one pass instead of copying the open call set per reply * fix(mistral): ignore dropped empty placeholders when tracking open tool calls * fix(mistral): drop an empty tool reply when another reply to the same call carries the result * refactor(anthropic): share tool_result text helpers to shrink the diff * fix(anthropic): answer 400 for a non-string tool_result text on the OpenAI Responses bridge --- litellm/litellm_core_utils/core_helpers.py | 5 + .../prompt_templates/factory.py | 16 +- .../chat/guardrail_translation/handler.py | 18 +- .../pass_through/adapters/handler.py | 4 + .../pass_through/adapters/transformation.py | 4 +- .../pass_through/convertible_blocks.py | 255 ++++- .../pass_through/messages/transformation.py | 5 +- .../responses_adapters/handler.py | 6 +- .../responses_adapters/transformation.py | 11 +- litellm/llms/gemini/common_utils.py | 5 +- litellm/llms/mistral/chat/transformation.py | 74 +- .../responses/count_tokens/token_counter.py | 6 +- .../responses/count_tokens/transformation.py | 7 +- litellm/llms/vertex_ai/common_utils.py | 5 +- .../llms/vertex_ai/count_tokens/context.py | 11 + litellm/proxy/proxy_server.py | 53 +- .../transformation.py | 6 +- .../test_tool_result_text_wire.py | 868 ++++++++++++++++++ ...llm_core_utils_prompt_templates_factory.py | 6 +- .../test_anthropic_guardrail_handler.py | 135 ++- ...al_pass_through_adapters_transformation.py | 153 +++ ...al_pass_through_messages_transformation.py | 64 ++ .../test_anthropic_messages_router.py | 85 ++ .../test_responses_adapters_handler.py | 85 ++ .../pass_through/test_convertible_blocks.py | 20 + .../llms/gemini/test_gemini_common_utils.py | 78 ++ .../test_mistral_chat_transformation.py | 205 +++++ .../count_tokens/test_token_counter.py | 87 ++ .../vertex_ai/test_vertex_ai_common_utils.py | 72 ++ tests/unit/proxy/test_proxy_token_counter.py | 200 +++- 30 files changed, 2484 insertions(+), 65 deletions(-) create mode 100644 tests/integration/messages_endpoint/test_tool_result_text_wire.py diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 65d0089304b..b7bfd5eeeb3 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_c import httpx from pydantic import TypeAdapter, ValidationError +import litellm from litellm._logging import verbose_logger from litellm.constants import KUBERNETES_POD_ROUTING_KEY from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id @@ -93,6 +94,10 @@ def normalize_drop_params(value: object) -> bool | None: return None +def drops_unsupported_params(drop_params: object) -> bool: + return litellm.drop_params is True or normalize_drop_params(drop_params) is True + + def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool: normalized: Final = normalize_drop_params(value) if normalized is None and value is not None: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index e2fb2be4ec2..481ec6a5076 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1669,7 +1669,7 @@ def _replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Patte return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK -def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: +def sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: """Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$.""" return _replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS) @@ -1815,7 +1815,7 @@ def convert_to_anthropic_tool_result( tool_message: Final[ChatCompletionToolMessage] = message tool_call_id: str = tool_message["tool_call_id"] # Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$ - sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id) + sanitized_tool_use_id = sanitize_anthropic_tool_use_id(tool_call_id) # We can't determine from openai message format whether it's a successful or # error call result so default to the successful result template @@ -1829,7 +1829,7 @@ def convert_to_anthropic_tool_result( function_message: Final[ChatCompletionFunctionMessage] = message tool_call_id = function_message.get("tool_call_id") or str(uuid.uuid4()) # Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$ - sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id) + sanitized_tool_use_id = sanitize_anthropic_tool_use_id(tool_call_id) anthropic_tool_result = AnthropicMessagesToolResultParam( type="tool_result", tool_use_id=sanitized_tool_use_id, @@ -2037,7 +2037,7 @@ def convert_to_anthropic_tool_invoke( elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids: continue else: - sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id) + sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id) _anthropic_tool_use_param = AnthropicMessagesToolUseParam( type="tool_use", id=sanitized_tool_id, @@ -5900,21 +5900,21 @@ def _optional_tool_name(value: object) -> str | None: def _split_tool_call_ids(calls: Sequence[tuple[str | None, int]]) -> tuple[tuple[str | None, ...], ...]: - taken: Final = frozenset(_sanitize_anthropic_tool_use_id(call_id) for call_id, _ in calls if call_id) + taken: Final = frozenset(sanitize_anthropic_tool_use_id(call_id) for call_id, _ in calls if call_id) def fresh(call_id: str) -> Iterator[str]: return filter( - lambda candidate: _sanitize_anthropic_tool_use_id(candidate) not in taken, + lambda candidate: sanitize_anthropic_tool_use_id(candidate) not in taken, (f"{call_id}__concat_{n}" for n in itertools.count(1)), ) suffixes: Final = MappingProxyType( - {_sanitize_anthropic_tool_use_id(call_id): fresh(call_id) for call_id, count in calls if call_id and count > 1} + {sanitize_anthropic_tool_use_id(call_id): fresh(call_id) for call_id, count in calls if call_id and count > 1} ) return tuple( ( call_id, - *(next(suffixes[_sanitize_anthropic_tool_use_id(call_id)]) for _ in range(count - 1)), + *(next(suffixes[sanitize_anthropic_tool_use_id(call_id)]) for _ in range(count - 1)), ) if call_id else (None,) * count diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 7654b56ad9a..aa2bee1f31d 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -30,6 +30,10 @@ from litellm.llms.anthropic.pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, is_provider_native_tool_dict, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import ( + keep_non_string_tool_result_texts, + scannable_tool_result_request, +) from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details from litellm.llms.anthropic.pass_through.stream_assembly import ( build_complete_streaming_response, @@ -493,8 +497,8 @@ class AnthropicMessagesHandler(BaseTranslation): ``get_streaming_string_so_far``): raw SSE ``bytes`` -- which may bundle several events separated by a blank line -- and an already-parsed event ``dict``.""" - if isinstance(item, dict): - return (_as_str_mapping(item),) + if (event := as_str_mapping(item)) is not None: + return (event,) if not isinstance(item, (bytes, bytearray)): return () return tuple( @@ -510,7 +514,7 @@ class AnthropicMessagesHandler(BaseTranslation): chat_completion_compatible_request, _tool_name_mapping, ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( - anthropic_message_request=cast(AnthropicMessagesRequest, data.copy()), + anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)), preserve_midturn_system=True, ) return chat_completion_compatible_request @@ -832,7 +836,7 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _write_back_structured_messages( - data: dict, # mutable-ok: API message payload + data: dict[str, object], # mutable-ok: API message payload structured_messages: list, # mutable-ok: API message payload hoisted_system_message: object = None, preserve_system_messages: bool = False, @@ -899,7 +903,7 @@ class AnthropicMessagesHandler(BaseTranslation): for block in content: if isinstance(block, dict) and block.get("type") == "thinking": block.pop("cache_control", None) - data["messages"] = converted + data["messages"] = keep_non_string_tool_result_texts(converted, data.get("messages")) @staticmethod def _extract_midturn_system_text( @@ -1601,8 +1605,8 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _rewrite_buffered_item(item: object, rewrite_event: _SSEEventRewriter) -> object: - if isinstance(item, dict): - return _rewritten_event(_as_str_mapping(item), rewrite_event) + if (event := as_str_mapping(item)) is not None: + return _rewritten_event(event, rewrite_event) if isinstance(item, (bytes, bytearray)): return AnthropicMessagesHandler._rewrite_sse_events(bytes(item), rewrite_event) return item diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index a9aa3de69ad..666823d6ec3 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -22,6 +22,7 @@ from litellm.llms.anthropic.pass_through.context_management import ( PolyfillResult, apply_context_management, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import reject_malformed_tool_result_text from litellm.llms.anthropic.pass_through.safeguards import ( SafeguardsEvaluator, build_safeguards_evaluator, @@ -731,6 +732,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs) request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs) + reject_malformed_tool_result_text(messages, model, request_kwargs) sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs) sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs) @@ -853,6 +855,8 @@ class LiteLLMMessagesToCompletionTransformationHandler: **kwargs, ) + reject_malformed_tool_result_text(messages, model, _REQUEST_KWARGS.validate_python(kwargs)) + # Run the context_management polyfill on the sync path too so that # ``litellm.messages.create()`` callers don't silently lose edits like # ``clear_tool_uses_20250919``. The dispatcher is async (so the diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index 0d3e48c49fd..de929acbfb2 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -9,6 +9,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.litellm_core_utils.hidden_params import get_hidden_params +from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_part from litellm.llms.anthropic.pass_through.utils import ( is_reasoning_auto_summary_enabled, json_copy_or_original, @@ -1334,7 +1335,8 @@ class LiteLLMAnthropicMessagesAdapter: block: Final = cast(Mapping[str, object], item) # cast-ok: untrusted client payload match block.get("type"): case "text": - return ChatCompletionTextObject(type="text", text=str(block.get("text") or "")) + text: Final = tool_result_text_part(block) + return None if text is None else ChatCompletionTextObject(type="text", text=text) case "image" | "document": return self._tool_result_image_part(block.get("source")) case "tool_reference": diff --git a/litellm/llms/anthropic/pass_through/convertible_blocks.py b/litellm/llms/anthropic/pass_through/convertible_blocks.py index 543a6efd4c4..fb17910fb97 100644 --- a/litellm/llms/anthropic/pass_through/convertible_blocks.py +++ b/litellm/llms/anthropic/pass_through/convertible_blocks.py @@ -1,11 +1,18 @@ -from collections.abc import Iterator, Mapping, Sequence -from itertools import chain +import json +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from functools import partial +from itertools import chain, groupby +from operator import itemgetter from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import assert_never +import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params +from litellm.litellm_core_utils.prompt_templates.factory import sanitize_anthropic_tool_use_id from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages from litellm.types.utils import TokenCountMessagesFormat @@ -104,3 +111,245 @@ def unconvertible_blocks_error( ) case _: assert_never(messages_format) + + +_OBJECTS: Final = TypeAdapter(tuple[object, ...]) + + +def _list_items(value: object) -> tuple[object, ...]: + return _OBJECTS.validate_python(value) if isinstance(value, list) else () + + +@dataclass(frozen=True, slots=True) +class _KeptPart: + block: object + scanned_text: str + + +_KeptParts: TypeAlias = Mapping[int, _KeptPart] +_NO_KEPT_PARTS: Final[_KeptParts] = MappingProxyType({}) + + +def _scannable_text(text: object) -> str: + return json.dumps(text, default=str, ensure_ascii=False) + + +def _non_str_text(block: object) -> _KeptPart | None: + match as_str_mapping(block): + case {"type": "text", "text": str()}: + return None + case {"type": "text", "text": text}: + return _KeptPart(block=block, scanned_text=_scannable_text(text)) + case _: + return None + + +def tool_result_text_part(block: object) -> str | None: + match block: + case {"type": "text", "text": str(text)}: + return text + case {"type": "text", "text": object()}: + return None + case {"type": "text"}: + return "" + case _: + return None + + +def tool_result_text_parts(tool_result: Mapping[str, object]) -> tuple[str, ...]: + parts: Final = map(tool_result_text_part, _list_items(tool_result.get("content"))) + return tuple(part for part in parts if part is not None) + + +def _user_content_blocks(message: object) -> tuple[object, ...]: + match as_str_mapping(message): + case {"role": "user", "content": content}: + return _list_items(content) + case _: + return () + + +def _with_user_blocks(message: object, transform: Callable[[object], object]) -> object: + match as_str_mapping(message): + case {"role": "user", "content": list()} as user_message: + return {**user_message, "content": [transform(block) for block in _user_content_blocks(user_message)]} + case _: + return message + + +def _non_string_text_errors(label: str, content: object) -> Iterator[str]: + return ( + f"{label} content.{index}.text: Input should be a valid string" + for index, text_block in enumerate(_list_items(content)) + if _non_str_text(text_block) is not None + ) + + +def _tool_result_text_errors(block: object) -> Iterator[str]: + match block: + case {"type": "tool_result", "tool_use_id": object() as tool_use_id, "content": object() as content}: + return _non_string_text_errors(f"tool_result {tool_use_id}", content) + case {"type": "tool_result", "content": object() as content}: + return _non_string_text_errors("tool_result", content) + case _: + return iter(()) + + +def malformed_tool_result_text_error(messages: Iterable[object]) -> str | None: + blocks: Final = chain.from_iterable(map(_user_content_blocks, messages)) + return next(chain.from_iterable(map(_tool_result_text_errors, blocks)), None) + + +def malformed_tool_result_count_error( + messages: Iterable[Mapping[str, object]] | None, + messages_format: TokenCountMessagesFormat, + deployment: Mapping[str, object] | None, +) -> str | None: + if messages_format != "anthropic" or deployment_drops_params(deployment): + return None + return malformed_tool_result_text_error(messages or ()) + + +def _scannable_part(part: object) -> object: + match as_str_mapping(part): + case {"type": "text", "text": str()}: + return part + case {"type": "text", "text": text} as text_block: + return {**text_block, "text": _scannable_text(text)} + case _: + return part + + +def _scannable_block(block: object) -> object: + match as_str_mapping(block): + case {"type": "tool_result", "content": list()} as tool_result: + return { + **tool_result, + "content": [_scannable_part(part) for part in _list_items(tool_result.get("content"))], + } + case _: + return block + + +def _scannable_messages(messages: object) -> object: + items: Final = _list_items(messages) + return [_with_user_blocks(message, _scannable_block) for message in items] if items else messages + + +def scannable_tool_result_request(request: object) -> object: + match as_str_mapping(request): + case {"messages": messages} as request_fields: + return {**request_fields, "messages": _scannable_messages(messages)} + case None: + return request + case request_fields: + return {**request_fields} + + +def _non_string_text_parts(content: object) -> _KeptParts: + return MappingProxyType( + {index: part for index, block in enumerate(_list_items(content)) if (part := _non_str_text(block)) is not None} + ) + + +def _tool_result_non_string_texts(block: object) -> tuple[tuple[str, _KeptParts], ...]: + match as_str_mapping(block): + case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result: + return ((sanitize_anthropic_tool_use_id(tool_use_id), _non_string_text_parts(tool_result.get("content"))),) + case _: + return () + + +def _content_blocks(content: object) -> tuple[object, ...]: + match content: + case str() if content: + return ({"type": "text", "text": content},) + case _: + return _list_items(content) + + +def _restored_part(block: object, kept: _KeptPart | None) -> object: + return kept.block if kept is not None and tool_result_text_part(block) == kept.scanned_text else block + + +def _with_kept_parts(block: object, tool_result: Mapping[str, object], parts: _KeptParts) -> object: + if not parts: + return block + blocks: Final = _content_blocks(tool_result.get("content")) + return {**tool_result, "content": [_restored_part(part, parts.get(index)) for index, part in enumerate(blocks)]} + + +def _restored_block(block: object, kept: Mapping[str, Iterator[_KeptParts]]) -> object: + match as_str_mapping(block): + case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result if ( + sanitize_anthropic_tool_use_id(tool_use_id) in kept + ): + return _with_kept_parts( + block, tool_result, next(kept[sanitize_anthropic_tool_use_id(tool_use_id)], _NO_KEPT_PARTS) + ) + case _: + return block + + +def keep_non_string_tool_result_texts(rewritten: Sequence[object], original: object) -> Sequence[object]: + blocks: Final = chain.from_iterable(map(_user_content_blocks, _list_items(original))) + by_id: Final = tuple(sorted(chain.from_iterable(map(_tool_result_non_string_texts, blocks)), key=itemgetter(0))) + if not any(parts for _, parts in by_id): + return rewritten + kept_in_order: Final = MappingProxyType( + { + tool_use_id: iter(tuple(map(itemgetter(1), group))) + for tool_use_id, group in groupby(by_id, key=itemgetter(0)) + } + ) + return [_with_user_blocks(message, partial(_restored_block, kept=kept_in_order)) for message in rewritten] + + +def _without_non_string_text(block: object) -> object: + match as_str_mapping(block): + case {"type": "tool_result", "content": list()} as tool_result: + parts: Final = _list_items(tool_result.get("content")) + return {**tool_result, "content": [part for part in parts if _non_str_text(part) is None]} + case _: + return block + + +def _without_non_string_texts( + message: dict[str, object], # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] +) -> dict[str, object]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + match message: + case {"role": "user", "content": list()}: + return {**message, "content": [_without_non_string_text(block) for block in _user_content_blocks(message)]} + case _: + return message + + +def drop_non_string_tool_result_texts( + messages: Sequence[dict[str, object]], # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + drop_params: object, +) -> Sequence[dict[str, object]]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + if not drops_unsupported_params(drop_params) or malformed_tool_result_text_error(messages) is None: + return messages + return [_without_non_string_texts(message) for message in messages] + + +def deployment_drops_params(deployment: Mapping[str, object] | None) -> bool: + match deployment: + case {"litellm_params": {"drop_params": object() as drop_params}}: + return drops_unsupported_params(drop_params) + case _: + return drops_unsupported_params(None) + + +def reject_malformed_tool_result_text( + messages: Iterable[Mapping[str, object]], model: str, forwarded_kwargs: Mapping[str, object] +) -> None: + if drops_unsupported_params(forwarded_kwargs.get("drop_params")): + return + error: Final = malformed_tool_result_text_error(messages) + if error is None: + return + provider: Final = forwarded_kwargs.get("custom_llm_provider") + raise litellm.BadRequestError( + message=error, model=model, llm_provider=provider if isinstance(provider, str) else "" + ) diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index 30d45f3cec1..0e8c1b49e8e 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -8,6 +8,7 @@ from litellm._logging import verbose_logger from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, @@ -692,7 +693,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): messages = strip_advisor_blocks_from_messages(messages) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( - messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(messages), + messages=strip_encrypted_reasoning_blocks_from_anthropic_messages( + drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None)) + ), max_tokens=max_tokens, model=model, **anthropic_messages_optional_request_params, diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index 05ea4e4301d..2cbba168ef4 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -14,7 +14,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, supports_openai_prompt_cache_breakpoint, ) -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS @@ -32,6 +32,7 @@ from litellm.types.llms.openai import ResponsesAPIResponse from litellm.utils import ProviderConfigManager from ..adapters.handler import request_safeguards_evaluator +from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper @@ -96,7 +97,7 @@ def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, objec return cache_control_as_prompt_cache_breakpoint if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs): return None - if normalize_drop_params(forwarded_kwargs.get("drop_params")) is True or litellm.drop_params is True: + if drops_unsupported_params(forwarded_kwargs.get("drop_params")): return _without_prompt_cache_breakpoint return None @@ -142,6 +143,7 @@ def _build_responses_kwargs( Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). """ forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) + reject_malformed_tool_result_text(messages, model, forwarded_kwargs) block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs) request_messages: Final[AnthropicRequestMessages] = ( [ diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index fb611b64dda..b14a0dc0756 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_parts from litellm.llms.anthropic.pass_through.messages.utils import ( refusal_stop_details, responses_output_refusal_text, @@ -371,10 +372,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif btype == "tool_result": tool_use_id = block.get("tool_use_id", "") inner = block.get("content") + tool_result_block = cast(Mapping[str, object], block) # cast-ok: TypedDict upcast tool_breakpoint, covered_image_parts, later_image_parts = ( - self._tool_result_images_by_breakpoint( - cast(Mapping[str, object], block) # cast-ok: TypedDict upcast - ) + self._tool_result_images_by_breakpoint(tool_result_block) ) document_candidates = ( tuple( @@ -391,10 +391,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif isinstance(inner, str): output_text = inner elif isinstance(inner, list): - parts = [ - c.get("text", "") for c in inner if isinstance(c, dict) and c.get("type") == "text" - ] - output_text = "\n".join(parts) + output_text = "\n".join(tool_result_text_parts(tool_result_block)) if covered_image_parts or later_image_parts: output_text = ( f"{output_text}\n{TOOL_RESULT_IMAGE_PLACEHOLDER}" diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 4006c89954d..c5ebd84e00f 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -508,6 +508,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter): count_context_error_response, gemini_count_contents, inference_request_view, + malformed_tool_result_text_count_error, provider_error_response, unconvertible_messages_error, vertex_gemini_count_context, @@ -524,7 +525,9 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter): count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools) if isinstance(count_context, VertexGeminiCountContextError): return count_context_error_response(count_context, request_model, model_to_use) - blocks_error: Final = unconvertible_messages_error(request_view.messages, messages_format) + blocks_error: Final = unconvertible_messages_error( + request_view.messages, messages_format + ) or malformed_tool_result_text_count_error(messages, messages_format, deployment) if blocks_error is not None: return count_context_error_response(blocks_error, request_model, model_to_use) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index a40970ca429..b8a6cb1beb9 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -7,6 +7,7 @@ Docs - https://docs.mistral.ai/api/ """ from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping +from itertools import accumulate, count from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload import httpx @@ -53,6 +54,9 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: return accepted +_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) + + class MistralConfig(OpenAIGPTConfig): """ Reference: https://docs.mistral.ai/api/ @@ -273,14 +277,30 @@ class MistralConfig(OpenAIGPTConfig): messages = handle_messages_with_content_list_to_str_conversion(messages) ## 3. Handle name in message - new_messages: Final[list[AllMessageValues]] = [] - for m in messages: - m = MistralConfig._handle_name_in_message(m) - m = MistralConfig._handle_tool_call_message(m) - if MistralConfig._is_empty_assistant_message(m): - continue - m = strip_none_values_from_message(m) # prevents 'extra_forbidden' error - new_messages.append(m) + prepared_messages: Final = tuple( + MistralConfig._handle_tool_call_message(MistralConfig._handle_name_in_message(m)) for m in messages + ) + empty_flags: Final = tuple(map(MistralConfig._is_empty_assistant_message, prepared_messages)) + tool_call_turns: Final = accumulate( + zip(count(), prepared_messages, empty_flags, strict=False), + MistralConfig._tool_call_turn, + initial=_NO_TOOL_CALL_TURN, + ) + tool_reply_keys: Final = tuple(map(MistralConfig._tool_reply_key, prepared_messages, tool_call_turns)) + first_reply_positions: Final = { + key: position for position, key in reversed(tuple(enumerate(tool_reply_keys))) if key is not None + } + keys_answered_with_content: Final = frozenset( + key for key, is_empty in zip(tool_reply_keys, empty_flags, strict=True) if not is_empty + ) + answering_positions: Final = frozenset( + position for key, position in first_reply_positions.items() if key not in keys_answered_with_content + ) + new_messages: Final[list[AllMessageValues]] = [ + strip_none_values_from_message(MistralConfig._with_tool_reply_content(m)) + for position, (m, is_empty) in enumerate(zip(prepared_messages, empty_flags, strict=True)) + if not is_empty or position in answering_positions + ] if is_async: return super()._transform_messages(new_messages, model, True) @@ -469,6 +489,44 @@ class MistralConfig(OpenAIGPTConfig): message["tool_calls"] = mistral_tool_calls return message + @staticmethod + def _tool_call_id(tool_call: object) -> str | None: + match tool_call: + case {"id": str(tool_call_id)}: + return tool_call_id + case _: + return None + + @staticmethod + def _tool_call_turn( + turn: tuple[int, frozenset[str]], message_entry: tuple[int, AllMessageValues, bool] + ) -> tuple[int, frozenset[str]]: + match message_entry: + case (position, {"role": "assistant", "tool_calls": list(tool_calls)}, _): + return position, frozenset(filter(None, map(MistralConfig._tool_call_id, tool_calls))) + case (_, {"role": "tool"}, _) | (_, _, True): + return turn + case _: + return _NO_TOOL_CALL_TURN + + @staticmethod + def _tool_reply_key(message: AllMessageValues, turn: tuple[int, frozenset[str]]) -> tuple[int, str] | None: + match message, turn: + case {"role": "tool", "tool_call_id": str(tool_call_id)}, (position, tool_call_ids) if ( + tool_call_id in tool_call_ids + ): + return position, tool_call_id + case _: + return None + + @staticmethod + def _with_tool_reply_content(message: AllMessageValues) -> AllMessageValues: + match message: + case {"role": "tool"} if not message.get("content"): + return {**message, "content": ""} + case _: + return message + @classmethod def _is_empty_assistant_message(cls, message: AllMessageValues) -> bool: """ diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 633f154c4cd..b12fdb95765 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -11,7 +11,7 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify -from litellm.llms.anthropic.pass_through.convertible_blocks import unconvertible_blocks_error +from litellm.llms.anthropic.pass_through.convertible_blocks import deployment_drops_params, unconvertible_blocks_error from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -110,7 +110,9 @@ class OpenAITokenCounter(BaseTokenCounter): return None malformed_error: Final = ( - OpenAICountTokensConfig.malformed_anthropic_messages_error(messages) + OpenAICountTokensConfig.malformed_anthropic_messages_error( + messages, drop_params=deployment_drops_params(deployment) + ) if messages_format == "anthropic" else None ) diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 90bfa4a8655..438aa05a4e9 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -10,6 +10,7 @@ from typing import Any, Final, Literal, TypeAlias from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict +from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, @@ -295,14 +296,16 @@ class OpenAICountTokensConfig: return input_items, instructions @staticmethod - def malformed_anthropic_messages_error(messages: Sequence[Mapping[str, object]]) -> str | None: + def malformed_anthropic_messages_error( + messages: Sequence[Mapping[str, object]], *, drop_params: bool + ) -> str | None: try: _ANTHROPIC_COUNT_MESSAGES.validate_python(messages) except ValidationError as e: return "; ".join( f"messages.{'.'.join(str(part) for part in error['loc'])}: {error['msg']}" for error in e.errors() ) - return None + return None if drop_params else malformed_tool_result_text_error(messages) @staticmethod def anthropic_messages_to_responses_input( diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 3c9a8429882..8298edd6744 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1375,6 +1375,7 @@ class VertexAITokenCounter(BaseTokenCounter): count_context_error_response, gemini_count_contents, inference_request_view, + malformed_tool_result_text_count_error, provider_error_response, unconvertible_messages_error, vertex_gemini_count_context, @@ -1392,7 +1393,9 @@ class VertexAITokenCounter(BaseTokenCounter): count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools) if isinstance(count_context, VertexGeminiCountContextError): return count_context_error_response(count_context, request_model, model_to_use) - blocks_error: Final = unconvertible_messages_error(request_view.messages, messages_format) + blocks_error: Final = unconvertible_messages_error( + request_view.messages, messages_format + ) or malformed_tool_result_text_count_error(messages, messages_format, deployment) if blocks_error is not None: return count_context_error_response(blocks_error, request_model, model_to_use) diff --git a/litellm/llms/vertex_ai/count_tokens/context.py b/litellm/llms/vertex_ai/count_tokens/context.py index c218b4710fd..b13f72e96fe 100644 --- a/litellm/llms/vertex_ai/count_tokens/context.py +++ b/litellm/llms/vertex_ai/count_tokens/context.py @@ -132,6 +132,17 @@ def unconvertible_messages_error( return None if error_message is None else VertexGeminiCountContextError(error_message=error_message) +def malformed_tool_result_text_count_error( + messages: Sequence[Mapping[str, object]] | None, + messages_format: TokenCountMessagesFormat, + deployment: Mapping[str, object] | None, +) -> VertexGeminiCountContextError | None: + from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_count_error + + error_message: Final = malformed_tool_result_count_error(messages, messages_format, deployment) + return None if error_message is None else VertexGeminiCountContextError(error_message=error_message) + + def count_context_error_response( error: VertexGeminiCountContextError, request_model: str, model_used: str ) -> TokenCountResponse: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 660d7d529cb..c1102cee8d1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -368,6 +368,11 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_keys, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import ( + deployment_drops_params, + drop_non_string_tool_result_texts, + malformed_tool_result_count_error, +) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.openai.live import parse_live_session_start from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS @@ -14716,6 +14721,32 @@ def _get_provider_token_counter( return None, None, None +_DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) + + +def _deployment_with_request_drop_params( + deployment: dict[str, object] | None, # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment + drop_params: object, +) -> dict[str, object] | None: # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment + if drop_params is None: + return deployment + resolved_deployment: Final = deployment or {} + litellm_params: Final = _DEPLOYMENT_LITELLM_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) + return {**resolved_deployment, "litellm_params": {**litellm_params, "drop_params": drop_params}} + + +def _counted_request_messages( + messages: Sequence[dict[str, object]] | None, # mutable-ok: TokenCountRequest.messages is typed list[dict] + messages_format: TokenCountMessagesFormat, + deployment: Mapping[str, object] | None, +) -> list[dict[str, object]] | None: # mutable-ok: BaseTokenCounter.count_tokens takes list messages + if messages is None: + return None + if messages_format != "anthropic": + return list(messages) + return list(drop_non_string_tool_result_texts(messages, deployment_drops_params(deployment))) + + async def _try_provider_token_count( provider_counter: "BaseTokenCounter", custom_llm_provider: str | None, @@ -15082,11 +15113,6 @@ _PROVIDERS_COUNTING_WITHOUT_MESSAGES: Final = frozenset( ) -def _drops_params(deployment: Mapping[str, object] | None) -> bool: - litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) - return litellm.drop_params is True or (litellm_params is not None and litellm_params.get("drop_params") is True) - - _ESTIMABLE_GOOGLE_PART_KEYS: Final = frozenset({"text", "functionCall", "functionResponse", "thoughtSignature"}) @@ -15095,7 +15121,7 @@ def _estimable_contents(contents: Sequence[Mapping[str, object]]) -> bool: return all(set(as_str_mapping(part) or ()) <= _ESTIMABLE_GOOGLE_PART_KEYS for part in parts) -def _router_default_litellm_params(router: Router) -> Mapping[str, object]: +def _router_default_litellm_params(router: Router | None) -> Mapping[str, object]: defaults: Final[Mapping[str, object] | None] = getattr(router, "default_litellm_params", None) if defaults is None: return {} @@ -15216,8 +15242,10 @@ async def count_request_tokens( if _model is not None: model_to_use = _model + sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params} + count_deployment: Final = _deployment_with_request_drop_params(deployment, sent_params.get("drop_params")) conversation_empty: Final = not messages and not contents - if prompt and not conversation_empty and not _drops_params(deployment): + if prompt and not conversation_empty and not deployment_drops_params(count_deployment): raise ProxyException( message="prompt cannot be sent with messages or contents. Set drop_params to count them without the prompt", type="invalid_request_error", @@ -15225,6 +15253,7 @@ async def count_request_tokens( code=400, ) prompt_only: Final = bool(prompt) and conversation_empty + request_messages: Final = _counted_request_messages(messages, messages_format, count_deployment) contents_estimable: Final = not contents or (bool(messages) and _estimable_contents(contents)) prompt_as_message: Final = prompt_only and custom_llm_provider in _PROVIDERS_COUNTING_WITHOUT_MESSAGES ladder: Final = ( @@ -15235,9 +15264,9 @@ async def count_request_tokens( _try_provider_token_count, provider_counter, custom_llm_provider, - messages=[{"role": "user", "content": prompt}] if prompt_as_message else messages, + messages=[{"role": "user", "content": prompt}] if prompt_as_message else request_messages, contents=None if prompt_as_message else contents, - deployment=deployment, + deployment=count_deployment, request_model=request.model, tools=tools, system=system, @@ -15253,6 +15282,10 @@ async def count_request_tokens( if isinstance(ladder, TokenCountResponse): return ladder + local_count_error: Final = malformed_tool_result_count_error(request_messages, messages_format, count_deployment) + if local_count_error is not None: + raise ProxyException(message=local_count_error, type="token_counting_error", param="messages", code=400) + # Check if token counter is disabled before fallback if litellm.disable_token_counter is True: raise ProxyException( @@ -15273,7 +15306,7 @@ async def count_request_tokens( prompt_turn: Final = prompt_only and not (system_message is None and not tools) typed_messages: Final = cast( # cast-ok: request messages are raw chat-shaped dicts that token_counter normalizes Sequence[AllMessageValues] | None, - [{"role": "user", "content": prompt}] if prompt_turn else (None if prompt_only else messages), + [{"role": "user", "content": prompt}] if prompt_turn else (None if prompt_only else request_messages), ) counted_messages: Final = ( typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 842230699b7..a2acbc09784 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -45,7 +45,7 @@ from litellm.constants import ( REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER, ) from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) @@ -326,10 +326,6 @@ def _dropped_cache_control(_: object) -> None: return None -def drops_unsupported_params(drop_params: object) -> bool: - return litellm.drop_params is True or normalize_drop_params(drop_params) is True - - def _without_cache_control_key(item: object) -> object: if not isinstance(item, dict): return item diff --git a/tests/integration/messages_endpoint/test_tool_result_text_wire.py b/tests/integration/messages_endpoint/test_tool_result_text_wire.py new file mode 100644 index 00000000000..f8c6eb6b0ca --- /dev/null +++ b/tests/integration/messages_endpoint/test_tool_result_text_wire.py @@ -0,0 +1,868 @@ +import json +import re +import uuid +from collections.abc import Callable, Mapping, Sequence +from email.parser import BytesParser +from email.policy import HTTP +from types import MappingProxyType +from typing import Final + +import anthropic +import httpx +import pytest +from integration._support.client import Gateway, Scenario, eventually, string_value +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_JSON_OBJECTS: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]) +_TOOL_USE_ID: Final = "toolu_01" +_WEATHER: Final = "sunny, 21C" +_FIELD_ERROR: Final = f"tool_result {_TOOL_USE_ID} content.0.text: Input should be a valid string" +_CLAUDE_CODE: Final = MappingProxyType({"user-agent": "claude-cli/2.1.0 (external, cli)"}) +_OPENAI_COUNT: Final = 4242 +_GOOGLE_COUNT: Final = 777 +_ANTHROPIC_COUNT: Final = 58 +_BEDROCK_COUNT: Final = 1345 +_GEMINI: Final = "gemini-4-argon" +_VERTEX_PROJECT: Final = "scripted-project" +_CITY: Final = "Paris" +_REDACTED_CITY: Final = "[CITY]" +_TOOLS: Final[list[JsonValue]] = [ + { + "name": "get_weather", + "description": "Get the weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } +] + + +def _messages(*blocks: JsonValue) -> list[JsonValue]: + return [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": _TOOL_USE_ID, "name": "get_weather", "input": {"city": "Paris"}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": _TOOL_USE_ID, "content": list(blocks)}]}, + ] + + +_NULL_TEXT: Final = _messages({"type": "text", "text": None}) +_NUMBER_THEN_WEATHER: Final = _messages({"type": "text", "text": 5}, {"type": "text", "text": _WEATHER}) +_WEATHER_ONLY: Final = _messages({"type": "text", "text": _WEATHER}) +_WEATHER_THEN_MISSING: Final = _messages({"type": "text", "text": _WEATHER}, {"type": "text"}) + + +def _body(model: str, messages: list[JsonValue], **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "messages": messages, "tools": _TOOLS, **extra} + + +def _responses_reply() -> bytes: + return json.dumps( + { + "id": f"resp_{uuid.uuid4().hex}", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-5.4-mini", + "output": [ + { + "type": "message", + "id": f"msg_{uuid.uuid4().hex}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "It is sunny.", "annotations": []}], + } + ], + "usage": {"input_tokens": 41, "output_tokens": 5, "total_tokens": 46}, + } + ).encode() + + +def _chat_reply() -> bytes: + return json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "mistral-small-latest", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "It is sunny."}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13}, + } + ).encode() + + +def _anthropic_reply() -> bytes: + return json.dumps( + { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "It is sunny."}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 9, "output_tokens": 4}, + } + ).encode() + + +def _openai_peer(request: Request) -> Reply: + if request.target.endswith("/responses/input_tokens"): + return Reply(body=json.dumps({"input_tokens": _OPENAI_COUNT}).encode()) + if request.target.endswith("/responses"): + return Reply(body=_responses_reply()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _chat_peer(request: Request) -> Reply: + if request.target.endswith("/chat/completions"): + return Reply(body=_chat_reply()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _google_peer(request: Request) -> Reply: + if request.target.endswith(":countTokens"): + return Reply(body=json.dumps({"totalTokens": _GOOGLE_COUNT}).encode()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _anthropic_peer(request: Request) -> Reply: + if request.target.endswith("/v1/messages/count_tokens"): + return Reply(body=json.dumps({"input_tokens": _ANTHROPIC_COUNT}).encode()) + if request.target.endswith("/v1/messages"): + return Reply(body=_anthropic_reply()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _bedrock_peer(request: Request) -> Reply: + if request.target.endswith("/count-tokens"): + return Reply(body=json.dumps({"inputTokens": _BEDROCK_COUNT}).encode()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _openai_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model(model="openai/gpt-5.4-mini", api_base=wire.url, api_key="synthetic-openai-key", **parameters) + + +def _mistral_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model( + model="mistral/mistral-small-latest", api_base=wire.url, api_key="synthetic-mistral-key", **parameters + ) + + +def _anthropic_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model( + model="anthropic/claude-sonnet-4-6", api_base=wire.url, api_key="synthetic-anthropic-key", **parameters + ) + + +def _google_model(gateway: Gateway, scenario: Scenario, wire: Wire, provider: str, **parameters: JsonValue) -> str: + if provider == "gemini": + return scenario.model(model=f"gemini/{_GEMINI}", api_base=wire.url, api_key="scripted-gemini-key", **parameters) + return scenario.model( + model=f"vertex_ai/{_GEMINI}", + api_base=wire.url, + api_key=None, + vertex_project=_VERTEX_PROJECT, + vertex_location="us-east5", + vertex_credentials=service_account_json(_VERTEX_PROJECT, gateway.upstream_url.rstrip("/")), + **parameters, + ) + + +def _json(request: Request) -> dict[str, JsonValue]: + return _JSON_OBJECT.validate_json(request.body) + + +def _only(requests: Sequence[Request], suffix: str) -> dict[str, JsonValue]: + matching: Final = tuple(request for request in requests if request.target.endswith(suffix)) + assert len(matching) == 1, [request.target for request in requests] + return _json(matching[0]) + + +def _function_call_output(body: Mapping[str, JsonValue]) -> JsonValue: + outputs: Final = tuple( + item["output"] + for item in _JSON_OBJECTS.validate_python(body["input"]) + if item.get("type") == "function_call_output" + ) + assert len(outputs) == 1, body + return outputs[0] + + +def _tool_message_text(body: Mapping[str, JsonValue]) -> str: + contents: Final = tuple( + message.get("content") + for message in _JSON_OBJECTS.validate_python(body["messages"]) + if message.get("role") == "tool" + ) + assert len(contents) == 1, body + content: Final = contents[0] + if isinstance(content, str): + return content + return "".join(str(part["text"]) for part in _JSON_OBJECTS.validate_python(content)) + + +def _anthropic_error_message(status: int, text: str, payload: bytes) -> str: + assert status == 400, text + error: Final = _JSON_OBJECT.validate_json(payload) + assert error["type"] == "error", text + detail: Final = _JSON_OBJECT.validate_python(error["error"]) + assert detail["type"] == "invalid_request_error", text + return str(detail["message"]) + + +def _assert_field_rejection(gateway: Gateway, path: str, body: Mapping[str, JsonValue], **headers: str) -> None: + response: Final = gateway.request("POST", path, body, headers=headers) + assert _FIELD_ERROR in _anthropic_error_message(response.status_code, response.text, response.content), ( + response.text + ) + + +def _sdk(gateway: Gateway) -> anthropic.Anthropic: + return anthropic.Anthropic( + api_key=gateway.key, base_url=str(gateway.client.base_url), max_retries=0, http_client=gateway.client + ) + + +def _assert_sdk_bad_request(raised: pytest.ExceptionInfo[anthropic.BadRequestError]) -> None: + error: Final = raised.value + assert error.status_code == 400, error + assert _FIELD_ERROR in _anthropic_error_message(error.status_code, error.message, error.response.content), error + + +def test_openai_rejects_a_non_string_tool_result_text_on_both_routes(gateway: Gateway) -> None: + client: Final = _sdk(gateway) + with wire_server(_openai_peer) as wire, gateway.scenario() as scenario: + model: Final = _openai_model(scenario, wire) + with pytest.raises(anthropic.BadRequestError) as created: + client.messages.create(model=model, max_tokens=64, messages=_NULL_TEXT, tools=_TOOLS) + _assert_sdk_bad_request(created) + with pytest.raises(anthropic.BadRequestError) as streamed: + client.messages.create(model=model, max_tokens=64, messages=_NULL_TEXT, tools=_TOOLS, stream=True) + _assert_sdk_bad_request(streamed) + with pytest.raises(anthropic.BadRequestError) as counted: + client.messages.count_tokens(model=model, messages=_NULL_TEXT, tools=_TOOLS) + _assert_sdk_bad_request(counted) + assert wire.drain() == () + + +_DROP_SOURCES: Final = MappingProxyType( + { + "deployment": (MappingProxyType({"drop_params": True}), MappingProxyType({}), MappingProxyType({})), + "body": (MappingProxyType({}), MappingProxyType({"drop_params": True}), MappingProxyType({})), + "claude_code": (MappingProxyType({}), MappingProxyType({}), _CLAUDE_CODE), + } +) + + +@pytest.mark.parametrize("source", sorted(_DROP_SOURCES)) +def test_openai_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None: + deployment, extra, headers = _DROP_SOURCES[source] + with wire_server(_openai_peer) as wire, gateway.scenario() as scenario: + model: Final = _openai_model(scenario, wire, **deployment) + body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _OPENAI_COUNT, counted.text + requests: Final = wire.drain() + assert _function_call_output(_only(requests, "/responses")) == _WEATHER + assert _function_call_output(_only(requests, "/responses/input_tokens")) == _WEATHER + + +def test_explicit_drop_params_false_wins_on_both_routes(gateway: Gateway) -> None: + with wire_server(_openai_peer) as wire, gateway.scenario() as scenario: + model: Final = _openai_model(scenario, wire, drop_params=True) + body: Final = _body(model, _NULL_TEXT, drop_params=False) + _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64}, **_CLAUDE_CODE) + _assert_field_rejection(gateway, "/v1/messages/count_tokens", body, **_CLAUDE_CODE) + assert wire.drain() == () + + +def test_chat_bridge_and_local_count_reject_a_non_string_tool_result_text(gateway: Gateway) -> None: + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + body: Final = _body(model, _NULL_TEXT) + _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64}) + _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64, "stream": True}) + _assert_field_rejection(gateway, "/v1/messages/count_tokens", body) + assert wire.drain() == () + + +@pytest.mark.parametrize("source", ("deployment", "body")) +def test_chat_bridge_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None: + deployment, extra, _ = _DROP_SOURCES[source] + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire, **deployment) + body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _tool_message_text(_only(wire.drain(), "/chat/completions")) == _WEATHER + + +def test_count_without_a_deployment_rejects_and_honors_body_drop_params(gateway: Gateway) -> None: + model: Final = f"integration-unrouted-{uuid.uuid4().hex}" + _assert_field_rejection(gateway, "/v1/messages/count_tokens", _body(model, _NULL_TEXT)) + dropped: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(model, _NULL_TEXT, drop_params=True)) + assert dropped.status_code == 200, dropped.text + tokens: Final = _JSON_OBJECT.validate_json(dropped.content)["input_tokens"] + assert isinstance(tokens, int) and tokens > 0, dropped.text + + +def _counted_contents(body: Mapping[str, JsonValue]) -> JsonValue: + if "contents" in body: + return body["contents"] + return _JSON_OBJECT.validate_python(body["generateContentRequest"])["contents"] + + +@pytest.mark.parametrize("provider", ("gemini", "vertex_ai")) +def test_google_count_rejects_and_drops_a_non_string_tool_result_text(gateway: Gateway, provider: str) -> None: + with wire_server(_google_peer) as wire, gateway.scenario() as scenario: + strict: Final = _google_model(gateway, scenario, wire, provider) + dropping: Final = _google_model(gateway, scenario, wire, provider, drop_params=True) + _assert_field_rejection(gateway, "/v1/messages/count_tokens", _body(strict, _NULL_TEXT)) + assert tuple(request for request in wire.drain() if request.target.endswith(":countTokens")) == () + dropped: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(dropping, _NUMBER_THEN_WEATHER)) + assert dropped.status_code == 200, dropped.text + dropped_body: Final = _only(wire.drain(), ":countTokens") + clean: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(dropping, _WEATHER_ONLY)) + assert clean.status_code == 200, clean.text + assert _JSON_OBJECT.validate_json(dropped.content)["input_tokens"] == _GOOGLE_COUNT, dropped.text + assert _counted_contents(dropped_body) == _counted_contents(_only(wire.drain(), ":countTokens")) + + +def _forwarded_tool_result(body: Mapping[str, JsonValue]) -> JsonValue: + last: Final = _JSON_OBJECTS.validate_python(body["messages"])[-1] + return _JSON_OBJECTS.validate_python(last["content"])[0]["content"] + + +@pytest.mark.parametrize( + ("headers", "forwarded"), + ((MappingProxyType({}), [{"type": "text", "text": None}]), (_CLAUDE_CODE, [])), + ids=("no-drop-params", "claude-code-drop-params"), +) +def test_native_anthropic_forwards_a_null_tool_result_text_unless_drop_params_drops_it( + gateway: Gateway, headers: Mapping[str, str], forwarded: list[JsonValue] +) -> None: + with wire_server(_anthropic_peer) as wire, gateway.scenario() as scenario: + model: Final = _anthropic_model(scenario, wire) + body: Final = _body(model, _NULL_TEXT) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _ANTHROPIC_COUNT, counted.text + requests: Final = wire.drain() + assert _forwarded_tool_result(_only(requests, "/v1/messages")) == forwarded + assert _forwarded_tool_result(_only(requests, "/v1/messages/count_tokens")) == forwarded + + +def test_bedrock_count_with_a_claude_code_user_agent_still_counts(gateway: Gateway) -> None: + with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="bedrock/anthropic.claude-sonnet-4-6", + api_base=wire.url, + api_key=None, + aws_access_key_id="AKIAINTEGRATIONTOOLRESULT", + aws_secret_access_key="integration-tool-result-secret", + aws_region_name="us-east-1", + ) + counted: Final = gateway.request( + "POST", "/v1/messages/count_tokens", _body(model, _WEATHER_ONLY), headers=_CLAUDE_CODE + ) + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _BEDROCK_COUNT, counted.text + assert len(tuple(request for request in wire.drain() if request.target.endswith("/count-tokens"))) == 1 + + +@pytest.mark.parametrize("provider", ("openai", "mistral")) +def test_valid_tool_result_text_reaches_the_upstream(gateway: Gateway, provider: str) -> None: + peer: Final = _openai_peer if provider == "openai" else _chat_peer + with wire_server(peer) as wire, gateway.scenario() as scenario: + model: Final = (_openai_model if provider == "openai" else _mistral_model)(scenario, wire) + body: Final = _body(model, _WEATHER_THEN_MISSING) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + requests: Final = wire.drain() + if provider == "openai": + assert _function_call_output(_only(requests, "/responses")) == f"{_WEATHER}\n" + assert _function_call_output(_only(requests, "/responses/input_tokens")) == f"{_WEATHER}\n" + else: + assert _tool_message_text(_only(requests, "/chat/completions")) == _WEATHER + + +def _rewritten(content: JsonValue, rewrite: Callable[[str], str]) -> JsonValue: + if isinstance(content, str): + return rewrite(content) + if isinstance(content, list): + return [_rewritten(part, rewrite) for part in content] + if isinstance(content, dict) and isinstance(text := content.get("text"), str): + return {**content, "text": rewrite(text)} + return content + + +def _redacted(content: JsonValue) -> JsonValue: + return _rewritten(content, lambda text: text.replace(_CITY, _REDACTED_CITY)) + + +def _redacted_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return {**row, "content": _redacted(row["content"])} if "content" in row else dict(row) + + +def _digits_masked_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + if "content" not in row: + return dict(row) + return {**row, "content": _rewritten(row["content"], lambda text: re.sub(r"\d", "#", text))} + + +def _joined_tool_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + redacted: Final = _redacted_row(row) + content: Final = redacted.get("content") + if row.get("role") != "tool" or not isinstance(content, list): + return redacted + texts: Final = ( + part["text"] for part in _JSON_OBJECTS.validate_python(content) if isinstance(part.get("text"), str) + ) + return {**redacted, "content": " | ".join(str(text) for text in texts)} + + +_ROW_REWRITES: Final = MappingProxyType({"join": _joined_tool_row, "digits": _digits_masked_row}) + + +def _guardrail_peer(request: Request) -> Reply: + rows: Final = _JSON_OBJECTS.validate_python(_json(request).get("structured_messages") or []) + if request.target.startswith("/none/") or not rows: + return Reply(body=json.dumps({"action": "NONE"}).encode()) + rewrite: Final = _ROW_REWRITES.get(request.target.split("/")[1], _redacted_row) + verdict: Final = {"action": "GUARDRAIL_INTERVENED", "structured_messages": [rewrite(row) for row in rows]} + return Reply(body=json.dumps(verdict).encode()) + + +def _delete_guardrail(gateway: Gateway, identity: str) -> None: + deleted: Final = gateway.request("DELETE", f"/guardrails/{identity}") + assert deleted.status_code == 200, deleted.text + + +def _guardrail(gateway: Gateway, scenario: Scenario, wire: Wire, mode: str) -> str: + name: Final = f"tool-result-{mode}-{uuid.uuid4().hex}" + params: Final[dict[str, JsonValue]] = { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": f"{wire.url}/{mode}", + "api_key": "synthetic-guardrail-key", + } + created: Final = gateway.post("/guardrails", {"guardrail": {"guardrail_name": name, "litellm_params": params}}) + scenario.cleanups.callback(_delete_guardrail, gateway, string_value(created["guardrail_id"])) + return name + + +def _first_turn(messages: JsonValue) -> str: + return json.dumps(_JSON_OBJECTS.validate_python(messages)[0]) + + +def test_rewriting_guardrail_leaves_the_rejection_to_the_openai_route(gateway: Gateway) -> None: + with ( + wire_server(_openai_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _openai_model(scenario, provider) + body: Final = _body(model, _NULL_TEXT, max_tokens=64, guardrails=[_guardrail(gateway, scenario, guard, "rows")]) + _assert_field_rejection(gateway, "/v1/messages", body) + _assert_field_rejection(gateway, "/v1/messages", {**body, "stream": True}) + assert provider.drain() == () + + +@pytest.mark.parametrize("source", sorted(_DROP_SOURCES)) +def test_rewriting_guardrail_then_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None: + deployment, extra, headers = _DROP_SOURCES[source] + with ( + wire_server(_openai_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _openai_model(scenario, provider, **deployment) + guardrail: Final = _guardrail(gateway, scenario, guard, "rows") + body: Final = _body(model, _NUMBER_THEN_WEATHER, max_tokens=64, guardrails=[guardrail], **extra) + created: Final = gateway.request("POST", "/v1/messages", body, headers=headers) + assert created.status_code == 200, created.text + forwarded: Final = _only(provider.drain(), "/responses") + first_turn: Final = _first_turn(forwarded["input"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + assert _function_call_output(forwarded) == _WEATHER, forwarded + + +@pytest.mark.parametrize( + ("deployment", "forwarded"), + ( + (MappingProxyType({}), [{"type": "text", "text": 5}, {"type": "text", "text": _WEATHER}]), + (MappingProxyType({"drop_params": True}), [{"type": "text", "text": _WEATHER}]), + ), + ids=("no-drop-params", "deployment-drop-params"), +) +def test_rewriting_guardrail_forwards_the_malformed_text_to_anthropic_unless_drop_params_drops_it( + gateway: Gateway, deployment: Mapping[str, JsonValue], forwarded: list[JsonValue] +) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider, **deployment) + guardrail: Final = _guardrail(gateway, scenario, guard, "rows") + body: Final = _body(model, _NUMBER_THEN_WEATHER, max_tokens=64, guardrails=[guardrail]) + created: Final = gateway.request("POST", "/v1/messages", body) + assert created.status_code == 200, created.text + forwarded_body: Final = _only(provider.drain(), "/v1/messages") + first_turn: Final = _first_turn(forwarded_body["messages"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + assert _forwarded_tool_result(forwarded_body) == forwarded + + +_MORNING: Final = f"{_CITY} morning: sunny" +_EVENING: Final = f"{_CITY} evening: 18C" +_TOMORROW: Final = f"{_CITY} tomorrow: rain" +_WEEKEND: Final = f"{_CITY} weekend: clear" + + +def _text(value: JsonValue) -> dict[str, JsonValue]: + return {"type": "text", "text": value} + + +def _tool_use(tool_use_id: str) -> dict[str, JsonValue]: + return {"type": "tool_use", "id": tool_use_id, "name": "get_weather", "input": {"city": _CITY}} + + +def _tool_result(tool_use_id: str, *parts: JsonValue) -> dict[str, JsonValue]: + return {"type": "tool_result", "tool_use_id": tool_use_id, "content": [_text(part) for part in parts]} + + +_SEVERAL_TOOL_RESULTS: Final[list[JsonValue]] = [ + {"role": "user", "content": f"What is the weather in {_CITY}?"}, + {"role": "assistant", "content": [_tool_use("toolu_01"), _tool_use("toolu_02"), _tool_use("toolu_03")]}, + { + "role": "user", + "content": [ + _tool_result("toolu_01", None, _MORNING, 5, _EVENING), + _tool_result("toolu_02", _TOMORROW, 7.5), + _tool_result("toolu_03", _WEEKEND), + ], + }, +] + + +def _forwarded_tool_results(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + last: Final = _JSON_OBJECTS.validate_python(body["messages"])[-1] + return { + string_value(block["tool_use_id"]): block["content"] + for block in _JSON_OBJECTS.validate_python(last["content"]) + if block.get("type") == "tool_result" + } + + +def _texts(content: JsonValue) -> tuple[JsonValue, ...]: + if isinstance(content, str): + return (content,) + return tuple(part.get("text") for part in _JSON_OBJECTS.validate_python(content)) + + +_RESTORED_TOOL_RESULTS: Final = MappingProxyType( + { + "rows": ( + [_text(None), _text(_redacted(_MORNING)), _text(5), _text(_redacted(_EVENING))], + [_text(_redacted(_TOMORROW)), _text(7.5)], + ), + "join": ( + [_text(f"null | {_redacted(_MORNING)} | 5 | {_redacted(_EVENING)}")], + [_text(f"{_redacted(_TOMORROW)} | 7.5")], + ), + } +) + + +@pytest.mark.parametrize("mode", sorted(_RESTORED_TOOL_RESULTS)) +def test_rewriting_guardrail_puts_back_each_malformed_text_it_left_unchanged(gateway: Gateway, mode: str) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, mode) + body: Final = _body(model, _SEVERAL_TOOL_RESULTS, max_tokens=64, guardrails=[guardrail]) + created: Final = gateway.request("POST", "/v1/messages", body) + assert created.status_code == 200, created.text + forwarded: Final = _only(provider.drain(), "/v1/messages") + first_turn: Final = _first_turn(forwarded["messages"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + tool_results: Final = _forwarded_tool_results(forwarded) + assert _CITY not in json.dumps(tool_results), tool_results + assert (tool_results["toolu_01"], tool_results["toolu_02"]) == _RESTORED_TOOL_RESULTS[mode], tool_results + assert _texts(tool_results["toolu_03"]) == (_redacted(_WEEKEND),), tool_results + + +def _null_text_answering(tool_use_id: str) -> list[JsonValue]: + return [ + {"role": "user", "content": f"What is the weather in {_CITY}?"}, + {"role": "assistant", "content": [_tool_use(tool_use_id)]}, + {"role": "user", "content": [_tool_result(tool_use_id, None)]}, + ] + + +@pytest.mark.parametrize("tool_use_id", ("toolu_01", "functions.get_weather:0")) +def test_rewriting_guardrail_leaves_the_rejection_to_the_chat_bridge_for_any_tool_use_id( + gateway: Gateway, tool_use_id: str +) -> None: + with ( + wire_server(_chat_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _mistral_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, "rows") + body: Final = _body(model, _null_text_answering(tool_use_id), max_tokens=64, guardrails=[guardrail]) + response: Final = gateway.request("POST", "/v1/messages", body) + message: Final = _anthropic_error_message(response.status_code, response.text, response.content) + assert "content.0.text: Input should be a valid string" in message, message + assert provider.drain() == () + + +def _files_peer(request: Request) -> Reply: + if request.method == "POST" and request.target.endswith("/files"): + uploaded: Final = { + "id": f"file-{uuid.uuid4().hex}", + "object": "file", + "bytes": len(request.body), + "created_at": 1789788253, + "filename": "batch.jsonl", + "purpose": "batch", + "status": "processed", + } + return Reply(body=json.dumps(uploaded).encode()) + if request.target.endswith("/models"): + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _uploaded_records(request: Request) -> tuple[dict[str, JsonValue], ...]: + envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body + parts: Final = BytesParser(policy=HTTP).parsebytes(envelope).iter_parts() + files: Final = tuple(part.get_payload(decode=True) for part in parts if part.get_filename() is not None) + assert len(files) == 1, request.headers + payload: Final = files[0] + assert isinstance(payload, bytes), payload + return tuple(_JSON_OBJECT.validate_json(line) for line in payload.splitlines() if line.strip()) + + +def _listed_models(gateway: Gateway) -> frozenset[str]: + with httpx.Client(base_url=gateway.client.base_url, timeout=15, trust_env=False) as client: + listed: Final = client.get("/v1/models", headers={"Authorization": f"Bearer {gateway.key}"}) + assert listed.status_code == 200, listed.text + entries: Final = _JSON_OBJECTS.validate_python(_JSON_OBJECT.validate_json(listed.content)["data"]) + return frozenset(string_value(entry["id"]) for entry in entries) + + +def _uploaded_batch_body( + gateway: Gateway, scenario: Scenario, guard: Wire, mode: str, messages: list[JsonValue] +) -> dict[str, JsonValue]: + with wire_server(_files_peer) as provider: + model: Final = _openai_model(scenario, provider) + key: Final = scenario.key(metadata={"guardrails": [_guardrail(gateway, scenario, guard, mode)]}) + eventually( + lambda: tuple(_listed_models(gateway) for _ in range(8)), + lambda listings: all(model in listing for listing in listings), + seconds=90, + ) + record: Final = { + "custom_id": f"weather-{uuid.uuid4().hex}", + "method": "POST", + "url": "/v1/messages", + "body": _body(model, messages, max_tokens=64), + } + uploaded: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": model}, + {"file": ("batch.jsonl", json.dumps(record).encode() + b"\n", "application/jsonl")}, + key=key, + ) + assert uploaded.status_code == 200, uploaded.text + uploads: Final = tuple(request for request in provider.drain() if request.target.endswith("/files")) + assert len(uploads) == 1, [request.target for request in uploads] + (forwarded,) = _uploaded_records(uploads[0]) + assert forwarded["custom_id"] == record["custom_id"], forwarded + return _JSON_OBJECT.validate_python(forwarded["body"]) + + +def test_batch_upload_behind_a_rewriting_guardrail_keeps_the_malformed_text(gateway: Gateway) -> None: + with wire_server(_guardrail_peer) as guard, gateway.scenario() as scenario: + forwarded_body: Final = _uploaded_batch_body(gateway, scenario, guard, "rows", _NUMBER_THEN_WEATHER) + first_turn: Final = _first_turn(forwarded_body["messages"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + assert _forwarded_tool_result(forwarded_body) == [_text(5), _text(_WEATHER)], forwarded_body + + +def test_guardrail_that_rewrites_nothing_forwards_a_null_tool_result_text_unchanged(gateway: Gateway) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, "none") + created: Final = gateway.request( + "POST", "/v1/messages", _body(model, _NULL_TEXT, max_tokens=64, guardrails=[guardrail]) + ) + assert created.status_code == 200, created.text + assert len(guard.drain()) == 1 + assert _forwarded_tool_result(_only(provider.drain(), "/v1/messages")) == [{"type": "text", "text": None}] + + +def _forwarded_chat_turns(wire: Wire) -> tuple[dict[str, JsonValue], ...]: + return _JSON_OBJECTS.validate_python(_only(wire.drain(), "/chat/completions")["messages"]) + + +def test_mistral_receives_the_tool_reply_whose_only_text_drop_params_dropped(gateway: Gateway) -> None: + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + created: Final = gateway.request( + "POST", "/v1/messages", _body(model, _NULL_TEXT, max_tokens=64, drop_params=True) + ) + assert created.status_code == 200, created.text + turns: Final = _forwarded_chat_turns(wire) + assert [turn["role"] for turn in turns] == ["user", "assistant", "tool"], turns + calls: Final = _JSON_OBJECTS.validate_python(turns[1]["tool_calls"]) + assert [call["id"] for call in calls] == [turns[2]["tool_call_id"]], turns + assert _tool_message_text({"messages": list(turns)}) == "", turns + + +_WEATHER_CALL: Final[dict[str, JsonValue]] = { + "id": "call_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"city": _CITY})}, +} + + +def test_mistral_chat_completions_keeps_an_empty_tool_reply(gateway: Gateway) -> None: + ask: Final = f"What is the weather in {_CITY}? {uuid.uuid4().hex}" + messages: Final[list[JsonValue]] = [ + {"role": "system", "content": ""}, + {"role": "user", "content": ask}, + {"role": "assistant", "content": ""}, + {"role": "assistant", "content": None, "tool_calls": [_WEATHER_CALL]}, + {"role": "tool", "tool_call_id": "call_weather", "content": ""}, + {"role": "user", "content": ""}, + ] + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + response: Final = gateway.request("POST", "/v1/chat/completions", {"model": model, "messages": messages}) + assert response.status_code == 200, response.text + turns: Final = _forwarded_chat_turns(wire) + assert [(turn["role"], turn.get("content")) for turn in turns] == [ + ("user", ask), + ("assistant", None), + ("tool", ""), + ], turns + assert turns[2]["tool_call_id"] == "call_weather", turns + + +def test_mistral_chat_completions_drops_an_orphan_tool_message_and_sends_a_null_tool_reply_as_empty_text( + gateway: Gateway, +) -> None: + ask: Final = f"What is the weather in {_CITY}? {uuid.uuid4().hex}" + messages: Final[list[JsonValue]] = [ + {"role": "user", "content": ask}, + {"role": "tool", "tool_call_id": "call_orphan", "content": ""}, + {"role": "assistant", "content": None, "tool_calls": [_WEATHER_CALL]}, + {"role": "tool", "tool_call_id": "call_weather", "content": None}, + ] + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + response: Final = gateway.request("POST", "/v1/chat/completions", {"model": model, "messages": messages}) + assert response.status_code == 200, response.text + turns: Final = _forwarded_chat_turns(wire) + assert [(turn["role"], turn.get("tool_call_id"), turn.get("content")) for turn in turns] == [ + ("user", None, ask), + ("assistant", None, None), + ("tool", "call_weather", ""), + ], turns + + +_NATIVE_DROP_PARAMS_CASES: Final = MappingProxyType( + { + "body": (_DROP_SOURCES["body"], [_text(_WEATHER)]), + "deployment": (_DROP_SOURCES["deployment"], [_text(_WEATHER)]), + "explicit-false": ( + (MappingProxyType({"drop_params": True}), MappingProxyType({"drop_params": False}), _CLAUDE_CODE), + [_text(5), _text(_WEATHER)], + ), + } +) + + +@pytest.mark.parametrize("case", sorted(_NATIVE_DROP_PARAMS_CASES)) +def test_native_anthropic_resolves_drop_params_the_same_way_on_both_routes(gateway: Gateway, case: str) -> None: + (deployment, extra, headers), forwarded = _NATIVE_DROP_PARAMS_CASES[case] + with wire_server(_anthropic_peer) as wire, gateway.scenario() as scenario: + model: Final = _anthropic_model(scenario, wire, **deployment) + body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _ANTHROPIC_COUNT, counted.text + requests: Final = wire.drain() + assert _forwarded_tool_result(_only(requests, "/v1/messages")) == forwarded + assert _forwarded_tool_result(_only(requests, "/v1/messages/count_tokens")) == forwarded + + +_CARD_NUMBER: Final = 4111111111111111 +_CARD_NULL_WEATHER: Final = _messages(_text(_CARD_NUMBER), _text(None), _text(_WEATHER)) +_SCANNED_CARD_NULL_WEATHER: Final = ("4111111111111111", "null", _WEATHER) +_MASKED_CARD_NULL_WEATHER: Final = [_text("################"), _text(None), _text("sunny, ##C")] + + +def _scanned_tool_texts(guard: Wire) -> tuple[JsonValue, ...]: + requests: Final = guard.drain() + assert len(requests) == 1, [request.target for request in requests] + rows: Final = _JSON_OBJECTS.validate_python(_json(requests[0])["structured_messages"]) + tool_rows: Final = tuple(row for row in rows if row.get("role") == "tool") + assert len(tool_rows) == 1, rows + return _texts(tool_rows[0]["content"]) + + +def test_masking_guardrail_scans_a_numeric_tool_result_text_before_anthropic_gets_it(gateway: Gateway) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, "digits") + created: Final = gateway.request( + "POST", "/v1/messages", _body(model, _CARD_NULL_WEATHER, max_tokens=64, guardrails=[guardrail]) + ) + assert created.status_code == 200, created.text + scanned: Final = _scanned_tool_texts(guard) + forwarded: Final = _forwarded_tool_result(_only(provider.drain(), "/v1/messages")) + assert scanned == _SCANNED_CARD_NULL_WEATHER, scanned + assert forwarded == _MASKED_CARD_NULL_WEATHER, forwarded + + +def test_masking_guardrail_scans_a_numeric_tool_result_text_in_a_batch_upload(gateway: Gateway) -> None: + with wire_server(_guardrail_peer) as guard, gateway.scenario() as scenario: + forwarded_body: Final = _uploaded_batch_body(gateway, scenario, guard, "digits", _CARD_NULL_WEATHER) + scanned: Final = _scanned_tool_texts(guard) + assert scanned == _SCANNED_CARD_NULL_WEATHER, scanned + assert _forwarded_tool_result(forwarded_body) == _MASKED_CARD_NULL_WEATHER, forwarded_body diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index ec6af7f27c3..046f448046b 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -25,7 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_tools_pt, _rename_duplicate_bedrock_document_names, _convert_to_bedrock_tool_call_invoke, - _sanitize_anthropic_tool_use_id, + sanitize_anthropic_tool_use_id, _convert_to_bedrock_tool_call_result, anthropic_messages_pt, convert_to_anthropic_tool_result, @@ -4503,7 +4503,7 @@ def test_get_tool_calls_from_response_keeps_sanitized_concat_ids_distinct(): ) ids: Final = [call["id"] for call in get_tool_calls_from_response(response)] - sanitized: Final = [_sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] + sanitized: Final = [sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] assert len(sanitized) == len(set(sanitized)) assert ids == ["a:b", "a:b__concat_2", "a_b__concat_1"] @@ -4517,7 +4517,7 @@ def test_get_tool_calls_from_response_bumps_suffix_when_sibling_sanitizes_onto_i ) ids: Final = [call["id"] for call in get_tool_calls_from_response(response)] - sanitized: Final = [_sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] + sanitized: Final = [sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] assert len(ids) == len(sanitized) assert len(sanitized) == len(set(sanitized)) diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 0aca60351bf..58e3b55773b 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -155,6 +155,17 @@ class MockStructuredMaskingGuardrail(CustomGuardrail): return masked +class MockNumberMaskingGuardrail(MockStructuredMaskingGuardrail): + @staticmethod + def _mask(text: str) -> str: + return ( + text.replace("bob@example.com", "") + .replace("5", "") + .replace("null", "") + .replace("山田太郎", "") + ) + + class TestAnthropicMessagesHandlerStreamingRequestData: """Post-call guardrails on streaming /v1/messages receive the response and identity metadata""" @@ -2499,7 +2510,7 @@ class TestStructuredWriteBackKeepsToolResults: """A guardrail rewrite must never leave a tool_use without its tool_result (Claude Code ToolSearch, LIT-6103).""" @staticmethod - def _claude_code_tool_search_turns(tool_result_content): + def _claude_code_tool_search_turns(tool_result_content, tool_use_id="toolu_01"): return [ {"role": "user", "content": "load WebFetch for bob@example.com"}, { @@ -2507,7 +2518,7 @@ class TestStructuredWriteBackKeepsToolResults: "content": [ { "type": "tool_use", - "id": "toolu_01", + "id": tool_use_id, "name": "ToolSearch", "input": {"query": "select:WebFetch"}, } @@ -2516,7 +2527,7 @@ class TestStructuredWriteBackKeepsToolResults: { "role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}, + {"type": "tool_result", "tool_use_id": tool_use_id, "content": tool_result_content}, {"type": "text", "text": "Now fetch the page."}, ], }, @@ -2563,6 +2574,124 @@ class TestStructuredWriteBackKeepsToolResults: later_blocks = [b for m in messages[tool_use_index + 1 :] for b in self._blocks(m)] assert {"type": "text", "text": "Now fetch the page."} in later_blocks + @pytest.mark.parametrize( + ("tool_result_content", "expected_written_back_content"), + [ + ( + [{"type": "text", "text": None}, {"type": "text", "text": "log for bob@example.com"}], + [{"type": "text", "text": None}, {"type": "text", "text": "log for "}], + ), + ( + [{"type": "text", "text": "log for bob@example.com"}, {"type": "text", "text": 73}], + [{"type": "text", "text": "log for "}, {"type": "text", "text": 73}], + ), + ([{"type": "text", "text": None}], [{"type": "text", "text": None}]), + ( + [ + {"type": "text", "text": None}, + {"type": "text", "text": "log for bob@example.com"}, + {"type": "text", "text": 5}, + {"type": "text", "text": "bob@example.com again"}, + ], + [ + {"type": "text", "text": None}, + {"type": "text", "text": "log for "}, + {"type": "text", "text": 5}, + {"type": "text", "text": " again"}, + ], + ), + ], + ids=["null-first", "number-last", "null-only", "interleaved"], + ) + async def test_rewrite_keeps_a_non_string_tool_result_text_in_place( + self, tool_result_content, expected_written_back_content + ): + data = {"model": "claude-fable-5", "messages": self._claude_code_tool_search_turns(tool_result_content)} + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockStructuredMaskingGuardrail() + ) + + assert "bob@example.com" not in json.dumps(data["messages"]) + assert data["messages"][2]["content"][0] == { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": expected_written_back_content, + } + + async def test_rewrite_keeps_a_non_string_tool_result_text_whose_tool_use_id_gets_sanitized(self): + tool_result_content = [{"type": "text", "text": None}, {"type": "text", "text": "log for bob@example.com"}] + data = { + "model": "claude-fable-5", + "messages": self._claude_code_tool_search_turns(tool_result_content, tool_use_id="functions.get_weather:0"), + } + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockStructuredMaskingGuardrail() + ) + + assert data["messages"][2]["content"][0]["content"] == [ + {"type": "text", "text": None}, + {"type": "text", "text": "log for "}, + ] + + async def test_rewrite_keeps_each_non_string_text_when_two_tool_use_ids_sanitize_alike(self): + data = { + "model": "claude-fable-5", + "messages": [ + {"role": "user", "content": "check both logs for bob@example.com"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "read:log", "name": "Read", "input": {"path": "a.log"}}, + {"type": "tool_use", "id": "read_log", "name": "Read", "input": {"path": "b.log"}}, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "read:log", + "content": [{"type": "text", "text": "ok"}, {"type": "text", "text": None}], + }, + { + "type": "tool_result", + "tool_use_id": "read_log", + "content": [{"type": "text", "text": 5}, {"type": "text", "text": "bob@example.com"}], + }, + ], + }, + ], + } + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockStructuredMaskingGuardrail() + ) + + assert [block["content"] for block in data["messages"][2]["content"]] == [ + [{"type": "text", "text": "ok"}, {"type": "text", "text": None}], + [{"type": "text", "text": 5}, {"type": "text", "text": ""}], + ] + + @pytest.mark.parametrize( + ("text", "masked"), + [(5, ""), (None, ""), ({"owner": "山田太郎"}, '{"owner": ""}')], + ids=["number", "null", "non-ascii-object"], + ) + async def test_guardrail_scans_and_masks_a_non_string_tool_result_text(self, text, masked): + tool_result_content = [{"type": "text", "text": text}, {"type": "text", "text": "log for bob@example.com"}] + data = {"model": "claude-fable-5", "messages": self._claude_code_tool_search_turns(tool_result_content)} + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockNumberMaskingGuardrail() + ) + + assert data["messages"][2]["content"][0]["content"] == [ + {"type": "text", "text": masked}, + {"type": "text", "text": "log for "}, + ] + class TestAnthropicMessagesHandlerStreamingScanKey: """get_streaming_scan_key mirrors what process_output_streaming_response would scan""" diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 9e0bd5c60b8..0e918cd49c5 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1,10 +1,13 @@ import base64 import functools import json +from collections.abc import Mapping from types import MappingProxyType from typing import Any, Final, cast +import httpx import pytest +import respx import litellm @@ -5235,3 +5238,153 @@ def test_eager_input_streaming_tool_reaches_bedrock_converse_as_beta(): assert data["additionalModelRequestFields"]["anthropic_beta"] == ["fine-grained-tool-streaming-2025-05-14"] assert data["toolConfig"]["tools"][0]["toolSpec"]["inputSchema"]["json"] == EAGER_INPUT_SCHEMA + + +_CHAT_BRIDGE_MODEL: Final = "azure/gpt-5.6" +_CHAT_BRIDGE_API_BASE: Final = "https://tool-result-text.openai.azure.com" +_CHAT_BRIDGE_URL: Final = r".*tool-result-text\.openai\.azure\.com/openai/deployments/gpt-5\.6/chat/completions.*" +_CHAT_BRIDGE_REPLY: Final = { + "id": "chatcmpl-tool-result-text", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "build passed"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 3, "total_tokens": 15}, +} +_MALFORMED_TOOL_RESULT_TEXTS: Final = (pytest.param(None, id="null"), pytest.param(5, id="number")) +_MALFORMED_TOOL_RESULT_TEXT_ERROR: Final = ( + "litellm.BadRequestError: tool_result toolu_log content.1.text: Input should be a valid string" +) + + +def _tool_result_with_text_blocks(text_block: Mapping[str, object]) -> list[dict[str, object]]: + blocks: Final = [{"type": "text", "text": "build ok"}, text_block, {"type": "text", "text": "done"}, text_block] + return [ + {"role": "user", "content": "run the build"}, + dict(_anthropic_tool_use_turn("toolu_log")), + dict(_anthropic_tool_result_turn({"toolu_log": blocks})), + ] + + +def _chat_bridge_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(url__regex=_CHAT_BRIDGE_URL).mock(return_value=httpx.Response(200, json=_CHAT_BRIDGE_REPLY)) + + +def _sent_tool_messages(route: respx.Route) -> list[object]: + sent: Final = json.loads(route.calls.last.request.read()) + return [message for message in sent["messages"] if message["role"] == "tool"] + + +@pytest.fixture +def _chat_bridge_httpx(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"]) +@pytest.mark.parametrize("text", _MALFORMED_TOOL_RESULT_TEXTS) +async def test_chat_bridge_async_malformed_tool_result_text_answers_400_before_any_provider_call( + respx_mock: respx.MockRouter, text: object, stream: bool +) -> None: + route: Final = _chat_bridge_route(respx_mock) + + with pytest.raises(litellm.BadRequestError) as rejected: + await litellm.anthropic.messages.acreate( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + stream=stream, + messages=_tool_result_with_text_blocks({"type": "text", "text": text}), + ) + + assert rejected.value.status_code == 400 + assert rejected.value.llm_provider == "azure" + assert rejected.value.message == _MALFORMED_TOOL_RESULT_TEXT_ERROR + assert route.call_count == 0 + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"]) +@pytest.mark.parametrize("text", _MALFORMED_TOOL_RESULT_TEXTS) +def test_chat_bridge_sync_malformed_tool_result_text_answers_400_before_any_provider_call( + respx_mock: respx.MockRouter, text: object, stream: bool +) -> None: + route: Final = _chat_bridge_route(respx_mock) + + with pytest.raises(litellm.BadRequestError) as rejected: + litellm.anthropic.messages.create( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + stream=stream, + messages=_tool_result_with_text_blocks({"type": "text", "text": text}), + ) + + assert rejected.value.status_code == 400 + assert rejected.value.llm_provider == "azure" + assert rejected.value.message == _MALFORMED_TOOL_RESULT_TEXT_ERROR + assert route.call_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize("drop_params_source", ["request", "global"]) +@pytest.mark.parametrize("text", _MALFORMED_TOOL_RESULT_TEXTS) +async def test_chat_bridge_drop_params_drops_a_malformed_tool_result_text( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, text: object, drop_params_source: str +) -> None: + route: Final = _chat_bridge_route(respx_mock) + if drop_params_source == "global": + monkeypatch.setattr(litellm, "drop_params", True) + + response: Final = await litellm.anthropic.messages.acreate( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + messages=_tool_result_with_text_blocks({"type": "text", "text": text}), + **({"drop_params": True} if drop_params_source == "request" else {}), + ) + + assert _sent_tool_messages(route) == [ + { + "role": "tool", + "tool_call_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": "done"}], + } + ] + assert isinstance(response, dict) + assert response["content"] == [{"type": "text", "text": "build passed"}] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_chat_bridge_tool_result_text_block_without_text_reaches_the_model_as_empty_text( + respx_mock: respx.MockRouter, +) -> None: + route: Final = _chat_bridge_route(respx_mock) + + await litellm.anthropic.messages.acreate( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + messages=_tool_result_with_text_blocks({"type": "text"}), + ) + + assert _sent_tool_messages(route) == [ + { + "role": "tool", + "tool_call_id": "toolu_log", + "content": [ + {"type": "text", "text": "build ok"}, + {"type": "text", "text": ""}, + {"type": "text", "text": "done"}, + {"type": "text", "text": ""}, + ], + } + ] diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py index c63bfbdd301..7fbec3ba12e 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py @@ -300,3 +300,67 @@ async def test_a_json_provider_gets_safeguards_and_the_beta_together_unless_drop assert sent.headers["anthropic-beta"] == forwarded_beta assert "safeguard_results" not in response assert response["content"][0]["text"] == "hello" + + +@pytest.mark.parametrize( + ("drop_params", "expected_tool_result_content"), + [ + (True, [{"type": "text", "text": "sunny"}]), + (None, [{"type": "text", "text": 5}, {"type": "text", "text": "sunny"}]), + ], + ids=["drop_params", "no_drop_params"], +) +def test_drop_params_drops_a_non_string_tool_result_text_on_the_native_request( + drop_params, expected_tool_result_content +): + messages = [ + {"role": "user", "content": "What is the weather?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": 5}, {"type": "text", "text": "sunny"}], + } + ], + }, + ] + + request = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-5", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + assert request["messages"][2]["content"][0]["content"] == expected_tool_result_content + + +def test_drop_params_forwards_a_non_object_message_item_as_sent_on_the_native_request(): + tool_result_turn = { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": None}, {"type": "text", "text": "sunny"}], + } + ], + } + + request = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "What is the weather?"}, 7, tool_result_turn], + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + + assert request["messages"][1] == 7 + assert request["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "sunny"}] diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py index c55fa476c06..409e64635f7 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py @@ -617,3 +617,88 @@ async def test_tool_search_forwards_deferred_tools_and_beta_header( assert [block for block in response["content"] if block["type"] == "tool_use"] == [ {"type": "tool_use", "id": "toolu_1", "name": tool_name, "input": tool_input} ] + + +_MIXED_GROUP: Final = "gpt-5.6-mixed" +_MIXED_AZURE_API_BASE: Final = "https://mixed-group.openai.azure.com" +_MIXED_AZURE_CHAT_URL: Final = r".*mixed-group\.openai\.azure\.com/openai/deployments/.*/chat/completions.*" +_MIXED_DEPLOYMENT_IDS: Final = ("mixed-openai-responses-bridge", "mixed-azure-chat-bridge") +_MIXED_AZURE_CHAT_BODY: Final = { + "id": "chatcmpl-mixed", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": _JOKE}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, +} + + +def _mixed_openai_azure_router() -> Router: + return Router( + model_list=[ + { + "model_name": _MIXED_GROUP, + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "fake-key"}, + "model_info": {"id": _MIXED_DEPLOYMENT_IDS[0]}, + }, + { + "model_name": _MIXED_GROUP, + "litellm_params": {"model": "azure/gpt-5.6", "api_key": "fake-key", "api_base": _MIXED_AZURE_API_BASE}, + "model_info": {"id": _MIXED_DEPLOYMENT_IDS[1]}, + }, + ], + num_retries=0, + ) + + +async def _null_tool_result_text_rejection(router: Router, deployment_id: str) -> litellm.BadRequestError: + with pytest.raises(litellm.BadRequestError) as rejected: + await router.aanthropic_messages( + model=deployment_id, + max_tokens=64, + messages=[ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_build", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_build", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": None}], + } + ], + }, + ], + ) + return rejected.value + + +@pytest.mark.asyncio +async def test_mixed_openai_azure_group_answers_the_same_400_for_a_null_tool_result_text_on_either_deployment( + respx_mock, +): + openai_route: Final = respx_mock.post(url__regex=_OPENAI_RESPONSES_URL).mock( + return_value=httpx.Response(200, json=_OPENAI_BODY) + ) + azure_route: Final = respx_mock.post(url__regex=_MIXED_AZURE_CHAT_URL).mock( + return_value=httpx.Response(200, json=_MIXED_AZURE_CHAT_BODY) + ) + router: Final = _mixed_openai_azure_router() + + rejections: Final = [ + await _null_tool_result_text_rejection(router, deployment_id) for deployment_id in _MIXED_DEPLOYMENT_IDS + ] + + assert [(rejection.status_code, rejection.llm_provider) for rejection in rejections] == [ + (400, "openai"), + (400, "azure"), + ] + assert {rejection.message.partition("\n\n")[0] for rejection in rejections} == { + "litellm.BadRequestError: tool_result toolu_build content.1.text: Input should be a valid string" + } + assert openai_route.call_count == 0 + assert azure_route.call_count == 0 diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index b6f831d4bef..8fe6bd31082 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -576,6 +576,91 @@ def test_client_cache_control_changes_nothing_for_a_deployment_that_does_not_rea assert marked == unmarked +MALFORMED_TEXT_BLOCKS: Final = [ + pytest.param({"type": "text", "text": None}, id="null"), + pytest.param({"type": "text", "text": 7}, id="number"), +] + + +def _tool_result_turn_with(text_block: Mapping[str, object]) -> list[dict[str, object]]: + return [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, text_block, {"type": "text", "text": "done"}], + } + ], + } + ] + + +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +def test_malformed_tool_result_text_answers_400_naming_the_block(text_block: Mapping[str, object]) -> None: + with pytest.raises(litellm.BadRequestError) as rejected: + _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs=OPENAI_API, + max_tokens=1024, + messages=_tool_result_turn_with(text_block), + ) + + assert rejected.value.status_code == 400 + assert "tool_result toolu_log content.1.text: Input should be a valid string" in rejected.value.message + + +def test_many_malformed_tool_result_texts_answer_one_bounded_error() -> None: + tool_use_id: Final = "toolu_" + "x" * 4096 + messages: Final = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_use_id, + "content": [{"type": "text", "text": None}] * 2048, + } + ], + } + ] + + with pytest.raises(litellm.BadRequestError) as rejected: + _build_responses_kwargs(model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, messages=messages) + + assert rejected.value.status_code == 400 + assert f"tool_result {tool_use_id} content.0.text: Input should be a valid string" in rejected.value.message + assert rejected.value.message.count(tool_use_id) == 1 + + +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +def test_drop_params_drops_a_malformed_tool_result_text(text_block: Mapping[str, object]) -> None: + responses_kwargs: Final = _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs={**OPENAI_API, "drop_params": True}, + max_tokens=1024, + messages=_tool_result_turn_with(text_block), + ) + + assert responses_kwargs["input"] == [ + {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\ndone"}, + ] + + +def test_tool_result_text_block_without_text_still_reaches_the_model_as_empty_text() -> None: + responses_kwargs: Final = _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs=OPENAI_API, + max_tokens=1024, + messages=_tool_result_turn_with({"type": "text"}), + ) + + assert responses_kwargs["input"] == [ + {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"}, + ] + + REASONING_MODEL: Final = "gpt-bridge-reasoning-cost" REASONING_RATES: Final = { "input_cost_per_token": 2e-06, diff --git a/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py b/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py index b5301b413ff..eac75486da4 100644 --- a/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py +++ b/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py @@ -8,6 +8,7 @@ from pydantic import JsonValue, TypeAdapter from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.llms.anthropic.pass_through.convertible_blocks import ( + malformed_tool_result_text_error, unconvertible_anthropic_blocks, unconvertible_blocks_error, ) @@ -190,3 +191,22 @@ def test_an_encrypted_web_search_inference_cannot_send_is_named() -> None: error: Final = unconvertible_blocks_error(_replayed_web_search("opaque"), "anthropic") assert error is not None assert error.endswith("server_tool_use, web_search_tool_result") + + +@pytest.mark.parametrize( + ("tool_result", "error"), + ( + ( + {"type": "tool_result", "tool_use_id": "toolu_01", "content": [{"type": "text", "text": None}]}, + "tool_result toolu_01 content.0.text: Input should be a valid string", + ), + ( + {"type": "tool_result", "content": [{"type": "text", "text": "ok"}, {"type": "text", "text": 5}]}, + "tool_result content.1.text: Input should be a valid string", + ), + ), +) +def test_a_non_string_tool_result_text_is_named_with_or_without_a_tool_use_id( + tool_result: Mapping[str, object], error: str +) -> None: + assert malformed_tool_result_text_error([{"role": "user", "content": [tool_result]}]) == error diff --git a/tests/unit/llms/gemini/test_gemini_common_utils.py b/tests/unit/llms/gemini/test_gemini_common_utils.py index e54023c6596..918a7475eec 100644 --- a/tests/unit/llms/gemini/test_gemini_common_utils.py +++ b/tests/unit/llms/gemini/test_gemini_common_utils.py @@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, patch import httpx import pytest +import respx from pydantic import JsonValue, TypeAdapter import litellm @@ -153,6 +154,83 @@ async def test_google_ai_studio_count_tokens_answers_with_the_status_google_reje assert "Invalid function name" in (result.error_message or "") +def _weather_tool_result_messages(text: JsonValue) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "Paris"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": text}, {"type": "text", "text": "partly cloudy"}], + } + ], + }, + ] + + +_COUNT_TOKENS_URL: Final = "https://count-tokens.test/v1beta/models/gemini-2.5-flash:countTokens" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text", (None, 5)) +async def test_google_ai_studio_count_tokens_rejects_a_non_string_tool_result_text_without_request( + text: JsonValue, respx_mock: respx.MockRouter +) -> None: + count_route: Final = respx_mock.post(_COUNT_TOKENS_URL).respond(200, json={"totalTokens": 11}) + + result: Final = await GoogleAIStudioTokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(text), + contents=None, + deployment={"litellm_params": {"api_key": "test-key", "api_base": "https://count-tokens.test"}}, + request_model="gemini/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is True + assert result.status_code == 400 + assert result.error_message == "tool_result toolu_01 content.0.text: Input should be a valid string" + assert not count_route.called + + +@pytest.mark.asyncio +async def test_google_ai_studio_count_tokens_leaves_out_a_numeric_tool_result_text_under_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + count_route: Final = respx_mock.post(_COUNT_TOKENS_URL).respond(200, json={"totalTokens": 11}) + + result: Final = await GoogleAIStudioTokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(5), + contents=None, + deployment={ + "litellm_params": {"api_key": "test-key", "api_base": "https://count-tokens.test", "drop_params": True} + }, + request_model="gemini/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is False + assert result.total_tokens == 11 + assert _JSON_BODY.validate_json(count_route.calls.last.request.content)["contents"] == [ + {"role": "user", "parts": [{"text": "What is the weather in Paris?"}]}, + {"role": "model", "parts": [{"function_call": {"name": "get_weather", "args": {"city": "Paris"}}}]}, + { + "role": "user", + "parts": [{"function_response": {"name": "get_weather", "response": {"content": "partly cloudy"}}}], + }, + ] + + @pytest.mark.asyncio @pytest.mark.parametrize( ("system_instruction", "tools", "expected_body"), diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index 2e88bccd49b..10f57be33fe 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -920,6 +920,85 @@ class TestMistralStripsOutputOnlyFields: assert "reasoning_content" not in result[-1] +def test_mistral_transform_request_keeps_an_empty_tool_message_after_its_tool_call(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "read the log"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + result = request["messages"] + assert [m.get("role") for m in result] == ["user", "assistant", "tool"] + assert result[2].get("tool_call_id") == "call_1" + + +@pytest.mark.parametrize("reply", [{"content": None}, {}], ids=["null", "missing"]) +def test_mistral_transform_request_sends_a_null_tool_reply_as_empty_text(reply): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "read the log"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", **reply}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"][2] == {"role": "tool", "tool_call_id": "call_1", "content": ""} + + +def test_mistral_transform_request_drops_an_empty_tool_message_no_tool_call_asked_for(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "hi"}, + {"role": "tool", "tool_call_id": "call_9", "content": ""}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"] == [{"role": "user", "content": "hi"}] + + +@pytest.mark.parametrize("role", ["system", "user"]) +def test_mistral_transform_request_drops_a_message_with_null_content(role): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [{"role": role, "content": None}, {"role": "user", "content": "hi"}], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"] == [{"role": "user", "content": "hi"}] + + def test_mistral_transform_request_hoists_tool_message_image(): """Images inside role:"tool" messages must be moved to a following user message (Mistral rejects/ignores non-text tool content), including when @@ -957,3 +1036,129 @@ def test_mistral_transform_request_hoists_tool_message_image(): {"type": "text", "text": TOOL_RESULT_IMAGE_BOUNDARY}, {"type": "image_url", "image_url": {"url": data_uri}}, ] + + +def test_mistral_transform_request_drops_an_empty_tool_message_for_a_call_already_answered_earlier(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + {"role": "assistant", "content": "It is sunny"}, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant", "tool", "assistant", "user"] + + +def test_mistral_transform_request_drops_an_empty_tool_message_when_a_later_reply_carries_the_result(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant", "tool", "user"] + assert request["messages"][2]["content"] == "sunny" + + +def test_mistral_transform_request_keeps_an_empty_tool_message_answering_a_call_in_the_first_message(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["assistant", "tool", "user"] + assert request["messages"][1]["content"] == "" + + +@pytest.mark.parametrize("placeholder_role", ["assistant", "user"]) +def test_mistral_transform_request_keeps_an_empty_tool_message_after_a_dropped_empty_placeholder(placeholder_role): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": placeholder_role, "content": ""}, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant", "tool", "user"] + assert request["messages"][2]["content"] == "" + + +def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_not_a_string(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "read the log"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": {"nested": "call_1"}, "type": "function", "function": {"name": "read", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": {"nested": "call_1"}, "content": ""}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index be394b71ec7..7495d98b42a 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -1,5 +1,6 @@ import json import operator +from collections.abc import Mapping from typing import Final import httpx @@ -395,6 +396,92 @@ async def test_undecodable_count_body_becomes_a_500_error_response_carrying_the_ ) +MALFORMED_TEXT_BLOCKS: Final = [ + pytest.param({"type": "text", "text": None}, id="null"), + pytest.param({"type": "text", "text": 7}, id="number"), +] + + +def _tool_result_turns_with(text_block: Mapping[str, object]) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_log", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, text_block, {"type": "text", "text": "done"}], + } + ], + }, + ] + + +async def _count_tool_result_turns( + monkeypatch: pytest.MonkeyPatch, text_block: Mapping[str, object], litellm_params: dict[str, object] +) -> tuple[TokenCountResponse | None, respx.Route]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9})) + result: Final = await OpenAITokenCounter().count_tokens( + model_to_use="gpt-4o", + messages=_tool_result_turns_with(text_block), + contents=None, + deployment={"litellm_params": litellm_params}, + request_model="openai/gpt-4o", + messages_format="anthropic", + ) + return result, route + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +async def test_anthropic_format_malformed_tool_result_text_answers_400_without_calling_openai( + monkeypatch: pytest.MonkeyPatch, text_block: Mapping[str, object] +) -> None: + result, route = await _count_tool_result_turns(monkeypatch, text_block, {"api_key": "sk-test"}) + + assert route.called is False + assert result is not None + assert (result.error, result.status_code, result.error_message) == ( + True, + 400, + "tool_result toolu_log content.1.text: Input should be a valid string", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +async def test_anthropic_format_drop_params_counts_the_tool_result_without_the_malformed_text( + monkeypatch: pytest.MonkeyPatch, text_block: Mapping[str, object] +) -> None: + result, route = await _count_tool_result_turns( + monkeypatch, text_block, {"api_key": "sk-test", "drop_params": True} + ) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 9) + sent: Final = json.loads(route.calls.last.request.content) + assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\ndone"} in sent["input"] + + +@pytest.mark.asyncio +async def test_anthropic_format_tool_result_text_block_without_text_still_counts_as_empty_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_tool_result_turns(monkeypatch, {"type": "text"}, {"api_key": "sk-test"}) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 9) + sent: Final = json.loads(route.calls.last.request.content) + assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"} in sent["input"] + + @pytest.mark.asyncio async def test_count_uses_the_global_openai_key_chat_would_use(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 20dba332c08..7d380205ca1 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -117,6 +117,7 @@ def _vertex_count_tokens_peer(status_code: int) -> Iterator[_VertexCountTokensPe _VERTEX_DEPLOYMENT: Final = {"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-central1"}} +_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue]) @pytest.mark.asyncio @@ -164,6 +165,77 @@ async def test_vertex_gemini_count_rejects_an_image_url_it_refuses_to_fetch_with assert peer.requests == () +def _weather_tool_result_messages(text: JsonValue) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "Paris"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": text}, {"type": "text", "text": "partly cloudy"}], + } + ], + }, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text", (None, 5)) +async def test_vertex_gemini_count_rejects_a_non_string_tool_result_text_without_request(text: JsonValue) -> None: + with _vertex_count_tokens_peer(200) as peer: + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(text), + contents=None, + deployment=_VERTEX_DEPLOYMENT, + request_model="vertex_ai/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is True + assert result.status_code == 400 + assert result.error_message == "tool_result toolu_01 content.0.text: Input should be a valid string" + assert peer.requests == () + + +@pytest.mark.asyncio +async def test_vertex_gemini_count_leaves_out_a_numeric_tool_result_text_when_the_deployment_drops_params() -> None: + with _vertex_count_tokens_peer(200) as peer: + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(5), + contents=None, + deployment={ + "litellm_params": { + "vertex_project": "test-project", + "vertex_location": "us-central1", + "drop_params": True, + } + }, + request_model="vertex_ai/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is False + assert result.total_tokens == 37 + assert _JSON_BODY.validate_json(peer.requests[0].content)["contents"] == [ + {"role": "user", "parts": [{"text": "What is the weather in Paris?"}]}, + {"role": "model", "parts": [{"function_call": {"name": "get_weather", "args": {"city": "Paris"}}}]}, + { + "role": "user", + "parts": [{"function_response": {"name": "get_weather", "response": {"content": "partly cloudy"}}}], + }, + ] + + _TOOL_OUTPUT: Final = "line of a long file that the agent read earlier. " * 180 diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 720739a730b..18fe5a2ad32 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -21,6 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import anyio import httpx import pytest +import respx import tiktoken from dotenv import load_dotenv from pydantic import BaseModel, ConfigDict, JsonValue @@ -36,6 +37,7 @@ load_dotenv() from fastapi import Request from fastapi.responses import JSONResponse +from starlette.types import Message import litellm from litellm import Router @@ -56,7 +58,7 @@ from litellm.llms.bedrock.count_tokens.kimi_k3 import ( from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import VertexAIPartnerModelsTokenCounter -from litellm.proxy._types import ProxyException, TokenCountRequest +from litellm.proxy._types import ProxyException, TokenCountRequest, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.endpoints import ( count_tokens as anthropic_count_tokens, ) @@ -1275,6 +1277,202 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): proxy_server.count_request_tokens = original_token_counter +def _json_request(body: Mapping[str, object], user_agent: str) -> Request: + payload: Final = json.dumps(body).encode() + + async def receive() -> Message: + return {"type": "http.request", "body": payload, "more_body": False} + + return Request( + { + "type": "http", + "method": "POST", + "path": "/v1/messages/count_tokens", + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"user-agent", user_agent.encode())], + }, + receive, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_drop_params", "user_agent"), + (({"drop_params": True}, "python-httpx/0.28.1"), ({}, "claude-cli/2.1.0 (external, cli)")), +) +async def test_anthropic_count_tokens_drops_a_malformed_tool_result_text_when_drop_params_applies( + monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, bool], user_agent: str +) -> None: + import litellm.proxy.proxy_server as proxy_server + + body: Final = { + "model": "gpt-tool-result", + **request_drop_params, + "messages": [ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_log", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": None}], + } + ], + }, + ], + } + router: Final = Router( + model_list=[ + {"model_name": "gpt-tool-result", "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-test"}} + ] + ) + request: Final = _json_request(body, user_agent) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.post("https://api.openai.com/v1/responses/input_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 9}) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert response["input_tokens"] == 9 + sent: Final = json.loads(route.calls.last.request.content) + assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok"} in sent["input"] + + +def _build_log_messages(log_text: JsonValue) -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_log", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": log_text}], + } + ], + }, + ] + + +async def _count_on_a_model_without_a_count_api( + monkeypatch: pytest.MonkeyPatch, body: Mapping[str, object], router_defaults: dict[str, JsonValue] | None = None +) -> JSONResponse | dict[str, JsonValue]: + import litellm.proxy.proxy_server as proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "mistral-tool-result", + "litellm_params": {"model": "mistral/mistral-small-latest", "api_key": "sk-test"}, + } + ], + default_litellm_params=router_defaults, + ) + request: Final = _json_request(body, "python-httpx/0.28.1") + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + with respx.mock: + return await anthropic_count_tokens(request, UserAPIKeyAuth()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("log_text", (None, 5)) +async def test_count_tokens_without_a_count_api_rejects_a_non_string_tool_result_text( + monkeypatch: pytest.MonkeyPatch, log_text: JsonValue +) -> None: + response: Final = await _count_on_a_model_without_a_count_api( + monkeypatch, {"model": "mistral-tool-result", "messages": _build_log_messages(log_text)} + ) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert json.loads(bytes(response.body))["error"] == { + "type": "invalid_request_error", + "message": "tool_result toolu_log content.1.text: Input should be a valid string", + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("mistral-tool-result", "model-not-in-the-router")) +async def test_count_tokens_without_a_count_api_counts_a_non_string_tool_result_text_under_drop_params( + monkeypatch: pytest.MonkeyPatch, model: str +) -> None: + response: Final = await _count_on_a_model_without_a_count_api( + monkeypatch, {"model": model, "drop_params": True, "messages": _build_log_messages(None)} + ) + + assert isinstance(response, dict) + assert isinstance(response["input_tokens"], int) and response["input_tokens"] > 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("body_drop_params", "expected_status"), (({}, None), ({"drop_params": False}, 400))) +async def test_count_tokens_honors_router_default_drop_params_unless_the_body_overrides_it( + monkeypatch: pytest.MonkeyPatch, body_drop_params: dict[str, JsonValue], expected_status: int | None +) -> None: + response: Final = await _count_on_a_model_without_a_count_api( + monkeypatch, + {"model": "mistral-tool-result", "messages": _build_log_messages(None), **body_drop_params}, + router_defaults={"drop_params": True}, + ) + + match response: + case JSONResponse(): + assert response.status_code == expected_status + case _: + assert expected_status is None + assert isinstance(response["input_tokens"], int) and response["input_tokens"] > 0 + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_tool_result_text_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "claude-native-drop", + "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", + "api_key": "sk-ant-test", + "drop_params": True, + }, + } + ] + ) + request: Final = _json_request( + {"model": "claude-native-drop", "messages": _build_log_messages(5)}, "python-httpx/0.28.1" + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 7}) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert response["input_tokens"] == 7 + sent: Final = json.loads(route.calls.last.request.content) + assert sent["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "build ok"}] + + class _ScriptedCounter(BaseTokenCounter): def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None: self._replies = replies