diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 6287dc17e8f..3090281127b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -3045,6 +3045,19 @@ def merge_consecutive_system_messages( ] +_TOOL_USE_ID_FALLBACK: Final = "tool_use_id" +_ANTHROPIC_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]") + + +def replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Pattern[str]) -> str: + return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK + + +def sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: + """Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$.""" + return replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS) + + _DECODED_JSON: Final = TypeAdapter(object) @@ -3107,7 +3120,7 @@ def _attempt_json_repair(s: str) -> object | None: def parse_tool_call_arguments( - arguments: str | None, + arguments: str | Mapping[str, object] | None, tool_name: str | None = None, context: str | None = None, ) -> Any: @@ -3120,7 +3133,7 @@ def parse_tool_call_arguments( callers are aware the arguments were not perfectly formed. Args: - arguments: The JSON string containing tool arguments, or None. + arguments: The JSON string containing tool arguments, an already-parsed mapping, or None. tool_name: Optional name of the tool (for error messages). context: Optional context string (e.g., "Anthropic Messages API"). @@ -3134,6 +3147,8 @@ def parse_tool_call_arguments( """ import json + if isinstance(arguments, Mapping): + return dict(arguments) if not arguments or not arguments.strip(): return {} diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d6d753635f6..829cfe6bcc0 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1,4 +1,5 @@ import base64 +import codecs import copy import hashlib import itertools @@ -8,7 +9,9 @@ import re import xml.etree.ElementTree as ET from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass, replace +from email.message import Message from enum import Enum +from functools import cache, partial from types import MappingProxyType from typing import Annotated, Any, Final, TypeAlias, TypedDict, TypeVar, cast, overload @@ -24,15 +27,18 @@ from litellm._uuid import uuid from litellm.constants import REDACTED_BY_LITELLM from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( + answered_ids_by_index, anthropic_system_messages, find_anthropic_server_tool_result, is_anthropic_server_tool_use, is_anthropic_tool_search_use, message_field, + native_tool_use_id, parts_of, place_mid_conversation_system, split_leading_system_run, string_field, + tool_use_is_answered, ) from litellm.litellm_core_utils.tool_fields import tools_without_hosted_mcp from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get @@ -75,7 +81,9 @@ from .common_utils import ( is_unsignable_thinking_block, native_redacted_thinking_data, parse_tool_call_arguments, + replace_invalid_tool_use_id_chars, salvage_concatenated_tool_arguments, + sanitize_anthropic_tool_use_id, ) from .image_handling import convert_url_to_base64 @@ -875,6 +883,18 @@ def convert_generic_image_chunk_to_openai_image_obj( return "data:{};{},{}".format(media_type, image_chunk["type"], image_chunk["data"]) +_CHARSET_AWARE_MEDIA_TYPE: Final = "text/plain" + + +def _media_type_with_original_parameters(format_override: str, original_media_type: str) -> str: + if format_override.strip().lower() != _CHARSET_AWARE_MEDIA_TYPE: + return format_override + base_type, separator, parameters = original_media_type.partition(";") + if not separator or base_type.strip().lower() != format_override.strip().lower(): + return format_override + return f"{format_override}{separator}{parameters}" + + def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) -> GenericImageParsingChunk: """ Input: @@ -893,7 +913,7 @@ def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) -> # Extract the media type and base64 data media_type, base64_data = openai_image_url.split("data:")[1].split(";base64,") - if format: + if isinstance(format, str) and format: media_type = format else: media_type = media_type.replace("\\/", "/") @@ -935,8 +955,14 @@ def create_anthropic_image_param( # as these providers don't support URL sources for images if is_bedrock_invoke or image_url.startswith("http://"): base64_url: Final = convert_url_to_base64(url=image_url) - image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) - return _anthropic_content_element_factory(image_chunk) + return anthropic_content_element_factory( + convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) + ) + elif isinstance(format, str) and format.partition(";")[0].strip().lower() == "application/pdf": + return AnthropicMessagesDocumentParam( + type="document", + source=AnthropicContentParamSourceUrl(type="url", url=image_url), + ) else: # HTTPS URL - pass directly for regular Anthropic return AnthropicMessagesImageParam( @@ -948,8 +974,9 @@ def create_anthropic_image_param( ) else: # Convert to base64 for data URIs or other formats - image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format) - return _anthropic_content_element_factory(image_chunk) + return anthropic_content_element_factory( + convert_to_anthropic_image_obj(openai_image_url=image_url, format=format) + ) # The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex @@ -1776,29 +1803,18 @@ def convert_to_gemini_tool_call_result( return _part -_TOOL_USE_ID_FALLBACK: Final = "tool_use_id" -_ANTHROPIC_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]") _BEDROCK_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_.:-]") _BEDROCK_TOOL_USE_ID_MAX_LEN: Final = 64 _BEDROCK_TOOL_USE_ID_HASH_LEN: Final = 8 -def _replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Pattern[str]) -> str: - return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK - - -def sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: - """Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$.""" - return _replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS) - - def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str: """ Bedrock Converse requires toolUseId to match [a-zA-Z0-9_.:-]+ and be at most 64 chars. Ids that need rewriting get a short hash of the original appended so two ids that only differ in a replaced char or past the cut still map to distinct values. """ - sanitized: Final = _replace_invalid_tool_use_id_chars(tool_use_id, _BEDROCK_TOOL_USE_ID_INVALID_CHARS) + sanitized: Final = replace_invalid_tool_use_id_chars(tool_use_id, _BEDROCK_TOOL_USE_ID_INVALID_CHARS) if sanitized == tool_use_id and len(sanitized) <= _BEDROCK_TOOL_USE_ID_MAX_LEN: return sanitized digest: Final = hashlib.sha256(tool_use_id.encode()).hexdigest()[:_BEDROCK_TOOL_USE_ID_HASH_LEN] @@ -1808,15 +1824,124 @@ def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str: _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES: Final = {"application/pdf", "text/plain"} -def _is_anthropic_document_data_uri(url: str) -> bool: - # Anthropic's base64 document source accepts only application/pdf and - # text/plain (see select_anthropic_content_block_type_for_file). Routing - # other mimes here would produce a document block the API rejects, so we - # leave them on the image code path. +_PDF_SIGNATURE: Final = b"%PDF-" + + +def _declared_media_type_for_data_uri(url: str, format_override: str | None) -> str | None: + if format_override and select_anthropic_content_block_type_for_file(format_override) != "container_upload": + return format_override match: Final = re.match(r"data:([^;,]+)", url) - if not match: + return match.group(1) if match else None + + +def _base64_holds_a_pdf(data: str) -> bool: + try: + return base64.b64decode(data[:8]).startswith(_PDF_SIGNATURE) + except ValueError: return False - return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES + + +def _data_uri_holds_a_pdf(url: str) -> bool: + _, _, payload = url.partition(",") + return _base64_holds_a_pdf(payload) + + +def _data_uri_media_type(url: str) -> str: + header, separator, _ = url.partition(";base64,") + return header.removeprefix("data:") if separator and url.startswith("data:") else "" + + +def anthropic_document_media_type_for_data_uri(url: str, format_override: str | None = None) -> str | None: + if not url.startswith("data:"): + return None + if _data_uri_holds_a_pdf(url): + return "application/pdf" + declared: Final = (_declared_media_type_for_data_uri(url, format_override) or "").lower() + return declared if declared in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES else None + + +_TEXT_ATTACHMENT_ENCODINGS: Final = frozenset( + codecs.lookup(label).name + for label in ( + "utf-8", + "utf-16", + "utf-16-le", + "utf-16-be", + "ascii", + "latin-1", + *(f"iso8859-{part}" for part in (*range(2, 12), *range(13, 17))), + "tis-620", + *(f"cp{code_page}" for code_page in (866, 874, *range(1250, 1259))), + "koi8-r", + "koi8-u", + "mac-roman", + "mac-cyrillic", + "gb2312", + "gbk", + "gb18030", + "big5", + "big5hkscs", + "euc-jp", + "iso2022-jp", + "shift-jis", + "cp932", + "euc-kr", + ) +) + + +def _text_attachment_encoding(charset: str) -> str: + encoding: Final = codecs.lookup(charset).name + if encoding not in _TEXT_ATTACHMENT_ENCODINGS: + raise LookupError(f"unsupported encoding: {charset}") + return encoding + + +def _anthropic_text_document_source( + media_type_with_parameters: str, data: str +) -> AnthropicContentParamSourceText | None: + media_type, _, _ = media_type_with_parameters.partition(";") + if media_type.strip().lower() != "text/plain": + return None + content_type_header: Final = Message() + content_type_header["content-type"] = media_type_with_parameters + charset: Final = content_type_header.get_content_charset("utf-8") + try: + text: Final = base64.b64decode(data).decode(_text_attachment_encoding(charset)) + except (LookupError, UnicodeDecodeError) as decode_error: + raise litellm.BadRequestError( + message=f"Text attachment could not be read as {charset!r}: {decode_error}. " + "Send it as utf-8, or name its encoding (text/plain;charset=...) in the data URI " + "or in a file block's format.", + model=None, + llm_provider="anthropic", + ) from decode_error + return AnthropicContentParamSourceText(type="text", media_type="text/plain", data=text) + + +def anthropic_document_block_from_data_uri( + url: str, + original_content_element: Mapping[str, object] | AllMessageValues, + media_type: str, +) -> AnthropicMessagesDocumentParam: + synth_file_message: Final[ChatCompletionFileObject] = { + "type": "file", + "file": {"file_data": url, "format": media_type}, + } + document_block: Final = anthropic_process_openai_file_message(synth_file_message) + return cast( + AnthropicMessagesDocumentParam, + add_cache_control_to_content( + anthropic_content_element=cast(AnthropicMessagesDocumentParam, document_block), + original_content_element=original_content_element, + ), + ) + + +def requires_inline_base64_media(model: str, llm_provider: str | None) -> bool: + if model.lower().startswith("invoke/"): + return True + return llm_provider is not None and llm_provider.startswith("vertex_ai") def convert_to_anthropic_tool_result( @@ -1863,10 +1988,13 @@ def convert_to_anthropic_tool_result( | ToolReference ] ) = "" - if isinstance(message["content"], str): - anthropic_content = message["content"] - elif isinstance(message["content"], list): - content_list: Final = message["content"] + tool_content: Final = message.get("content") + if isinstance(tool_content, str): + anthropic_content = tool_content + elif isinstance(tool_content, (Mapping, int, float)): + anthropic_content = json.dumps(tool_content) + elif isinstance(tool_content, list): + content_list: Final = tool_content anthropic_content_list: list[ AnthropicMessagesToolResultContent | AnthropicMessagesImageParam @@ -1889,20 +2017,13 @@ def convert_to_anthropic_tool_result( image_url_value = content["image_url"] format = image_url_value.get("format") if isinstance(image_url_value, dict) else None url_str = image_url_value.get("url") if isinstance(image_url_value, dict) else image_url_value - # Data URIs with non-image mime types (e.g. application/pdf) must - # translate to Anthropic document blocks, not image blocks — - # wrapping a PDF in `type: "image"` is rejected by the API. - if isinstance(url_str, str) and _is_anthropic_document_data_uri(url_str): - synth_file_message: ChatCompletionFileObject = { - "type": "file", - "file": {"file_data": url_str}, - } - _document_block = anthropic_process_openai_file_message(synth_file_message) - _document_block = add_cache_control_to_content( - anthropic_content_element=cast(AnthropicMessagesDocumentParam, _document_block), - original_content_element=content, + if ( + isinstance(url_str, str) + and (document_media_type := anthropic_document_media_type_for_data_uri(url_str, format)) is not None + ): + anthropic_content_list.append( + anthropic_document_block_from_data_uri(url_str, content, document_media_type) ) - anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _document_block)) else: _anthropic_image_param = create_anthropic_image_param( image_url_value, @@ -2129,6 +2250,7 @@ def convert_to_anthropic_tool_invoke( web_search_results: Sequence[object] | None = None, tool_results: Sequence[object] | None = None, answered_tool_ids: Container[str] = frozenset(), + drop_params: bool = False, ) -> list[_AnthropicToolInvokeItem]: """ OpenAI tool invokes: @@ -2183,10 +2305,8 @@ def convert_to_anthropic_tool_invoke( str, get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"), ) - tool_input = parse_tool_call_arguments( - get_attribute_or_key(get_attribute_or_key(tool, "function"), "arguments"), - tool_name=tool_name, - context="Anthropic tool invoke", + tool_input = _tool_call_input( + get_attribute_or_key(get_attribute_or_key(tool, "function"), "arguments"), tool_name, drop_params ) server_tool_result = ( @@ -2244,7 +2364,7 @@ def add_cache_control_to_content( | AnthropicMessagesDocumentParam | AnthropicMessagesToolUseParam | ChatCompletionThinkingBlock, - original_content_element: dict | AllMessageValues | ChatCompletionFileObject, + original_content_element: Mapping[str, object] | AllMessageValues | ChatCompletionFileObject, ): cache_control_param: Final = original_content_element.get("cache_control") if cache_control_param is not None and isinstance(cache_control_param, dict): @@ -2255,16 +2375,91 @@ def add_cache_control_to_content( return anthropic_content_element -def _anthropic_content_element_factory( +_TOOL_USE_INPUT: Final = TypeAdapter(dict[str, object]) + + +def _object_unless_dropped(value: object, drop_params: bool) -> object: + if not drop_params: + return value + try: + return _TOOL_USE_INPUT.validate_python(value) + except ValidationError: + return {} + + +def _tool_call_input(arguments: str | Mapping[str, object] | None, tool_name: str, drop_params: bool) -> object: + try: + return _object_unless_dropped( + parse_tool_call_arguments(arguments, tool_name=tool_name, context="Anthropic tool invoke"), drop_params + ) + except ValueError: + if drop_params: + return {} + raise + + +def _tool_use_input_object(value: object, tool_name: str) -> dict[str, object]: + try: + return _TOOL_USE_INPUT.validate_python(value) + except ValidationError as error: + raise ValueError( + f"Tool input for tool '{tool_name}' (Anthropic assistant tool_use block) must be a JSON object, " + f"got {type(value).__name__}. Pass drop_params=True to send an empty object in its place" + ) from error + + +def _parsed_native_tool_use_input(raw: object, tool_name: str) -> object: + if not isinstance(raw, str): + return raw + return parse_tool_call_arguments( + "{}" if raw == REDACTED_BY_LITELLM else raw, + tool_name=tool_name, + context="Anthropic assistant tool_use block", + ) + + +def _native_tool_use_input(raw: object, tool_name: str, drop_params: bool) -> dict[str, object]: + try: + return _tool_use_input_object(_parsed_native_tool_use_input(raw, tool_name), tool_name) + except ValueError: + if drop_params: + return {} + raise + + +def _tool_call_ids(message: Mapping[str, object]) -> frozenset[str]: + return frozenset(native_tool_use_id(tool_call) for tool_call in parts_of(message.get("tool_calls"))) + + +def _anthropic_native_tool_use_block( + block: Mapping[str, object], tool_use_id: str, drop_params: bool +) -> AnthropicMessagesToolUseParam: + tool_name: Final = str(block.get("name", "")) + tool_use_param: Final = AnthropicMessagesToolUseParam( + type="tool_use", + id=tool_use_id, + name=tool_name, + input=_native_tool_use_input(block.get("input"), tool_name, drop_params), + ) + add_cache_control_to_content( + anthropic_content_element=tool_use_param, + original_content_element=dict(block), + ) + return tool_use_param + + +def anthropic_content_element_factory( image_chunk: GenericImageParsingChunk, ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: - if image_chunk["media_type"] == "application/pdf": + if image_chunk["media_type"].partition(";")[0].strip().lower() == "application/pdf" or _base64_holds_a_pdf( + image_chunk["data"] + ): _anthropic_content_element: AnthropicMessagesDocumentParam | AnthropicMessagesImageParam = ( AnthropicMessagesDocumentParam( type="document", source=AnthropicContentParamSource( type="base64", - media_type=image_chunk["media_type"], + media_type="application/pdf", data=image_chunk["data"], ), ) @@ -2283,11 +2478,14 @@ def _anthropic_content_element_factory( def select_anthropic_content_block_type_for_file( - format: str, + format: object, ) -> Literal["document", "image", "container_upload"]: - if format == "application/pdf" or format == "text/plain": + if not isinstance(format, str): + return "container_upload" + media_type: Final = format.strip().lower() + if media_type == "application/pdf" or media_type == "text/plain": return "document" - elif format in ["image/jpeg", "image/png", "image/gif", "image/webp"]: + elif media_type in ["image/jpeg", "image/png", "image/gif", "image/webp"]: return "image" else: return "container_upload" @@ -2327,11 +2525,27 @@ def anthropic_process_openai_file_message( openai_image_url=file_data, format=format, ) + if _base64_holds_a_pdf(image_chunk["data"]): + return AnthropicMessagesDocumentParam( + type="document", + source=AnthropicContentParamSource( + type="base64", + media_type="application/pdf", + data=image_chunk["data"], + ), + ) + text_source: Final = _anthropic_text_document_source( + _media_type_with_original_parameters(image_chunk["media_type"], _data_uri_media_type(file_data)), + image_chunk["data"], + ) + if text_source is not None: + return AnthropicMessagesDocumentParam(type="document", source=text_source) + media_type, _, _ = image_chunk["media_type"].partition(";") anthropic_document_param: Final = AnthropicMessagesDocumentParam( type="document", source=AnthropicContentParamSource( type="base64", - media_type=image_chunk["media_type"], + media_type=media_type.strip().lower(), data=image_chunk["data"], ), ) @@ -2404,13 +2618,14 @@ def _with_placeholder_text(block: object) -> object: return {**fields, "text": _EMPTY_TEXT_PLACEHOLDER} -def _survives_assistant_conversion(block: object) -> bool: +def _survives_assistant_conversion(block: object, answered: Callable[[], frozenset[str]]) -> bool: """Whether ``anthropic_messages_pt`` keeps this list-content block of an assistant turn. Mirrors the assistant converter: a ``thinking`` / ``redacted_thinking`` block is kept - unless it is unsignable (a signed block is kept even with empty text), and - ``server_tool_use`` and ``*_tool_result`` blocks pass through. A redacted block with - blank data is dropped under ``drop_params``, which is not known here, so it never counts. + unless it is unsignable (a signed block is kept even with empty text), + ``server_tool_use`` and ``*_tool_result`` blocks pass through, and a native ``tool_use`` + is kept when the next turn answers it. A redacted block with blank data is dropped under + ``drop_params``, which is not known here, so it never counts. """ fields: Final = _dict_block(block) if fields is None: @@ -2420,6 +2635,8 @@ def _survives_assistant_conversion(block: object) -> bool: match fields: case {"type": "server_tool_use"}: return True + case {"type": "tool_use"}: + return tool_use_is_answered(native_tool_use_id(fields), answered()) case {"type": str() as block_type}: return block_type.endswith("_tool_result") case _: @@ -2441,12 +2658,16 @@ def _tool_call_survives_conversion(tool_call: object) -> bool: return True -def _carries_non_text_content(message: AllMessageValues) -> bool: +def _carries_non_text_content(message: AllMessageValues, answered: Callable[[], frozenset[str]]) -> bool: if any(_tool_call_survives_conversion(tool_call) for tool_call in parts_of(message_field(message, "tool_calls"))): return True content: Final = message_field(message, "content") blocks: Final = parts_of(content) - return isinstance(content, list) and any(_survives_assistant_conversion(block) for block in blocks) + return isinstance(content, list) and any(_survives_assistant_conversion(block, answered) for block in blocks) + + +def _answered_at(answered_by_index: Callable[[], Mapping[int, frozenset[str]]], index: int) -> frozenset[str]: + return answered_by_index().get(index, frozenset()) def _with_content(message: AllMessageValues, content: object, action: str) -> AllMessageValues: @@ -2457,6 +2678,7 @@ def _with_content(message: AllMessageValues, content: object, action: str) -> Al def _sanitize_empty_text_content( message: AllMessageValues, + answered: Callable[[], frozenset[str]], ) -> AllMessageValues: """ Case C: Sanitize empty text content @@ -2464,8 +2686,8 @@ def _sanitize_empty_text_content( Anthropic rejects empty text blocks, so empty or whitespace-only text must never be forwarded verbatim. What replaces it depends on what else the turn carries. - Assistant turns that already carry substantive content (``tool_calls``, or non-text - blocks such as ``server_tool_use``) get the empty text dropped. Anthropic accepts an + Assistant turns that already carry substantive content (``tool_calls``, an answered + ``tool_use``, or other non-text blocks such as ``server_tool_use``) get the empty text dropped. Anthropic accepts an assistant turn whose content is just ``tool_use``, so there is nothing to substitute, and substituting anyway leaks a protocol diagnostic into the conversation as though the assistant had said it, which the model then echoes on later turns. @@ -2480,7 +2702,9 @@ def _sanitize_empty_text_content( return message content: Final = message_field(message, "content") - drop_empty_text: Final = message_field(message, "role") == "assistant" and _carries_non_text_content(message) + drop_empty_text: Final = message_field(message, "role") == "assistant" and _carries_non_text_content( + message, answered + ) if isinstance(content, str): if content.strip(): @@ -2641,8 +2865,13 @@ def _tool_call_ids_of_the_latest_assistant( if message.get("role") != "assistant": return latest_ids tool_calls: Final = message.get("tool_calls") or () + native_tool_uses: Final = ( + part for part in parts_of(message.get("content")) if message_field(part, "type") == "tool_use" + ) return frozenset( - tool_call_id for tool_call_id in map(_tool_call_id, cast(Iterable[object], tool_calls)) if tool_call_id + tool_call_id + for tool_call_id in map(_tool_call_id, (*cast(Iterable[object], tool_calls), *native_tool_uses)) + if tool_call_id ) @@ -2761,6 +2990,7 @@ def _pair_tool_calls_with_results(messages: Sequence[AllMessageValues]) -> tuple latest_assistant_tool_call_ids: Final = tuple( itertools.accumulate(messages, _tool_call_ids_of_the_latest_assistant, initial=frozenset[object]()) ) + answered_by_index: Final = cache(partial(answered_ids_by_index, messages)) sanitized_messages: list[AllMessageValues] = [] i = 0 @@ -2768,7 +2998,7 @@ def _pair_tool_calls_with_results(messages: Sequence[AllMessageValues]) -> tuple current_message = messages[i] # Case C: Sanitize empty text content - current_message = _sanitize_empty_text_content(current_message) + current_message = _sanitize_empty_text_content(current_message, partial(_answered_at, answered_by_index, i)) # Case A: Check if assistant message has tool_calls without following tool results if current_message.get("role") == "assistant": @@ -2978,6 +3208,7 @@ def anthropic_messages_pt( model: str, llm_provider: str, drop_params: bool = False, + force_base64: bool = False, ) -> _AnthropicMessageList: """ format messages for anthropic @@ -3003,7 +3234,11 @@ def anthropic_messages_pt( # request will always 400 otherwise. The richer tool-call sanitization # (Cases A/B/D in `sanitize_messages_for_tool_calling`) remains gated on # `modify_params` because it actually mutates conversation structure. - messages = [_sanitize_empty_text_content(m) for m in messages] + answered_by_assistant_index: Final = cache(partial(answered_ids_by_index, messages)) + messages = [ + _sanitize_empty_text_content(m, partial(_answered_at, answered_by_assistant_index, i)) + for i, m in enumerate(messages) + ] replaced_server_tool_answers: Final = server_tool_answers_replaced_by_their_result(messages) client_answered_tool_ids: Final = _tool_call_ids_in_tool_messages(messages) - replaced_server_tool_answers @@ -3022,11 +3257,7 @@ def anthropic_messages_pt( else: messages.append(DEFAULT_USER_CONTINUE_MESSAGE_TYPED) - # Bedrock invoke models have format: invoke/... - # Vertex AI Anthropic also doesn't support URL sources for images - is_bedrock_invoke = model.lower().startswith("invoke/") - is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False - force_base64 = is_bedrock_invoke or is_vertex_ai + inline_base64_media: Final = force_base64 or requires_inline_base64_media(model, llm_provider) tool_result_later_in_turn: Final = _tool_result_later_in_turn(messages, replaced_server_tool_answers) msg_i = 0 @@ -3049,26 +3280,27 @@ def anthropic_messages_pt( for m in user_block_content: if m.get("type", "") == "image_url": m = cast(ChatCompletionImageObject, m) - format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None - # Convert ChatCompletionImageUrlObject to dict if needed image_url_value = m["image_url"] + url_str = image_url_value if isinstance(image_url_value, str) else image_url_value["url"] + format = image_url_value.get("format") if isinstance(image_url_value, dict) else None + if ( + document_media_type := anthropic_document_media_type_for_data_uri(url_str, format) + ) is not None: + user_content.append( + anthropic_document_block_from_data_uri(url_str, dict(m), document_media_type) + ) + continue if isinstance(image_url_value, str): image_url_input: str | dict[str, object] = image_url_value else: - # ChatCompletionImageUrlObject or dict case - convert to dict image_url_input = { "url": image_url_value["url"], "format": image_url_value.get("format"), } - # Bedrock invoke models have format: invoke/... - # Vertex AI Anthropic also doesn't support URL sources for images - is_bedrock_invoke = model.lower().startswith("invoke/") - is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False - force_base64 = is_bedrock_invoke or is_vertex_ai _anthropic_content_element = create_anthropic_image_param( image_url_input, format=format, - is_bedrock_invoke=force_base64, + is_bedrock_invoke=inline_base64_media, ) _content_element = add_cache_control_to_content( anthropic_content_element=_anthropic_content_element, @@ -3137,7 +3369,7 @@ def anthropic_messages_pt( ) and message_field(user_message_types_block, "tool_call_id") not in replaced_server_tool_answers: # OpenAI's tool message content will always be a string user_content.append( - convert_to_anthropic_tool_result(user_message_types_block, force_base64=force_base64) + convert_to_anthropic_tool_result(user_message_types_block, force_base64=inline_base64_media) ) msg_i += 1 @@ -3198,6 +3430,7 @@ def anthropic_messages_pt( web_search_results=_provider_specific_fields.get("web_search_results"), tool_results=_provider_specific_fields.get("tool_results"), answered_tool_ids=client_answered_tool_ids, + drop_params=drop_params, ) if assistant_tool_calls is not None else () @@ -3323,9 +3556,11 @@ def anthropic_messages_pt( if not _content_is_list: # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR assistant_content.extend(replayed_thinking_blocks) if _content_is_list and _content_list is not None: + tool_call_ids = _tool_call_ids(assistant_content_block) for m in _with_missing_thinking_blocks(_content_list, replayed_thinking_blocks): if not isinstance(m, dict): continue + block_type: str = str(m.get("type", "")) # handle thinking blocks if (thinking_block := _as_thinking_block(m)) is not None: assistant_content.extend( @@ -3337,7 +3572,7 @@ def anthropic_messages_pt( text_block = cast(str, m.get("text", "")) # handle text if ( - m.get("type", "") == "text" and len(text_block) > 0 + block_type == "text" and len(text_block) > 0 ): # don't pass empty text blocks. anthropic api raises errors. anthropic_message = AnthropicMessagesTextParam(type="text", text=text_block) _cached_message = add_cache_control_to_content( @@ -3348,8 +3583,20 @@ def anthropic_messages_pt( assistant_content.append(cast(AnthropicMessagesTextParam, _cached_message)) # handle server_tool_use blocks (tool search, web search, etc.) # Pass through as-is since these are Anthropic-native content types - elif m.get("type", "") == "server_tool_use" or m.get("type", "").endswith("_tool_result"): + elif block_type == "server_tool_use" or block_type.endswith("_tool_result"): assistant_content.append(m) + elif block_type == "tool_use": + tool_use_id = native_tool_use_id(m) + if ( + tool_use_id in unique_tool_ids + or tool_use_id in tool_call_ids + or not tool_use_is_answered( + tool_use_id, answered_by_assistant_index().get(msg_i, frozenset()) + ) + ): + continue + unique_tool_ids.add(tool_use_id) + assistant_content.append(_anthropic_native_tool_use_block(m, tool_use_id, drop_params)) elif ( "content" in assistant_content_block and isinstance(assistant_content_block["content"], str) @@ -3978,8 +4225,6 @@ def stringify_json_tool_call_content(messages: list) -> list: ###### AMAZON BEDROCK ####### -from email.message import Message - import httpx from litellm.types.llms.bedrock import ( diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py index ec6de4e206d..1d6d4276ad2 100644 --- a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py @@ -49,7 +49,7 @@ from litellm.types.llms.openai import ( ChatCompletionUserMessage, ) -from .common_utils import has_blank_redacted_data, is_unsignable_thinking_block +from .common_utils import has_blank_redacted_data, is_unsignable_thinking_block, sanitize_anthropic_tool_use_id CONVERTED_SYSTEM_NOTE: Final = ( "Operator note (not from the user): the following was originally a mid-conversation system-role reminder." @@ -277,11 +277,25 @@ def _thinking_block_renders(block: object) -> bool: ) -def _assistant_part_renders(part: object) -> bool: - """A text part always renders: the converter pads empty text with a placeholder.""" +def native_tool_use_id(block: object) -> str: + raw_id: Final = message_field(block, "id") + return sanitize_anthropic_tool_use_id(raw_id if isinstance(raw_id, str) else "") + + +def tool_use_is_answered(tool_use_id: str, answered: frozenset[str]) -> bool: + return tool_use_id in answered + + +def _assistant_part_renders(part: object, answered: frozenset[str]) -> bool: + """A text part always renders: the converter pads empty text with a placeholder. + + A native ``tool_use`` renders only when the next turn answers it, since the converter drops an orphan. + """ part_type: Final = message_field(part, "type") if part_type in _THINKING_BLOCK_TYPES: return _thinking_block_renders(part) + if part_type == "tool_use": + return tool_use_is_answered(native_tool_use_id(part), answered) return part_type in _RENDERED_ASSISTANT_PART_TYPES or ( isinstance(part_type, str) and part_type.endswith("_tool_result") ) @@ -317,6 +331,7 @@ def _tool_result_ids(answers: Iterable[object]) -> Iterator[str]: for message in answers: if isinstance(tool_call_id := message_field(message, "tool_call_id"), str) and tool_call_id: yield tool_call_id + yield sanitize_anthropic_tool_use_id(tool_call_id) def answered_tool_call_ids(messages: Sequence[object], assistant_index: int) -> frozenset[str]: @@ -343,7 +358,7 @@ def _answered_by_assistant_index( return ((index, answered) for index, _ in groups[group_index][1]) -def _answered_ids_by_index(messages: Sequence[object]) -> Mapping[int, frozenset[str]]: +def answered_ids_by_index(messages: Sequence[object]) -> Mapping[int, frozenset[str]]: """``answered_tool_call_ids`` for every assistant message, computed in one pass over the messages.""" groups: Final = tuple( (role_group, tuple(group)) @@ -407,7 +422,7 @@ def _assistant_renders(message: object, answered: frozenset[str]) -> bool: return True parts: Final = parts_of(content) return ( - any(_assistant_part_renders(part) for part in parts) + any(_assistant_part_renders(part, answered) for part in parts) or _separate_thinking_blocks_render(message) or _tool_calls_render(message, answered) or bool(message_field(message, "function_call")) @@ -438,7 +453,7 @@ def _renders(message: object, answered: frozenset[str]) -> bool: def _rendered_indices(messages: Sequence[object]) -> frozenset[int]: - answered: Final = _answered_ids_by_index(messages) + answered: Final = answered_ids_by_index(messages) return frozenset(index for index, message in enumerate(messages) if _renders(message, answered.get(index, _NO_IDS))) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 2e908e66626..d149cc5c89c 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2481,6 +2481,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): llm_provider=self._resolved_provider, drop_params=drop_params, ) + except litellm.BadRequestError as e: + raise non_string_message_field_error(messages, e) or AnthropicError(status_code=400, message=str(e)) except Exception as e: conversion_field_error: Final = non_string_message_field_error(messages, e) raise conversion_field_error or AnthropicError( diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index 7a7d1d776d3..166896d52e4 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -118,6 +118,8 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): except AnthropicError: # Re-raise Anthropic exceptions as-is raise + except litellm.BadRequestError: + raise except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code verbose_logger.error("HTTP error in CountTokens handler: %s", e) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 99b1cd7f241..ddf5e797e20 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -144,6 +144,17 @@ class AnthropicTokenCounter(BaseTokenCounter): tokenizer_type="anthropic_api", original_response=result, ) + except litellm.BadRequestError as e: + return TokenCountResponse( + total_tokens=0, + request_model=request_model, + model_used=model_to_use, + tokenizer_type="anthropic_api", + error=True, + error_message=e.message, + status_code=400, + rejected_before_provider_call=True, + ) except (AnthropicError, AuthenticationError) as e: verbose_logger.warning("Anthropic CountTokens error: status=%s, message=%s", e.status_code, e.message) return TokenCountResponse( diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 1c8f0c206cd..8d61287eeb8 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -49,16 +49,25 @@ _PLAIN_MESSAGE_KEYS: Final = frozenset({"role", "content"}) def _count_image(block: JsonValue) -> JsonValue: if not isinstance(block, dict) or block.get("type") != "image_url": return block - from litellm.litellm_core_utils.prompt_templates.factory import convert_to_anthropic_image_obj + from litellm.litellm_core_utils.prompt_templates.factory import ( + anthropic_content_element_factory, + anthropic_document_block_from_data_uri, + anthropic_document_media_type_for_data_uri, + convert_to_anthropic_image_obj, + ) image_block: Final = _IMAGE_BLOCK.validate_python(block) image_url: Final = image_block["image_url"] - source: Final = convert_to_anthropic_image_obj( - openai_image_url=image_url if isinstance(image_url, str) else image_url["url"], - format=image_url.get("format") if isinstance(image_url, dict) else None, - ) - image: Final = _COUNT_REQUEST.validate_python({"type": "image", "source": source}) - return {**{key: value for key, value in block.items() if key not in {"type", "image_url"}}, **image} + url: Final = image_url if isinstance(image_url, str) else image_url["url"] + format_override: Final = image_url.get("format") if isinstance(image_url, dict) else None + other_fields: Final = {key: value for key, value in block.items() if key not in {"type", "image_url"}} + document_media_type: Final = anthropic_document_media_type_for_data_uri(url, format_override) + if document_media_type is not None: + document: Final = anthropic_document_block_from_data_uri(url, image_block, document_media_type) + return {**other_fields, **_COUNT_REQUEST.validate_python(document)} + source: Final = convert_to_anthropic_image_obj(openai_image_url=url, format=format_override) + media_block: Final = anthropic_content_element_factory(source) + return {**other_fields, **_COUNT_REQUEST.validate_python(media_block)} def _count_block(block: JsonValue) -> JsonValue: diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 97ee44bd3bb..a0c4d767692 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -135,6 +135,8 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): except AnthropicError: # Re-raise Anthropic exceptions as-is raise + except litellm.BadRequestError: + raise except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code verbose_logger.error("HTTP error in CountTokens handler: %s", e) diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index 14f7d4bfd2b..58063fcdc34 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -148,6 +148,17 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): "original_response": result, } ) + except litellm.BadRequestError as e: + return TokenCountResponse( + total_tokens=0, + request_model=request_model, + model_used=model_to_use, + tokenizer_type="azure_ai_anthropic_api", + error=True, + error_message=e.message, + status_code=400, + rejected_before_provider_call=True, + ) except AnthropicError as e: verbose_logger.warning( "Azure AI Anthropic CountTokens API error: status=%s, message=%s", e.status_code, e.message diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index dec099f37c0..e22f2df4f03 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -54,6 +54,7 @@ class CountTokensFailure: status_code: int message: str tokenizer_type: str + rejected_before_provider_call: bool = False CountTokensOutcome = CountedTokens | CountTokensFailure @@ -162,6 +163,10 @@ class BedrockTokenCounter(BaseTokenCounter): client=self._client, ) reply: Final = _CountTokensReply.model_validate(result) + except litellm.BadRequestError as e: + return CountTokensFailure( + status_code=400, message=e.message, tokenizer_type=tokenizer_type, rejected_before_provider_call=True + ) except BedrockError as e: verbose_logger.debug( "%s CountTokens API error: status=%s, message=%s", tokenizer_type, e.status_code, e.message @@ -268,6 +273,7 @@ class BedrockTokenCounter(BaseTokenCounter): error=True, error_message=outcome.message, status_code=outcome.status_code, + rejected_before_provider_call=outcome.rejected_before_provider_call, ) case _: assert_never(outcome) diff --git a/litellm/llms/bedrock/count_tokens/mantle_handler.py b/litellm/llms/bedrock/count_tokens/mantle_handler.py index 5e5c5516b94..2a6d4d53cb2 100644 --- a/litellm/llms/bedrock/count_tokens/mantle_handler.py +++ b/litellm/llms/bedrock/count_tokens/mantle_handler.py @@ -106,7 +106,7 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM): response=response, ) return _COUNT_RESPONSE.validate_json(response.content) - except BedrockError: + except (BedrockError, litellm.BadRequestError): raise except httpx.HTTPStatusError as e: raise BedrockError( diff --git a/litellm/llms/snowflake/chat/transformation.py b/litellm/llms/snowflake/chat/transformation.py index 928dcc3bc52..a300bd9d161 100644 --- a/litellm/llms/snowflake/chat/transformation.py +++ b/litellm/llms/snowflake/chat/transformation.py @@ -8,28 +8,24 @@ Routes to native Cortex REST API endpoints based on model: Ref: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api """ -import copy -import json -import re from collections.abc import Mapping, Sequence +from itertools import chain from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict import httpx from typing_extensions import ReadOnly -from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.litellm_core_utils.prompt_templates.factory import ( - anthropic_process_openai_file_message, - convert_to_anthropic_tool_result, - create_anthropic_image_param, - select_anthropic_content_block_type_for_file, -) +import litellm +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt from litellm.litellm_core_utils.prompt_templates.image_handling import async_inline_remote_media +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.chat.handler import ModelResponseIterator as AnthropicStreamParser from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolMessage +from litellm.types.llms.anthropic import AllAnthropicPassThroughMessageValues, AnthropicSystemMessageContent +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk from litellm.types.utils import ( Choices, GenericStreamingChunk, @@ -40,7 +36,7 @@ from litellm.types.utils import ( from ...base_llm.base_model_iterator import BaseModelResponseIterator from ...openai_like.chat.transformation import OpenAIGPTConfig -from ..utils import SnowflakeBaseConfig +from ..utils import SnowflakeBaseConfig, SnowflakeException if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -104,115 +100,21 @@ def is_snowflake_claude_model(model: str) -> bool: return any(name.startswith(p) for p in _CLAUDE_MODEL_PREFIXES) -def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object: - """One OpenAI ``image_url`` block in the native shape Cortex accepts. - - Cortex documents base64 sources only, so remote URLs are inlined the way every - other base64-only Anthropic dialect (Bedrock invoke, Vertex) inlines them, and - pdf/text data URIs become document blocks rather than malformed image blocks. - """ - image_url: Final = block.get("image_url") - url: Final = image_url if isinstance(image_url, str) else _str_field(image_url, "url") - if not url: - return block - - converted: Final = ( - anthropic_process_openai_file_message({"type": "file", "file": {"file_data": url}}) - if select_anthropic_content_block_type_for_file(_data_uri_media_type(url)) == "document" - else create_anthropic_image_param( - image_url if isinstance(image_url, dict) else url, - format=_str_field(image_url, "format"), - is_bedrock_invoke=True, - ) - ) - return _with_cache_control(converted, block) +_CORTEX_CLAUDE_USER_PART_TYPES: Final = frozenset({"text", "image_url", "document", "file"}) -def _convert_file_to_anthropic(block: Mapping[str, object]) -> object: - file_data: Final = _str_field(block.get("file"), "file_data") - if not file_data: - return block - return _with_cache_control( - anthropic_process_openai_file_message({"type": "file", "file": {"file_data": file_data}}), block - ) +def _user_parts(message: AllMessageValues) -> tuple[object, ...]: + return parts_of(message.get("content")) if message["role"] == "user" else () -def _with_cache_control(converted: Mapping[str, object], block: Mapping[str, object]) -> Mapping[str, object]: - cache_control: Final = block.get("cache_control") - if cache_control is None: - return converted - return {**converted, "cache_control": cache_control} +def _part_type(part: object) -> str: + part_type: Final = message_field(part, "type") + return f"untyped {type(part).__name__}" if part_type is None else str(part_type) -def _str_field(container: object, key: str) -> str | None: - value: Final = container.get(key) if isinstance(container, dict) else None - return value if isinstance(value, str) else None - - -def _data_uri_media_type(url: str) -> str: - match: Final = re.match(r"data:([^;,]+)", url) - return match.group(1) if match else "" - - -def _convert_media_block_to_anthropic(block: object) -> object: - mapping: Final = as_str_mapping(block) - if mapping is None: - return block - match mapping.get("type"): - case "image_url": - return _convert_image_url_to_anthropic(mapping) - case "file": - return _convert_file_to_anthropic(mapping) - case _: - return block - - -def _convert_media_blocks_to_anthropic(content: object) -> object: - if not isinstance(content, list): - return content - return [_convert_media_block_to_anthropic(block) for block in content] - - -def _convert_tool_result_to_anthropic( - content: object, tool_call_id: str, cache_control: object -) -> Mapping[str, object]: - """The Anthropic ``tool_result`` block for one OpenAI tool message. - - Delegating to the shared converter keeps image, document and per-block cache - breakpoints identical to every other Anthropic dialect; only the plain-string - and non-list shapes it does not model are handled here. - """ - if not isinstance(content, list): - plain: Final[dict[str, object]] = { # mutable-ok: JSON wire block - "type": "tool_result", - "tool_use_id": tool_call_id, - "content": content if isinstance(content, str) else json.dumps(content), - } - return {**plain, "cache_control": cache_control} if cache_control is not None else plain - converted: Final = convert_to_anthropic_tool_result( - ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content), - force_base64=True, - ) - if cache_control is None: - return converted - return {**converted, "cache_control": cache_control} - - -def _signed_thinking_blocks(msg: object) -> list[dict[str, object]]: # mutable-ok: JSON wire blocks - """The assistant turn's thinking blocks that can legally be echoed back. - - Only signed blocks round-trip: Cortex rejects a thinking block whose signature is - missing, which is what an unsigned block from a non-thinking turn would produce. - """ - blocks: Final = msg.get("thinking_blocks") if isinstance(msg, dict) else getattr(msg, "thinking_blocks", None) - if not isinstance(blocks, list): - return [] - mappings: Final = tuple(filter(None, map(as_str_mapping, blocks))) - return [ - {key: value for key, value in block.items() if key != "index"} - for block in mappings - if block.get("signature") or block.get("type") == "redacted_thinking" - ] +def _untranslatable_user_part_types(messages: Sequence[AllMessageValues]) -> frozenset[str]: + part_types: Final = frozenset(map(_part_type, chain.from_iterable(map(_user_parts, messages)))) + return part_types - _CORTEX_CLAUDE_USER_PART_TYPES def _clean_input_schema(schema: object) -> object: @@ -319,105 +221,34 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): ) return anthropic_tools - def _extract_system_and_messages(self, messages: list[AllMessageValues]) -> tuple[list[dict] | None, list[dict]]: - """ - Split messages into system prompt and conversation turns for Anthropic format. - - - system messages → collected and joined (preserves guardrail prompts) - - assistant messages with tool_calls → tool_use content blocks - - tool role messages → user role with tool_result content blocks - """ - system_parts: Final[list[dict]] = [] # mutable-ok: JSON wire messages - conversation: Final[list[dict]] = [] # mutable-ok: JSON wire messages - - for msg in messages: - if isinstance(msg, dict): - role = msg.get("role", "") - content: object = msg.get("content", "") - msg_cache_control: object = msg.get("cache_control") - else: - role = getattr(msg, "role", "") - content = getattr(msg, "content", "") - msg_cache_control = getattr(msg, "cache_control", None) - - if role == "system": - if isinstance(content, str) and content: - system_parts.append({"type": "text", "text": content}) - elif isinstance(content, list): - system_parts.extend( - { - "type": "text", - "text": block.get("text", ""), - **({"cache_control": block["cache_control"]} if "cache_control" in block else {}), - } - for block in content - if isinstance(block, Mapping) and block.get("type") == "text" - ) - elif role == "assistant": - tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None) - thinking_blocks = _signed_thinking_blocks(msg) - if tool_calls: - content_blocks: list[dict[str, object]] = list(thinking_blocks) # mutable-ok: JSON wire blocks - if content: - content_blocks.append({"type": "text", "text": content}) - for tc in tool_calls: - func = tc.get("function", {}) if isinstance(tc, dict) else getattr(tc, "function", {}) - tc_id = tc.get("id", "") if isinstance(tc, dict) else getattr(tc, "id", "") - func_name = func.get("name", "") if isinstance(func, dict) else getattr(func, "name", "") - func_args = ( - func.get("arguments", "{}") if isinstance(func, dict) else getattr(func, "arguments", "{}") - ) - try: - input_data = json.loads(func_args) if isinstance(func_args, str) else func_args - except (json.JSONDecodeError, TypeError): - input_data = {} - content_blocks.append( - { - "type": "tool_use", - "id": tc_id, - "name": func_name, - "input": input_data, - } - ) - conversation.append({"role": "assistant", "content": content_blocks}) - elif thinking_blocks: - thinking_content = ( - [ - *thinking_blocks, - *copy.deepcopy(content), - ] - if isinstance(content, list) - else [*thinking_blocks, *([{"type": "text", "text": content}] if content else [])] - ) - conversation.append({"role": "assistant", "content": thinking_content}) - else: - conversation.append({"role": "assistant", "content": content}) - elif role == "tool": - tool_call_id_value = ( - msg.get("tool_call_id", "") if isinstance(msg, dict) else getattr(msg, "tool_call_id", "") - ) - tool_call_id = tool_call_id_value if isinstance(tool_call_id_value, str) else "" - tool_result_block = _convert_tool_result_to_anthropic(content, tool_call_id, msg_cache_control) - if ( - conversation - and conversation[-1]["role"] == "user" - and isinstance(conversation[-1]["content"], list) - and conversation[-1]["content"] - and conversation[-1]["content"][0].get("type") == "tool_result" - ): - conversation[-1]["content"].append(tool_result_block) - else: - conversation.append({"role": "user", "content": [tool_result_block]}) - else: - conversation.append( - { - "role": role, - "content": _convert_media_blocks_to_anthropic(content), - } - ) - - system: Final[list[dict] | None] = system_parts if system_parts else None # mutable-ok: JSON wire messages - return system, conversation + def _extract_system_and_messages( + self, model: str, messages: list[AllMessageValues], drop_params: bool + ) -> tuple[list[AnthropicSystemMessageContent] | None, list[AllAnthropicPassThroughMessageValues]]: + """Cortex's /messages endpoint speaks Anthropic's dialect, so the shared translators own the shape.""" + conversation_messages: Final = list(messages) + untranslatable: Final = _untranslatable_user_part_types(conversation_messages) + if untranslatable and not drop_params: + raise SnowflakeException( + status_code=400, + message=f"Snowflake Cortex Claude cannot take {', '.join(sorted(untranslatable))} parts in a user " + "message. Remove them, or pass drop_params=True to drop them", + ) + system: Final = [ + part + for part in AnthropicConfig().translate_system_message(messages=conversation_messages) + if part.get("type") == "text" + ] + try: + conversation: Final = anthropic_messages_pt( + messages=conversation_messages, + model=model, + llm_provider="snowflake", + drop_params=drop_params, + force_base64=True, + ) + except (AttributeError, LookupError, TypeError, ValueError, BadRequestError) as e: + raise SnowflakeException(status_code=400, message=str(e)) from e + return (system or None), conversation def transform_request( self, @@ -431,7 +262,8 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): extra_body: Final = optional_params.pop("extra_body", {}) if is_snowflake_claude_model(model): - return self._transform_request_anthropic(model, messages, optional_params, stream, extra_body) + drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True + return self._transform_request_anthropic(model, messages, optional_params, stream, extra_body, drop_params) return self._transform_request_openai(model, messages, optional_params, stream, extra_body) @property @@ -506,10 +338,11 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): optional_params: dict, stream: bool, extra_body: dict, + drop_params: bool, ) -> dict: """Anthropic Messages format for /messages endpoint.""" passthrough_system: Final = optional_params.pop("system", None) - extracted_system, conversation = self._extract_system_and_messages(messages) + extracted_system, conversation = self._extract_system_and_messages(model, messages, drop_params) system: Final = passthrough_system if passthrough_system is not None else extracted_system if "tools" in optional_params: diff --git a/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py b/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py index 4b25275285e..1d3551fe549 100644 --- a/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py +++ b/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py @@ -962,10 +962,10 @@ def _shaped_requests(model: str, shape: str) -> tuple[dict[str, JsonValue], dict pytest.param(f"openrouter/anthropic/{ANTHROPIC_MODEL}", "system", True, id="openrouter-claude"), pytest.param("openrouter/google/gemini-3.8-flash", "system", True, id="openrouter-gemini"), pytest.param("snowflake/claude-4-sonnet", "tool", True, id="snowflake-claude-tool-output"), + pytest.param("snowflake/claude-4-sonnet", "system", True, id="snowflake-claude-string-system"), pytest.param("vertex-claude", "system", True, id="vertex-claude"), pytest.param("sap-claude", "system", True, id="sap-claude"), pytest.param("anthropic/my-gateway-model", "system", True, id="anthropic-non-claude-name"), - pytest.param("snowflake/claude-4-sonnet", "system", False, id="snowflake-claude-string-system"), pytest.param("openrouter/openai/gpt-5.4", "system", False, id="openrouter-gpt"), pytest.param("snowflake/llama3.1-70b", "system", False, id="snowflake-llama"), pytest.param("snowflake/llama-claude-distill", "system", False, id="snowflake-claude-inside-the-name"), diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 146bb08a4fd..ae031367823 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -5,6 +5,7 @@ import json import os import sys from collections.abc import Mapping +from types import MappingProxyType from typing import Final, cast import pytest @@ -2561,6 +2562,23 @@ def test_schema_size_counts_every_written_object_once(schema, properties, branch assert schema_grammar_branch_count(schema) == branches +@pytest.mark.parametrize( + "arguments", + [{"already": "parsed"}, MappingProxyType({"already": "parsed"})], + ids=["dict", "mapping_proxy"], +) +def test_parse_tool_call_arguments_returns_already_parsed_mapping(arguments): + parsed = parse_tool_call_arguments(arguments, tool_name="dict_tool", context="test") + + assert parsed == {"already": "parsed"} + assert json.loads(json.dumps(parsed)) == {"already": "parsed"} + + +def test_parse_tool_call_arguments_rejects_unrepairable_json(): + with pytest.raises(ValueError, match="broken_tool"): + parse_tool_call_arguments("not valid json{{{", tool_name="broken_tool", context="test") + + @pytest.mark.parametrize("part_type", ["image_url", "input_image"]) @pytest.mark.parametrize( "fields", diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 8a755d47817..c8d53c12aaf 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -16,6 +16,7 @@ import pytest import respx import litellm +from litellm.constants import REDACTED_BY_LITELLM from litellm.litellm_core_utils.prompt_templates.factory import ( BEDROCK_DOCUMENT_PLACEHOLDER_TEXT, _EMPTY_TEXT_PLACEHOLDER, @@ -38,6 +39,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( make_valid_bedrock_tool_name, ollama_pt, parse_mime_type, + requires_inline_base64_media, sanitize_messages_for_tool_calling, THOUGHT_SIGNATURE_SEPARATOR, ) @@ -3189,6 +3191,35 @@ def test_convert_to_anthropic_tool_result_image_with_cache_control(): assert result["content"][1]["cache_control"]["type"] == "ephemeral" +@pytest.mark.parametrize("tool_output", [{"result_key": "result_value"}, 42, 0, 3.5, True, False]) +def test_convert_to_anthropic_tool_result_serializes_non_string_content(tool_output): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + result = convert_to_anthropic_tool_result({"role": "tool", "tool_call_id": "call_value", "content": tool_output}) + + assert result["type"] == "tool_result" + assert json.loads(result["content"]) == tool_output + + +@pytest.mark.parametrize( + "tool_message", + [ + {"role": "tool", "tool_call_id": "call_value", "content": None}, + {"role": "tool", "tool_call_id": "call_value"}, + ], +) +def test_convert_to_anthropic_tool_result_sends_an_empty_result_for_null_or_missing_content(tool_message): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + result = convert_to_anthropic_tool_result(tool_message) + + assert result == {"type": "tool_result", "tool_use_id": "call_value", "content": ""} + + def test_convert_to_anthropic_tool_result_image_without_cache_control(): """ Test that images without cache_control in tool results work correctly. @@ -5203,6 +5234,7 @@ def test_group_tool_exchanges_is_linear_in_message_count(): _PDF_DATA_URI = "data:application/pdf;base64," + base64.b64encode(b"%PDF-1.4 regression fixture").decode() +_PNG_B64 = base64.b64encode(b"\x89PNG\r\n\x1a\n regression fixture").decode() _PNG_DATA_URI = ( "data:image/png;base64," "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" @@ -5235,6 +5267,132 @@ def test_bedrock_converse_pdf_only_user_message_gets_text_block(): assert _text_blocks(result[0]) == [BEDROCK_DOCUMENT_PLACEHOLDER_TEXT] +@pytest.mark.parametrize( + ("model", "llm_provider"), + [ + ("claude-sonnet-5", "anthropic"), + ("invoke/us.anthropic.claude-sonnet-5", "bedrock"), + ("claude-sonnet-5", "vertex_ai"), + ("claude-sonnet-5", "snowflake"), + ], +) +def test_anthropic_messages_pt_user_pdf_data_uri_becomes_document_block(model, llm_provider): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What word is in this document?"}, + { + "type": "image_url", + "image_url": {"url": _PDF_DATA_URI}, + "cache_control": {"type": "ephemeral"}, + }, + ], + } + ] + + result = anthropic_messages_pt(messages=messages, model=model, llm_provider=llm_provider) + + assert result[0]["content"][1] == { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": _PDF_DATA_URI.split(",", 1)[1], + }, + "cache_control": {"type": "ephemeral"}, + } + assert all(block["type"] != "image" for block in result[0]["content"]) + + +_TEXT_B64 = base64.b64encode("the secret code word is MANGO\n".encode()).decode() +_TEXT_DATA_URI = "data:text/plain;base64," + _TEXT_B64 +_TEXT_DATA_URI_WITH_CHARSET = "data:text/plain;charset=utf-8;base64," + _TEXT_B64 +_LATIN1_TEXT_DATA_URI = "data:text/plain;charset=iso-8859-1;base64," + base64.b64encode( + "the secret code word is MANGÓ\n".encode("latin-1") +).decode() +_OCTET_TEXT_DATA_URI = "data:application/octet-stream;base64," + _TEXT_B64 + + +@pytest.mark.parametrize( + "block, expected_text", + [ + ({"type": "image_url", "image_url": {"url": _TEXT_DATA_URI}}, "the secret code word is MANGO\n"), + ({"type": "file", "file": {"file_data": _TEXT_DATA_URI}}, "the secret code word is MANGO\n"), + ({"type": "image_url", "image_url": {"url": _TEXT_DATA_URI_WITH_CHARSET}}, "the secret code word is MANGO\n"), + ({"type": "file", "file": {"file_data": _TEXT_DATA_URI_WITH_CHARSET}}, "the secret code word is MANGO\n"), + ({"type": "file", "file": {"file_data": _LATIN1_TEXT_DATA_URI}}, "the secret code word is MANGÓ\n"), + ( + {"type": "image_url", "image_url": {"url": "data:TEXT/PLAIN;base64," + _TEXT_B64}}, + "the secret code word is MANGO\n", + ), + ( + {"type": "image_url", "image_url": {"url": _LATIN1_TEXT_DATA_URI.replace("text/plain", "TEXT/PLAIN", 1)}}, + "the secret code word is MANGÓ\n", + ), + ({"type": "file", "file": {"file_data": "data:TEXT/PLAIN;base64," + _TEXT_B64}}, "the secret code word is MANGO\n"), + ( + {"type": "file", "file": {"file_data": _OCTET_TEXT_DATA_URI, "format": "Text/Plain"}}, + "the secret code word is MANGO\n", + ), + ( + {"type": "image_url", "image_url": {"url": _OCTET_TEXT_DATA_URI, "format": "Text/Plain"}}, + "the secret code word is MANGO\n", + ), + ], + ids=[ + "image_url", + "file", + "image_url_charset", + "file_charset", + "file_latin1", + "image_url_uppercase", + "uppercase_latin1", + "file_uppercase", + "file_mixed_case_format", + "image_url_mixed_case_format", + ], +) +def test_anthropic_messages_pt_text_plain_data_uri_becomes_text_source_document(block, expected_text): + messages = [{"role": "user", "content": [{"type": "text", "text": "What is the code word?"}, block]}] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"][1] == { + "type": "document", + "source": {"type": "text", "media_type": "text/plain", "data": expected_text}, + } + + +def test_anthropic_messages_pt_user_png_data_uri_stays_an_image_block(): + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": _PNG_DATA_URI}}]}] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"] == [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": _PNG_DATA_URI.split(",", 1)[1]}, + } + ] + + +@pytest.mark.parametrize( + ("model", "llm_provider", "expected"), + [ + ("invoke/us.anthropic.claude-sonnet-5", "bedrock", True), + ("claude-sonnet-5", "vertex_ai", True), + ("claude-sonnet-5", "vertex_ai_beta", True), + ("claude-sonnet-5", "snowflake", False), + ("claude-sonnet-5", "anthropic", False), + ("us.anthropic.claude-sonnet-5", "bedrock", False), + ("claude-sonnet-5", None, False), + ], +) +def test_requires_inline_base64_media(model, llm_provider, expected): + assert requires_inline_base64_media(model, llm_provider) is expected + + def test_bedrock_converse_document_with_text_gets_no_extra_text_block(): messages = [ { @@ -9008,3 +9166,694 @@ def test_wrong_type_template_field_in_requests_leaves_the_deployments_in_rotatio response = router.completion(model="template-group", messages=TEMPLATE_MESSAGES, mock_response="Blue") assert response.choices[0].message.content == "Blue" + + +def test_anthropic_messages_pt_keeps_assistant_tool_use_content_block(): + messages = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "let me look that up"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "paris"}}, + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + {"type": "text", "text": "let me look that up"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "paris"}}, + ] + assert result[2]["content"] == [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "sunny"}] + + +def test_anthropic_messages_pt_does_not_duplicate_a_tool_use_present_in_both_shapes(): + messages = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "functions.Bash:0", "name": "get_weather", "input": {"city": "paris"}} + ], + "tool_calls": [ + { + "id": "functions.Bash:0", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "paris"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "functions.Bash:0", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + {"type": "tool_use", "id": "functions_Bash_0", "name": "get_weather", "input": {"city": "paris"}} + ] + + +def test_anthropic_messages_pt_sanitizes_a_native_tool_use_id_like_its_tool_result(): + messages = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "functions.Bash:0", "name": "get_weather", "input": {"city": "paris"}} + ], + }, + {"role": "tool", "tool_call_id": "functions.Bash:0", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + {"type": "tool_use", "id": "functions_Bash_0", "name": "get_weather", "input": {"city": "paris"}} + ] + assert result[2]["content"] == [ + {"type": "tool_result", "tool_use_id": "functions_Bash_0", "content": "sunny"} + ] + + +def test_anthropic_messages_pt_normalizes_a_native_tool_use_block(): + messages = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "index": 0, + "id": "toolu_01", + "name": "get_weather", + "input": '{"city": "paris"}', + "cache_control": {"type": "ephemeral"}, + } + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + { + "type": "tool_use", + "id": "toolu_01", + "name": "get_weather", + "input": {"city": "paris"}, + "cache_control": {"type": "ephemeral"}, + } + ] + + +@pytest.mark.parametrize( + ("url", "format_override", "expected"), + [ + ( + "data:application/pdf;base64," + _PNG_B64, + "image/png", + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": _PNG_B64}}, + ), + ( + "data:image/png;base64,JVBERi0=", + "application/pdf", + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0="}}, + ), + ( + _PDF_DATA_URI, + "image/png", + { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": _PDF_DATA_URI.split(",", 1)[1]}, + }, + ), + ], + ids=["png_bytes_under_a_pdf_uri_declared_png", "png_uri_declared_pdf", "pdf_bytes_declared_png"], +) +def test_anthropic_messages_pt_honours_an_explicit_format_over_the_data_uri(url, format_override, expected): + messages = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": url, "format": format_override}}]} + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"] == [expected] + + +def test_anthropic_messages_pt_collapses_native_tool_use_blocks_sharing_an_id(): + messages = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "functions.Bash:0", "name": "get_weather", "input": {"city": "paris"}}, + {"type": "tool_use", "id": "functions_Bash_0", "name": "get_weather", "input": {"city": "london"}}, + ], + }, + {"role": "tool", "tool_call_id": "functions.Bash:0", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + {"type": "tool_use", "id": "functions_Bash_0", "name": "get_weather", "input": {"city": "paris"}} + ] + + +@pytest.mark.parametrize( + "image_url", + [{"url": _PDF_DATA_URI}, {"url": _PDF_DATA_URI, "format": "image/png"}], + ids=["no-format", "declared-png"], +) +def test_anthropic_messages_pt_tool_result_pdf_data_uri_becomes_a_document_block(image_url): + messages = [ + {"role": "user", "content": "fetch the invoice"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "fetch_invoice", "input": {}}], + }, + { + "role": "tool", + "tool_call_id": "toolu_01", + "content": [{"type": "image_url", "image_url": image_url}], + }, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[2]["content"] == [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": _PDF_DATA_URI.split(",", 1)[1], + }, + } + ], + } + ] + + +def test_anthropic_messages_pt_text_document_keeps_charset_named_by_the_data_uri(): + latin1_data_uri: Final = "data:text/plain;charset=iso-8859-1;base64," + base64.b64encode( + "café".encode("iso-8859-1") + ).decode() + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": latin1_data_uri, "format": "text/plain"}}], + } + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"] == [ + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "café"}} + ] + + +@pytest.mark.parametrize( + ("charset", "text"), + [ + ("shift_jis", "日本語"), + ("windows-1252", "café"), + ("KOI8-R", "привет"), + ("utf-16", "naïve"), + ("iso-8859-9", "İstanbul"), + ("iso-8859-11", "ไทย"), + ("tis-620", "ไทย"), + ("gb2312", "中文"), + ("big5-hkscs", "香港"), + ("ms932", "日本語"), + ], +) +def test_anthropic_messages_pt_reads_a_text_attachment_in_a_legacy_encoding_its_data_uri_names(charset, text): + data_uri: Final = f"data:text/plain;charset={charset};base64," + base64.b64encode(text.encode(charset)).decode() + messages = [{"role": "user", "content": [{"type": "file", "file": {"file_data": data_uri}}]}] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"] == [ + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": text}} + ] + + +@pytest.mark.parametrize("charset", ["punycode", "utf-7"]) +def test_anthropic_messages_pt_rejects_a_text_attachment_charset_that_is_not_a_supported_text_encoding(charset): + data_uri: Final = f"data:text/plain;charset={charset};base64," + base64.b64encode(b"abc").decode() + messages = [{"role": "user", "content": [{"type": "file", "file": {"file_data": data_uri}}]}] + + with pytest.raises(litellm.BadRequestError, match=f"unsupported encoding: {charset}"): + anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + +def test_anthropic_messages_pt_native_tool_use_input_survives_guardrail_redaction(): + messages = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": REDACTED_BY_LITELLM}], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}] + + +def test_anthropic_messages_pt_drops_a_native_tool_use_nobody_answered(): + messages = [ + {"role": "user", "content": "what is the weather"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "checking"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}, + ], + }, + {"role": "user", "content": "never mind, tell me a joke"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [{"type": "text", "text": "checking"}] + + +def _empty_text_beside_native_tool_use(follow_up: dict[str, object]) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "what is the weather"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": " "}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}, + ], + }, + follow_up, + ] + + +@pytest.mark.parametrize("modify_params", [False, True]) +def test_anthropic_messages_pt_drops_empty_text_beside_an_answered_native_tool_use(monkeypatch, modify_params): + monkeypatch.setattr(litellm, "modify_params", modify_params) + messages: Final = _empty_text_beside_native_tool_use({"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}) + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}] + + +@pytest.mark.parametrize("modify_params", [False, True]) +def test_anthropic_messages_pt_keeps_the_placeholder_beside_an_unanswered_native_tool_use(monkeypatch, modify_params): + monkeypatch.setattr(litellm, "modify_params", modify_params) + messages: Final = _empty_text_beside_native_tool_use({"role": "user", "content": "never mind"}) + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}] + + +def _answered_native_tool_use(tool_use: dict[str, object]) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "what is the weather"}, + {"role": "assistant", "content": [tool_use]}, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + +@pytest.mark.parametrize( + "tool_use", + [ + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": "[1, 2]"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": [1, 2]}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": "{not json"}, + ], + ids=["json-array-string", "list", "missing", "unparseable"], +) +def test_anthropic_messages_pt_native_tool_use_input_that_is_not_an_object_follows_drop_params(tool_use): + messages: Final = _answered_native_tool_use(tool_use) + + with pytest.raises(ValueError, match="get_weather"): + anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + dropped: Final = anthropic_messages_pt( + messages=messages, model="claude-sonnet-5", llm_provider="anthropic", drop_params=True + ) + + assert dropped[1]["content"] == [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}] + + +def test_anthropic_messages_pt_unparseable_tool_call_arguments_follow_drop_params(): + messages: Final = [ + {"role": "user", "content": "what is the weather"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "toolu_01", "type": "function", "function": {"name": "get_weather", "arguments": "{not json"}} + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + with pytest.raises(ValueError, match="get_weather"): + anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + dropped: Final = anthropic_messages_pt( + messages=messages, model="claude-sonnet-5", llm_provider="anthropic", drop_params=True + ) + + assert dropped[1]["content"] == [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}] + + +_NAMED_PDF_DATA_URI = "data:application/pdf;name=report.pdf;base64," + base64.b64encode(b"%PDF-1.4 named").decode() + + +@pytest.mark.parametrize( + "block", + [ + {"type": "image_url", "image_url": {"url": _NAMED_PDF_DATA_URI}}, + {"type": "file", "file": {"file_data": _NAMED_PDF_DATA_URI}}, + ], + ids=["image_url", "file"], +) +def test_anthropic_messages_pt_sends_a_named_pdf_under_its_bare_media_type(block): + result = anthropic_messages_pt( + messages=[{"role": "user", "content": [block]}], model="claude-sonnet-5", llm_provider="anthropic" + ) + + assert result[0]["content"] == [ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": _NAMED_PDF_DATA_URI.split(",", 1)[1], + }, + } + ] + + +def test_anthropic_messages_pt_keeps_a_pdf_a_document_when_the_format_is_generic(): + messages = [ + {"role": "user", "content": "fetch the invoice"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "fetch_invoice", "input": {}}], + }, + { + "role": "tool", + "tool_call_id": "toolu_01", + "content": [ + { + "type": "image_url", + "image_url": {"url": _PDF_DATA_URI, "format": "application/octet-stream"}, + } + ], + }, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[2]["content"] == [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": _PDF_DATA_URI.split(",", 1)[1], + }, + } + ], + } + ] + + +def test_anthropic_messages_pt_drops_data_uri_parameters_the_format_replaces(): + png_data_uri: Final = "data:image/png;name=diagram.png;base64," + base64.b64encode(b"\x89PNG\r\n\x1a\n").decode() + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": png_data_uri, "format": "image/png"}}], + } + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"] == [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": png_data_uri.split(",", 1)[1]}, + } + ] + + +def test_anthropic_messages_pt_rejects_a_text_attachment_it_cannot_decode(): + undeclared_latin1: Final = "data:text/plain;base64," + base64.b64encode( + "the secret code word is caf\xe9".encode("iso-8859-1") * 200 + ).decode() + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": undeclared_latin1, "format": "text/plain"}}], + } + ] + + with pytest.raises(litellm.BadRequestError) as raised: + anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert "utf-8" in str(raised.value) + assert undeclared_latin1.split(",", 1)[1] not in str(raised.value) + assert "file block's format" in str(raised.value) + + +def test_anthropic_messages_pt_reads_a_text_file_block_in_the_charset_its_format_names(): + undeclared_latin1: Final = "data:text/plain;base64," + base64.b64encode("caf\xe9".encode("iso-8859-1")).decode() + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"file_data": undeclared_latin1, "format": "text/plain;charset=iso-8859-1"}} + ], + } + ] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"] == [ + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "café"}} + ] + + +@pytest.mark.parametrize("native_input", [{"city": "lyon"}, "[1, 2]"]) +def test_anthropic_messages_pt_sends_tool_calls_arguments_over_a_native_tool_use_with_the_same_id(native_input): + messages: Final = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": native_input}], + "tool_calls": [ + { + "id": "toolu_01", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "paris"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-5-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "paris"}} + ] + + +@pytest.mark.parametrize(("drop_params", "expected_input"), [(True, {}), (False, [1, 2])]) +def test_anthropic_messages_pt_tool_call_arguments_that_are_not_an_object_follow_drop_params( + drop_params, expected_input +): + messages: Final = [ + {"role": "user", "content": "what is the weather"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "toolu_01", "type": "function", "function": {"name": "get_weather", "arguments": "[1, 2]"}} + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + result: Final = anthropic_messages_pt( + messages=messages, model="claude-sonnet-5-5", llm_provider="anthropic", drop_params=drop_params + ) + + assert result[1]["content"][0]["input"] == expected_input + + +_PDF_PAYLOAD: Final = _PDF_DATA_URI.split(",", 1)[1] + + +@pytest.mark.parametrize( + "block", + [ + {"type": "file", "file": {"file_data": "data:application/octet-stream;base64," + _PDF_PAYLOAD}}, + {"type": "file", "file": {"file_data": "data:image/png;base64," + _PDF_PAYLOAD}}, + {"type": "file", "file": {"file_data": _PDF_DATA_URI, "format": "text/plain"}}, + {"type": "image_url", "image_url": {"url": _PDF_DATA_URI, "format": "text/plain"}}, + ], + ids=["file-octet-stream", "file-image-png", "file-format-text", "image-url-format-text"], +) +def test_anthropic_messages_pt_sends_pdf_bytes_as_a_pdf_document_whatever_the_label(block): + result: Final = anthropic_messages_pt( + messages=[{"role": "user", "content": [block]}], model="claude-sonnet-5-5", llm_provider="anthropic" + ) + + assert result[0]["content"] == [ + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": _PDF_PAYLOAD}} + ] + + +def test_gemini_inline_data_carries_the_format_it_was_given(): + text_data_uri: Final = "data:text/plain;charset=utf-8;base64," + base64.b64encode(b"MANGO").decode() + messages: Final = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": text_data_uri, "format": "text/plain"}}]} + ] + + contents: Final = gemini_convert_messages_with_history(messages=messages, model="gemini-3.8-flash") + + assert contents[0]["parts"][0]["inline_data"]["mime_type"] == "text/plain" + + +def _parallel_native_tool_use_seconds(calls: int) -> float: + answers: Final = tuple( + {"role": "tool", "tool_call_id": f"functions.get_weather:{index}", "content": "sunny"} for index in range(calls) + ) + messages: Final = [ + {"role": "user", "content": "weather everywhere"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": f"functions.get_weather:{index}", "name": "get_weather", "input": {}} + for index in range(calls) + ], + }, + *answers, + ] + + def once() -> float: + started: Final = time.perf_counter() + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-5-5", llm_provider="anthropic") + elapsed: Final = time.perf_counter() - started + assert [block["id"] for block in result[1]["content"]] == [ + f"functions_get_weather_{index}" for index in range(calls) + ] + return elapsed + + return min(once() for _ in range(3)) + + +def test_anthropic_messages_pt_cost_grows_linearly_with_parallel_native_tool_uses(): + growth: Final = _parallel_native_tool_use_seconds(1600) / _parallel_native_tool_use_seconds(400) + + assert growth < 11.0, f"cost grew {growth:.1f}x for 4x the tool uses" + + +def test_anthropic_messages_pt_keeps_a_native_tool_use_answered_by_a_tool_message_under_modify_params(monkeypatch): + monkeypatch.setattr(litellm, "modify_params", True) + messages = [ + {"role": "user", "content": "what is the weather"}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}]}, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1:] == [ + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "sunny"}]}, + ] + + +@pytest.mark.parametrize( + "pdf_format, content_type", + ((None, "application/pdf"), ("Application/PDF", "application/pdf"), (None, "image/png"), ("image/png", "application/pdf")), +) +def test_create_anthropic_image_param_sends_a_downloaded_pdf_as_a_document(monkeypatch, pdf_format, content_type): + from litellm.litellm_core_utils.prompt_templates.image_handling import in_memory_cache + + pdf_bytes = b"%PDF-1.4\n1 0 obj\n" + mock_client = MagicMock() + mock_client.get.return_value = httpx.Response(200, content=pdf_bytes, headers={"Content-Type": content_type}) + monkeypatch.setattr(litellm, "user_url_validation", False, raising=False) + monkeypatch.setattr(litellm, "module_level_client", mock_client, raising=False) + in_memory_cache.flush_cache() + + block = create_anthropic_image_param( + {"url": "https://example.com/report.pdf", "format": pdf_format}, is_bedrock_invoke=True + ) + + assert block == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": base64.b64encode(pdf_bytes).decode()}, + } + + +def test_create_anthropic_image_param_links_an_https_pdf_as_a_document_only_when_format_says_pdf(): + url = "https://example.com/report.pdf" + + assert create_anthropic_image_param({"url": url, "format": "application/pdf"}) == { + "type": "document", + "source": {"type": "url", "url": url}, + } + assert create_anthropic_image_param({"url": url, "format": "Application/PDF"}) == { + "type": "document", + "source": {"type": "url", "url": url}, + } + assert create_anthropic_image_param({"url": url, "format": "image/png"}) == { + "type": "image", + "source": {"type": "url", "url": url}, + } + + +@pytest.mark.parametrize("bad_format", (7, ["image/png"]), ids=("int", "list")) +@pytest.mark.parametrize( + "block, expected", + ( + ( + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + {"type": "image", "source": {"type": "url", "url": "https://example.com/cat.png"}}, + ), + ( + {"type": "image_url", "image_url": {"url": _PNG_DATA_URI}}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": _PNG_DATA_URI.split(",", 1)[1]}}, + ), + ( + {"type": "file", "file": {"file_id": "file_abc"}}, + {"type": "container_upload", "file_id": "file_abc"}, + ), + ), + ids=("https_link", "data_uri", "file_id"), +) +def test_anthropic_messages_pt_ignores_a_format_that_is_not_a_string(bad_format, block, expected): + payload_key = "image_url" if block["type"] == "image_url" else "file" + messages = [{"role": "user", "content": [{**block, payload_key: {**block[payload_key], "format": bad_format}}]}] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"] == [expected] diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py index aab57b87a90..74c488abeb5 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py @@ -536,6 +536,23 @@ def test_flagged_placement_keeps_a_system_before_an_assistant_turn_whose_empty_t assert _roles(placed) == ["user", "system", "assistant", "user"] +def test_flagged_placement_keeps_a_system_before_an_answered_native_tool_use(): + placed = place_mid_conversation_system( + [ + {"role": "user", "content": "weather?"}, + {"role": "system", "content": "Be terse."}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "functions.get_weather:0", "name": "get_weather", "input": {}}], + }, + {"role": "tool", "tool_call_id": "functions.get_weather:0", "content": "sunny"}, + ], + supports_mid_conversation_system=True, + ) + + assert _roles(placed) == ["user", "system", "assistant", "tool"] + + def test_anthropic_system_blocks_keeps_text_parts_with_their_cache_control_and_drops_the_rest(): run = [ {"role": "system", "content": "one", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 13ff11e8a3f..e574eef26b5 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,3 +1,4 @@ +import base64 import copy import json import logging @@ -10465,14 +10466,14 @@ def test_chat_transform_request_keeps_a_conversion_error_that_a_dropped_orphan_a "tool_calls": [ { **_WEATHER_CALL["tool_calls"][0], - "function": {"name": "get_weather", "arguments": {"city": "Paris"}}, + "function": {"name": "get_weather", "arguments": ["Paris"]}, } ], }, {"role": "tool", "tool_call_id": "toolu_01", "content": "rain"}, ], - "'dict' object has no attribute 'strip'", - id="object_tool_call_arguments", + "'list' object has no attribute 'strip'", + id="list_tool_call_arguments", ), pytest.param( [{"role": "user", "content": [{"type": "image_url", "image_url": ["x"]}]}], @@ -10726,3 +10727,27 @@ def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_ assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"] assert json.dumps(body).count('"cache_control"') == 4 + + +def test_completion_rejecting_a_text_attachment_does_not_echo_the_request_back(respx_mock: respx.MockRouter): + route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(return_value=httpx.Response(500)) + attachment_data: Final = base64.b64encode(b"attachment body that stays out of the error " * 64).decode() + + with pytest.raises(litellm.BadRequestError) as exc_info: + litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + api_key="mock_api_key", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this file"}, + {"type": "file", "file": {"file_data": f"data:text/plain;charset=punycode;base64,{attachment_data}"}}, + ], + } + ], + ) + + assert "unsupported encoding: punycode" in str(exc_info.value) + assert attachment_data not in str(exc_info.value) + assert route.call_count == 0 diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index aa7fa2727b4..973c1a29111 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -88,6 +88,56 @@ def test_count_translates_openai_images_without_mutating_input( assert messages == original +@pytest.mark.parametrize( + "config_type", (AnthropicCountTokensConfig, AzureAIAnthropicCountTokensConfig) +) +def test_count_sends_a_pdf_image_url_as_the_document_the_chat_request_sends( + config_type: type[AnthropicCountTokensConfig], +) -> None: + pdf_url: Final = "data:application/pdf;base64," + b64encode(b"%PDF-1.4 count fixture").decode() + messages: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": pdf_url}}]} + ] + + counted: Final = config_type().transform_request_to_count_tokens(model="claude-opus-5-5", messages=messages) + sent: Final = AnthropicConfig().transform_request( + model="claude-opus-5-5", messages=deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert counted["messages"] == sent["messages"] + assert counted["messages"][0]["content"][0]["type"] == "document" + + +@pytest.mark.usefixtures("httpx_transport_clients") +@pytest.mark.parametrize( + "config_type", (AnthropicCountTokensConfig, AzureAIAnthropicCountTokensConfig) +) +@pytest.mark.parametrize("pdf_format", (None, "application/pdf")) +def test_count_sends_a_linked_pdf_as_a_document( + config_type: type[AnthropicCountTokensConfig], + pdf_format: str | None, +) -> None: + pdf_bytes: Final = b"%PDF-1.4 linked count fixture" + pdf_url: Final = f"https://1.1.1.1/{config_type.__name__}-{pdf_format}.pdf" + image_url: Final[dict[str, JsonValue]] = {"url": pdf_url} if pdf_format is None else { + "url": pdf_url, "format": pdf_format + } + messages: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": [{"type": "image_url", "image_url": image_url}]} + ] + + with respx.mock: + respx.get(pdf_url).mock( + return_value=httpx.Response(200, content=pdf_bytes, headers={"Content-Type": "application/pdf"}) + ) + counted: Final = config_type().transform_request_to_count_tokens(model="claude-opus-5-5", messages=messages) + + assert counted["messages"][0]["content"][0] == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": b64encode(pdf_bytes).decode()}, + } + + @pytest.mark.parametrize( "config_type", (AnthropicCountTokensConfig, AzureAIAnthropicCountTokensConfig) ) diff --git a/tests/unit/llms/anthropic/test_message_sanitization.py b/tests/unit/llms/anthropic/test_message_sanitization.py index 0afd443d07a..1a0007ba1e1 100644 --- a/tests/unit/llms/anthropic/test_message_sanitization.py +++ b/tests/unit/llms/anthropic/test_message_sanitization.py @@ -449,7 +449,7 @@ class TestMessageSanitization: ], } - sanitized = _sanitize_empty_text_content(message) + sanitized = _sanitize_empty_text_content(message, frozenset) assert [b["type"] for b in sanitized["content"]] == ["thinking"] assert PLACEHOLDER not in json.dumps(sanitized) diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index 30063936b2b..72cf544b95a 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -392,6 +392,24 @@ async def test_claude_count_rejects_a_lone_responses_tool_without_calling_bedroc assert bedrock.posted_hosts() == () +@pytest.mark.asyncio +async def test_claude_count_rejects_a_text_attachment_it_cannot_decode_without_calling_bedrock() -> None: + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + text_attachment: Final = "data:text/plain;charset=bogus-cs;base64," + base64.b64encode(b"hello").decode() + + result: Final = await _counter(bedrock).count_tokens( + model_to_use="bedrock_mantle/anthropic.claude-opus-5-5", + messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": text_attachment}}]}], + contents=None, + deployment=DEPLOYMENT, + ) + + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) + assert "bogus-cs" in (result.error_message or "") + assert bedrock.posted_hosts() == () + + _CLAUDE_REPLY: Final = { "id": "msg_1", "type": "message", diff --git a/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py b/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py index a31f15551e9..f907e92e4ab 100644 --- a/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py +++ b/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py @@ -14,12 +14,17 @@ from unittest.mock import AsyncMock, patch, Mock, MagicMock import httpx import pytest +import respx import litellm from litellm import completion, acompletion from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter -from litellm.llms.snowflake.chat.transformation import SnowflakeConfig, SnowflakeStreamingHandler +from litellm.llms.snowflake.chat.transformation import ( + SnowflakeConfig, + SnowflakeException, + SnowflakeStreamingHandler, +) from litellm.types.utils import ModelResponse @@ -354,6 +359,15 @@ class TestSnowflakeCortexClaudeFixes: ) assert body["system"] == [{"type": "text", "text": "You are helpful", "cache_control": {"type": "ephemeral"}}] + def test_string_system_prompt_keeps_its_message_cache_control_without_ttl(self): + body = self._transform( + [ + {"role": "system", "content": "You are helpful", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"role": "user", "content": "hi"}, + ] + ) + assert body["system"] == [{"type": "text", "text": "You are helpful", "cache_control": {"type": "ephemeral"}}] + def test_direct_system_param_is_normalized(self): body = self._transform( [{"role": "user", "content": "hi"}], @@ -422,6 +436,23 @@ class TestSnowflakeCortexClaudeFixes: {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": "ZmFrZQ=="}} ] + def test_remote_image_urls_are_inlined_as_base64(self): + with respx.mock(assert_all_called=True) as respx_mock: + respx_mock.get("https://example.com/cat.png").respond( + content=b"fake", headers={"Content-Type": "image/png"} + ) + body = self._transform( + [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + } + ] + ) + assert body["messages"][0]["content"] == [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "ZmFrZQ=="}} + ] + def test_tool_result_image_list_is_converted(self): body = self._transform( [ @@ -458,7 +489,7 @@ class TestSnowflakeCortexClaudeFixes: }, ] ) - tool_result = body["messages"][1]["content"][0] + tool_result = body["messages"][0]["content"][1] assert tool_result["cache_control"] == {"type": "ephemeral"}, tool_content def test_pdf_data_uri_becomes_a_document_block(self): @@ -524,7 +555,7 @@ class TestSnowflakeCortexClaudeFixes: }, ] ) - assert body["messages"][1]["content"][0]["content"] == [ + assert body["messages"][0]["content"][1]["content"] == [ {"type": "text", "text": "first"}, {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "ZmFrZQ=="}}, {"type": "text", "text": "last"}, @@ -534,7 +565,10 @@ class TestSnowflakeCortexClaudeFixes: body = self._transform( [{"role": "user", "content": "look"}, {"role": "tool", "tool_call_id": "call_1", "content": "done"}] ) - assert body["messages"][1]["content"][0]["content"] == "done" + assert body["messages"][0]["content"] == [ + {"type": "text", "text": "look"}, + {"type": "tool_result", "tool_use_id": "call_1", "content": "done"}, + ] def test_anthropic_tool_schema_strips_only_top_level_schema_key(self): tools = [ @@ -687,6 +721,123 @@ class TestSnowflakeCortexClaudeFixes: ], } + def test_assistant_tool_use_content_block_survives_the_transform(self): + body = self._transform( + [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "let me look that up"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "paris"}}, + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + ) + assert body["messages"][1]["content"] == [ + {"type": "text", "text": "let me look that up"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "paris"}}, + ] + assert body["messages"][2]["content"] == [ + {"type": "tool_result", "tool_use_id": "toolu_01", "content": "sunny"} + ] + + def test_native_tool_use_id_is_sanitized_like_its_tool_result(self): + body = self._transform( + [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "functions.Bash:0", + "name": "get_weather", + "input": {"city": "paris"}, + } + ], + }, + {"role": "tool", "tool_call_id": "functions.Bash:0", "content": "sunny"}, + ] + ) + assert body["messages"][1]["content"] == [ + {"type": "tool_use", "id": "functions_Bash_0", "name": "get_weather", "input": {"city": "paris"}} + ] + assert body["messages"][2]["content"] == [ + {"type": "tool_result", "tool_use_id": "functions_Bash_0", "content": "sunny"} + ] + + def test_system_prompt_keeps_only_its_text_parts(self): + body = self._transform( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "be terse"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}, + ], + }, + {"role": "user", "content": "hi"}, + ] + ) + assert body["system"] == [{"type": "text", "text": "be terse"}] + + _AUDIO_TURN = ( + { + "role": "user", + "content": [ + {"type": "text", "text": "hear"}, + {"type": "input_audio", "input_audio": {"data": "UklGRg==", "format": "wav"}}, + ], + }, + ) + + def test_user_part_cortex_claude_cannot_take_is_a_400_naming_it(self): + with pytest.raises(SnowflakeException) as exc_info: + self._transform(list(self._AUDIO_TURN)) + assert exc_info.value.status_code == 400 + assert "input_audio" in str(exc_info.value) + + @pytest.mark.parametrize( + ("part", "named"), + (("hello", "untyped str"), ({"type": "image_url", "image_url": {"detail": "low"}}, "url")), + ) + def test_malformed_user_part_is_a_400_naming_it(self, part, named): + with pytest.raises(SnowflakeException) as exc_info: + self._transform([{"role": "user", "content": [part]}]) + assert exc_info.value.status_code == 400 + assert named in str(exc_info.value) + + def test_drop_params_drops_a_user_part_cortex_claude_cannot_take(self): + body = SnowflakeConfig().transform_request( + model="snowflake/claude-sonnet-4-6", + messages=list(self._AUDIO_TURN), + optional_params={}, + litellm_params={"drop_params": True}, + headers={}, + ) + assert body["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hear"}]}] + + def test_unknown_charset_on_a_text_attachment_is_a_400(self): + with pytest.raises(SnowflakeException) as exc_info: + self._transform( + [ + { + "role": "user", + "content": [ + { + "type": "file", + "file": {"file_data": "data:text/plain;charset=not-a-charset;base64,aGVsbG8="}, + } + ], + } + ] + ) + + assert exc_info.value.status_code == 400 + assert "not-a-charset" in str(exc_info.value) + def test_thinking_only_assistant_turn_sends_no_empty_text_block(self): """Anthropic-shaped APIs reject empty text blocks, so a content-less thinking turn is thinking only.""" body = self._transform( diff --git a/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py b/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py index 8ddbb7c1a6b..c70128d3225 100644 --- a/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py +++ b/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py @@ -20,6 +20,7 @@ from litellm.llms.snowflake.chat.transformation import ( SnowflakeConfig, is_snowflake_claude_model, ) +from litellm.llms.snowflake.utils import SnowflakeException from litellm.types.utils import ModelResponse @@ -342,7 +343,7 @@ class TestAnthropicConfigRequest: ) assert body["system"] == [{"type": "text", "text": "You are helpful."}] assert all(m["role"] != "system" for m in body["messages"]) - assert body["messages"][0] == {"role": "user", "content": "Hello"} + assert body["messages"][0] == {"role": "user", "content": [{"type": "text", "text": "Hello"}]} def test_model_prefix_stripped(self): body = self.cfg.transform_request( @@ -637,7 +638,7 @@ class TestAnthropicMultiTurnToolMessages: headers={}, ) msgs = body["messages"] - assert msgs[0] == {"role": "user", "content": "What's the weather in Paris?"} + assert msgs[0] == {"role": "user", "content": [{"type": "text", "text": "What's the weather in Paris?"}]} assistant_msg = msgs[1] assert assistant_msg["role"] == "assistant" @@ -647,13 +648,14 @@ class TestAnthropicMultiTurnToolMessages: assert assistant_msg["content"][0]["name"] == "get_weather" assert assistant_msg["content"][0]["input"] == {"city": "Paris"} - tool_result_msg = msgs[2] - assert tool_result_msg["role"] == "user" - assert tool_result_msg["content"][0]["type"] == "tool_result" - assert tool_result_msg["content"][0]["tool_use_id"] == "call_123" - assert tool_result_msg["content"][0]["content"] == "Sunny, 22°C" - - assert msgs[3] == {"role": "user", "content": "Thanks!"} + assert msgs[2] == { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "call_123", "content": "Sunny, 22°C"}, + {"type": "text", "text": "Thanks!"}, + ], + } + assert len(msgs) == 3 def test_assistant_with_text_and_tool_calls(self): messages = [ @@ -705,7 +707,32 @@ class TestAnthropicMultiTurnToolMessages: for msg in body["messages"]: assert msg["role"] != "tool" - def test_malformed_json_in_tool_arguments_handled_gracefully(self): + def test_malformed_json_in_tool_arguments_is_rejected_as_bad_request(self): + messages = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_bad", + "type": "function", + "function": {"name": "broken_tool", "arguments": "not valid json{{{"}, + } + ], + }, + ] + with pytest.raises(SnowflakeException, match="broken_tool") as excinfo: + self.cfg.transform_request( + model="snowflake/claude-sonnet-4-5", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + assert excinfo.value.status_code == 400 + + def test_malformed_json_in_tool_arguments_is_dropped_under_drop_params(self): messages = [ {"role": "user", "content": "hi"}, { @@ -724,14 +751,12 @@ class TestAnthropicMultiTurnToolMessages: model="snowflake/claude-sonnet-4-5", messages=messages, optional_params={}, - litellm_params={}, + litellm_params={"drop_params": True}, headers={}, ) - assistant_msg = body["messages"][1] - tool_use_block = assistant_msg["content"][0] - assert tool_use_block["type"] == "tool_use" - assert tool_use_block["name"] == "broken_tool" - assert tool_use_block["input"] == {} + assert body["messages"][1]["content"] == [ + {"type": "tool_use", "id": "call_bad", "name": "broken_tool", "input": {}} + ] def test_non_string_tool_arguments_pass_through(self): messages = [ diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 2f85d1c7516..1a6d105af0f 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2115,6 +2115,47 @@ async def test_framing_keeps_the_400_for_blocks_the_provider_counter_cannot_conv assert "search_result" in raised.value.message +@pytest.mark.parametrize( + "litellm_params", + [ + {"model": "anthropic/claude-opus-5-5", "api_key": "sk-unused-framed"}, + { + "model": "azure_ai/claude-opus-5-5", + "api_key": "sk-unused-framed", + "api_base": "https://framed.services.ai.azure.com/anthropic", + }, + ], + ids=["anthropic", "azure_ai"], +) +@pytest.mark.asyncio +async def test_framing_keeps_the_400_for_a_text_attachment_the_claude_counter_cannot_decode( + monkeypatch: pytest.MonkeyPatch, litellm_params: dict[str, str] +): + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + {"model_name": "framed", "litellm_params": litellm_params, "model_info": {"token_count_framing": 100}} + ] + ), + ) + text_attachment: Final = "data:text/plain;charset=bogus-cs;base64," + base64.b64encode(b"hello").decode() + + with pytest.raises(ProxyException) as raised: + await count_request_tokens( + request=TokenCountRequest( + model="framed", + messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": text_attachment}}]}], + ), + call_endpoint=True, + messages_format="anthropic", + ) + + assert raised.value.code == "400" + assert "bogus-cs" in raised.value.message + + @pytest.mark.asyncio @pytest.mark.parametrize( "unreadable_body",