diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 101f069c0f8..742b948b6df 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2838,6 +2838,19 @@ def merge_consecutive_system_messages( ] +_TOOL_USE_ID_FALLBACK: Final = "tool_use_id" +_ANTHROPIC_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]") + + +def replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Pattern[str]) -> str: + return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK + + +def sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: + """Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$.""" + return replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS) + + _DECODED_JSON: Final = TypeAdapter(object) @@ -2900,7 +2913,7 @@ def _attempt_json_repair(s: str) -> object | None: def parse_tool_call_arguments( - arguments: str | None, + arguments: str | Mapping[str, object] | None, tool_name: str | None = None, context: str | None = None, ) -> Any: @@ -2913,7 +2926,7 @@ def parse_tool_call_arguments( callers are aware the arguments were not perfectly formed. Args: - arguments: The JSON string containing tool arguments, or None. + arguments: The JSON string containing tool arguments, an already-parsed mapping, or None. tool_name: Optional name of the tool (for error messages). context: Optional context string (e.g., "Anthropic Messages API"). @@ -2927,6 +2940,8 @@ def parse_tool_call_arguments( """ import json + if isinstance(arguments, Mapping): + return dict(arguments) if not arguments or not arguments.strip(): return {} diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fad4caa59a0..9c4e1bd8767 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -8,7 +8,9 @@ import re import xml.etree.ElementTree as ET from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass, replace +from email.message import Message from enum import Enum +from functools import cache, partial from types import MappingProxyType from typing import Annotated, Any, Final, TypeAlias, TypedDict, TypeVar, cast, overload @@ -24,13 +26,16 @@ from litellm._uuid import uuid from litellm.constants import REDACTED_BY_LITELLM from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( + answered_ids_by_index, anthropic_system_messages, find_anthropic_server_tool_result, is_anthropic_tool_search_use, message_field, + native_tool_use_id, parts_of, place_mid_conversation_system, split_leading_system_run, + tool_use_is_answered, ) from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value @@ -72,7 +77,9 @@ from .common_utils import ( is_unsignable_thinking_block, native_redacted_thinking_data, parse_tool_call_arguments, + replace_invalid_tool_use_id_chars, salvage_concatenated_tool_arguments, + sanitize_anthropic_tool_use_id, ) from .image_handling import convert_url_to_base64 @@ -872,6 +879,18 @@ def convert_generic_image_chunk_to_openai_image_obj( return "data:{};{},{}".format(media_type, image_chunk["type"], image_chunk["data"]) +_CHARSET_AWARE_MEDIA_TYPE: Final = "text/plain" + + +def _media_type_with_original_parameters(format_override: str, original_media_type: str) -> str: + if format_override.strip() != _CHARSET_AWARE_MEDIA_TYPE: + return format_override + base_type, separator, parameters = original_media_type.partition(";") + if not separator or base_type.strip() != format_override.strip(): + return format_override + return f"{format_override}{separator}{parameters}" + + def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) -> GenericImageParsingChunk: """ Input: @@ -1773,29 +1792,18 @@ def convert_to_gemini_tool_call_result( return _part -_TOOL_USE_ID_FALLBACK: Final = "tool_use_id" -_ANTHROPIC_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]") _BEDROCK_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_.:-]") _BEDROCK_TOOL_USE_ID_MAX_LEN: Final = 64 _BEDROCK_TOOL_USE_ID_HASH_LEN: Final = 8 -def _replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Pattern[str]) -> str: - return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK - - -def sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: - """Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$.""" - return _replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS) - - def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str: """ Bedrock Converse requires toolUseId to match [a-zA-Z0-9_.:-]+ and be at most 64 chars. Ids that need rewriting get a short hash of the original appended so two ids that only differ in a replaced char or past the cut still map to distinct values. """ - sanitized: Final = _replace_invalid_tool_use_id_chars(tool_use_id, _BEDROCK_TOOL_USE_ID_INVALID_CHARS) + sanitized: Final = replace_invalid_tool_use_id_chars(tool_use_id, _BEDROCK_TOOL_USE_ID_INVALID_CHARS) if sanitized == tool_use_id and len(sanitized) <= _BEDROCK_TOOL_USE_ID_MAX_LEN: return sanitized digest: Final = hashlib.sha256(tool_use_id.encode()).hexdigest()[:_BEDROCK_TOOL_USE_ID_HASH_LEN] @@ -1805,15 +1813,86 @@ def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str: _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES: Final = {"application/pdf", "text/plain"} -def _is_anthropic_document_data_uri(url: str) -> bool: - # Anthropic's base64 document source accepts only application/pdf and - # text/plain (see select_anthropic_content_block_type_for_file). Routing - # other mimes here would produce a document block the API rejects, so we - # leave them on the image code path. +_PDF_SIGNATURE: Final = b"%PDF-" + + +def _declared_media_type_for_data_uri(url: str, format_override: str | None) -> str | None: + if format_override and select_anthropic_content_block_type_for_file(format_override) != "container_upload": + return format_override match: Final = re.match(r"data:([^;,]+)", url) - if not match: + return match.group(1) if match else None + + +def _base64_holds_a_pdf(data: str) -> bool: + try: + return base64.b64decode(data[:8]).startswith(_PDF_SIGNATURE) + except ValueError: return False - return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES + + +def _data_uri_holds_a_pdf(url: str) -> bool: + _, _, payload = url.partition(",") + return _base64_holds_a_pdf(payload) + + +def _data_uri_media_type(url: str) -> str: + header, separator, _ = url.partition(";base64,") + return header.removeprefix("data:") if separator and url.startswith("data:") else "" + + +def anthropic_document_media_type_for_data_uri(url: str, format_override: str | None = None) -> str | None: + if not url.startswith("data:"): + return None + if _data_uri_holds_a_pdf(url): + return "application/pdf" + declared: Final = _declared_media_type_for_data_uri(url, format_override) + return declared if declared in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES else None + + +def _anthropic_text_document_source( + media_type_with_parameters: str, data: str +) -> AnthropicContentParamSourceText | None: + media_type, _, _ = media_type_with_parameters.partition(";") + if media_type.strip() != "text/plain": + return None + content_type_header: Final = Message() + content_type_header["content-type"] = media_type_with_parameters + charset: Final = content_type_header.get_content_charset("utf-8") + try: + text: Final = base64.b64decode(data).decode(charset) + except (LookupError, UnicodeDecodeError) as decode_error: + raise litellm.BadRequestError( + message=f"Text attachment could not be read as {charset!r}: {decode_error}. " + "Send it as utf-8, or name its encoding in the data URI (data:text/plain;charset=...).", + model=None, + llm_provider="anthropic", + ) from decode_error + return AnthropicContentParamSourceText(type="text", media_type="text/plain", data=text) + + +def anthropic_document_block_from_data_uri( + url: str, + original_content_element: Mapping[str, object] | AllMessageValues, + media_type: str, +) -> AnthropicMessagesDocumentParam: + synth_file_message: Final[ChatCompletionFileObject] = { + "type": "file", + "file": {"file_data": url, "format": media_type}, + } + document_block: Final = anthropic_process_openai_file_message(synth_file_message) + return cast( + AnthropicMessagesDocumentParam, + add_cache_control_to_content( + anthropic_content_element=cast(AnthropicMessagesDocumentParam, document_block), + original_content_element=original_content_element, + ), + ) + + +def requires_inline_base64_media(model: str, llm_provider: str | None) -> bool: + if model.lower().startswith("invoke/"): + return True + return llm_provider is not None and llm_provider.startswith("vertex_ai") def convert_to_anthropic_tool_result( @@ -1862,6 +1941,8 @@ def convert_to_anthropic_tool_result( ) = "" if isinstance(message["content"], str): anthropic_content = message["content"] + elif isinstance(message["content"], Mapping): + anthropic_content = json.dumps(message["content"]) elif isinstance(message["content"], list): content_list: Final = message["content"] anthropic_content_list: list[ @@ -1886,20 +1967,13 @@ def convert_to_anthropic_tool_result( image_url_value = content["image_url"] format = image_url_value.get("format") if isinstance(image_url_value, dict) else None url_str = image_url_value.get("url") if isinstance(image_url_value, dict) else image_url_value - # Data URIs with non-image mime types (e.g. application/pdf) must - # translate to Anthropic document blocks, not image blocks — - # wrapping a PDF in `type: "image"` is rejected by the API. - if isinstance(url_str, str) and _is_anthropic_document_data_uri(url_str): - synth_file_message: ChatCompletionFileObject = { - "type": "file", - "file": {"file_data": url_str}, - } - _document_block = anthropic_process_openai_file_message(synth_file_message) - _document_block = add_cache_control_to_content( - anthropic_content_element=cast(AnthropicMessagesDocumentParam, _document_block), - original_content_element=content, + if ( + isinstance(url_str, str) + and (document_media_type := anthropic_document_media_type_for_data_uri(url_str, format)) is not None + ): + anthropic_content_list.append( + anthropic_document_block_from_data_uri(url_str, content, document_media_type) ) - anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _document_block)) else: _anthropic_image_param = create_anthropic_image_param( image_url_value, @@ -2073,6 +2147,7 @@ def convert_to_anthropic_tool_invoke( web_search_results: Sequence[object] | None = None, tool_results: Sequence[object] | None = None, answered_tool_ids: Container[str] = frozenset(), + drop_params: bool = False, ) -> list[_AnthropicToolInvokeItem]: """ OpenAI tool invokes: @@ -2127,10 +2202,8 @@ def convert_to_anthropic_tool_invoke( str, get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"), ) - tool_input = parse_tool_call_arguments( - get_attribute_or_key(get_attribute_or_key(tool, "function"), "arguments"), - tool_name=tool_name, - context="Anthropic tool invoke", + tool_input = _tool_call_input( + get_attribute_or_key(get_attribute_or_key(tool, "function"), "arguments"), tool_name, drop_params ) server_tool_result = ( @@ -2179,7 +2252,7 @@ def add_cache_control_to_content( | AnthropicMessagesDocumentParam | AnthropicMessagesToolUseParam | ChatCompletionThinkingBlock, - original_content_element: dict | AllMessageValues | ChatCompletionFileObject, + original_content_element: Mapping[str, object] | AllMessageValues | ChatCompletionFileObject, ): cache_control_param: Final = original_content_element.get("cache_control") if cache_control_param is not None and isinstance(cache_control_param, dict): @@ -2190,6 +2263,79 @@ def add_cache_control_to_content( return anthropic_content_element +_TOOL_USE_INPUT: Final = TypeAdapter(dict[str, object]) + + +def _object_unless_dropped(value: object, drop_params: bool) -> object: + if not drop_params: + return value + try: + return _TOOL_USE_INPUT.validate_python(value) + except ValidationError: + return {} + + +def _tool_call_input(arguments: str | Mapping[str, object] | None, tool_name: str, drop_params: bool) -> object: + try: + return _object_unless_dropped( + parse_tool_call_arguments(arguments, tool_name=tool_name, context="Anthropic tool invoke"), drop_params + ) + except ValueError: + if drop_params: + return {} + raise + + +def _tool_use_input_object(value: object, tool_name: str) -> dict[str, object]: + try: + return _TOOL_USE_INPUT.validate_python(value) + except ValidationError as error: + raise ValueError( + f"Tool input for tool '{tool_name}' (Anthropic assistant tool_use block) must be a JSON object, " + f"got {type(value).__name__}. Pass drop_params=True to send an empty object in its place" + ) from error + + +def _parsed_native_tool_use_input(raw: object, tool_name: str) -> object: + if not isinstance(raw, str): + return raw + return parse_tool_call_arguments( + "{}" if raw == REDACTED_BY_LITELLM else raw, + tool_name=tool_name, + context="Anthropic assistant tool_use block", + ) + + +def _native_tool_use_input(raw: object, tool_name: str, drop_params: bool) -> dict[str, object]: + try: + return _tool_use_input_object(_parsed_native_tool_use_input(raw, tool_name), tool_name) + except ValueError: + if drop_params: + return {} + raise + + +def _tool_call_ids(message: Mapping[str, object]) -> frozenset[str]: + return frozenset(native_tool_use_id(tool_call) for tool_call in parts_of(message.get("tool_calls"))) + + +def _anthropic_native_tool_use_block( + block: Mapping[str, object], tool_use_id: str, drop_params: bool +) -> AnthropicMessagesToolUseParam: + tool_name: Final = str(block.get("name", "")) + tool_use_param: Final = AnthropicMessagesToolUseParam( + type="tool_use", + id=tool_use_id, + name=tool_name, + input=_native_tool_use_input(block.get("input"), tool_name, drop_params), + ) + add_cache_control_to_content( + anthropic_content_element=tool_use_param, + original_content_element=dict(block), + ) + return tool_use_param + + def _anthropic_content_element_factory( image_chunk: GenericImageParsingChunk, ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: @@ -2262,11 +2408,27 @@ def anthropic_process_openai_file_message( openai_image_url=file_data, format=format, ) + if _base64_holds_a_pdf(image_chunk["data"]): + return AnthropicMessagesDocumentParam( + type="document", + source=AnthropicContentParamSource( + type="base64", + media_type="application/pdf", + data=image_chunk["data"], + ), + ) + text_source: Final = _anthropic_text_document_source( + _media_type_with_original_parameters(image_chunk["media_type"], _data_uri_media_type(file_data)), + image_chunk["data"], + ) + if text_source is not None: + return AnthropicMessagesDocumentParam(type="document", source=text_source) + media_type, _, _ = image_chunk["media_type"].partition(";") anthropic_document_param: Final = AnthropicMessagesDocumentParam( type="document", source=AnthropicContentParamSource( type="base64", - media_type=image_chunk["media_type"], + media_type=media_type.strip(), data=image_chunk["data"], ), ) @@ -2913,6 +3075,7 @@ def anthropic_messages_pt( model: str, llm_provider: str, drop_params: bool = False, + force_base64: bool = False, ) -> _AnthropicMessageList: """ format messages for anthropic @@ -2957,13 +3120,10 @@ def anthropic_messages_pt( else: messages.append(DEFAULT_USER_CONTINUE_MESSAGE_TYPED) - # Bedrock invoke models have format: invoke/... - # Vertex AI Anthropic also doesn't support URL sources for images - is_bedrock_invoke = model.lower().startswith("invoke/") - is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False - force_base64 = is_bedrock_invoke or is_vertex_ai + inline_base64_media: Final = force_base64 or requires_inline_base64_media(model, llm_provider) tool_result_later_in_turn: Final = _tool_result_later_in_turn(messages, replaced_server_tool_answers) + answered_by_assistant_index: Final = cache(partial(answered_ids_by_index, messages)) msg_i = 0 while msg_i < len(messages): user_content: list[AnthropicMessagesUserMessageValues] = [] @@ -2984,26 +3144,27 @@ def anthropic_messages_pt( for m in user_block_content: if m.get("type", "") == "image_url": m = cast(ChatCompletionImageObject, m) - format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None - # Convert ChatCompletionImageUrlObject to dict if needed image_url_value = m["image_url"] + url_str = image_url_value if isinstance(image_url_value, str) else image_url_value["url"] + format = image_url_value.get("format") if isinstance(image_url_value, dict) else None + if ( + document_media_type := anthropic_document_media_type_for_data_uri(url_str, format) + ) is not None: + user_content.append( + anthropic_document_block_from_data_uri(url_str, dict(m), document_media_type) + ) + continue if isinstance(image_url_value, str): image_url_input: str | dict[str, object] = image_url_value else: - # ChatCompletionImageUrlObject or dict case - convert to dict image_url_input = { "url": image_url_value["url"], "format": image_url_value.get("format"), } - # Bedrock invoke models have format: invoke/... - # Vertex AI Anthropic also doesn't support URL sources for images - is_bedrock_invoke = model.lower().startswith("invoke/") - is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False - force_base64 = is_bedrock_invoke or is_vertex_ai _anthropic_content_element = create_anthropic_image_param( image_url_input, format=format, - is_bedrock_invoke=force_base64, + is_bedrock_invoke=inline_base64_media, ) _content_element = add_cache_control_to_content( anthropic_content_element=_anthropic_content_element, @@ -3072,7 +3233,7 @@ def anthropic_messages_pt( ) and message_field(user_message_types_block, "tool_call_id") not in replaced_server_tool_answers: # OpenAI's tool message content will always be a string user_content.append( - convert_to_anthropic_tool_result(user_message_types_block, force_base64=force_base64) + convert_to_anthropic_tool_result(user_message_types_block, force_base64=inline_base64_media) ) msg_i += 1 @@ -3142,6 +3303,7 @@ def anthropic_messages_pt( web_search_results=_web_search_results_tc, tool_results=_tool_results_tc, answered_tool_ids=client_answered_tool_ids, + drop_params=drop_params, ) # Group tool invoke results into (server_tool_use, result) pairs @@ -3247,9 +3409,11 @@ def anthropic_messages_pt( ): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR assistant_content.extend(thinking_blocks) if _content_is_list and _content_list is not None: + tool_call_ids = _tool_call_ids(assistant_content_block) for m in _with_missing_thinking_blocks(_content_list, thinking_blocks or ()): if not isinstance(m, dict): continue + block_type: str = str(m.get("type", "")) # handle thinking blocks if (thinking_block := _as_thinking_block(m)) is not None: assistant_content.extend( @@ -3259,7 +3423,7 @@ def anthropic_messages_pt( text_block = cast(str, m.get("text", "")) # handle text if ( - m.get("type", "") == "text" and len(text_block) > 0 + block_type == "text" and len(text_block) > 0 ): # don't pass empty text blocks. anthropic api raises errors. anthropic_message = AnthropicMessagesTextParam(type="text", text=text_block) _cached_message = add_cache_control_to_content( @@ -3270,8 +3434,20 @@ def anthropic_messages_pt( assistant_content.append(cast(AnthropicMessagesTextParam, _cached_message)) # handle server_tool_use blocks (tool search, web search, etc.) # Pass through as-is since these are Anthropic-native content types - elif m.get("type", "") == "server_tool_use" or m.get("type", "").endswith("_tool_result"): + elif block_type == "server_tool_use" or block_type.endswith("_tool_result"): assistant_content.append(m) + elif block_type == "tool_use": + tool_use_id = native_tool_use_id(m) + if ( + tool_use_id in unique_tool_ids + or tool_use_id in tool_call_ids + or not tool_use_is_answered( + tool_use_id, answered_by_assistant_index().get(msg_i, frozenset()) + ) + ): + continue + unique_tool_ids.add(tool_use_id) + assistant_content.append(_anthropic_native_tool_use_block(m, tool_use_id, drop_params)) elif ( "content" in assistant_content_block and isinstance(assistant_content_block["content"], str) @@ -3307,6 +3483,7 @@ def anthropic_messages_pt( web_search_results=_web_search_results, tool_results=_tool_results, answered_tool_ids=client_answered_tool_ids, + drop_params=drop_params, ) # Prevent "tool_use ids must be unique" errors by filtering duplicates @@ -3916,8 +4093,6 @@ def stringify_json_tool_call_content(messages: list) -> list: ###### AMAZON BEDROCK ####### -from email.message import Message - import httpx from litellm.types.llms.bedrock import ( diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py index eb3400aee8b..36a522c7e20 100644 --- a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py @@ -49,7 +49,7 @@ from litellm.types.llms.openai import ( ChatCompletionUserMessage, ) -from .common_utils import has_blank_redacted_data, is_unsignable_thinking_block +from .common_utils import has_blank_redacted_data, is_unsignable_thinking_block, sanitize_anthropic_tool_use_id CONVERTED_SYSTEM_NOTE: Final = ( "Operator note (not from the user): the following was originally a mid-conversation system-role reminder." @@ -272,11 +272,25 @@ def _thinking_block_renders(block: object) -> bool: ) -def _assistant_part_renders(part: object) -> bool: - """A text part always renders: the converter pads empty text with a placeholder.""" +def native_tool_use_id(block: object) -> str: + raw_id: Final = message_field(block, "id") + return sanitize_anthropic_tool_use_id(raw_id if isinstance(raw_id, str) else "") + + +def tool_use_is_answered(tool_use_id: str, answered: frozenset[str]) -> bool: + return tool_use_id in answered + + +def _assistant_part_renders(part: object, answered: frozenset[str]) -> bool: + """A text part always renders: the converter pads empty text with a placeholder. + + A native ``tool_use`` renders only when the next turn answers it, since the converter drops an orphan. + """ part_type: Final = message_field(part, "type") if part_type in _THINKING_BLOCK_TYPES: return _thinking_block_renders(part) + if part_type == "tool_use": + return tool_use_is_answered(native_tool_use_id(part), answered) return part_type in _RENDERED_ASSISTANT_PART_TYPES or ( isinstance(part_type, str) and part_type.endswith("_tool_result") ) @@ -308,6 +322,7 @@ def _tool_result_ids(answers: Iterable[object]) -> Iterator[str]: for message in answers: if isinstance(tool_call_id := message_field(message, "tool_call_id"), str) and tool_call_id: yield tool_call_id + yield sanitize_anthropic_tool_use_id(tool_call_id) def answered_tool_call_ids(messages: Sequence[object], assistant_index: int) -> frozenset[str]: @@ -334,7 +349,7 @@ def _answered_by_assistant_index( return ((index, answered) for index, _ in groups[group_index][1]) -def _answered_ids_by_index(messages: Sequence[object]) -> Mapping[int, frozenset[str]]: +def answered_ids_by_index(messages: Sequence[object]) -> Mapping[int, frozenset[str]]: """``answered_tool_call_ids`` for every assistant message, computed in one pass over the messages.""" groups: Final = tuple( (role_group, tuple(group)) @@ -398,7 +413,7 @@ def _assistant_renders(message: object, answered: frozenset[str]) -> bool: return True parts: Final = parts_of(content) return ( - any(_assistant_part_renders(part) for part in parts) + any(_assistant_part_renders(part, answered) for part in parts) or _separate_thinking_blocks_render(message) or _tool_calls_render(message, answered) or bool(message_field(message, "function_call")) @@ -429,7 +444,7 @@ def _renders(message: object, answered: frozenset[str]) -> bool: def _rendered_indices(messages: Sequence[object]) -> frozenset[int]: - answered: Final = _answered_ids_by_index(messages) + answered: Final = answered_ids_by_index(messages) return frozenset(index for index, message in enumerate(messages) if _renders(message, answered.get(index, _NO_IDS))) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 0967c22f93a..7f892cd46d7 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -41,16 +41,24 @@ _PLAIN_MESSAGE_KEYS: Final = frozenset({"role", "content"}) def _count_image(block: JsonValue) -> JsonValue: if not isinstance(block, dict) or block.get("type") != "image_url": return block - from litellm.litellm_core_utils.prompt_templates.factory import convert_to_anthropic_image_obj + from litellm.litellm_core_utils.prompt_templates.factory import ( + anthropic_document_block_from_data_uri, + anthropic_document_media_type_for_data_uri, + convert_to_anthropic_image_obj, + ) image_block: Final = _IMAGE_BLOCK.validate_python(block) image_url: Final = image_block["image_url"] - source: Final = convert_to_anthropic_image_obj( - openai_image_url=image_url if isinstance(image_url, str) else image_url["url"], - format=image_url.get("format") if isinstance(image_url, dict) else None, - ) + url: Final = image_url if isinstance(image_url, str) else image_url["url"] + format_override: Final = image_url.get("format") if isinstance(image_url, dict) else None + other_fields: Final = {key: value for key, value in block.items() if key not in {"type", "image_url"}} + document_media_type: Final = anthropic_document_media_type_for_data_uri(url, format_override) + if document_media_type is not None: + document: Final = anthropic_document_block_from_data_uri(url, image_block, document_media_type) + return {**other_fields, **_COUNT_REQUEST.validate_python(document)} + source: Final = convert_to_anthropic_image_obj(openai_image_url=url, format=format_override) image: Final = _COUNT_REQUEST.validate_python({"type": "image", "source": source}) - return {**{key: value for key, value in block.items() if key not in {"type", "image_url"}}, **image} + return {**other_fields, **image} def _count_block(block: JsonValue) -> JsonValue: diff --git a/litellm/llms/snowflake/chat/transformation.py b/litellm/llms/snowflake/chat/transformation.py index 030413030eb..7b0ea78ba96 100644 --- a/litellm/llms/snowflake/chat/transformation.py +++ b/litellm/llms/snowflake/chat/transformation.py @@ -8,28 +8,22 @@ Routes to native Cortex REST API endpoints based on model: Ref: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api """ -import copy -import json -import re from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict import httpx from typing_extensions import ReadOnly -from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.litellm_core_utils.prompt_templates.factory import ( - anthropic_process_openai_file_message, - convert_to_anthropic_tool_result, - create_anthropic_image_param, - select_anthropic_content_block_type_for_file, -) +import litellm +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt from litellm.litellm_core_utils.prompt_templates.image_handling import async_inline_remote_media from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.chat.handler import ModelResponseIterator as AnthropicStreamParser from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolMessage +from litellm.types.llms.anthropic import AllAnthropicPassThroughMessageValues, AnthropicSystemMessageContent +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk from litellm.types.utils import ( Choices, GenericStreamingChunk, @@ -40,7 +34,7 @@ from litellm.types.utils import ( from ...base_llm.base_model_iterator import BaseModelResponseIterator from ...openai_like.chat.transformation import OpenAIGPTConfig -from ..utils import SnowflakeBaseConfig +from ..utils import SnowflakeBaseConfig, SnowflakeException if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -104,96 +98,6 @@ def is_snowflake_claude_model(model: str) -> bool: return any(name.startswith(p) for p in _CLAUDE_MODEL_PREFIXES) -def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object: - """One OpenAI ``image_url`` block in the native shape Cortex accepts. - - Cortex documents base64 sources only, so remote URLs are inlined the way every - other base64-only Anthropic dialect (Bedrock invoke, Vertex) inlines them, and - pdf/text data URIs become document blocks rather than malformed image blocks. - """ - image_url: Final = block.get("image_url") - url: Final = image_url if isinstance(image_url, str) else _image_url_field(image_url, "url") - if not url: - return block - - converted: Final = ( - anthropic_process_openai_file_message({"type": "file", "file": {"file_data": url}}) - if select_anthropic_content_block_type_for_file(_data_uri_media_type(url)) == "document" - else create_anthropic_image_param( - image_url if isinstance(image_url, dict) else url, - format=_image_url_field(image_url, "format"), - is_bedrock_invoke=True, - ) - ) - cache_control: Final = block.get("cache_control") - if cache_control is None: - return converted - return {**converted, "cache_control": cache_control} - - -def _image_url_field(image_url: object, key: str) -> str | None: - value: Final = image_url.get(key) if isinstance(image_url, dict) else None - return value if isinstance(value, str) else None - - -def _data_uri_media_type(url: str) -> str: - match: Final = re.match(r"data:([^;,]+)", url) - return match.group(1) if match else "" - - -def _convert_image_url_blocks_to_anthropic(content: object) -> object: - if not isinstance(content, list): - return content - return [ - _convert_image_url_to_anthropic(block) - if isinstance(block, Mapping) and block.get("type") == "image_url" - else block - for block in content - ] - - -def _convert_tool_result_to_anthropic( - content: object, tool_call_id: str, cache_control: object -) -> Mapping[str, object]: - """The Anthropic ``tool_result`` block for one OpenAI tool message. - - Delegating to the shared converter keeps image, document and per-block cache - breakpoints identical to every other Anthropic dialect; only the plain-string - and non-list shapes it does not model are handled here. - """ - if not isinstance(content, list): - plain: Final[dict[str, object]] = { # mutable-ok: JSON wire block - "type": "tool_result", - "tool_use_id": tool_call_id, - "content": content if isinstance(content, str) else json.dumps(content), - } - return {**plain, "cache_control": cache_control} if cache_control is not None else plain - converted: Final = convert_to_anthropic_tool_result( - ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content), - force_base64=True, - ) - if cache_control is None: - return converted - return {**converted, "cache_control": cache_control} - - -def _signed_thinking_blocks(msg: object) -> list[dict[str, object]]: # mutable-ok: JSON wire blocks - """The assistant turn's thinking blocks that can legally be echoed back. - - Only signed blocks round-trip: Cortex rejects a thinking block whose signature is - missing, which is what an unsigned block from a non-thinking turn would produce. - """ - blocks: Final = msg.get("thinking_blocks") if isinstance(msg, dict) else getattr(msg, "thinking_blocks", None) - if not isinstance(blocks, list): - return [] - mappings: Final = tuple(filter(None, map(as_str_mapping, blocks))) - return [ - {key: value for key, value in block.items() if key != "index"} - for block in mappings - if block.get("signature") or block.get("type") == "redacted_thinking" - ] - - def _clean_input_schema(schema: object) -> object: return {key: value for key, value in schema.items() if key != "$schema"} if isinstance(schema, Mapping) else schema @@ -298,105 +202,23 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): ) return anthropic_tools - def _extract_system_and_messages(self, messages: list[AllMessageValues]) -> tuple[list[dict] | None, list[dict]]: - """ - Split messages into system prompt and conversation turns for Anthropic format. - - - system messages → collected and joined (preserves guardrail prompts) - - assistant messages with tool_calls → tool_use content blocks - - tool role messages → user role with tool_result content blocks - """ - system_parts: Final[list[dict]] = [] # mutable-ok: JSON wire messages - conversation: Final[list[dict]] = [] # mutable-ok: JSON wire messages - - for msg in messages: - if isinstance(msg, dict): - role = msg.get("role", "") - content: object = msg.get("content", "") - msg_cache_control: object = msg.get("cache_control") - else: - role = getattr(msg, "role", "") - content = getattr(msg, "content", "") - msg_cache_control = getattr(msg, "cache_control", None) - - if role == "system": - if isinstance(content, str) and content: - system_parts.append({"type": "text", "text": content}) - elif isinstance(content, list): - system_parts.extend( - { - "type": "text", - "text": block.get("text", ""), - **({"cache_control": block["cache_control"]} if "cache_control" in block else {}), - } - for block in content - if isinstance(block, Mapping) and block.get("type") == "text" - ) - elif role == "assistant": - tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None) - thinking_blocks = _signed_thinking_blocks(msg) - if tool_calls: - content_blocks: list[dict[str, object]] = list(thinking_blocks) # mutable-ok: JSON wire blocks - if content: - content_blocks.append({"type": "text", "text": content}) - for tc in tool_calls: - func = tc.get("function", {}) if isinstance(tc, dict) else getattr(tc, "function", {}) - tc_id = tc.get("id", "") if isinstance(tc, dict) else getattr(tc, "id", "") - func_name = func.get("name", "") if isinstance(func, dict) else getattr(func, "name", "") - func_args = ( - func.get("arguments", "{}") if isinstance(func, dict) else getattr(func, "arguments", "{}") - ) - try: - input_data = json.loads(func_args) if isinstance(func_args, str) else func_args - except (json.JSONDecodeError, TypeError): - input_data = {} - content_blocks.append( - { - "type": "tool_use", - "id": tc_id, - "name": func_name, - "input": input_data, - } - ) - conversation.append({"role": "assistant", "content": content_blocks}) - elif thinking_blocks: - thinking_content = ( - [ - *thinking_blocks, - *copy.deepcopy(content), - ] - if isinstance(content, list) - else [*thinking_blocks, *([{"type": "text", "text": content}] if content else [])] - ) - conversation.append({"role": "assistant", "content": thinking_content}) - else: - conversation.append({"role": "assistant", "content": content}) - elif role == "tool": - tool_call_id_value = ( - msg.get("tool_call_id", "") if isinstance(msg, dict) else getattr(msg, "tool_call_id", "") - ) - tool_call_id = tool_call_id_value if isinstance(tool_call_id_value, str) else "" - tool_result_block = _convert_tool_result_to_anthropic(content, tool_call_id, msg_cache_control) - if ( - conversation - and conversation[-1]["role"] == "user" - and isinstance(conversation[-1]["content"], list) - and conversation[-1]["content"] - and conversation[-1]["content"][0].get("type") == "tool_result" - ): - conversation[-1]["content"].append(tool_result_block) - else: - conversation.append({"role": "user", "content": [tool_result_block]}) - else: - conversation.append( - { - "role": role, - "content": _convert_image_url_blocks_to_anthropic(content), - } - ) - - system: Final[list[dict] | None] = system_parts if system_parts else None # mutable-ok: JSON wire messages - return system, conversation + def _extract_system_and_messages( + self, model: str, messages: list[AllMessageValues], drop_params: bool + ) -> tuple[list[AnthropicSystemMessageContent] | None, list[AllAnthropicPassThroughMessageValues]]: + """Cortex's /messages endpoint speaks Anthropic's dialect, so the shared translators own the shape.""" + conversation_messages: Final = list(messages) + system: Final = AnthropicConfig().translate_system_message(messages=conversation_messages) + try: + conversation: Final = anthropic_messages_pt( + messages=conversation_messages, + model=model, + llm_provider="snowflake", + drop_params=drop_params, + force_base64=True, + ) + except (ValueError, BadRequestError) as e: + raise SnowflakeException(status_code=400, message=str(e)) from e + return (system or None), conversation def transform_request( self, @@ -410,7 +232,8 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): extra_body: Final = optional_params.pop("extra_body", {}) if is_snowflake_claude_model(model): - return self._transform_request_anthropic(model, messages, optional_params, stream, extra_body) + drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True + return self._transform_request_anthropic(model, messages, optional_params, stream, extra_body, drop_params) return self._transform_request_openai(model, messages, optional_params, stream, extra_body) @property @@ -485,10 +308,11 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): optional_params: dict, stream: bool, extra_body: dict, + drop_params: bool, ) -> dict: """Anthropic Messages format for /messages endpoint.""" passthrough_system: Final = optional_params.pop("system", None) - extracted_system, conversation = self._extract_system_and_messages(messages) + extracted_system, conversation = self._extract_system_and_messages(model, messages, drop_params) system: Final = passthrough_system if passthrough_system is not None else extracted_system if "tools" in optional_params: diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index b9675f6fd5c..f45eb965377 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -5,6 +5,7 @@ import json import os import sys from collections.abc import Mapping +from types import MappingProxyType from typing import Final, cast import pytest @@ -2493,6 +2494,23 @@ def test_schema_size_counts_every_written_object_once(schema, properties, branch assert schema_grammar_branch_count(schema) == branches +@pytest.mark.parametrize( + "arguments", + [{"already": "parsed"}, MappingProxyType({"already": "parsed"})], + ids=["dict", "mapping_proxy"], +) +def test_parse_tool_call_arguments_returns_already_parsed_mapping(arguments): + parsed = parse_tool_call_arguments(arguments, tool_name="dict_tool", context="test") + + assert parsed == {"already": "parsed"} + assert json.loads(json.dumps(parsed)) == {"already": "parsed"} + + +def test_parse_tool_call_arguments_rejects_unrepairable_json(): + with pytest.raises(ValueError, match="broken_tool"): + parse_tool_call_arguments("not valid json{{{", tool_name="broken_tool", context="test") + + @pytest.mark.parametrize("part_type", ["image_url", "input_image"]) @pytest.mark.parametrize( "fields", diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 53be3608979..42b185fa48c 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -16,6 +16,7 @@ import pytest import respx import litellm +from litellm.constants import REDACTED_BY_LITELLM from litellm.litellm_core_utils.prompt_templates.factory import ( BEDROCK_DOCUMENT_PLACEHOLDER_TEXT, _EMPTY_TEXT_PLACEHOLDER, @@ -38,6 +39,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( make_valid_bedrock_tool_name, ollama_pt, parse_mime_type, + requires_inline_base64_media, sanitize_messages_for_tool_calling, THOUGHT_SIGNATURE_SEPARATOR, ) @@ -3115,6 +3117,19 @@ def test_convert_to_anthropic_tool_result_image_with_cache_control(): assert result["content"][1]["cache_control"]["type"] == "ephemeral" +def test_convert_to_anthropic_tool_result_serializes_mapping_content(): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + result = convert_to_anthropic_tool_result( + {"role": "tool", "tool_call_id": "call_dict", "content": {"result_key": "result_value"}} + ) + + assert result["type"] == "tool_result" + assert json.loads(result["content"]) == {"result_key": "result_value"} + + def test_convert_to_anthropic_tool_result_image_without_cache_control(): """ Test that images without cache_control in tool results work correctly. @@ -5129,6 +5144,7 @@ def test_group_tool_exchanges_is_linear_in_message_count(): _PDF_DATA_URI = "data:application/pdf;base64," + base64.b64encode(b"%PDF-1.4 regression fixture").decode() +_PNG_B64 = base64.b64encode(b"\x89PNG\r\n\x1a\n regression fixture").decode() _PNG_DATA_URI = ( "data:image/png;base64," "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" @@ -5161,6 +5177,116 @@ def test_bedrock_converse_pdf_only_user_message_gets_text_block(): assert _text_blocks(result[0]) == [BEDROCK_DOCUMENT_PLACEHOLDER_TEXT] +@pytest.mark.parametrize( + ("model", "llm_provider"), + [ + ("claude-sonnet-5", "anthropic"), + ("invoke/us.anthropic.claude-sonnet-5", "bedrock"), + ("claude-sonnet-5", "vertex_ai"), + ("claude-sonnet-5", "snowflake"), + ], +) +def test_anthropic_messages_pt_user_pdf_data_uri_becomes_document_block(model, llm_provider): + """ + Regression for LIT-6778: an OpenAI client attaches a PDF as an image_url part + whose url is a pdf data URI. Every Anthropic-shaped API rejects + `{"type": "image", "media_type": "application/pdf"}`, so the user content must + carry a document block, with the block's cache breakpoint intact. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What word is in this document?"}, + { + "type": "image_url", + "image_url": {"url": _PDF_DATA_URI}, + "cache_control": {"type": "ephemeral"}, + }, + ], + } + ] + + result = anthropic_messages_pt(messages=messages, model=model, llm_provider=llm_provider) + + assert result[0]["content"][1] == { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": _PDF_DATA_URI.split(",", 1)[1], + }, + "cache_control": {"type": "ephemeral"}, + } + assert all(block["type"] != "image" for block in result[0]["content"]) + + +_TEXT_B64 = base64.b64encode("the secret code word is MANGO\n".encode()).decode() +_TEXT_DATA_URI = "data:text/plain;base64," + _TEXT_B64 +_TEXT_DATA_URI_WITH_CHARSET = "data:text/plain;charset=utf-8;base64," + _TEXT_B64 +_LATIN1_TEXT_DATA_URI = "data:text/plain;charset=iso-8859-1;base64," + base64.b64encode( + "the secret code word is MANGÓ\n".encode("latin-1") +).decode() + + +@pytest.mark.parametrize( + "block, expected_text", + [ + ({"type": "image_url", "image_url": {"url": _TEXT_DATA_URI}}, "the secret code word is MANGO\n"), + ({"type": "file", "file": {"file_data": _TEXT_DATA_URI}}, "the secret code word is MANGO\n"), + ({"type": "image_url", "image_url": {"url": _TEXT_DATA_URI_WITH_CHARSET}}, "the secret code word is MANGO\n"), + ({"type": "file", "file": {"file_data": _TEXT_DATA_URI_WITH_CHARSET}}, "the secret code word is MANGO\n"), + ({"type": "file", "file": {"file_data": _LATIN1_TEXT_DATA_URI}}, "the secret code word is MANGÓ\n"), + ], + ids=["image_url", "file", "image_url_charset", "file_charset", "file_latin1"], +) +def test_anthropic_messages_pt_text_plain_data_uri_becomes_text_source_document(block, expected_text): + """ + Anthropic's base64 document source only takes application/pdf; a text/plain + document has to travel as a `text` source carrying the decoded text, or the + API answers 400 on every Anthropic-shaped provider. The data URI may carry a + charset parameter, which must neither hide the text/plain type nor be ignored + when decoding the bytes. + """ + messages = [{"role": "user", "content": [{"type": "text", "text": "What is the code word?"}, block]}] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"][1] == { + "type": "document", + "source": {"type": "text", "media_type": "text/plain", "data": expected_text}, + } + + +def test_anthropic_messages_pt_user_png_data_uri_stays_an_image_block(): + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": _PNG_DATA_URI}}]}] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"] == [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": _PNG_DATA_URI.split(",", 1)[1]}, + } + ] + + +@pytest.mark.parametrize( + ("model", "llm_provider", "expected"), + [ + ("invoke/us.anthropic.claude-sonnet-5", "bedrock", True), + ("claude-sonnet-5", "vertex_ai", True), + ("claude-sonnet-5", "vertex_ai_beta", True), + ("claude-sonnet-5", "snowflake", False), + ("claude-sonnet-5", "anthropic", False), + ("us.anthropic.claude-sonnet-5", "bedrock", False), + ("claude-sonnet-5", None, False), + ], +) +def test_requires_inline_base64_media(model, llm_provider, expected): + assert requires_inline_base64_media(model, llm_provider) is expected + + def test_bedrock_converse_document_with_text_gets_no_extra_text_block(): messages = [ { @@ -8771,3 +8897,602 @@ def test_wrong_type_template_field_in_requests_leaves_the_deployments_in_rotatio response = router.completion(model="template-group", messages=TEMPLATE_MESSAGES, mock_response="Blue") assert response.choices[0].message.content == "Blue" + + +def test_anthropic_messages_pt_keeps_assistant_tool_use_content_block(): + """ + A conversation replayed from an Anthropic-shaped client carries the tool call + as a `tool_use` block inside the assistant content list. Dropping it orphans + the `tool_result` that answers it, and the API rejects the whole request. + """ + messages = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "let me look that up"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "paris"}}, + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + {"type": "text", "text": "let me look that up"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "paris"}}, + ] + assert result[2]["content"] == [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "sunny"}] + + +def test_anthropic_messages_pt_does_not_duplicate_a_tool_use_present_in_both_shapes(): + """ + The same call can arrive as a `tool_use` block and as a `tool_calls` entry. + Both shapes sanitize the id the same way, so the deduplication has to compare + the sanitized ids, not the raw ones. + """ + messages = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "functions.Bash:0", "name": "get_weather", "input": {"city": "paris"}} + ], + "tool_calls": [ + { + "id": "functions.Bash:0", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "paris"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "functions.Bash:0", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + {"type": "tool_use", "id": "functions_Bash_0", "name": "get_weather", "input": {"city": "paris"}} + ] + + +def test_anthropic_messages_pt_sanitizes_a_native_tool_use_id_like_its_tool_result(): + """ + Anthropic only accepts tool ids matching ^[a-zA-Z0-9_-]+$, and the `tool_result` + side already sanitizes. A replayed `tool_use` block whose id came from another + client (`functions.Bash:0`) has to get the same treatment, or the two ids stop + matching and the request is rejected. + """ + messages = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "functions.Bash:0", "name": "get_weather", "input": {"city": "paris"}} + ], + }, + {"role": "tool", "tool_call_id": "functions.Bash:0", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + {"type": "tool_use", "id": "functions_Bash_0", "name": "get_weather", "input": {"city": "paris"}} + ] + assert result[2]["content"] == [ + {"type": "tool_result", "tool_use_id": "functions_Bash_0", "content": "sunny"} + ] + + +def test_anthropic_messages_pt_normalizes_a_native_tool_use_block(): + """ + A block accumulated from a stream carries a streaming `index` key and can hold + the arguments as a JSON string. Anthropic rejects both, so the block is rebuilt + from its id, name, and parsed input, keeping any cache_control the caller set. + """ + messages = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "index": 0, + "id": "toolu_01", + "name": "get_weather", + "input": '{"city": "paris"}', + "cache_control": {"type": "ephemeral"}, + } + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + { + "type": "tool_use", + "id": "toolu_01", + "name": "get_weather", + "input": {"city": "paris"}, + "cache_control": {"type": "ephemeral"}, + } + ] + + +@pytest.mark.parametrize( + ("url", "format_override", "expected"), + [ + ( + "data:application/pdf;base64," + _PNG_B64, + "image/png", + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": _PNG_B64}}, + ), + ( + "data:image/png;base64,JVBERi0=", + "application/pdf", + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0="}}, + ), + ( + _PDF_DATA_URI, + "image/png", + { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": _PDF_DATA_URI.split(",", 1)[1]}, + }, + ), + ], + ids=["png_bytes_under_a_pdf_uri_declared_png", "png_uri_declared_pdf", "pdf_bytes_declared_png"], +) +def test_anthropic_messages_pt_honours_an_explicit_format_over_the_data_uri(url, format_override, expected): + """ + `format` is how a caller corrects a media type the data URI got wrong, so the + routing between image and document blocks has to read it before the URI's own + declared type. Bytes that are a PDF stay a document whatever the labels say, + since Anthropic rejects a PDF sent as an image. + """ + messages = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": url, "format": format_override}}]} + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"] == [expected] + + +def test_anthropic_messages_pt_collapses_native_tool_use_blocks_sharing_an_id(): + """ + A replayed history can carry the same call twice, once with the raw id and once + with the sanitized one, and both land on the same `tool_use_id`. Anthropic 400s + a request whose tool ids repeat, so only the first block may survive. + """ + messages = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "functions.Bash:0", "name": "get_weather", "input": {"city": "paris"}}, + {"type": "tool_use", "id": "functions_Bash_0", "name": "get_weather", "input": {"city": "london"}}, + ], + }, + {"role": "tool", "tool_call_id": "functions.Bash:0", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + {"type": "tool_use", "id": "functions_Bash_0", "name": "get_weather", "input": {"city": "paris"}} + ] + + +@pytest.mark.parametrize( + "image_url", + [{"url": _PDF_DATA_URI}, {"url": _PDF_DATA_URI, "format": "image/png"}], + ids=["no-format", "declared-png"], +) +def test_anthropic_messages_pt_tool_result_pdf_data_uri_becomes_a_document_block(image_url): + """ + A tool can answer with a PDF it just fetched, which arrives as an `image_url` + entry inside the tool message's content list. Anthropic only accepts a PDF in a + document block, so the tool_result has to route it the same way user content does, + whatever image format the tool labelled it with. + """ + messages = [ + {"role": "user", "content": "fetch the invoice"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "fetch_invoice", "input": {}}], + }, + { + "role": "tool", + "tool_call_id": "toolu_01", + "content": [{"type": "image_url", "image_url": image_url}], + }, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[2]["content"] == [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": _PDF_DATA_URI.split(",", 1)[1], + }, + } + ], + } + ] + + +def test_anthropic_messages_pt_text_document_keeps_charset_named_by_the_data_uri(): + """ + `format` is meant to name the media type when the data URI is vague, so naming + the type the URI already declares must not throw away its `charset`. Decoding + latin-1 bytes as utf-8 raises before the request ever leaves litellm. + """ + latin1_data_uri: Final = "data:text/plain;charset=iso-8859-1;base64," + base64.b64encode( + "café".encode("iso-8859-1") + ).decode() + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": latin1_data_uri, "format": "text/plain"}}], + } + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"] == [ + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "café"}} + ] + + +def test_anthropic_messages_pt_native_tool_use_input_survives_guardrail_redaction(): + """ + A guardrail replaces tool input with the redaction sentinel, which is not JSON. + Rejecting it would turn every redacted replay into a 400. + """ + messages = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": REDACTED_BY_LITELLM}], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}] + + +def test_anthropic_messages_pt_drops_a_native_tool_use_nobody_answered(): + """ + A stored conversation can leave a tool call without its result, the user moving + on with a plain message instead. Anthropic rejects a `tool_use` with no + `tool_result` right after it, and these replays were answered before native + blocks were kept, so an unanswered one is still left out. + """ + messages = [ + {"role": "user", "content": "what is the weather"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "checking"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}, + ], + }, + {"role": "user", "content": "never mind, tell me a joke"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[1]["content"] == [{"type": "text", "text": "checking"}] + + +def _answered_native_tool_use(tool_use: dict[str, object]) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "what is the weather"}, + {"role": "assistant", "content": [tool_use]}, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + +@pytest.mark.parametrize( + "tool_use", + [ + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": "[1, 2]"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": [1, 2]}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": "{not json"}, + ], + ids=["json-array-string", "list", "missing", "unparseable"], +) +def test_anthropic_messages_pt_native_tool_use_input_that_is_not_an_object_follows_drop_params(tool_use): + """ + Anthropic only takes an object as tool input. Sending `{}` in place of what the + caller wrote changes the conversation, so that is only done when the caller asked + for unsupported values to be dropped, and is a client error otherwise. + """ + messages: Final = _answered_native_tool_use(tool_use) + + with pytest.raises(ValueError, match="get_weather"): + anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + dropped: Final = anthropic_messages_pt( + messages=messages, model="claude-sonnet-5", llm_provider="anthropic", drop_params=True + ) + + assert dropped[1]["content"] == [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}] + + +def test_anthropic_messages_pt_unparseable_tool_call_arguments_follow_drop_params(): + """ + Broken `tool_calls` arguments are a client error, unless the caller asked for + unsupported values to be dropped, in which case the call goes out with no input. + """ + messages: Final = [ + {"role": "user", "content": "what is the weather"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "toolu_01", "type": "function", "function": {"name": "get_weather", "arguments": "{not json"}} + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + with pytest.raises(ValueError, match="get_weather"): + anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + dropped: Final = anthropic_messages_pt( + messages=messages, model="claude-sonnet-5", llm_provider="anthropic", drop_params=True + ) + + assert dropped[1]["content"] == [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}] + + +_NAMED_PDF_DATA_URI = "data:application/pdf;name=report.pdf;base64," + base64.b64encode(b"%PDF-1.4 named").decode() + + +@pytest.mark.parametrize( + "block", + [ + {"type": "image_url", "image_url": {"url": _NAMED_PDF_DATA_URI}}, + {"type": "file", "file": {"file_data": _NAMED_PDF_DATA_URI}}, + ], + ids=["image_url", "file"], +) +def test_anthropic_messages_pt_sends_a_named_pdf_under_its_bare_media_type(block): + """ + A data URI built from a file picker carries the file name in its media type, and + Anthropic's document media_type only takes `application/pdf` itself. + """ + result = anthropic_messages_pt( + messages=[{"role": "user", "content": [block]}], model="claude-sonnet-5", llm_provider="anthropic" + ) + + assert result[0]["content"] == [ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": _NAMED_PDF_DATA_URI.split(",", 1)[1], + }, + } + ] + + +def test_anthropic_messages_pt_keeps_a_pdf_a_document_when_the_format_is_generic(): + """ + A browser reports `application/octet-stream` for any file it cannot identify, so + that `format` says nothing about the bytes. Letting it override the data URI's own + `application/pdf` routes a real PDF into an image block Anthropic rejects. + """ + messages = [ + {"role": "user", "content": "fetch the invoice"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "fetch_invoice", "input": {}}], + }, + { + "role": "tool", + "tool_call_id": "toolu_01", + "content": [ + { + "type": "image_url", + "image_url": {"url": _PDF_DATA_URI, "format": "application/octet-stream"}, + } + ], + }, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[2]["content"] == [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": _PDF_DATA_URI.split(",", 1)[1], + }, + } + ], + } + ] + + +def test_anthropic_messages_pt_drops_data_uri_parameters_the_format_replaces(): + """ + A data URI built from a file picker carries the file name in its media type, which + Anthropic's media_type enum rejects. Naming the type in `format` is how a caller + cleans that up, so only the charset of a text attachment may survive it. + """ + png_data_uri: Final = "data:image/png;name=diagram.png;base64," + base64.b64encode(b"\x89PNG\r\n\x1a\n").decode() + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": png_data_uri, "format": "image/png"}}], + } + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert result[0]["content"] == [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": png_data_uri.split(",", 1)[1]}, + } + ] + + +def test_anthropic_messages_pt_rejects_a_text_attachment_it_cannot_decode(): + """ + A text file a browser turned into a data URI names no charset, so bytes that are + not utf-8 cannot be read. The caller has to get a client error naming the problem, + never a decode failure carrying the whole attachment back in its message. + """ + undeclared_latin1: Final = "data:text/plain;base64," + base64.b64encode( + "the secret code word is caf\xe9".encode("iso-8859-1") * 200 + ).decode() + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": undeclared_latin1, "format": "text/plain"}}], + } + ] + + with pytest.raises(litellm.BadRequestError) as raised: + anthropic_messages_pt(messages=messages, model="claude-sonnet-5", llm_provider="anthropic") + + assert "utf-8" in str(raised.value) + assert undeclared_latin1.split(",", 1)[1] not in str(raised.value) + + +@pytest.mark.parametrize("native_input", [{"city": "lyon"}, "[1, 2]"]) +def test_anthropic_messages_pt_sends_tool_calls_arguments_over_a_native_tool_use_with_the_same_id(native_input): + messages: Final = [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": native_input}], + "tool_calls": [ + { + "id": "toolu_01", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "paris"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-5-5", llm_provider="anthropic") + + assert result[1]["content"] == [ + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "paris"}} + ] + + +@pytest.mark.parametrize(("drop_params", "expected_input"), [(True, {}), (False, [1, 2])]) +def test_anthropic_messages_pt_tool_call_arguments_that_are_not_an_object_follow_drop_params( + drop_params, expected_input +): + messages: Final = [ + {"role": "user", "content": "what is the weather"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "toolu_01", "type": "function", "function": {"name": "get_weather", "arguments": "[1, 2]"}} + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + + result: Final = anthropic_messages_pt( + messages=messages, model="claude-sonnet-5-5", llm_provider="anthropic", drop_params=drop_params + ) + + assert result[1]["content"][0]["input"] == expected_input + + +_PDF_PAYLOAD: Final = _PDF_DATA_URI.split(",", 1)[1] + + +@pytest.mark.parametrize( + "block", + [ + {"type": "file", "file": {"file_data": "data:application/octet-stream;base64," + _PDF_PAYLOAD}}, + {"type": "file", "file": {"file_data": "data:image/png;base64," + _PDF_PAYLOAD}}, + {"type": "file", "file": {"file_data": _PDF_DATA_URI, "format": "text/plain"}}, + {"type": "image_url", "image_url": {"url": _PDF_DATA_URI, "format": "text/plain"}}, + ], + ids=["file-octet-stream", "file-image-png", "file-format-text", "image-url-format-text"], +) +def test_anthropic_messages_pt_sends_pdf_bytes_as_a_pdf_document_whatever_the_label(block): + result: Final = anthropic_messages_pt( + messages=[{"role": "user", "content": [block]}], model="claude-sonnet-5-5", llm_provider="anthropic" + ) + + assert result[0]["content"] == [ + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": _PDF_PAYLOAD}} + ] + + +def test_gemini_inline_data_carries_the_format_it_was_given(): + text_data_uri: Final = "data:text/plain;charset=utf-8;base64," + base64.b64encode(b"MANGO").decode() + messages: Final = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": text_data_uri, "format": "text/plain"}}]} + ] + + contents: Final = gemini_convert_messages_with_history(messages=messages, model="gemini-3.8-flash") + + assert contents[0]["parts"][0]["inline_data"]["mime_type"] == "text/plain" + + +def _parallel_native_tool_use_seconds(calls: int) -> float: + answers: Final = tuple( + {"role": "tool", "tool_call_id": f"functions.get_weather:{index}", "content": "sunny"} for index in range(calls) + ) + messages: Final = [ + {"role": "user", "content": "weather everywhere"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": f"functions.get_weather:{index}", "name": "get_weather", "input": {}} + for index in range(calls) + ], + }, + *answers, + ] + + def once() -> float: + started: Final = time.perf_counter() + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-5-5", llm_provider="anthropic") + elapsed: Final = time.perf_counter() - started + assert [block["id"] for block in result[1]["content"]] == [ + f"functions_get_weather_{index}" for index in range(calls) + ] + return elapsed + + return min(once() for _ in range(3)) + + +def test_anthropic_messages_pt_cost_grows_linearly_with_parallel_native_tool_uses(): + growth: Final = _parallel_native_tool_use_seconds(1600) / _parallel_native_tool_use_seconds(400) + + assert growth < 11.0, f"cost grew {growth:.1f}x for 4x the tool uses" diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py index aab57b87a90..74c488abeb5 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py @@ -536,6 +536,23 @@ def test_flagged_placement_keeps_a_system_before_an_assistant_turn_whose_empty_t assert _roles(placed) == ["user", "system", "assistant", "user"] +def test_flagged_placement_keeps_a_system_before_an_answered_native_tool_use(): + placed = place_mid_conversation_system( + [ + {"role": "user", "content": "weather?"}, + {"role": "system", "content": "Be terse."}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "functions.get_weather:0", "name": "get_weather", "input": {}}], + }, + {"role": "tool", "tool_call_id": "functions.get_weather:0", "content": "sunny"}, + ], + supports_mid_conversation_system=True, + ) + + assert _roles(placed) == ["user", "system", "assistant", "tool"] + + def test_anthropic_system_blocks_keeps_text_parts_with_their_cache_control_and_drops_the_rest(): run = [ {"role": "system", "content": "one", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 89352ce8702..bd9a3da1357 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -71,6 +71,26 @@ def test_count_translates_openai_images_without_mutating_input( assert messages == original +@pytest.mark.parametrize( + "config_type", (AnthropicCountTokensConfig, AzureAIAnthropicCountTokensConfig) +) +def test_count_sends_a_pdf_image_url_as_the_document_the_chat_request_sends( + config_type: type[AnthropicCountTokensConfig], +) -> None: + pdf_url: Final = "data:application/pdf;base64," + b64encode(b"%PDF-1.4 count fixture").decode() + messages: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": pdf_url}}]} + ] + + counted: Final = config_type().transform_request_to_count_tokens(model="claude-opus-5-5", messages=messages) + sent: Final = AnthropicConfig().transform_request( + model="claude-opus-5-5", messages=deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert counted["messages"] == sent["messages"] + assert counted["messages"][0]["content"][0]["type"] == "document" + + @pytest.mark.parametrize( "config_type", (AnthropicCountTokensConfig, AzureAIAnthropicCountTokensConfig) ) diff --git a/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py b/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py index 36c319d9524..d7ba87b20ef 100644 --- a/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py +++ b/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py @@ -14,11 +14,16 @@ from unittest.mock import AsyncMock, patch, Mock, MagicMock import httpx import pytest +import respx import litellm from litellm import completion, acompletion from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.llms.snowflake.chat.transformation import SnowflakeConfig, SnowflakeStreamingHandler +from litellm.llms.snowflake.chat.transformation import ( + SnowflakeConfig, + SnowflakeException, + SnowflakeStreamingHandler, +) from litellm.types.utils import ModelResponse @@ -421,6 +426,23 @@ class TestSnowflakeCortexClaudeFixes: {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": "ZmFrZQ=="}} ] + def test_remote_image_urls_are_inlined_as_base64(self): + with respx.mock(assert_all_called=True) as respx_mock: + respx_mock.get("https://example.com/cat.png").respond( + content=b"fake", headers={"Content-Type": "image/png"} + ) + body = self._transform( + [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + } + ] + ) + assert body["messages"][0]["content"] == [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "ZmFrZQ=="}} + ] + def test_tool_result_image_list_is_converted(self): body = self._transform( [ @@ -457,7 +479,7 @@ class TestSnowflakeCortexClaudeFixes: }, ] ) - tool_result = body["messages"][1]["content"][0] + tool_result = body["messages"][0]["content"][1] assert tool_result["cache_control"] == {"type": "ephemeral"}, tool_content def test_pdf_data_uri_becomes_a_document_block(self): @@ -494,7 +516,7 @@ class TestSnowflakeCortexClaudeFixes: }, ] ) - assert body["messages"][1]["content"][0]["content"] == [ + assert body["messages"][0]["content"][1]["content"] == [ {"type": "text", "text": "first"}, {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "ZmFrZQ=="}}, {"type": "text", "text": "last"}, @@ -504,7 +526,10 @@ class TestSnowflakeCortexClaudeFixes: body = self._transform( [{"role": "user", "content": "look"}, {"role": "tool", "tool_call_id": "call_1", "content": "done"}] ) - assert body["messages"][1]["content"][0]["content"] == "done" + assert body["messages"][0]["content"] == [ + {"type": "text", "text": "look"}, + {"type": "tool_result", "tool_use_id": "call_1", "content": "done"}, + ] def test_anthropic_tool_schema_strips_only_top_level_schema_key(self): tools = [ @@ -657,6 +682,80 @@ class TestSnowflakeCortexClaudeFixes: ], } + def test_assistant_tool_use_content_block_survives_the_transform(self): + """A client replaying an Anthropic-shaped history sends the tool call as a + tool_use block inside assistant content; dropping it orphans the tool_result + that answers it and Cortex rejects the request.""" + body = self._transform( + [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "let me look that up"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "paris"}}, + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ] + ) + assert body["messages"][1]["content"] == [ + {"type": "text", "text": "let me look that up"}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "paris"}}, + ] + assert body["messages"][2]["content"] == [ + {"type": "tool_result", "tool_use_id": "toolu_01", "content": "sunny"} + ] + + def test_native_tool_use_id_is_sanitized_like_its_tool_result(self): + """Cortex takes Anthropic's id pattern, and the tool_result side already sanitizes, + so a replayed id like functions.Bash:0 has to be sanitized on both sides or the two + stop matching.""" + body = self._transform( + [ + {"role": "user", "content": "what is the weather in paris"}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "functions.Bash:0", + "name": "get_weather", + "input": {"city": "paris"}, + } + ], + }, + {"role": "tool", "tool_call_id": "functions.Bash:0", "content": "sunny"}, + ] + ) + assert body["messages"][1]["content"] == [ + {"type": "tool_use", "id": "functions_Bash_0", "name": "get_weather", "input": {"city": "paris"}} + ] + assert body["messages"][2]["content"] == [ + {"type": "tool_result", "tool_use_id": "functions_Bash_0", "content": "sunny"} + ] + + def test_unknown_charset_on_a_text_attachment_is_a_400(self): + """An unknown encoding label is a client mistake, so it has to come back as a 400 + naming the encoding instead of a 500 from the decode.""" + with pytest.raises(SnowflakeException) as exc_info: + self._transform( + [ + { + "role": "user", + "content": [ + { + "type": "file", + "file": {"file_data": "data:text/plain;charset=not-a-charset;base64,aGVsbG8="}, + } + ], + } + ] + ) + + assert exc_info.value.status_code == 400 + assert "not-a-charset" in str(exc_info.value) + def test_thinking_only_assistant_turn_sends_no_empty_text_block(self): """Anthropic-shaped APIs reject empty text blocks, so a content-less thinking turn is thinking only.""" body = self._transform( diff --git a/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py b/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py index 8ddbb7c1a6b..c70128d3225 100644 --- a/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py +++ b/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py @@ -20,6 +20,7 @@ from litellm.llms.snowflake.chat.transformation import ( SnowflakeConfig, is_snowflake_claude_model, ) +from litellm.llms.snowflake.utils import SnowflakeException from litellm.types.utils import ModelResponse @@ -342,7 +343,7 @@ class TestAnthropicConfigRequest: ) assert body["system"] == [{"type": "text", "text": "You are helpful."}] assert all(m["role"] != "system" for m in body["messages"]) - assert body["messages"][0] == {"role": "user", "content": "Hello"} + assert body["messages"][0] == {"role": "user", "content": [{"type": "text", "text": "Hello"}]} def test_model_prefix_stripped(self): body = self.cfg.transform_request( @@ -637,7 +638,7 @@ class TestAnthropicMultiTurnToolMessages: headers={}, ) msgs = body["messages"] - assert msgs[0] == {"role": "user", "content": "What's the weather in Paris?"} + assert msgs[0] == {"role": "user", "content": [{"type": "text", "text": "What's the weather in Paris?"}]} assistant_msg = msgs[1] assert assistant_msg["role"] == "assistant" @@ -647,13 +648,14 @@ class TestAnthropicMultiTurnToolMessages: assert assistant_msg["content"][0]["name"] == "get_weather" assert assistant_msg["content"][0]["input"] == {"city": "Paris"} - tool_result_msg = msgs[2] - assert tool_result_msg["role"] == "user" - assert tool_result_msg["content"][0]["type"] == "tool_result" - assert tool_result_msg["content"][0]["tool_use_id"] == "call_123" - assert tool_result_msg["content"][0]["content"] == "Sunny, 22°C" - - assert msgs[3] == {"role": "user", "content": "Thanks!"} + assert msgs[2] == { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "call_123", "content": "Sunny, 22°C"}, + {"type": "text", "text": "Thanks!"}, + ], + } + assert len(msgs) == 3 def test_assistant_with_text_and_tool_calls(self): messages = [ @@ -705,7 +707,32 @@ class TestAnthropicMultiTurnToolMessages: for msg in body["messages"]: assert msg["role"] != "tool" - def test_malformed_json_in_tool_arguments_handled_gracefully(self): + def test_malformed_json_in_tool_arguments_is_rejected_as_bad_request(self): + messages = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_bad", + "type": "function", + "function": {"name": "broken_tool", "arguments": "not valid json{{{"}, + } + ], + }, + ] + with pytest.raises(SnowflakeException, match="broken_tool") as excinfo: + self.cfg.transform_request( + model="snowflake/claude-sonnet-4-5", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + assert excinfo.value.status_code == 400 + + def test_malformed_json_in_tool_arguments_is_dropped_under_drop_params(self): messages = [ {"role": "user", "content": "hi"}, { @@ -724,14 +751,12 @@ class TestAnthropicMultiTurnToolMessages: model="snowflake/claude-sonnet-4-5", messages=messages, optional_params={}, - litellm_params={}, + litellm_params={"drop_params": True}, headers={}, ) - assistant_msg = body["messages"][1] - tool_use_block = assistant_msg["content"][0] - assert tool_use_block["type"] == "tool_use" - assert tool_use_block["name"] == "broken_tool" - assert tool_use_block["input"] == {} + assert body["messages"][1]["content"] == [ + {"type": "tool_use", "id": "call_bad", "name": "broken_tool", "input": {}} + ] def test_non_string_tool_arguments_pass_through(self): messages = [