From 9771edb30223f000ce553e93e67ddc71f9aed01f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 16 Jul 2024 19:58:27 -0700 Subject: [PATCH 1/4] fix(factory.py): use stronger typing for anthropic translation Fixes https://github.com/BerriAI/litellm/issues/4738 --- litellm/llms/prompt_templates/factory.py | 119 ++++++++++++++--------- litellm/tests/test_prompt_factory.py | 34 +++++++ 2 files changed, 106 insertions(+), 47 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 3a43e8bb0c5..1daf5e6fd9d 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -1081,7 +1081,7 @@ def convert_to_gemini_tool_call_result( return _part -def convert_to_anthropic_tool_result(message: dict) -> dict: +def convert_to_anthropic_tool_result(message: dict) -> AnthropicMessagesToolResultParam: """ OpenAI message with a tool result looks like: { @@ -1114,44 +1114,50 @@ def convert_to_anthropic_tool_result(message: dict) -> dict: } """ if message["role"] == "tool": - tool_call_id = message.get("tool_call_id") - content = message.get("content") + tool_call_id: str = message.get("tool_call_id") # type: ignore + content: str = message.get("content") # type: ignore # We can't determine from openai message format whether it's a successful or # error call result so default to the successful result template - anthropic_tool_result = { - "type": "tool_result", - "tool_use_id": tool_call_id, - "content": content, - } + anthropic_tool_result = AnthropicMessagesToolResultParam( + type="tool_result", tool_use_id=tool_call_id, content=content + ) return anthropic_tool_result - elif message["role"] == "function": - content = message.get("content") - anthropic_tool_result = { - "type": "tool_result", - "tool_use_id": str(uuid.uuid4()), - "content": content, - } + if message["role"] == "function": + content = message.get("content") # type: ignore + anthropic_tool_result = AnthropicMessagesToolResultParam( + type="tool_result", tool_use_id=str(uuid.uuid4()), content=content + ) + return anthropic_tool_result - return {} + else: + raise Exception( + "Invalid role={}. Only 'tool' or 'function' are accepted for tool result blocks.".format( + message.get("content") + ) + ) -def convert_function_to_anthropic_tool_invoke(function_call): +def convert_function_to_anthropic_tool_invoke( + function_call, +) -> List[AnthropicMessagesToolUseParam]: try: anthropic_tool_invoke = [ - { - "type": "tool_use", - "id": str(uuid.uuid4()), - "name": get_attribute_or_key(function_call, "name"), - "input": json.loads(get_attribute_or_key(function_call, "arguments")), - } + AnthropicMessagesToolUseParam( + type="tool_use", + id=str(uuid.uuid4()), + name=get_attribute_or_key(function_call, "name"), + input=json.loads(get_attribute_or_key(function_call, "arguments")), + ) ] return anthropic_tool_invoke except Exception as e: raise e -def convert_to_anthropic_tool_invoke(tool_calls: list) -> list: +def convert_to_anthropic_tool_invoke( + tool_calls: list, +) -> List[AnthropicMessagesToolUseParam]: """ OpenAI tool invokes: { @@ -1189,18 +1195,16 @@ def convert_to_anthropic_tool_invoke(tool_calls: list) -> list: } """ anthropic_tool_invoke = [ - { - "type": "tool_use", - "id": get_attribute_or_key(tool, "id"), - "name": get_attribute_or_key( - get_attribute_or_key(tool, "function"), "name" - ), - "input": json.loads( + AnthropicMessagesToolUseParam( + type="tool_use", + id=get_attribute_or_key(tool, "id"), + name=get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"), + input=json.loads( get_attribute_or_key( get_attribute_or_key(tool, "function"), "arguments" ) ), - } + ) for tool in tool_calls if get_attribute_or_key(tool, "type") == "function" ] @@ -1212,7 +1216,12 @@ def anthropic_messages_pt( messages: list, model: str, llm_provider: str, -): +) -> List[ + Union[ + AnthropicMessagesUserMessageParam, + AnthopicMessagesAssistantMessageParam, + ] +]: """ format messages for anthropic 1. Anthropic supports roles like "user" and "assistant" (system prompt sent separately) @@ -1225,24 +1234,33 @@ def anthropic_messages_pt( # add role=tool support to allow function call result/error submission user_message_types = {"user", "tool", "function"} # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them. - new_messages: list = [] + new_messages: List[ + Union[ + AnthropicMessagesUserMessageParam, + AnthopicMessagesAssistantMessageParam, + ] + ] = [] msg_i = 0 - tool_use_param = False while msg_i < len(messages): - user_content = [] + user_content: List[AnthropicMessagesUserMessageValues] = [] init_msg_i = msg_i ## MERGE CONSECUTIVE USER CONTENT ## while msg_i < len(messages) and messages[msg_i]["role"] in user_message_types: if isinstance(messages[msg_i]["content"], list): for m in messages[msg_i]["content"]: if m.get("type", "") == "image_url": + image_chunk = convert_to_anthropic_image_obj( + m["image_url"]["url"] + ) user_content.append( - { - "type": "image", - "source": convert_to_anthropic_image_obj( - m["image_url"]["url"] + AnthropicMessagesImageParam( + type="image", + source=AnthropicImageParamSource( + type="base64", + media_type=image_chunk["media_type"], + data=image_chunk["data"], ), - } + ) ) elif m.get("type", "") == "text": user_content.append({"type": "text", "text": m["text"]}) @@ -1262,14 +1280,21 @@ def anthropic_messages_pt( if user_content: new_messages.append({"role": "user", "content": user_content}) - assistant_content = [] + assistant_content: List[AnthropicMessagesAssistantMessageValues] = [] ## MERGE CONSECUTIVE ASSISTANT CONTENT ## while msg_i < len(messages) and messages[msg_i]["role"] == "assistant": - assistant_text = ( - messages[msg_i].get("content") or "" - ) # either string or none - if assistant_text: - assistant_content.append({"type": "text", "text": assistant_text}) + if isinstance(messages[msg_i]["content"], list): + for m in messages[msg_i]["content"]: + # handle text + if m.get("type", "") == "text": + anthropic_message = AnthropicMessagesTextParam( + type="text", text=m.get("text") + ) + assistant_content.append(anthropic_message) + elif isinstance(messages[msg_i]["content"], str): + assistant_content.append( + {"type": "text", "text": messages[msg_i]["content"]} + ) if messages[msg_i].get( "tool_calls", [] diff --git a/litellm/tests/test_prompt_factory.py b/litellm/tests/test_prompt_factory.py index 60a297203ab..8d079280300 100644 --- a/litellm/tests/test_prompt_factory.py +++ b/litellm/tests/test_prompt_factory.py @@ -7,6 +7,8 @@ import pytest sys.path.insert(0, os.path.abspath("../..")) +from typing import Union + # from litellm.llms.prompt_templates.factory import prompt_factory import litellm from litellm import completion @@ -138,6 +140,38 @@ def test_anthropic_messages_pt(): assert "Invalid first message" in str(err.value) +def test_anthropic_messages_nested_pt(): + from litellm.types.llms.anthropic import ( + AnthopicMessagesAssistantMessageParam, + AnthropicMessagesUserMessageParam, + ) + + messages = [ + {"content": [{"text": "here is a task", "type": "text"}], "role": "user"}, + { + "content": [{"text": "sure happy to help", "type": "text"}], + "role": "assistant", + }, + { + "content": [ + { + "text": "Here is a screenshot of the current desktop with the " + "mouse coordinates (500, 350). Please select an action " + "from the provided schema.", + "type": "text", + } + ], + "role": "user", + }, + ] + + new_messages = anthropic_messages_pt( + messages, model="claude-3-sonnet-20240229", llm_provider="anthropic" + ) + + assert isinstance(new_messages[1]["content"][0]["text"], str) + + # codellama_prompt_format() def test_bedrock_tool_calling_pt(): tools = [ From 7d9f715c98c1a81841d7c141a5d0c587618f63bc Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 16 Jul 2024 20:06:01 -0700 Subject: [PATCH 2/4] fix(utils.py): fix linting errors --- litellm/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 10dc64c00d8..8f15f1c08a3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1878,8 +1878,8 @@ def token_counter( text: Optional[Union[str, List[str]]] = None, messages: Optional[List] = None, count_response_tokens: Optional[bool] = False, - tools: list[ChatCompletionToolParam] | None = None, - tool_choice: ChatCompletionNamedToolChoiceParam | None = None, + tools: Optional[list[ChatCompletionToolParam]] = None, + tool_choice: Optional[ChatCompletionNamedToolChoiceParam] = None, ) -> int: """ Count the number of tokens in a given text using a specified model. From c2cb9502aeaefcda88d45392541bae905d3222c6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 16 Jul 2024 20:17:52 -0700 Subject: [PATCH 3/4] fix(utils.py): fix linting error --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 8f15f1c08a3..48fdf80c59f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1878,7 +1878,7 @@ def token_counter( text: Optional[Union[str, List[str]]] = None, messages: Optional[List] = None, count_response_tokens: Optional[bool] = False, - tools: Optional[list[ChatCompletionToolParam]] = None, + tools: Optional[List[ChatCompletionToolParam]] = None, tool_choice: Optional[ChatCompletionNamedToolChoiceParam] = None, ) -> int: """ From bc9f5eb6284317b015d5b052a14ca9966f2aad25 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 16 Jul 2024 21:51:30 -0700 Subject: [PATCH 4/4] fix(factory.py): handle content not being set --- litellm/llms/prompt_templates/factory.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 1daf5e6fd9d..0299f37d530 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -1283,7 +1283,9 @@ def anthropic_messages_pt( assistant_content: List[AnthropicMessagesAssistantMessageValues] = [] ## MERGE CONSECUTIVE ASSISTANT CONTENT ## while msg_i < len(messages) and messages[msg_i]["role"] == "assistant": - if isinstance(messages[msg_i]["content"], list): + if "content" in messages[msg_i] and isinstance( + messages[msg_i]["content"], list + ): for m in messages[msg_i]["content"]: # handle text if m.get("type", "") == "text": @@ -1291,7 +1293,9 @@ def anthropic_messages_pt( type="text", text=m.get("text") ) assistant_content.append(anthropic_message) - elif isinstance(messages[msg_i]["content"], str): + elif "content" in messages[msg_i] and isinstance( + messages[msg_i]["content"], str + ): assistant_content.append( {"type": "text", "text": messages[msg_i]["content"]} )