From 676d9df7fa6f55cb052d82fb0ef666def9090e8d Mon Sep 17 00:00:00 2001 From: Zihao Li Date: Fri, 15 Mar 2024 11:34:54 +0800 Subject: [PATCH 1/3] Add function call result submission support for Claude 3 models --- litellm/llms/prompt_templates/factory.py | 84 ++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 9be8970489f..21a86a9f799 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -551,6 +551,58 @@ def convert_to_anthropic_image_obj(openai_image_url: str): ) +def convert_openai_message_to_anthropic_tool_result(message): + """ + OpenAI message with a tool result looks like: + { + "tool_call_id": "tool_1", + "role": "tool", + "name": "get_current_weather", + "content": "function result goes here", + }, + """ + + """ + Anthropic tool_results look like: + + [Successful results] + + + get_current_weather + + function result goes here + + + + + [Error results] + + + error message goes here + + + """ + name = message.get("name") + content = message.get("content") + + # We can't determine from openai message format whether it's a successful or + # error call result so default to the successful result template + anthropic_tool_result = { + "role": "user", + "content": ( + "\n" + "\n" + f"{name}\n" + "\n" + f"{content}\n" + "\n" + "\n" + "" + ), + } + return anthropic_tool_result + + def anthropic_messages_pt(messages: list): """ format messages for anthropic @@ -563,6 +615,8 @@ def anthropic_messages_pt(messages: list): """ ## Ensure final assistant message has no trailing whitespace last_assistant_message_idx: Optional[int] = None + # add role=tool support to allow function call result/error submission + user_message_types = {"user", "tool"} # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, add a blank 'user' or 'assistant' message to ensure compatibility new_messages = [] if len(messages) == 1: @@ -591,6 +645,11 @@ def anthropic_messages_pt(messages: list): {"role": messages[0]["role"], "content": messages[0]["content"]} ) + if new_messages[-1]["role"] == "tool": # function call result or error + new_messages[-1] = convert_openai_message_to_anthropic_tool_result( + new_messages[-1] + ) + return new_messages for i in range(len(messages) - 1): # type: ignore @@ -616,16 +675,29 @@ def anthropic_messages_pt(messages: list): {"role": messages[i]["role"], "content": messages[i]["content"]} ) - if messages[i]["role"] == messages[i + 1]["role"]: - if messages[i]["role"] == "user": - new_messages.append({"role": "assistant", "content": ""}) - else: - new_messages.append({"role": "user", "content": ""}) + if new_messages[-1]["role"] == "tool": # function call result or error + new_messages[-1] = convert_openai_message_to_anthropic_tool_result( + new_messages[-1] + ) + + if ( + messages[i]["role"] in user_message_types + and messages[i + 1]["role"] in user_message_types + ): + new_messages.append({"role": "assistant", "content": ""}) if messages[i]["role"] == "assistant": + if messages[i + 1]["role"] == "assistant": + new_messages.append({"role": "user", "content": ""}) + last_assistant_message_idx = i - new_messages.append(messages[-1]) + new_messages.append( + convert_openai_message_to_anthropic_tool_result(messages[-1]) + if messages[-1]["role"] == "tool" + else messages[-1] + ) + if last_assistant_message_idx is not None: new_messages[last_assistant_message_idx]["content"] = new_messages[ last_assistant_message_idx From 807f999a6604ff7f96871ef99524045fe281a268 Mon Sep 17 00:00:00 2001 From: Zihao Li Date: Fri, 15 Mar 2024 22:47:11 +0800 Subject: [PATCH 2/3] Refactor to allow tool invoke conversion and tool results submission; Allow alternating consecutuve message roles and fix a bug --- litellm/llms/prompt_templates/factory.py | 188 +++++++++++------------ 1 file changed, 94 insertions(+), 94 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 21a86a9f799..4ebb48dbae2 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -551,7 +551,7 @@ def convert_to_anthropic_image_obj(openai_image_url: str): ) -def convert_openai_message_to_anthropic_tool_result(message): +def convert_to_anthropic_tool_result(message: dict) -> str: """ OpenAI message with a tool result looks like: { @@ -587,22 +587,45 @@ def convert_openai_message_to_anthropic_tool_result(message): # We can't determine from openai message format whether it's a successful or # error call result so default to the successful result template - anthropic_tool_result = { - "role": "user", - "content": ( - "\n" - "\n" - f"{name}\n" - "\n" - f"{content}\n" - "\n" - "\n" - "" - ), - } + anthropic_tool_result = ( + "\n" + "\n" + f"{name}\n" + "\n" + f"{content}\n" + "\n" + "\n" + "" + ) + return anthropic_tool_result +def convert_to_anthropic_tool_invoke(tool_calls: list) -> str: + invokes = "" + for tool in tool_calls: + if tool["type"] != "function": + continue + + tool_name = tool["function"]["name"] + parameters = "".join( + f"<{param}>{val}\n" + for param, val in json.loads(tool["function"]["arguments"]).items() + ) + invokes += ( + "\n" + f"{tool_name}\n" + "\n" + f"{parameters}" + "\n" + "\n" + ) + + anthropic_tool_invoke = f"\n{invokes}" + + return anthropic_tool_invoke + + def anthropic_messages_pt(messages: list): """ format messages for anthropic @@ -613,97 +636,74 @@ def anthropic_messages_pt(messages: list): 5. System messages are a separate param to the Messages API (used for tool calling) 6. Ensure we only accept role, content. (message.name is not supported) """ - ## Ensure final assistant message has no trailing whitespace - last_assistant_message_idx: Optional[int] = None # add role=tool support to allow function call result/error submission user_message_types = {"user", "tool"} # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, add a blank 'user' or 'assistant' message to ensure compatibility new_messages = [] - if len(messages) == 1: - # check if the message is a user message - if messages[0]["role"] == "assistant": - new_messages.append({"role": "user", "content": ""}) + msg_i = 0 + while msg_i < len(messages): + user_content = [] + while msg_i < len(messages) and messages[msg_i]["role"] in user_message_types: + if isinstance(messages[msg_i]["content"], list): + for m in messages[msg_i]["content"]: + if m.get("type", "") == "image_url": + user_content.append( + { + "type": "image", + "source": convert_to_anthropic_image_obj( + m["image_url"]["url"] + ), + } + ) + elif m.get("type", "") == "text": + user_content.append({"type": "text", "text": m["text"]}) + else: + # Tool message content will always be a string + user_content.append( + { + "type": "text", + "text": ( + convert_to_anthropic_tool_result(messages[msg_i]) + if messages[msg_i]["role"] == "tool" + else messages[msg_i]["content"] + ), + } + ) - # check if content is a list (vision) - if isinstance(messages[0]["content"], list): # vision input - new_content = [] - for m in messages[0]["content"]: - if m.get("type", "") == "image_url": - new_content.append( - { - "type": "image", - "source": convert_to_anthropic_image_obj( - m["image_url"]["url"] - ), - } - ) - elif m.get("type", "") == "text": - new_content.append({"type": "text", "text": m["text"]}) - new_messages.append({"role": messages[0]["role"], "content": new_content}) # type: ignore - else: - new_messages.append( - {"role": messages[0]["role"], "content": messages[0]["content"]} - ) + msg_i += 1 - if new_messages[-1]["role"] == "tool": # function call result or error - new_messages[-1] = convert_openai_message_to_anthropic_tool_result( - new_messages[-1] - ) + if user_content: + new_messages.append({"role": "user", "content": user_content}) - return new_messages + assistant_content = [] + while msg_i < len(messages) and messages[msg_i]["role"] == "assistant": + assistant_text = ( + messages[msg_i].get("content") or "" + ) # either string or none + if messages[msg_i].get( + "tool_calls", [] + ): # support assistant tool invoke convertion + assistant_text += convert_to_anthropic_tool_invoke( + messages[msg_i]["tool_calls"] + ) - for i in range(len(messages) - 1): # type: ignore - if i == 0 and messages[i]["role"] == "assistant": - new_messages.append({"role": "user", "content": ""}) - if isinstance(messages[i]["content"], list): # vision input - new_content = [] - for m in messages[i]["content"]: - if m.get("type", "") == "image_url": - new_content.append( - { - "type": "image", - "source": convert_to_anthropic_image_obj( - m["image_url"]["url"] - ), - } - ) - elif m.get("type", "") == "text": - new_content.append({"type": "text", "content": m["text"]}) - new_messages.append({"role": messages[i]["role"], "content": new_content}) # type: ignore - else: - new_messages.append( - {"role": messages[i]["role"], "content": messages[i]["content"]} - ) + assistant_content.append({"type": "text", "text": assistant_text}) + msg_i += 1 - if new_messages[-1]["role"] == "tool": # function call result or error - new_messages[-1] = convert_openai_message_to_anthropic_tool_result( - new_messages[-1] - ) + if assistant_content: + new_messages.append({"role": "assistant", "content": assistant_content}) - if ( - messages[i]["role"] in user_message_types - and messages[i + 1]["role"] in user_message_types - ): - new_messages.append({"role": "assistant", "content": ""}) + if new_messages[0]["role"] != "user": + new_messages.insert( + 0, {"role": "user", "content": [{"type": "text", "text": "."}]} + ) - if messages[i]["role"] == "assistant": - if messages[i + 1]["role"] == "assistant": - new_messages.append({"role": "user", "content": ""}) - - last_assistant_message_idx = i - - new_messages.append( - convert_openai_message_to_anthropic_tool_result(messages[-1]) - if messages[-1]["role"] == "tool" - else messages[-1] - ) - - if last_assistant_message_idx is not None: - new_messages[last_assistant_message_idx]["content"] = new_messages[ - last_assistant_message_idx - ][ - "content" - ].strip() # no trailing whitespace for final assistant message + if new_messages[-1]["role"] == "assistant": + for content in new_messages[-1]["content"]: + if content["type"] == "text": + content["text"] = content[ + "text" + ].rstrip() # no trailing whitespace for final assistant message return new_messages From 91f467f55d2a9596d3f3f5da98e9f3b53d69d82b Mon Sep 17 00:00:00 2001 From: Zihao Li Date: Sat, 16 Mar 2024 01:40:36 +0800 Subject: [PATCH 3/3] Add tool result submission to claude 3 function call test and claude 3 multi-turn conversion to ensure alternating message roles --- litellm/tests/test_completion.py | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 0b69cdc19b1..85f1139fa05 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -152,6 +152,52 @@ def test_completion_claude_3_function_call(): assert isinstance( response.choices[0].message.tool_calls[0].function.arguments, str ) + + messages.append( + response.choices[0].message.model_dump() + ) # Add assistant tool invokes + tool_result = ( + '{"location": "Boston", "temperature": "72", "unit": "fahrenheit"}' + ) + # Add user submitted tool results in OpenAI format + messages.append( + { + "tool_call_id": response.choices[0].message.tool_calls[0].id, + "role": "tool", + "name": response.choices[0].message.tool_calls[0].function.name, + "content": tool_result, + } + ) + # In the second response, Claude should deduce answer from tool results + second_response = completion( + model="anthropic/claude-3-opus-20240229", + messages=messages, + tools=tools, + tool_choice="auto", + ) + print(second_response) + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + +def test_completion_claude_3_multi_turn_conversations(): + litellm.set_verbose = True + messages = [ + {"role": "assistant", "content": "?"}, # test first user message auto injection + {"role": "user", "content": "Hi!"}, + { + "role": "user", + "content": [{"type": "text", "text": "What is the weather like today?"}], + }, + {"role": "assistant", "content": "Hi! I am Claude. "}, + {"role": "assistant", "content": "Today is a sunny "}, + ] + try: + response = completion( + model="anthropic/claude-3-opus-20240229", + messages=messages, + ) + print(response) except Exception as e: pytest.fail(f"Error occurred: {e}")