From f16d0c06fd9841ac2b48d072b9729d861680363a Mon Sep 17 00:00:00 2001 From: Zihao Li Date: Sat, 6 Apr 2024 00:34:33 +0800 Subject: [PATCH] Add backward compatibility to support xml tool use for bedrock and vertex --- litellm/llms/bedrock.py | 4 +- litellm/llms/prompt_templates/factory.py | 173 ++++++++++++++++++++++- litellm/llms/vertex_ai_anthropic.py | 2 +- 3 files changed, 176 insertions(+), 3 deletions(-) diff --git a/litellm/llms/bedrock.py b/litellm/llms/bedrock.py index 8d5669b3881..2a50739a0e1 100644 --- a/litellm/llms/bedrock.py +++ b/litellm/llms/bedrock.py @@ -746,7 +746,7 @@ def completion( ] # Format rest of message according to anthropic guidelines messages = prompt_factory( - model=model, messages=messages, custom_llm_provider="anthropic" + model=model, messages=messages, custom_llm_provider="anthropic_xml" ) ## LOAD CONFIG config = litellm.AmazonAnthropicClaude3Config.get_config() @@ -1108,6 +1108,7 @@ def completion( raise BedrockError(status_code=500, message=traceback.format_exc()) + class ModelResponseIterator: def __init__(self, model_response): self.model_response = model_response @@ -1133,6 +1134,7 @@ class ModelResponseIterator: self.is_done = True return self.model_response + def _embedding_func_single( model: str, input: str, diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index b895573b471..cc984a48425 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -556,6 +556,175 @@ def convert_to_anthropic_image_obj(openai_image_url: str): ) +# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex +# ------------------------------------------------------------------------------ +def convert_to_anthropic_tool_result_xml(message: dict) -> str: + """ + OpenAI message with a tool result looks like: + { + "tool_call_id": "tool_1", + "role": "tool", + "name": "get_current_weather", + "content": "function result goes here", + }, + """ + + """ + Anthropic tool_results look like: + + [Successful results] + + + get_current_weather + + function result goes here + + + + + [Error results] + + + error message goes here + + + """ + name = message.get("name") + content = message.get("content") + + # We can't determine from openai message format whether it's a successful or + # error call result so default to the successful result template + anthropic_tool_result = ( + "\n" + "\n" + f"{name}\n" + "\n" + f"{content}\n" + "\n" + "\n" + "" + ) + + return anthropic_tool_result + + +def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str: + invokes = "" + for tool in tool_calls: + if tool["type"] != "function": + continue + + tool_name = tool["function"]["name"] + parameters = "".join( + f"<{param}>{val}\n" + for param, val in json.loads(tool["function"]["arguments"]).items() + ) + invokes += ( + "\n" + f"{tool_name}\n" + "\n" + f"{parameters}" + "\n" + "\n" + ) + + anthropic_tool_invoke = f"\n{invokes}" + + return anthropic_tool_invoke + + +def anthropic_messages_pt_xml(messages: list): + """ + format messages for anthropic + 1. Anthropic supports roles like "user" and "assistant", (here litellm translates system-> assistant) + 2. The first message always needs to be of role "user" + 3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm) + 4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise) + 5. System messages are a separate param to the Messages API (used for tool calling) + 6. Ensure we only accept role, content. (message.name is not supported) + """ + # add role=tool support to allow function call result/error submission + user_message_types = {"user", "tool"} + # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them. + new_messages = [] + msg_i = 0 + while msg_i < len(messages): + user_content = [] + ## MERGE CONSECUTIVE USER CONTENT ## + while msg_i < len(messages) and messages[msg_i]["role"] in user_message_types: + if isinstance(messages[msg_i]["content"], list): + for m in messages[msg_i]["content"]: + if m.get("type", "") == "image_url": + user_content.append( + { + "type": "image", + "source": convert_to_anthropic_image_obj( + m["image_url"]["url"] + ), + } + ) + elif m.get("type", "") == "text": + user_content.append({"type": "text", "text": m["text"]}) + else: + # Tool message content will always be a string + user_content.append( + { + "type": "text", + "text": ( + convert_to_anthropic_tool_result(messages[msg_i]) + if messages[msg_i]["role"] == "tool" + else messages[msg_i]["content"] + ), + } + ) + + msg_i += 1 + + if user_content: + new_messages.append({"role": "user", "content": user_content}) + + assistant_content = [] + ## MERGE CONSECUTIVE ASSISTANT CONTENT ## + while msg_i < len(messages) and messages[msg_i]["role"] == "assistant": + assistant_text = ( + messages[msg_i].get("content") or "" + ) # either string or none + if messages[msg_i].get( + "tool_calls", [] + ): # support assistant tool invoke convertion + assistant_text += convert_to_anthropic_tool_invoke( + messages[msg_i]["tool_calls"] + ) + + assistant_content.append({"type": "text", "text": assistant_text}) + msg_i += 1 + + if assistant_content: + new_messages.append({"role": "assistant", "content": assistant_content}) + + if new_messages[0]["role"] != "user": + if litellm.modify_params: + new_messages.insert( + 0, {"role": "user", "content": [{"type": "text", "text": "."}]} + ) + else: + raise Exception( + "Invalid first message. Should always start with 'role'='user' for Anthropic. System prompt is sent separately for Anthropic. set 'litellm.modify_params = True' or 'litellm_settings:modify_params = True' on proxy, to insert a placeholder user message - '.' as the first message, " + ) + + if new_messages[-1]["role"] == "assistant": + for content in new_messages[-1]["content"]: + if isinstance(content, dict) and content["type"] == "text": + content["text"] = content[ + "text" + ].rstrip() # no trailing whitespace for final assistant message + + return new_messages + + +# ------------------------------------------------------------------------------ + + def convert_to_anthropic_tool_result(message: dict) -> dict: """ OpenAI message with a tool result looks like: @@ -653,7 +822,7 @@ def anthropic_messages_pt(messages: list): 2. The first message always needs to be of role "user" 3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm) 4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise) - 5. System messages are a separate param to the Messages API (used for tool calling) + 5. System messages are a separate param to the Messages API 6. Ensure we only accept role, content. (message.name is not supported) """ # add role=tool support to allow function call result/error submission @@ -1093,6 +1262,8 @@ def prompt_factory( if model == "claude-instant-1" or model == "claude-2": return anthropic_pt(messages=messages) return anthropic_messages_pt(messages=messages) + elif custom_llm_provider == "anthropic_xml": + return anthropic_messages_pt_xml(messages=messages) elif custom_llm_provider == "together_ai": prompt_format, chat_template = get_model_info(token=api_key, model=model) return format_prompt_togetherai( diff --git a/litellm/llms/vertex_ai_anthropic.py b/litellm/llms/vertex_ai_anthropic.py index e1ab527b7e0..fe96b20a341 100644 --- a/litellm/llms/vertex_ai_anthropic.py +++ b/litellm/llms/vertex_ai_anthropic.py @@ -187,7 +187,7 @@ def completion( # Format rest of message according to anthropic guidelines try: messages = prompt_factory( - model=model, messages=messages, custom_llm_provider="anthropic" + model=model, messages=messages, custom_llm_provider="anthropic_xml" ) except Exception as e: raise VertexAIError(status_code=400, message=str(e))