Add backward compatibility to support xml tool use for bedrock and vertex

litellm/llms/bedrock.py

@@ -746,7 +746,7 @@ def completion(
                     ]
                 # Format rest of message according to anthropic guidelines
                 messages = prompt_factory(
-                    model=model, messages=messages, custom_llm_provider="anthropic"
+                    model=model, messages=messages, custom_llm_provider="anthropic_xml"
                 )
                 ## LOAD CONFIG
                 config = litellm.AmazonAnthropicClaude3Config.get_config()
@@ -1108,6 +1108,7 @@ def completion(
 
             raise BedrockError(status_code=500, message=traceback.format_exc())
 
+
 class ModelResponseIterator:
     def __init__(self, model_response):
         self.model_response = model_response
@@ -1133,6 +1134,7 @@ class ModelResponseIterator:
         self.is_done = True
         return self.model_response
 
+
 def _embedding_func_single(
     model: str,
     input: str,

litellm/llms/prompt_templates/factory.py

@@ -556,6 +556,175 @@ def convert_to_anthropic_image_obj(openai_image_url: str):
         )
 
 
+# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
+# ------------------------------------------------------------------------------
+def convert_to_anthropic_tool_result_xml(message: dict) -> str:
+    """
+    OpenAI message with a tool result looks like:
+    {
+        "tool_call_id": "tool_1",
+        "role": "tool",
+        "name": "get_current_weather",
+        "content": "function result goes here",
+    },
+    """
+
+    """
+    Anthropic tool_results look like:
+    
+    [Successful results]
+    <function_results>
+    <result>
+    <tool_name>get_current_weather</tool_name>
+    <stdout>
+    function result goes here
+    </stdout>
+    </result>
+    </function_results>
+
+    [Error results]
+    <function_results>
+    <error>
+    error message goes here
+    </error>
+    </function_results>
+    """
+    name = message.get("name")
+    content = message.get("content")
+
+    # We can't determine from openai message format whether it's a successful or
+    # error call result so default to the successful result template
+    anthropic_tool_result = (
+        "<function_results>\n"
+        "<result>\n"
+        f"<tool_name>{name}</tool_name>\n"
+        "<stdout>\n"
+        f"{content}\n"
+        "</stdout>\n"
+        "</result>\n"
+        "</function_results>"
+    )
+
+    return anthropic_tool_result
+
+
+def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str:
+    invokes = ""
+    for tool in tool_calls:
+        if tool["type"] != "function":
+            continue
+
+        tool_name = tool["function"]["name"]
+        parameters = "".join(
+            f"<{param}>{val}</{param}>\n"
+            for param, val in json.loads(tool["function"]["arguments"]).items()
+        )
+        invokes += (
+            "<invoke>\n"
+            f"<tool_name>{tool_name}</tool_name>\n"
+            "<parameters>\n"
+            f"{parameters}"
+            "</parameters>\n"
+            "</invoke>\n"
+        )
+
+    anthropic_tool_invoke = f"<function_calls>\n{invokes}</function_calls>"
+
+    return anthropic_tool_invoke
+
+
+def anthropic_messages_pt_xml(messages: list):
+    """
+    format messages for anthropic
+    1. Anthropic supports roles like "user" and "assistant", (here litellm translates system-> assistant)
+    2. The first message always needs to be of role "user"
+    3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm)
+    4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise)
+    5. System messages are a separate param to the Messages API (used for tool calling)
+    6. Ensure we only accept role, content. (message.name is not supported)
+    """
+    # add role=tool support to allow function call result/error submission
+    user_message_types = {"user", "tool"}
+    # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them.
+    new_messages = []
+    msg_i = 0
+    while msg_i < len(messages):
+        user_content = []
+        ## MERGE CONSECUTIVE USER CONTENT ##
+        while msg_i < len(messages) and messages[msg_i]["role"] in user_message_types:
+            if isinstance(messages[msg_i]["content"], list):
+                for m in messages[msg_i]["content"]:
+                    if m.get("type", "") == "image_url":
+                        user_content.append(
+                            {
+                                "type": "image",
+                                "source": convert_to_anthropic_image_obj(
+                                    m["image_url"]["url"]
+                                ),
+                            }
+                        )
+                    elif m.get("type", "") == "text":
+                        user_content.append({"type": "text", "text": m["text"]})
+            else:
+                # Tool message content will always be a string
+                user_content.append(
+                    {
+                        "type": "text",
+                        "text": (
+                            convert_to_anthropic_tool_result(messages[msg_i])
+                            if messages[msg_i]["role"] == "tool"
+                            else messages[msg_i]["content"]
+                        ),
+                    }
+                )
+
+            msg_i += 1
+
+        if user_content:
+            new_messages.append({"role": "user", "content": user_content})
+
+        assistant_content = []
+        ## MERGE CONSECUTIVE ASSISTANT CONTENT ##
+        while msg_i < len(messages) and messages[msg_i]["role"] == "assistant":
+            assistant_text = (
+                messages[msg_i].get("content") or ""
+            )  # either string or none
+            if messages[msg_i].get(
+                "tool_calls", []
+            ):  # support assistant tool invoke convertion
+                assistant_text += convert_to_anthropic_tool_invoke(
+                    messages[msg_i]["tool_calls"]
+                )
+
+            assistant_content.append({"type": "text", "text": assistant_text})
+            msg_i += 1
+
+        if assistant_content:
+            new_messages.append({"role": "assistant", "content": assistant_content})
+
+    if new_messages[0]["role"] != "user":
+        if litellm.modify_params:
+            new_messages.insert(
+                0, {"role": "user", "content": [{"type": "text", "text": "."}]}
+            )
+        else:
+            raise Exception(
+                "Invalid first message. Should always start with 'role'='user' for Anthropic. System prompt is sent separately for Anthropic. set 'litellm.modify_params = True' or 'litellm_settings:modify_params = True' on proxy, to insert a placeholder user message - '.' as the first message, "
+            )
+
+    if new_messages[-1]["role"] == "assistant":
+        for content in new_messages[-1]["content"]:
+            if isinstance(content, dict) and content["type"] == "text":
+                content["text"] = content[
+                    "text"
+                ].rstrip()  # no trailing whitespace for final assistant message
+
+    return new_messages
+
+
+# ------------------------------------------------------------------------------
+
+
 def convert_to_anthropic_tool_result(message: dict) -> dict:
     """
     OpenAI message with a tool result looks like:
@@ -653,7 +822,7 @@ def anthropic_messages_pt(messages: list):
     2. The first message always needs to be of role "user"
     3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm)
     4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise)
-    5. System messages are a separate param to the Messages API (used for tool calling)
+    5. System messages are a separate param to the Messages API
     6. Ensure we only accept role, content. (message.name is not supported)
     """
     # add role=tool support to allow function call result/error submission
@@ -1093,6 +1262,8 @@ def prompt_factory(
         if model == "claude-instant-1" or model == "claude-2":
             return anthropic_pt(messages=messages)
         return anthropic_messages_pt(messages=messages)
+    elif custom_llm_provider == "anthropic_xml":
+        return anthropic_messages_pt_xml(messages=messages)
     elif custom_llm_provider == "together_ai":
         prompt_format, chat_template = get_model_info(token=api_key, model=model)
         return format_prompt_togetherai(

litellm/llms/vertex_ai_anthropic.py

@@ -187,7 +187,7 @@ def completion(
         # Format rest of message according to anthropic guidelines
         try:
             messages = prompt_factory(
-                model=model, messages=messages, custom_llm_provider="anthropic"
+                model=model, messages=messages, custom_llm_provider="anthropic_xml"
             )
         except Exception as e:
             raise VertexAIError(status_code=400, message=str(e))