From d2cf9d2cf1fec3f1fbd9a090c350b659a59a03a2 Mon Sep 17 00:00:00 2001 From: Zihao Li Date: Fri, 5 Apr 2024 16:01:40 +0800 Subject: [PATCH 1/4] Move tool definitions from system prompt to parameter and refactor tool calling parse --- litellm/llms/anthropic.py | 81 +++++++++++++++++---------------------- litellm/utils.py | 2 + 2 files changed, 38 insertions(+), 45 deletions(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 864ad658fbb..4c5e1a2ea42 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -118,7 +118,6 @@ def completion( ): headers = validate_environment(api_key, headers) _is_function_call = False - json_schemas: dict = {} messages = copy.deepcopy(messages) optional_params = copy.deepcopy(optional_params) if model in custom_prompt_dict: @@ -162,17 +161,15 @@ def completion( ## Handle Tool Calling if "tools" in optional_params: _is_function_call = True + headers["anthropic-beta"] = "tools-2024-04-04" + + anthropic_tools = [] for tool in optional_params["tools"]: - json_schemas[tool["function"]["name"]] = tool["function"].get( - "parameters", None - ) - tool_calling_system_prompt = construct_tool_use_system_prompt( - tools=optional_params["tools"] - ) - optional_params["system"] = ( - optional_params.get("system", "\n") + tool_calling_system_prompt - ) # add the anthropic tool calling prompt to the system prompt - optional_params.pop("tools") + new_tool = tool["function"] + new_tool["input_schema"] = new_tool.pop("parameters") # rename key + anthropic_tools.append(new_tool) + + optional_params["tools"] = anthropic_tools stream = optional_params.pop("stream", None) @@ -195,9 +192,9 @@ def completion( print_verbose(f"_is_function_call: {_is_function_call}") ## COMPLETION CALL if ( - stream is not None and stream == True and _is_function_call == False + stream and not _is_function_call ): # if function call - fake the streaming (need complete blocks for output parsing in openai format) - print_verbose(f"makes anthropic streaming POST request") + print_verbose("makes anthropic streaming POST request") data["stream"] = stream response = requests.post( api_base, @@ -245,46 +242,40 @@ def completion( status_code=response.status_code, ) else: - text_content = completion_response["content"][0].get("text", None) - ## TOOL CALLING - OUTPUT PARSE - if text_content is not None and contains_tag("invoke", text_content): - function_name = extract_between_tags("tool_name", text_content)[0] - function_arguments_str = extract_between_tags("invoke", text_content)[ - 0 - ].strip() - function_arguments_str = f"{function_arguments_str}" - function_arguments = parse_xml_params( - function_arguments_str, - json_schema=json_schemas.get( - function_name, None - ), # check if we have a json schema for this function name - ) - _message = litellm.Message( - tool_calls=[ + text_content = "" + tool_calls = [] + for content in completion_response["content"]: + if content["type"] == "text": + text_content += content["text"] + ## TOOL CALLING + elif content["type"] == "tool_use": + tool_calls.append( { - "id": f"call_{uuid.uuid4()}", + "id": content["id"], "type": "function", "function": { - "name": function_name, - "arguments": json.dumps(function_arguments), + "name": content["name"], + "arguments": json.dumps(content["input"]), }, } - ], - content=None, - ) - model_response.choices[0].message = _message # type: ignore - model_response._hidden_params["original_response"] = ( - text_content # allow user to access raw anthropic tool calling response - ) - else: - model_response.choices[0].message.content = text_content # type: ignore + ) + + _message = litellm.Message( + tool_calls=tool_calls, + content=text_content or None, + ) + model_response.choices[0].message = _message # type: ignore + model_response._hidden_params["original_response"] = completion_response[ + "content" + ] # allow user to access raw anthropic tool calling response + model_response.choices[0].finish_reason = map_finish_reason( completion_response["stop_reason"] ) print_verbose(f"_is_function_call: {_is_function_call}; stream: {stream}") - if _is_function_call == True and stream is not None and stream == True: - print_verbose(f"INSIDE ANTHROPIC STREAMING TOOL CALLING CONDITION BLOCK") + if _is_function_call and stream: + print_verbose("INSIDE ANTHROPIC STREAMING TOOL CALLING CONDITION BLOCK") # return an iterator streaming_model_response = ModelResponse(stream=True) streaming_model_response.choices[0].finish_reason = model_response.choices[ @@ -318,7 +309,7 @@ def completion( model_response=streaming_model_response ) print_verbose( - f"Returns anthropic CustomStreamWrapper with 'cached_response' streaming object" + "Returns anthropic CustomStreamWrapper with 'cached_response' streaming object" ) return CustomStreamWrapper( completion_stream=completion_stream, @@ -337,7 +328,7 @@ def completion( usage = Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, + total_tokens=total_tokens, ) model_response.usage = usage return model_response diff --git a/litellm/utils.py b/litellm/utils.py index 17a31751d96..99da1b183f2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -207,6 +207,8 @@ def map_finish_reason( return "stop" elif finish_reason == "max_tokens": # anthropic return "length" + elif finish_reason == "tool_use": # anthropic + return "tool_calls" return finish_reason From 71fdf3179099134842f80d18f2cfe9af357f5287 Mon Sep 17 00:00:00 2001 From: Zihao Li Date: Fri, 5 Apr 2024 17:11:35 +0800 Subject: [PATCH 2/4] Refactor tool result submission and tool invoke conversion --- litellm/llms/prompt_templates/factory.py | 134 ++++++++++++----------- 1 file changed, 73 insertions(+), 61 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 06faaf55746..b895573b471 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -556,7 +556,7 @@ def convert_to_anthropic_image_obj(openai_image_url: str): ) -def convert_to_anthropic_tool_result(message: dict) -> str: +def convert_to_anthropic_tool_result(message: dict) -> dict: """ OpenAI message with a tool result looks like: { @@ -569,64 +569,79 @@ def convert_to_anthropic_tool_result(message: dict) -> str: """ Anthropic tool_results look like: - - [Successful results] - - - get_current_weather - - function result goes here - - - - - [Error results] - - - error message goes here - - + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01A09q90qw90lq917835lq9", + "content": "ConnectionError: the weather service API is not available (HTTP 500)", + # "is_error": true + } + ] + } """ - name = message.get("name") + tool_call_id = message.get("tool_call_id") content = message.get("content") # We can't determine from openai message format whether it's a successful or # error call result so default to the successful result template - anthropic_tool_result = ( - "\n" - "\n" - f"{name}\n" - "\n" - f"{content}\n" - "\n" - "\n" - "" - ) + anthropic_tool_result = { + "type": "tool_result", + "tool_use_id": tool_call_id, + "content": content, + } return anthropic_tool_result -def convert_to_anthropic_tool_invoke(tool_calls: list) -> str: - invokes = "" - for tool in tool_calls: - if tool["type"] != "function": - continue +def convert_to_anthropic_tool_invoke(tool_calls: list) -> list: + """ + OpenAI tool invokes: + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\n\"location\": \"Boston, MA\"\n}" + } + } + ] + }, + """ - tool_name = tool["function"]["name"] - parameters = "".join( - f"<{param}>{val}\n" - for param, val in json.loads(tool["function"]["arguments"]).items() - ) - invokes += ( - "\n" - f"{tool_name}\n" - "\n" - f"{parameters}" - "\n" - "\n" - ) - - anthropic_tool_invoke = f"\n{invokes}" + """ + Anthropic tool invokes: + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "To answer this question, I will: 1. Use the get_weather tool to get the current weather in San Francisco. 2. Use the get_time tool to get the current time in the America/Los_Angeles timezone, which covers San Francisco, CA." + }, + { + "type": "tool_use", + "id": "toolu_01A09q90qw90lq917835lq9", + "name": "get_weather", + "input": {"location": "San Francisco, CA"} + } + ] + } + """ + anthropic_tool_invoke = [ + { + "type": "tool_use", + "id": tool["id"], + "name": tool["function"]["name"], + "input": json.loads(tool["function"]["arguments"]), + } + for tool in tool_calls + if tool["type"] == "function" + ] return anthropic_tool_invoke @@ -663,17 +678,12 @@ def anthropic_messages_pt(messages: list): ) elif m.get("type", "") == "text": user_content.append({"type": "text", "text": m["text"]}) + elif messages[msg_i]["role"] == "tool": + # OpenAI's tool message content will always be a string + user_content.append(convert_to_anthropic_tool_result(messages[msg_i])) else: - # Tool message content will always be a string user_content.append( - { - "type": "text", - "text": ( - convert_to_anthropic_tool_result(messages[msg_i]) - if messages[msg_i]["role"] == "tool" - else messages[msg_i]["content"] - ), - } + {"type": "text", "text": messages[msg_i]["content"]} ) msg_i += 1 @@ -687,14 +697,16 @@ def anthropic_messages_pt(messages: list): assistant_text = ( messages[msg_i].get("content") or "" ) # either string or none + if assistant_text: + assistant_content.append({"type": "text", "text": assistant_text}) + if messages[msg_i].get( "tool_calls", [] ): # support assistant tool invoke convertion - assistant_text += convert_to_anthropic_tool_invoke( - messages[msg_i]["tool_calls"] + assistant_content.extend( + convert_to_anthropic_tool_invoke(messages[msg_i]["tool_calls"]) ) - assistant_content.append({"type": "text", "text": assistant_text}) msg_i += 1 if assistant_content: From 342073c212e9e2bf21c5dad031075f9d414470e5 Mon Sep 17 00:00:00 2001 From: Zihao Li Date: Fri, 5 Apr 2024 22:36:18 +0800 Subject: [PATCH 3/4] Clean up imports of XML processing functions --- litellm/llms/anthropic.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 4c5e1a2ea42..cb8e4f8d86c 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -2,18 +2,12 @@ import os, types import json from enum import Enum import requests, copy -import time, uuid +import time from typing import Callable, Optional, List from litellm.utils import ModelResponse, Usage, map_finish_reason, CustomStreamWrapper import litellm -from .prompt_templates.factory import ( - contains_tag, - prompt_factory, - custom_prompt, - construct_tool_use_system_prompt, - extract_between_tags, - parse_xml_params, -) +from .prompt_templates.factory import prompt_factory, custom_prompt + import httpx From f16d0c06fd9841ac2b48d072b9729d861680363a Mon Sep 17 00:00:00 2001 From: Zihao Li Date: Sat, 6 Apr 2024 00:34:33 +0800 Subject: [PATCH 4/4] Add backward compatibility to support xml tool use for bedrock and vertex --- litellm/llms/bedrock.py | 4 +- litellm/llms/prompt_templates/factory.py | 173 ++++++++++++++++++++++- litellm/llms/vertex_ai_anthropic.py | 2 +- 3 files changed, 176 insertions(+), 3 deletions(-) diff --git a/litellm/llms/bedrock.py b/litellm/llms/bedrock.py index 8d5669b3881..2a50739a0e1 100644 --- a/litellm/llms/bedrock.py +++ b/litellm/llms/bedrock.py @@ -746,7 +746,7 @@ def completion( ] # Format rest of message according to anthropic guidelines messages = prompt_factory( - model=model, messages=messages, custom_llm_provider="anthropic" + model=model, messages=messages, custom_llm_provider="anthropic_xml" ) ## LOAD CONFIG config = litellm.AmazonAnthropicClaude3Config.get_config() @@ -1108,6 +1108,7 @@ def completion( raise BedrockError(status_code=500, message=traceback.format_exc()) + class ModelResponseIterator: def __init__(self, model_response): self.model_response = model_response @@ -1133,6 +1134,7 @@ class ModelResponseIterator: self.is_done = True return self.model_response + def _embedding_func_single( model: str, input: str, diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index b895573b471..cc984a48425 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -556,6 +556,175 @@ def convert_to_anthropic_image_obj(openai_image_url: str): ) +# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex +# ------------------------------------------------------------------------------ +def convert_to_anthropic_tool_result_xml(message: dict) -> str: + """ + OpenAI message with a tool result looks like: + { + "tool_call_id": "tool_1", + "role": "tool", + "name": "get_current_weather", + "content": "function result goes here", + }, + """ + + """ + Anthropic tool_results look like: + + [Successful results] + + + get_current_weather + + function result goes here + + + + + [Error results] + + + error message goes here + + + """ + name = message.get("name") + content = message.get("content") + + # We can't determine from openai message format whether it's a successful or + # error call result so default to the successful result template + anthropic_tool_result = ( + "\n" + "\n" + f"{name}\n" + "\n" + f"{content}\n" + "\n" + "\n" + "" + ) + + return anthropic_tool_result + + +def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str: + invokes = "" + for tool in tool_calls: + if tool["type"] != "function": + continue + + tool_name = tool["function"]["name"] + parameters = "".join( + f"<{param}>{val}\n" + for param, val in json.loads(tool["function"]["arguments"]).items() + ) + invokes += ( + "\n" + f"{tool_name}\n" + "\n" + f"{parameters}" + "\n" + "\n" + ) + + anthropic_tool_invoke = f"\n{invokes}" + + return anthropic_tool_invoke + + +def anthropic_messages_pt_xml(messages: list): + """ + format messages for anthropic + 1. Anthropic supports roles like "user" and "assistant", (here litellm translates system-> assistant) + 2. The first message always needs to be of role "user" + 3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm) + 4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise) + 5. System messages are a separate param to the Messages API (used for tool calling) + 6. Ensure we only accept role, content. (message.name is not supported) + """ + # add role=tool support to allow function call result/error submission + user_message_types = {"user", "tool"} + # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them. + new_messages = [] + msg_i = 0 + while msg_i < len(messages): + user_content = [] + ## MERGE CONSECUTIVE USER CONTENT ## + while msg_i < len(messages) and messages[msg_i]["role"] in user_message_types: + if isinstance(messages[msg_i]["content"], list): + for m in messages[msg_i]["content"]: + if m.get("type", "") == "image_url": + user_content.append( + { + "type": "image", + "source": convert_to_anthropic_image_obj( + m["image_url"]["url"] + ), + } + ) + elif m.get("type", "") == "text": + user_content.append({"type": "text", "text": m["text"]}) + else: + # Tool message content will always be a string + user_content.append( + { + "type": "text", + "text": ( + convert_to_anthropic_tool_result(messages[msg_i]) + if messages[msg_i]["role"] == "tool" + else messages[msg_i]["content"] + ), + } + ) + + msg_i += 1 + + if user_content: + new_messages.append({"role": "user", "content": user_content}) + + assistant_content = [] + ## MERGE CONSECUTIVE ASSISTANT CONTENT ## + while msg_i < len(messages) and messages[msg_i]["role"] == "assistant": + assistant_text = ( + messages[msg_i].get("content") or "" + ) # either string or none + if messages[msg_i].get( + "tool_calls", [] + ): # support assistant tool invoke convertion + assistant_text += convert_to_anthropic_tool_invoke( + messages[msg_i]["tool_calls"] + ) + + assistant_content.append({"type": "text", "text": assistant_text}) + msg_i += 1 + + if assistant_content: + new_messages.append({"role": "assistant", "content": assistant_content}) + + if new_messages[0]["role"] != "user": + if litellm.modify_params: + new_messages.insert( + 0, {"role": "user", "content": [{"type": "text", "text": "."}]} + ) + else: + raise Exception( + "Invalid first message. Should always start with 'role'='user' for Anthropic. System prompt is sent separately for Anthropic. set 'litellm.modify_params = True' or 'litellm_settings:modify_params = True' on proxy, to insert a placeholder user message - '.' as the first message, " + ) + + if new_messages[-1]["role"] == "assistant": + for content in new_messages[-1]["content"]: + if isinstance(content, dict) and content["type"] == "text": + content["text"] = content[ + "text" + ].rstrip() # no trailing whitespace for final assistant message + + return new_messages + + +# ------------------------------------------------------------------------------ + + def convert_to_anthropic_tool_result(message: dict) -> dict: """ OpenAI message with a tool result looks like: @@ -653,7 +822,7 @@ def anthropic_messages_pt(messages: list): 2. The first message always needs to be of role "user" 3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm) 4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise) - 5. System messages are a separate param to the Messages API (used for tool calling) + 5. System messages are a separate param to the Messages API 6. Ensure we only accept role, content. (message.name is not supported) """ # add role=tool support to allow function call result/error submission @@ -1093,6 +1262,8 @@ def prompt_factory( if model == "claude-instant-1" or model == "claude-2": return anthropic_pt(messages=messages) return anthropic_messages_pt(messages=messages) + elif custom_llm_provider == "anthropic_xml": + return anthropic_messages_pt_xml(messages=messages) elif custom_llm_provider == "together_ai": prompt_format, chat_template = get_model_info(token=api_key, model=model) return format_prompt_togetherai( diff --git a/litellm/llms/vertex_ai_anthropic.py b/litellm/llms/vertex_ai_anthropic.py index e1ab527b7e0..fe96b20a341 100644 --- a/litellm/llms/vertex_ai_anthropic.py +++ b/litellm/llms/vertex_ai_anthropic.py @@ -187,7 +187,7 @@ def completion( # Format rest of message according to anthropic guidelines try: messages = prompt_factory( - model=model, messages=messages, custom_llm_provider="anthropic" + model=model, messages=messages, custom_llm_provider="anthropic_xml" ) except Exception as e: raise VertexAIError(status_code=400, message=str(e))