From 1c4028262790c7808c2040b4826c2bfc9932d7b1 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 4 Mar 2024 10:11:29 -0800 Subject: [PATCH 1/6] fix(anthropic.py): support anthropic system prompt --- litellm/llms/anthropic.py | 45 ++++++++---------------- litellm/llms/prompt_templates/factory.py | 45 +++++++++++++++++++++--- 2 files changed, 55 insertions(+), 35 deletions(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 44a1b128a96..184bc7153f0 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -41,6 +41,7 @@ class AnthropicConfig: top_p: Optional[int] = None top_k: Optional[int] = None metadata: Optional[dict] = None + system: Optional[str] = None def __init__( self, @@ -50,6 +51,7 @@ class AnthropicConfig: top_p: Optional[int] = None, top_k: Optional[int] = None, metadata: Optional[dict] = None, + system: Optional[str] = None, ) -> None: locals_ = locals() for key, value in locals_.items(): @@ -118,38 +120,19 @@ def completion( messages=messages, ) else: - prompt = prompt_factory( + # Separate system prompt from rest of message + system_prompt_idx: Optional[int] = None + for idx, message in enumerate(messages): + if message["role"] == "system": + optional_params["system"] = message["content"] + system_prompt_idx = idx + break + if system_prompt_idx is not None: + messages.pop(system_prompt_idx) + # Format rest of message according to anthropic guidelines + messages = prompt_factory( model=model, messages=messages, custom_llm_provider="anthropic" ) - """ - format messages for anthropic - 1. Anthropic supports roles like "user" and "assistant", (here litellm translates system-> assistant) - 2. The first message always needs to be of role "user" - 3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm) - 4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise) - """ - # 1. Anthropic only supports roles like "user" and "assistant" - for idx, message in enumerate(messages): - if message["role"] == "system": - message["role"] = "assistant" - - # if this is the final assistant message, remove trailing whitespace - # TODO: only do this if it's the final assistant message - if message["role"] == "assistant": - message["content"] = message["content"].strip() - - # 2. The first message always needs to be of role "user" - if len(messages) > 0: - if messages[0]["role"] != "user": - # find the index of the first user message - for i, message in enumerate(messages): - if message["role"] == "user": - break - - # remove the user message at existing position and add it to the front - messages.pop(i) - # move the first user message to the front - messages = [message] + messages ## Load Config config = litellm.AnthropicConfig.get_config() @@ -167,7 +150,7 @@ def completion( ## LOGGING logging_obj.pre_call( - input=prompt, + input=messages, api_key=api_key, additional_args={ "complete_input_dict": data, diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 103eb5977ac..6b0deb2ee7f 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -424,6 +424,46 @@ def anthropic_pt( return prompt +def anthropic_messages_pt(messages: list): + """ + format messages for anthropic + 1. Anthropic supports roles like "user" and "assistant", (here litellm translates system-> assistant) + 2. The first message always needs to be of role "user" + 3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm) + 4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise) + 5. System messages are a separate param to the Messages API (used for tool calling) + """ + ## Ensure final assistant message has no trailing whitespace + last_assistant_message_idx: Optional[int] = None + # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, add a blank 'user' or 'assistant' message to ensure compatibility + new_messages = [] + for i in range(len(messages) - 1): # type: ignore + if i == 0 and messages[i]["role"] == "assistant": + new_messages.append({"role": "user", "content": ""}) + + new_messages.append(messages[i]) + + if messages[i]["role"] == messages[i + 1]["role"]: + if messages[i]["role"] == "user": + new_messages.append({"role": "assistant", "content": ""}) + else: + new_messages.append({"role": "user", "content": ""}) + + if messages[i]["role"] == "assistant": + last_assistant_message_idx = i + + new_messages.append(messages[-1]) + + if last_assistant_message_idx is not None: + new_messages[last_assistant_message_idx]["content"] = new_messages[ + last_assistant_message_idx + ][ + "content" + ].strip() # no trailing whitespace for final assistant message + + return new_messages + + def amazon_titan_pt( messages: list, ): # format - https://github.com/BerriAI/litellm/issues/1896 @@ -650,10 +690,7 @@ def prompt_factory( if custom_llm_provider == "ollama": return ollama_pt(model=model, messages=messages) elif custom_llm_provider == "anthropic": - if any(_ in model for _ in ["claude-2.1", "claude-v2:1"]): - return claude_2_1_pt(messages=messages) - else: - return anthropic_pt(messages=messages) + return anthropic_messages_pt(messages=messages) elif custom_llm_provider == "together_ai": prompt_format, chat_template = get_model_info(token=api_key, model=model) return format_prompt_togetherai( From ae82b3f31a3b535cb48d51bcd5de6932acfa12c6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 4 Mar 2024 10:42:28 -0800 Subject: [PATCH 2/6] feat(anthropic.py): adds tool calling support --- litellm/llms/anthropic.py | 15 +++++- litellm/llms/prompt_templates/factory.py | 63 +++++++++++++++++++++++- litellm/utils.py | 15 +++++- 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 184bc7153f0..ce413be65ab 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -6,7 +6,11 @@ import time from typing import Callable, Optional from litellm.utils import ModelResponse, Usage import litellm -from .prompt_templates.factory import prompt_factory, custom_prompt +from .prompt_templates.factory import ( + prompt_factory, + custom_prompt, + construct_tool_use_system_prompt, +) import httpx @@ -142,6 +146,15 @@ def completion( ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in optional_params[k] = v + ## Handle Tool Calling + if "tools" in optional_params: + tool_calling_system_prompt = construct_tool_use_system_prompt( + tools=optional_params["tools"] + ) + optional_params["system"] = ( + optional_params("system", "\n") + tool_calling_system_prompt + ) # add the anthropic tool calling prompt to the system prompt + data = { "model": model, "messages": messages, diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 6b0deb2ee7f..cc75237e05d 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -390,7 +390,7 @@ def format_prompt_togetherai(messages, prompt_format, chat_template): return prompt -### +### ANTHROPIC ### def anthropic_pt( @@ -424,6 +424,62 @@ def anthropic_pt( return prompt +def construct_format_parameters_prompt(parameters: dict): + parameter_str = "\n" + for k, v in parameters.items(): + parameter_str += f"<{k}>" + parameter_str += f"{v}" + parameter_str += f"" + parameter_str += "\n" + return parameter_str + + +def construct_format_tool_for_claude_prompt(name, description, parameters): + constructed_prompt = ( + "\n" + f"{name}\n" + "\n" + f"{description}\n" + "\n" + "\n" + f"{construct_format_parameters_prompt(parameters)}\n" + "\n" + "" + ) + return constructed_prompt + + +def construct_tool_use_system_prompt( + tools, +): # from https://github.com/anthropics/anthropic-cookbook/blob/main/function_calling/function_calling.ipynb + tool_str_list = [] + for tool in tools: + tool_str = construct_format_tool_for_claude_prompt( + tool["function"]["name"], + tool["function"].get("description", ""), + tool["function"].get("parameters", {}), + ) + tool_str_list.append(tool_str) + tool_use_system_prompt = ( + "In this environment you have access to a set of tools you can use to answer the user's question.\n" + "\n" + "You may call them like this:\n" + "\n" + "\n" + "$TOOL_NAME\n" + "\n" + "<$PARAMETER_NAME>$PARAMETER_VALUE\n" + "...\n" + "\n" + "\n" + "\n" + "\n" + "Here are the tools available:\n" + "\n" + "\n".join([tool_str for tool_str in tool_str_list]) + "\n" + ) + return tool_use_system_prompt + + def anthropic_messages_pt(messages: list): """ format messages for anthropic @@ -464,6 +520,9 @@ def anthropic_messages_pt(messages: list): return new_messages +### + + def amazon_titan_pt( messages: list, ): # format - https://github.com/BerriAI/litellm/issues/1896 @@ -690,6 +749,8 @@ def prompt_factory( if custom_llm_provider == "ollama": return ollama_pt(model=model, messages=messages) elif custom_llm_provider == "anthropic": + if model == "claude-instant-1" or model == "claude-2.1": + return anthropic_pt(messages=messages) return anthropic_messages_pt(messages=messages) elif custom_llm_provider == "together_ai": prompt_format, chat_template = get_model_info(token=api_key, model=model) diff --git a/litellm/utils.py b/litellm/utils.py index 233fd6bae7f..69f324589af 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4106,6 +4106,7 @@ def get_optional_params( and custom_llm_provider != "anyscale" and custom_llm_provider != "together_ai" and custom_llm_provider != "mistral" + and custom_llm_provider != "anthropic" ): if custom_llm_provider == "ollama" or custom_llm_provider == "ollama_chat": # ollama actually supports json output @@ -4186,7 +4187,15 @@ def get_optional_params( ## raise exception if provider doesn't support passed in param if custom_llm_provider == "anthropic": ## check if unsupported param passed in - supported_params = ["stream", "stop", "temperature", "top_p", "max_tokens"] + supported_params = [ + "stream", + "stop", + "temperature", + "top_p", + "max_tokens", + "tools", + "tool_choice", + ] _check_valid_arg(supported_params=supported_params) # handle anthropic params if stream: @@ -4201,6 +4210,8 @@ def get_optional_params( optional_params["top_p"] = top_p if max_tokens is not None: optional_params["max_tokens"] = max_tokens + if tools is not None: + optional_params["tools"] = tools elif custom_llm_provider == "cohere": ## check if unsupported param passed in supported_params = [ @@ -9704,4 +9715,4 @@ def _get_base_model_from_metadata(model_call_details=None): base_model = model_info.get("base_model", None) if base_model is not None: return base_model - return None \ No newline at end of file + return None From c53563a1fe8bf0f67dcdb77a5d0a7837ca778163 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 4 Mar 2024 11:31:56 -0800 Subject: [PATCH 3/6] fix(test_completion.py): testing for anthropic function calling --- litellm/llms/anthropic.py | 40 ++++++++++++++++++++--- litellm/llms/prompt_templates/factory.py | 17 +++++++++- litellm/tests/test_completion.py | 41 ++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 6 deletions(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index ce413be65ab..6cea6450b71 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -2,14 +2,16 @@ import os, types import json from enum import Enum import requests -import time +import time, uuid from typing import Callable, Optional -from litellm.utils import ModelResponse, Usage +from litellm.utils import ModelResponse, Usage, map_finish_reason import litellm from .prompt_templates.factory import ( prompt_factory, custom_prompt, construct_tool_use_system_prompt, + extract_between_tags, + parse_xml_params, ) import httpx @@ -114,6 +116,7 @@ def completion( headers={}, ): headers = validate_environment(api_key, headers) + _is_function_call = False if model in custom_prompt_dict: # check if the model has a registered custom prompt model_prompt_details = custom_prompt_dict[model] @@ -148,12 +151,14 @@ def completion( ## Handle Tool Calling if "tools" in optional_params: + _is_function_call = True tool_calling_system_prompt = construct_tool_use_system_prompt( tools=optional_params["tools"] ) optional_params["system"] = ( - optional_params("system", "\n") + tool_calling_system_prompt + optional_params.get("system", "\n") + tool_calling_system_prompt ) # add the anthropic tool calling prompt to the system prompt + optional_params.pop("tools") data = { "model": model, @@ -221,8 +226,33 @@ def completion( ) else: text_content = completion_response["content"][0].get("text", None) - model_response.choices[0].message.content = text_content # type: ignore - model_response.choices[0].finish_reason = completion_response["stop_reason"] + ## TOOL CALLING - OUTPUT PARSE + if _is_function_call == True: + function_name = extract_between_tags("tool_name", text_content)[0] + function_arguments_str = extract_between_tags("invoke", text_content)[ + 0 + ].strip() + function_arguments_str = f"{function_arguments_str}" + function_arguments = parse_xml_params(function_arguments_str) + _message = litellm.Message( + tool_calls=[ + { + "id": f"call_{uuid.uuid4()}", + "type": "function", + "function": { + "name": function_name, + "arguments": json.dumps(function_arguments), + }, + } + ], + content=None, + ) + model_response.choices[0].message = _message + else: + model_response.choices[0].message.content = text_content # type: ignore + model_response.choices[0].finish_reason = map_finish_reason( + completion_response["stop_reason"] + ) ## CALCULATING USAGE prompt_tokens = completion_response["usage"]["input_tokens"] diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index cc75237e05d..2b0f4a2cf15 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -1,6 +1,6 @@ from enum import Enum import requests, traceback -import json +import json, re, xml.etree.ElementTree as ET from jinja2 import Template, exceptions, Environment, meta from typing import Optional, Any @@ -520,6 +520,21 @@ def anthropic_messages_pt(messages: list): return new_messages +def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]: + ext_list = re.findall(f"<{tag}>(.+?)", string, re.DOTALL) + if strip: + ext_list = [e.strip() for e in ext_list] + return ext_list + + +def parse_xml_params(xml_content): + root = ET.fromstring(xml_content) + params = {} + for child in root.findall(".//parameters/*"): + params[child.tag] = child.text + return params + + ### diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index a9d41be8d14..068ddc78f01 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -99,6 +99,47 @@ def test_completion_claude_3(): pytest.fail(f"Error occurred: {e}") +def test_completion_claude_3_function_call(): + litellm.set_verbose = True + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ] + messages = [{"role": "user", "content": "What's the weather like in Boston today?"}] + try: + # test without max tokens + response = completion( + model="anthropic/claude-3-opus-20240229", + messages=messages, + tools=tools, + tool_choice="auto", + ) + # Add any assertions, here to check response args + print(response) + assert isinstance(response.choices[0].message.tool_calls[0].function.name, str) + assert isinstance( + response.choices[0].message.tool_calls[0].function.arguments, str + ) + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + def test_completion_claude_3_stream(): litellm.set_verbose = False messages = [{"role": "user", "content": "Hello, world"}] From 33afa53353d2d522cd5631b424359c2a5c123b43 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 4 Mar 2024 13:08:25 -0800 Subject: [PATCH 4/6] fix(factory.py): support anthropic vision calling --- litellm/llms/prompt_templates/factory.py | 40 +++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 2b0f4a2cf15..79fc59069ef 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -3,6 +3,7 @@ import requests, traceback import json, re, xml.etree.ElementTree as ET from jinja2 import Template, exceptions, Environment, meta from typing import Optional, Any +import imghdr, base64 def default_pt(messages): @@ -480,6 +481,27 @@ def construct_tool_use_system_prompt( return tool_use_system_prompt +def convert_to_anthropic_image_obj(openai_image_url: str): + """ + Input: + "image_url": "data:image/jpeg;base64,{base64_image}", + + Return: + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": {base64_image}, + } + """ + # Extract the base64 image data + base64_data = openai_image_url.split("data:image/")[1].split(";base64,")[1] + + # Infer image format from the URL + image_format = openai_image_url.split("data:image/")[1].split(";base64,")[0] + + return {"type": "base64", "media_type": image_format, "data": base64_data} + + def anthropic_messages_pt(messages: list): """ format messages for anthropic @@ -497,7 +519,23 @@ def anthropic_messages_pt(messages: list): if i == 0 and messages[i]["role"] == "assistant": new_messages.append({"role": "user", "content": ""}) - new_messages.append(messages[i]) + if isinstance(messages[i]["content"], list): # vision input + new_content = [] + for m in messages[i]["content"]: + if m.get("type", "") == "image_url": + new_content.append( + { + "type": "image", + "source": convert_to_anthropic_image_obj( + m["image_url"]["url"] + ), + } + ) + elif m.get("type", "") == "text": + new_content.append({"type": "text", "content": m["text"]}) + new_messages.append({"role": messages[i]["role"], "content": new_content}) # type: ignore + else: + new_messages.append(messages[i]) if messages[i]["role"] == messages[i + 1]["role"]: if messages[i]["role"] == "user": From edda2d9293f70bce37369560eb852bcb8ee4b433 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 4 Mar 2024 13:34:49 -0800 Subject: [PATCH 5/6] test(test_completion.py): add testing for anthropic vision calling --- litellm/llms/prompt_templates/factory.py | 35 +++++++++++++-- .../tests/test_amazing_vertex_completion.py | 2 +- litellm/tests/test_completion.py | 45 +++++++++++++++++++ litellm/utils.py | 4 ++ 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 79fc59069ef..baf2c3a2a7f 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -499,7 +499,11 @@ def convert_to_anthropic_image_obj(openai_image_url: str): # Infer image format from the URL image_format = openai_image_url.split("data:image/")[1].split(";base64,")[0] - return {"type": "base64", "media_type": image_format, "data": base64_data} + return { + "type": "base64", + "media_type": f"image/{image_format}", + "data": base64_data, + } def anthropic_messages_pt(messages: list): @@ -515,10 +519,35 @@ def anthropic_messages_pt(messages: list): last_assistant_message_idx: Optional[int] = None # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, add a blank 'user' or 'assistant' message to ensure compatibility new_messages = [] + if len(messages) == 1: + # check if the message is a user message + if messages[0]["role"] == "assistant": + new_messages.append({"role": "user", "content": ""}) + + # check if content is a list (vision) + if isinstance(messages[0]["content"], list): # vision input + new_content = [] + for m in messages[0]["content"]: + if m.get("type", "") == "image_url": + new_content.append( + { + "type": "image", + "source": convert_to_anthropic_image_obj( + m["image_url"]["url"] + ), + } + ) + elif m.get("type", "") == "text": + new_content.append({"type": "text", "text": m["text"]}) + new_messages.append({"role": messages[0]["role"], "content": new_content}) # type: ignore + else: + new_messages.append(messages[0]) + + return new_messages + for i in range(len(messages) - 1): # type: ignore if i == 0 and messages[i]["role"] == "assistant": new_messages.append({"role": "user", "content": ""}) - if isinstance(messages[i]["content"], list): # vision input new_content = [] for m in messages[i]["content"]: @@ -546,8 +575,6 @@ def anthropic_messages_pt(messages: list): if messages[i]["role"] == "assistant": last_assistant_message_idx = i - new_messages.append(messages[-1]) - if last_assistant_message_idx is not None: new_messages[last_assistant_message_idx]["content"] = new_messages[ last_assistant_message_idx diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index 35d66907c34..8ed15db65fa 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -351,7 +351,7 @@ def test_gemini_pro_vision_base64(): load_vertex_ai_credentials() litellm.set_verbose = True litellm.num_retries = 3 - image_path = "cached_logo.jpg" + image_path = "../proxy/cached_logo.jpg" # Getting the base64 string base64_image = encode_image(image_path) resp = litellm.completion( diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 068ddc78f01..1677e04cfd7 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -159,6 +159,51 @@ def test_completion_claude_3_stream(): pytest.fail(f"Error occurred: {e}") +def encode_image(image_path): + import base64 + + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode("utf-8") + + +@pytest.mark.skip( + reason="we already test claude-3, this is just another way to pass images" +) +def test_completion_claude_3_base64(): + try: + litellm.set_verbose = True + litellm.num_retries = 3 + image_path = "../proxy/cached_logo.jpg" + # Getting the base64 string + base64_image = encode_image(image_path) + resp = litellm.completion( + model="anthropic/claude-3-opus-20240229", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Whats in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64," + base64_image + }, + }, + ], + } + ], + ) + print(f"\nResponse: {resp}") + + prompt_tokens = resp.usage.prompt_tokens + raise Exception("it worked!") + except Exception as e: + if "500 Internal error encountered.'" in str(e): + pass + else: + pytest.fail(f"An exception occurred - {str(e)}") + + def test_completion_mistral_api(): try: litellm.set_verbose = True diff --git a/litellm/utils.py b/litellm/utils.py index 69f324589af..1aa1d37673c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -200,6 +200,10 @@ def map_finish_reason( return "content_filter" elif finish_reason == "STOP": # vertex ai return "stop" + elif finish_reason == "end_turn" or finish_reason == "stop_sequence": # anthropic + return "stop" + elif finish_reason == "max_tokens": # anthropic + return "length" return finish_reason From 78efe027b28d661769f0e3956d306efa8cd52c40 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 4 Mar 2024 13:58:43 -0800 Subject: [PATCH 6/6] refactor(anthropic.py): fix linting error --- litellm/llms/anthropic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 6cea6450b71..047616cfe51 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -247,7 +247,7 @@ def completion( ], content=None, ) - model_response.choices[0].message = _message + model_response.choices[0].message = _message # type: ignore else: model_response.choices[0].message.content = text_content # type: ignore model_response.choices[0].finish_reason = map_finish_reason(