diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index a99a2677e8f..5e665a6e25c 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -582,3 +582,24 @@ def is_function_call(optional_params: dict) -> bool: if "functions" in optional_params and optional_params.get("functions"): return True return False + + +def get_file_ids_from_messages(messages: List[AllMessageValues]) -> List[str]: + """ + Gets file ids from messages + """ + file_ids = [] + for message in messages: + if message.get("role") == "user": + content = message.get("content") + if content: + if isinstance(content, str): + continue + for c in content: + if c["type"] == "file": + file_object = cast(ChatCompletionFileObject, c) + file_object_file_field = file_object["file"] + file_id = file_object_file_field.get("file_id") + if file_id: + file_ids.append(file_id) + return file_ids diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 7d862af6ae6..c39739953e5 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1385,6 +1385,84 @@ def _anthropic_content_element_factory( return _anthropic_content_element +def select_anthropic_content_block_type_for_file( + format: str, +) -> Literal["document", "image", "container_upload"]: + if format == "application/pdf" or format == "text/plain": + return "document" + elif format in ["image/jpeg", "image/png", "image/gif", "image/webp"]: + return "image" + else: + return "container_upload" + + +def anthropic_process_openai_file_message( + message: ChatCompletionFileObject, +) -> Union[ + AnthropicMessagesDocumentParam, + AnthropicMessagesImageParam, + AnthropicMessagesContainerUploadParam, +]: + file_message = cast(ChatCompletionFileObject, message) + file_data = file_message["file"].get("file_data") + file_id = file_message["file"].get("file_id") + format = file_message["file"].get("format") + if file_data: + image_chunk = convert_to_anthropic_image_obj( + openai_image_url=file_data, + format=format, + ) + anthropic_document_param = AnthropicMessagesDocumentParam( + type="document", + source=AnthropicContentParamSource( + type="base64", + media_type=image_chunk["media_type"], + data=image_chunk["data"], + ), + ) + return anthropic_document_param + elif file_id: + content_block_type = ( + select_anthropic_content_block_type_for_file(format) + if format + else "container_upload" + ) + return_block_param: Optional[ + Union[ + AnthropicMessagesDocumentParam, + AnthropicMessagesImageParam, + AnthropicMessagesContainerUploadParam, + ] + ] = None + if content_block_type == "document": + return_block_param = AnthropicMessagesDocumentParam( + type="document", + source=AnthropicContentParamSourceFileId( + type="file", + file_id=file_id, + ), + ) + elif content_block_type == "image": + return_block_param = AnthropicMessagesImageParam( + type="image", + source=AnthropicContentParamSourceFileId( + type="file", + file_id=file_id, + ), + ) + elif content_block_type == "container_upload": + return_block_param = AnthropicMessagesContainerUploadParam( + type="container_upload", file_id=file_id + ) + + if return_block_param is None: + raise Exception(f"Unable to parse anthropic file message: {message}") + return return_block_param + raise Exception( + f"Either file_data or file_id must be present in the file message: {message}" + ) + + def anthropic_messages_pt( # noqa: PLR0915 messages: List[AllMessageValues], model: str, @@ -1489,24 +1567,11 @@ def anthropic_messages_pt( # noqa: PLR0915 elif m.get("type", "") == "document": user_content.append(cast(AnthropicMessagesDocumentParam, m)) elif m.get("type", "") == "file": - file_message = cast(ChatCompletionFileObject, m) - file_data = file_message["file"].get("file_data") - if file_data: - image_chunk = convert_to_anthropic_image_obj( - openai_image_url=file_data, - format=file_message["file"].get("format"), + user_content.append( + anthropic_process_openai_file_message( + cast(ChatCompletionFileObject, m) ) - anthropic_document_param = ( - AnthropicMessagesDocumentParam( - type="document", - source=AnthropicContentParamSource( - type="base64", - media_type=image_chunk["media_type"], - data=image_chunk["data"], - ), - ) - ) - user_content.append(anthropic_document_param) + ) elif isinstance(user_message_types_block["content"], str): _anthropic_content_text_element: AnthropicMessagesTextParam = { "type": "text", diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index e7f421e75bf..bb3495ab825 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -18,7 +18,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messag from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.llms.anthropic import ( + AllAnthropicMessageValues, AllAnthropicToolsValues, + AnthropicCodeExecutionTool, AnthropicComputerTool, AnthropicHostedTools, AnthropicInputSchema, @@ -530,6 +532,40 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return anthropic_system_message_list + def add_code_execution_tool( + self, + messages: List[AllAnthropicMessageValues], + tools: List[Union[AllAnthropicToolsValues, Dict]], + ) -> List[Union[AllAnthropicToolsValues, Dict]]: + """if 'container_upload' in messages, add code_execution tool""" + add_code_execution_tool = False + for message in messages: + message_content = message.get("content", None) + if message_content and isinstance(message_content, list): + for content in message_content: + content_type = content.get("type", None) + if content_type == "container_upload": + add_code_execution_tool = True + break + + if add_code_execution_tool: + ## check if code_execution tool is already in tools + for tool in tools: + tool_type = tool.get("type", None) + if ( + tool_type + and isinstance(tool_type, str) + and tool_type.startswith("code_execution") + ): + return tools + tools.append( + AnthropicCodeExecutionTool( + name="code_execution", + type="code_execution_20250522", + ) + ) + return tools + def transform_request( self, model: str, @@ -579,6 +615,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): message="{}\nReceived Messages={}".format(str(e), messages), ) # don't use verbose_logger.exception, if exception is raised + ## Add code_execution tool if container_upload is in messages + _tools = ( + cast( + Optional[List[Union[AllAnthropicToolsValues, Dict]]], + optional_params.get("tools"), + ) + or [] + ) + tools = self.add_code_execution_tool(messages=anthropic_messages, tools=_tools) + if len(tools) > 1: + optional_params["tools"] = tools + ## Load Config config = litellm.AnthropicConfig.get_config() for k, v in config.items(): diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index bacd2a54d06..a531e9b6089 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -7,6 +7,9 @@ from typing import Dict, List, Optional, Union import httpx import litellm +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + get_file_ids_from_messages, +) from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -42,6 +45,13 @@ class AnthropicModelInfo(BaseLLMModelInfo): return False + def is_file_id_used(self, messages: List[AllMessageValues]) -> bool: + """ + Return if {"source": {"type": "file", "file_id": ..}} in message content block + """ + file_ids = get_file_ids_from_messages(messages) + return len(file_ids) > 0 + def is_computer_tool_used( self, tools: Optional[List[AllAnthropicToolsValues]] ) -> bool: @@ -82,6 +92,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): computer_tool_used: bool = False, prompt_caching_set: bool = False, pdf_used: bool = False, + file_id_used: bool = False, is_vertex_request: bool = False, user_anthropic_beta_headers: Optional[List[str]] = None, ) -> dict: @@ -90,8 +101,11 @@ class AnthropicModelInfo(BaseLLMModelInfo): betas.add("prompt-caching-2024-07-31") if computer_tool_used: betas.add("computer-use-2024-10-22") - if pdf_used: - betas.add("pdfs-2024-09-25") + # if pdf_used: + # betas.add("pdfs-2024-09-25") + if file_id_used: + betas.add("files-api-2025-04-14") + betas.add("code-execution-2025-05-22") headers = { "anthropic-version": anthropic_version or "2023-06-01", "x-api-key": api_key, @@ -131,6 +145,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): prompt_caching_set = self.is_cache_control_set(messages=messages) computer_tool_used = self.is_computer_tool_used(tools=tools) pdf_used = self.is_pdf_used(messages=messages) + file_id_used = self.is_file_id_used(messages=messages) user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( anthropic_beta_header=headers.get("anthropic-beta") ) @@ -139,6 +154,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): prompt_caching_set=prompt_caching_set, pdf_used=pdf_used, api_key=api_key, + file_id_used=file_id_used, is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, ) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 6a12bc98a71..5ec557cd480 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -15,5 +15,6 @@ model_list: drop_params: true additional_drop_params: - red + litellm_settings: cache: true \ No newline at end of file diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 21df55c0840..5782c8e2cd1 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -19,6 +19,10 @@ from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import * from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.http_parsing_utils import ( + _read_request_body, + get_form_data, +) from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( create_pass_through_route, @@ -319,6 +323,18 @@ async def mistral_proxy_route( return received_value +async def is_streaming_request_fn(request: Request) -> bool: + if request.method == "POST": + content_type = request.headers.get("content-type", None) + if content_type and "multipart/form-data" in content_type: + _request_body = await get_form_data(request) + else: + _request_body = await _read_request_body(request) + if _request_body.get("stream"): + return True + return False + + @router.api_route( "/anthropic/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], @@ -351,12 +367,7 @@ async def anthropic_proxy_route( ) ## check for streaming - is_streaming_request = False - # anthropic is streaming when 'stream' = True is in the body - if request.method == "POST": - _request_body = await request.json() - if _request_body.get("stream"): - is_streaming_request = True + is_streaming_request = await is_streaming_request_fn(request) ## CREATE PASS-THROUGH endpoint_func = create_pass_through_route( diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 468a1e0ffca..5c37b05f579 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -57,11 +57,18 @@ class AnthropicHostedTools(TypedDict, total=False): # for bash_tool and text_ed cache_control: Optional[Union[dict, ChatCompletionCachedContent]] +class AnthropicCodeExecutionTool(TypedDict, total=False): + type: Required[str] + name: Required[Literal["code_execution"]] + cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + + AllAnthropicToolsValues = Union[ AnthropicComputerTool, AnthropicHostedTools, AnthropicMessagesTool, AnthropicWebSearchTool, + AnthropicCodeExecutionTool, ] @@ -107,9 +114,22 @@ class AnthropicContentParamSource(TypedDict): data: str +class AnthropicContentParamSourceFileId(TypedDict): + type: Literal["file"] + file_id: str + + +class AnthropicMessagesContainerUploadParam(TypedDict, total=False): + type: Required[Literal["container_upload"]] + file_id: str + cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + + class AnthropicMessagesImageParam(TypedDict, total=False): type: Required[Literal["image"]] - source: Required[AnthropicContentParamSource] + source: Required[ + Union[AnthropicContentParamSource, AnthropicContentParamSourceFileId] + ] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] @@ -119,7 +139,9 @@ class CitationsObject(TypedDict): class AnthropicMessagesDocumentParam(TypedDict, total=False): type: Required[Literal["document"]] - source: Required[AnthropicContentParamSource] + source: Required[ + Union[AnthropicContentParamSource, AnthropicContentParamSourceFileId] + ] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] title: str context: str @@ -149,6 +171,7 @@ AnthropicMessagesUserMessageValues = Union[ AnthropicMessagesImageParam, AnthropicMessagesToolResultParam, AnthropicMessagesDocumentParam, + AnthropicMessagesContainerUploadParam, ] diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index d35115e5241..e86430cdaeb 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -185,3 +185,25 @@ def test_web_search_tool_transformation_with_search_context_size( assert anthropic_web_search_tool["user_location"]["type"] == "approximate" assert anthropic_web_search_tool["user_location"]["city"] == "San Francisco" assert anthropic_web_search_tool["max_uses"] == expected_max_uses + + +def test_add_code_execution_tool(): + config = AnthropicConfig() + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this sheet?"}, + { + "type": "container_upload", + "file_id": "file_011CPd1KVEsbD8MjfZSwBd1u", + }, + ], + } + ] + tools = [] + tools = config.add_code_execution_tool(messages=messages, tools=tools) + assert tools is not None + assert len(tools) == 1 + assert tools[0]["type"] == "code_execution_20250522" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 501b4364dac..9dcde87a594 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -573,3 +573,16 @@ class TestVertexAIDiscoveryPassThroughHandler: mock_auth.assert_called_once() call_args = mock_auth.call_args[1] assert call_args["api_key"] == "Bearer test-key-123" + + +@pytest.mark.asyncio +async def test_is_streaming_request_fn(): + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + is_streaming_request_fn, + ) + + mock_request = Mock() + mock_request.method = "POST" + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.form = AsyncMock(return_value={"stream": "true"}) + assert await is_streaming_request_fn(mock_request) is True