scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

Anthropic - Files API w/ form-data support on passthrough + File ID support on /chat/completion (#11256)

Browse source 
* fix(anthropic/chat): support passing 'file_id' param to anthropic

Partial fix for LIT-200

* feat(anthropic/chat): use correct anthropic content block based on file object

* fix(anthropic/chat): fix file id for container_upload message type

* fix(anthropic/chat/transformation.py): fix check for adding code execution to tool calls - needed for 'container_upload' message type

* fix(llm_passthrough_endpoints.py): support reading form data for anthropic passthrough

* refactor(llm_passthrough_endpoints.py): refactor block into function for easier testing

* test: add unit test

* fix: don't pass in empty tools list

* [Fix] Responses API - Session management  (#11254)

* fix: import session handling

* fix: imports for session handler

* tests: tests for session handler

* Update enterprise/litellm_enterprise/enterprise_callbacks/session_handler.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* bump: bump litellm enterprise

* fixes: test_create_user_default_budget

* fix: fix linting error

---------

Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Krish Dholakia 2025-05-29 23:54:24 -07:00 • committed by GitHub
parent 5d4ae9aa4d
commit 44a69421ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 247 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -582,3 +582,24 @@ def is_function_call(optional_params: dict) -> bool:
if "functions" in optional_params and optional_params.get("functions"):
return True
return False
def get_file_ids_from_messages(messages: List[AllMessageValues]) -> List[str]:
"""
Gets file ids from messages
"""
file_ids = []
for message in messages:
if message.get("role") == "user":
content = message.get("content")
if content:
if isinstance(content, str):
continue
for c in content:
if c["type"] == "file":
file_object = cast(ChatCompletionFileObject, c)
file_object_file_field = file_object["file"]
file_id = file_object_file_field.get("file_id")
if file_id:
file_ids.append(file_id)
return file_ids

99
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1385,6 +1385,84 @@ def _anthropic_content_element_factory(
return _anthropic_content_element
def select_anthropic_content_block_type_for_file(
format: str,
) -> Literal["document", "image", "container_upload"]:
if format == "application/pdf" or format == "text/plain":
return "document"
elif format in ["image/jpeg", "image/png", "image/gif", "image/webp"]:
return "image"
else:
return "container_upload"
def anthropic_process_openai_file_message(
message: ChatCompletionFileObject,
) -> Union[
AnthropicMessagesDocumentParam,
AnthropicMessagesImageParam,
AnthropicMessagesContainerUploadParam,
]:
file_message = cast(ChatCompletionFileObject, message)
file_data = file_message["file"].get("file_data")
file_id = file_message["file"].get("file_id")
format = file_message["file"].get("format")
if file_data:
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=file_data,
format=format,
)
anthropic_document_param = AnthropicMessagesDocumentParam(
type="document",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
return anthropic_document_param
elif file_id:
content_block_type = (
select_anthropic_content_block_type_for_file(format)
if format
else "container_upload"
)
return_block_param: Optional[
Union[
AnthropicMessagesDocumentParam,
AnthropicMessagesImageParam,
AnthropicMessagesContainerUploadParam,
]
] = None
if content_block_type == "document":
return_block_param = AnthropicMessagesDocumentParam(
type="document",
source=AnthropicContentParamSourceFileId(
type="file",
file_id=file_id,
),
)
elif content_block_type == "image":
return_block_param = AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSourceFileId(
type="file",
file_id=file_id,
),
)
elif content_block_type == "container_upload":
return_block_param = AnthropicMessagesContainerUploadParam(
type="container_upload", file_id=file_id
)
if return_block_param is None:
raise Exception(f"Unable to parse anthropic file message: {message}")
return return_block_param
raise Exception(
f"Either file_data or file_id must be present in the file message: {message}"
)
def anthropic_messages_pt( # noqa: PLR0915
messages: List[AllMessageValues],
model: str,
@ -1489,24 +1567,11 @@ def anthropic_messages_pt( # noqa: PLR0915
elif m.get("type", "") == "document":
user_content.append(cast(AnthropicMessagesDocumentParam, m))
elif m.get("type", "") == "file":
file_message = cast(ChatCompletionFileObject, m)
file_data = file_message["file"].get("file_data")
if file_data:
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=file_data,
format=file_message["file"].get("format"),
user_content.append(
anthropic_process_openai_file_message(
cast(ChatCompletionFileObject, m)
)
anthropic_document_param = (
AnthropicMessagesDocumentParam(
type="document",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
)
user_content.append(anthropic_document_param)
)
elif isinstance(user_message_types_block["content"], str):
_anthropic_content_text_element: AnthropicMessagesTextParam = {
"type": "text",

48
litellm/llms/anthropic/chat/transformation.py
View file

@ -18,7 +18,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messag
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.types.llms.anthropic import (
AllAnthropicMessageValues,
AllAnthropicToolsValues,
AnthropicCodeExecutionTool,
AnthropicComputerTool,
AnthropicHostedTools,
AnthropicInputSchema,
@ -530,6 +532,40 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return anthropic_system_message_list
def add_code_execution_tool(
self,
messages: List[AllAnthropicMessageValues],
tools: List[Union[AllAnthropicToolsValues, Dict]],
) -> List[Union[AllAnthropicToolsValues, Dict]]:
"""if 'container_upload' in messages, add code_execution tool"""
add_code_execution_tool = False
for message in messages:
message_content = message.get("content", None)
if message_content and isinstance(message_content, list):
for content in message_content:
content_type = content.get("type", None)
if content_type == "container_upload":
add_code_execution_tool = True
break
if add_code_execution_tool:
## check if code_execution tool is already in tools
for tool in tools:
tool_type = tool.get("type", None)
if (
tool_type
and isinstance(tool_type, str)
and tool_type.startswith("code_execution")
):
return tools
tools.append(
AnthropicCodeExecutionTool(
name="code_execution",
type="code_execution_20250522",
)
)
return tools
def transform_request(
self,
model: str,
@ -579,6 +615,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
message="{}\nReceived Messages={}".format(str(e), messages),
) # don't use verbose_logger.exception, if exception is raised
## Add code_execution tool if container_upload is in messages
_tools = (
cast(
Optional[List[Union[AllAnthropicToolsValues, Dict]]],
optional_params.get("tools"),
)
or []
)
tools = self.add_code_execution_tool(messages=anthropic_messages, tools=_tools)
if len(tools) > 1:
optional_params["tools"] = tools
## Load Config
config = litellm.AnthropicConfig.get_config()
for k, v in config.items():

20
litellm/llms/anthropic/common_utils.py
View file

@ -7,6 +7,9 @@ from typing import Dict, List, Optional, Union
import httpx
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_file_ids_from_messages,
)
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
@ -42,6 +45,13 @@ class AnthropicModelInfo(BaseLLMModelInfo):
return False
def is_file_id_used(self, messages: List[AllMessageValues]) -> bool:
"""
Return if {"source": {"type": "file", "file_id": ..}} in message content block
"""
file_ids = get_file_ids_from_messages(messages)
return len(file_ids) > 0
def is_computer_tool_used(
self, tools: Optional[List[AllAnthropicToolsValues]]
) -> bool:
@ -82,6 +92,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
computer_tool_used: bool = False,
prompt_caching_set: bool = False,
pdf_used: bool = False,
file_id_used: bool = False,
is_vertex_request: bool = False,
user_anthropic_beta_headers: Optional[List[str]] = None,
) -> dict:
@ -90,8 +101,11 @@ class AnthropicModelInfo(BaseLLMModelInfo):
betas.add("prompt-caching-2024-07-31")
if computer_tool_used:
betas.add("computer-use-2024-10-22")
if pdf_used:
betas.add("pdfs-2024-09-25")
# if pdf_used:
# betas.add("pdfs-2024-09-25")
if file_id_used:
betas.add("files-api-2025-04-14")
betas.add("code-execution-2025-05-22")
headers = {
"anthropic-version": anthropic_version or "2023-06-01",
"x-api-key": api_key,
@ -131,6 +145,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
prompt_caching_set = self.is_cache_control_set(messages=messages)
computer_tool_used = self.is_computer_tool_used(tools=tools)
pdf_used = self.is_pdf_used(messages=messages)
file_id_used = self.is_file_id_used(messages=messages)
user_anthropic_beta_headers = self._get_user_anthropic_beta_headers(
anthropic_beta_header=headers.get("anthropic-beta")
)
@ -139,6 +154,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
prompt_caching_set=prompt_caching_set,
pdf_used=pdf_used,
api_key=api_key,
file_id_used=file_id_used,
is_vertex_request=optional_params.get("is_vertex_request", False),
user_anthropic_beta_headers=user_anthropic_beta_headers,
)

1
litellm/proxy/_new_secret_config.yaml
View file

@ -15,5 +15,6 @@ model_list:
drop_params: true
additional_drop_params:
- red
litellm_settings:
cache: true

23
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -19,6 +19,10 @@ from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.proxy._types import *
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.http_parsing_utils import (
_read_request_body,
get_form_data,
)
from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
create_pass_through_route,
@ -319,6 +323,18 @@ async def mistral_proxy_route(
return received_value
async def is_streaming_request_fn(request: Request) -> bool:
if request.method == "POST":
content_type = request.headers.get("content-type", None)
if content_type and "multipart/form-data" in content_type:
_request_body = await get_form_data(request)
else:
_request_body = await _read_request_body(request)
if _request_body.get("stream"):
return True
return False
@router.api_route(
"/anthropic/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
@ -351,12 +367,7 @@ async def anthropic_proxy_route(
)
## check for streaming
is_streaming_request = False
# anthropic is streaming when 'stream' = True is in the body
if request.method == "POST":
_request_body = await request.json()
if _request_body.get("stream"):
is_streaming_request = True
is_streaming_request = await is_streaming_request_fn(request)
## CREATE PASS-THROUGH
endpoint_func = create_pass_through_route(

27
litellm/types/llms/anthropic.py
View file

@ -57,11 +57,18 @@ class AnthropicHostedTools(TypedDict, total=False): # for bash_tool and text_ed
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
class AnthropicCodeExecutionTool(TypedDict, total=False):
type: Required[str]
name: Required[Literal["code_execution"]]
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
AllAnthropicToolsValues = Union[
AnthropicComputerTool,
AnthropicHostedTools,
AnthropicMessagesTool,
AnthropicWebSearchTool,
AnthropicCodeExecutionTool,
]
@ -107,9 +114,22 @@ class AnthropicContentParamSource(TypedDict):
data: str
class AnthropicContentParamSourceFileId(TypedDict):
type: Literal["file"]
file_id: str
class AnthropicMessagesContainerUploadParam(TypedDict, total=False):
type: Required[Literal["container_upload"]]
file_id: str
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
class AnthropicMessagesImageParam(TypedDict, total=False):
type: Required[Literal["image"]]
source: Required[AnthropicContentParamSource]
source: Required[
Union[AnthropicContentParamSource, AnthropicContentParamSourceFileId]
]
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
@ -119,7 +139,9 @@ class CitationsObject(TypedDict):
class AnthropicMessagesDocumentParam(TypedDict, total=False):
type: Required[Literal["document"]]
source: Required[AnthropicContentParamSource]
source: Required[
Union[AnthropicContentParamSource, AnthropicContentParamSourceFileId]
]
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
title: str
context: str
@ -149,6 +171,7 @@ AnthropicMessagesUserMessageValues = Union[
AnthropicMessagesImageParam,
AnthropicMessagesToolResultParam,
AnthropicMessagesDocumentParam,
AnthropicMessagesContainerUploadParam,
]

22
tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -185,3 +185,25 @@ def test_web_search_tool_transformation_with_search_context_size(
assert anthropic_web_search_tool["user_location"]["type"] == "approximate"
assert anthropic_web_search_tool["user_location"]["city"] == "San Francisco"
assert anthropic_web_search_tool["max_uses"] == expected_max_uses
def test_add_code_execution_tool():
config = AnthropicConfig()
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is in this sheet?"},
{
"type": "container_upload",
"file_id": "file_011CPd1KVEsbD8MjfZSwBd1u",
},
],
}
]
tools = []
tools = config.add_code_execution_tool(messages=messages, tools=tools)
assert tools is not None
assert len(tools) == 1
assert tools[0]["type"] == "code_execution_20250522"

13
tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
View file

@ -573,3 +573,16 @@ class TestVertexAIDiscoveryPassThroughHandler:
mock_auth.assert_called_once()
call_args = mock_auth.call_args[1]
assert call_args["api_key"] == "Bearer test-key-123"
@pytest.mark.asyncio
async def test_is_streaming_request_fn():
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
is_streaming_request_fn,
)
mock_request = Mock()
mock_request.method = "POST"
mock_request.headers = {"content-type": "multipart/form-data"}
mock_request.form = AsyncMock(return_value={"stream": "true"})
assert await is_streaming_request_fn(mock_request) is True