From 1e5b79d887ca01c3ac477db837e9d830697d12b4 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Thu, 2 Apr 2026 21:13:56 -0700 Subject: [PATCH] =?UTF-8?q?[Bedrock]=20Fix=20Anthropic=20file=5Fid=20suppo?= =?UTF-8?q?rt=20-=20async=20path=20+=20document=20URL=E2=86=92base64=20+?= =?UTF-8?q?=20beta=20header=20filtering=20(#25047)=20(#25050)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ishaan Jaffer --- litellm/llms/bedrock/chat/invoke_handler.py | 115 +++++++++++ .../anthropic_claude3_transformation.py | 188 ++++++++++++++++-- .../anthropic_claude3_transformation.py | 13 +- 3 files changed, 291 insertions(+), 25 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 1077731779d..67bba28e4c5 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -855,6 +855,32 @@ class BedrockLLM(BaseAWSLLM): endpoint_url = f"{endpoint_url}/model/{modelId}/invoke" proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke" + if acompletion and provider == "anthropic" and self.is_claude_messages_api_model( + model + ): + if isinstance(client, HTTPHandler): + client = None + return self._async_anthropic_messages_completion( + model=model, + messages=messages, + endpoint_url=endpoint_url, + proxy_endpoint_url=proxy_endpoint_url, + credentials=credentials, + aws_region_name=aws_region_name, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, + litellm_params=litellm_params, + logger_fn=logger_fn, + extra_headers=extra_headers, + timeout=timeout, + client=client, + stream_chunk_size=stream_chunk_size, + ) # type: ignore[return-value] + prompt, chat_history = self.convert_messages_to_prompt( model, messages, provider, custom_prompt_dict ) @@ -1148,6 +1174,95 @@ class BedrockLLM(BaseAWSLLM): encoding=encoding, ) + async def _async_anthropic_messages_completion( + self, + model: str, + messages: list, + endpoint_url: str, + proxy_endpoint_url: str, + credentials, + aws_region_name: str, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + logging_obj: Logging, + optional_params: dict, + stream, + litellm_params=None, + logger_fn=None, + extra_headers: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[AsyncHTTPHandler] = None, + stream_chunk_size: int = 1024, + ) -> Union[ModelResponse, CustomStreamWrapper]: + transformed_request = await litellm.AmazonAnthropicClaudeConfig().async_transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params or {}, + headers=extra_headers or {}, + ) + data = json.dumps(transformed_request) + + headers = {"Content-Type": "application/json"} + if extra_headers is not None: + headers = {"Content-Type": "application/json", **extra_headers} + prepped = self.get_request_headers( + credentials=credentials, + aws_region_name=aws_region_name, + extra_headers=extra_headers, + endpoint_url=endpoint_url, + data=data, + headers=headers, + ) + + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": proxy_endpoint_url, + "headers": prepped.headers, + }, + ) + + if stream is True: + return await self.async_streaming( + model=model, + messages=messages, + data=data, + api_base=proxy_endpoint_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=True, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + stream_chunk_size=stream_chunk_size, + ) + return await self.async_completion( + model=model, + messages=messages, + data=data, + api_base=proxy_endpoint_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, # type: ignore + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + ) + async def async_completion( self, model: str, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 7936b6ea644..2b28473ad3e 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -2,6 +2,14 @@ from typing import TYPE_CHECKING, Any, List, Optional import httpx +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers +from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_image_obj, +) +from litellm.litellm_core_utils.prompt_templates.image_handling import ( + async_convert_url_to_base64, + convert_url_to_base64, +) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, @@ -85,8 +93,62 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): litellm_params: dict, headers: dict, ) -> dict: - # Filter out AWS authentication parameters before passing to Anthropic transformation - # AWS params should only be used for signing requests, not included in request body + _anthropic_request = self._build_bedrock_anthropic_request_base( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + self._convert_document_url_sources_to_base64(_anthropic_request) + beta_list = self._compute_bedrock_invoke_beta_headers( + model=model, + messages=messages, + optional_params=optional_params, + headers=headers, + ) + if beta_list: + _anthropic_request["anthropic_beta"] = beta_list + + return _anthropic_request + + async def async_transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + _anthropic_request = self._build_bedrock_anthropic_request_base( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + await self._async_convert_document_url_sources_to_base64(_anthropic_request) + beta_list = self._compute_bedrock_invoke_beta_headers( + model=model, + messages=messages, + optional_params=optional_params, + headers=headers, + ) + if beta_list: + _anthropic_request["anthropic_beta"] = beta_list + + return _anthropic_request + + def _build_bedrock_anthropic_request_base( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: filtered_params = { k: v for k, v in optional_params.items() @@ -94,7 +156,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): } filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params) - _anthropic_request = AnthropicConfig.transform_request( + anthropic_request = AnthropicConfig.transform_request( self, model=model, messages=messages, @@ -103,28 +165,31 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): headers=headers, ) - _anthropic_request.pop("model", None) - _anthropic_request.pop("stream", None) - # Bedrock Invoke doesn't support output_format parameter - _anthropic_request.pop("output_format", None) - # Bedrock Invoke doesn't support output_config parameter - # Fixes: https://github.com/BerriAI/litellm/issues/22797 - _anthropic_request.pop("output_config", None) - if "anthropic_version" not in _anthropic_request: - _anthropic_request["anthropic_version"] = self.anthropic_version + anthropic_request.pop("model", None) + anthropic_request.pop("stream", None) + anthropic_request.pop("output_format", None) + anthropic_request.pop("output_config", None) + if "anthropic_version" not in anthropic_request: + anthropic_request["anthropic_version"] = self.anthropic_version # Remove `custom` field from tools (Bedrock doesn't support it) - # Claude Code sends `custom: {defer_loading: true}` on tool definitions, - # which causes Bedrock to reject the request with "Extra inputs are not permitted" - # Ref: https://github.com/BerriAI/litellm/issues/22847 - remove_custom_field_from_tools(_anthropic_request) + remove_custom_field_from_tools(anthropic_request) + return anthropic_request + def _compute_bedrock_invoke_beta_headers( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + headers: dict, + ) -> List[str]: tools = optional_params.get("tools") tool_search_used = self.is_tool_search_used(tools) programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools) input_examples_used = self.is_input_examples_used(tools) - beta_set = set(get_anthropic_beta_from_headers(headers)) + user_beta_set = set(get_anthropic_beta_from_headers(headers)) + beta_set = set(user_beta_set) auto_betas = self.get_anthropic_beta_list( model=model, optional_params=optional_params, @@ -142,12 +207,91 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): if "opus-4" in model.lower() or "opus_4" in model.lower(): beta_set.add("tool-search-tool-2025-10-19") - # Filter out beta headers that Bedrock Invoke doesn't support - # Uses centralized configuration from anthropic_beta_headers_config.json - beta_list = list(beta_set) - _anthropic_request["anthropic_beta"] = beta_list + auto_beta_list = filter_and_transform_beta_headers( + beta_headers=list(beta_set - user_beta_set), + provider="bedrock", + ) + return sorted(user_beta_set.union(set(auto_beta_list))) - return _anthropic_request + def _convert_document_url_sources_to_base64(self, anthropic_request: dict) -> None: + """ + Bedrock Invoke does not accept document URL sources. Convert to base64 payloads. + """ + messages = anthropic_request.get("messages") + if not isinstance(messages, list): + return + + for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") + if not isinstance(content, list): + continue + + for block in content: + if not isinstance(block, dict) or block.get("type") != "document": + continue + source = block.get("source") + if not isinstance(source, dict) or source.get("type") != "url": + continue + source_url = source.get("url") + if not isinstance(source_url, str): + continue + + inferred_format: Optional[str] = None + if source_url.lower().endswith(".pdf"): + inferred_format = "application/pdf" + base64_url = convert_url_to_base64(url=source_url) + image_chunk = convert_to_anthropic_image_obj( + openai_image_url=base64_url, + format=inferred_format, + ) + block["source"] = { + "type": "base64", + "media_type": image_chunk["media_type"], + "data": image_chunk["data"], + } + + async def _async_convert_document_url_sources_to_base64( + self, anthropic_request: dict + ) -> None: + """ + Async version of document URL conversion for async completion paths. + """ + messages = anthropic_request.get("messages") + if not isinstance(messages, list): + return + + for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") + if not isinstance(content, list): + continue + + for block in content: + if not isinstance(block, dict) or block.get("type") != "document": + continue + source = block.get("source") + if not isinstance(source, dict) or source.get("type") != "url": + continue + source_url = source.get("url") + if not isinstance(source_url, str): + continue + + inferred_format: Optional[str] = None + if source_url.lower().endswith(".pdf"): + inferred_format = "application/pdf" + base64_url = await async_convert_url_to_base64(url=source_url) + image_chunk = convert_to_anthropic_image_obj( + openai_image_url=base64_url, + format=inferred_format, + ) + block["source"] = { + "type": "base64", + "media_type": image_chunk["media_type"], + "data": image_chunk["data"], + } def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict: """ diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index a890cf2a04d..784c6a50cfd 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -12,6 +12,7 @@ from typing import ( import httpx +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -436,7 +437,8 @@ class AmazonAnthropicClaudeMessagesConfig( ) input_examples_used = anthropic_model_info.is_input_examples_used(tools) - beta_set = set(get_anthropic_beta_from_headers(headers)) + user_beta_set = set(get_anthropic_beta_from_headers(headers)) + beta_set = set(user_beta_set) auto_betas = anthropic_model_info.get_anthropic_beta_list( model=model, optional_params=anthropic_messages_optional_request_params, @@ -460,8 +462,13 @@ class AmazonAnthropicClaudeMessagesConfig( if "tool-search-tool-2025-10-19" in beta_set: beta_set.add("tool-examples-2025-10-29") - if beta_set: - anthropic_messages_request["anthropic_beta"] = list(beta_set) + filtered_auto_betas = filter_and_transform_beta_headers( + beta_headers=list(beta_set - user_beta_set), + provider="bedrock", + ) + filtered_betas = sorted(user_beta_set.union(set(filtered_auto_betas))) + if filtered_betas: + anthropic_messages_request["anthropic_beta"] = filtered_betas return anthropic_messages_request