mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 29c6bebae0 into 356b8d4074
This commit is contained in:
commit
a86ae4edc3
9 changed files with 901 additions and 29 deletions
|
|
@ -1755,6 +1755,9 @@ if TYPE_CHECKING:
|
|||
from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import (
|
||||
AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig,
|
||||
)
|
||||
from .llms.bedrock.chat.openai_chat_completions_transformation import (
|
||||
AmazonBedrockOpenAIChatCompletionsConfig as AmazonBedrockOpenAIChatCompletionsConfig,
|
||||
)
|
||||
from .llms.bedrock.image_generation.amazon_stability1_transformation import (
|
||||
AmazonStabilityConfig as AmazonStabilityConfig,
|
||||
)
|
||||
|
|
@ -1861,6 +1864,9 @@ if TYPE_CHECKING:
|
|||
from .llms.bedrock_mantle.responses.transformation import (
|
||||
BedrockMantleResponsesAPIConfig as BedrockMantleResponsesAPIConfig,
|
||||
)
|
||||
from .llms.bedrock.responses.transformation import (
|
||||
AmazonBedrockResponsesAPIConfig as AmazonBedrockResponsesAPIConfig,
|
||||
)
|
||||
from .llms.gemini.interactions.transformation import (
|
||||
GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -204,6 +204,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"AmazonTwelveLabsPegasusConfig",
|
||||
"AmazonInvokeConfig",
|
||||
"AmazonBedrockOpenAIConfig",
|
||||
"AmazonBedrockOpenAIChatCompletionsConfig",
|
||||
"AmazonStabilityConfig",
|
||||
"AmazonStability3Config",
|
||||
"AmazonNovaCanvasConfig",
|
||||
|
|
@ -244,6 +245,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"DatabricksResponsesAPIConfig",
|
||||
"OpenRouterResponsesAPIConfig",
|
||||
"BedrockMantleResponsesAPIConfig",
|
||||
"AmazonBedrockResponsesAPIConfig",
|
||||
"GoogleAIStudioInteractionsConfig",
|
||||
"VertexAIInteractionsConfig",
|
||||
"OpenAIOSeriesConfig",
|
||||
|
|
@ -848,6 +850,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
".llms.bedrock.chat.invoke_transformations.amazon_openai_transformation",
|
||||
"AmazonBedrockOpenAIConfig",
|
||||
),
|
||||
"AmazonBedrockOpenAIChatCompletionsConfig": (
|
||||
".llms.bedrock.chat.openai_chat_completions_transformation",
|
||||
"AmazonBedrockOpenAIChatCompletionsConfig",
|
||||
),
|
||||
"AmazonStabilityConfig": (
|
||||
".llms.bedrock.image_generation.amazon_stability1_transformation",
|
||||
"AmazonStabilityConfig",
|
||||
|
|
@ -991,6 +997,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
".llms.bedrock_mantle.responses.transformation",
|
||||
"BedrockMantleResponsesAPIConfig",
|
||||
),
|
||||
"AmazonBedrockResponsesAPIConfig": (
|
||||
".llms.bedrock.responses.transformation",
|
||||
"AmazonBedrockResponsesAPIConfig",
|
||||
),
|
||||
"GoogleAIStudioInteractionsConfig": (
|
||||
".llms.gemini.interactions.transformation",
|
||||
"GoogleAIStudioInteractionsConfig",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.request_metadata import (
|
||||
bedrock_request_metadata_headers,
|
||||
merge_bedrock_invoke_headers,
|
||||
)
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
|
||||
class AmazonBedrockOpenAIChatCompletionsConfig(OpenAIGPTConfig, BaseAWSLLM):
|
||||
def __init__(self, **kwargs: object) -> None: # kwargs-ok: provider interface
|
||||
OpenAIGPTConfig.__init__(self, **kwargs)
|
||||
BaseAWSLLM.__init__(self, **kwargs)
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> str:
|
||||
return "bedrock"
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
api_key: str | None,
|
||||
model: str,
|
||||
optional_params: dict, # mutable-ok: provider interface
|
||||
litellm_params: dict, # mutable-ok: provider interface
|
||||
stream: bool | None = None,
|
||||
) -> str:
|
||||
aws_region_name: Final = self._get_aws_region_name(optional_params=optional_params, model=model)
|
||||
endpoint_url: Final = self.get_runtime_endpoint(
|
||||
api_base=api_base,
|
||||
aws_bedrock_runtime_endpoint=optional_params.get("aws_bedrock_runtime_endpoint"),
|
||||
aws_region_name=aws_region_name,
|
||||
)[0]
|
||||
return f"{endpoint_url.rstrip('/')}/openai/v1/chat/completions"
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[AllMessageValues], # mutable-ok: provider interface
|
||||
optional_params: dict, # mutable-ok: provider interface
|
||||
litellm_params: dict, # mutable-ok: provider interface
|
||||
headers: dict, # mutable-ok: provider interface
|
||||
) -> dict: # mutable-ok: provider interface
|
||||
inference_params: Final = { # mutable-ok: provider interface
|
||||
key: value for key, value in optional_params.items() if key not in self.aws_authentication_params
|
||||
}
|
||||
return super().transform_request(
|
||||
model=model.removeprefix("bedrock/"),
|
||||
messages=messages,
|
||||
optional_params=inference_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict, # mutable-ok: provider interface
|
||||
model: str,
|
||||
messages: list[AllMessageValues], # mutable-ok: provider interface
|
||||
optional_params: dict, # mutable-ok: provider interface
|
||||
litellm_params: dict, # mutable-ok: provider interface
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict: # mutable-ok: provider interface
|
||||
owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
|
||||
return merge_bedrock_invoke_headers(
|
||||
{
|
||||
**{key: value for key, value in headers.items() if key.lower() != "content-type"},
|
||||
"Content-Type": next(
|
||||
(value for key, value in headers.items() if key.lower() == "content-type"),
|
||||
"application/json",
|
||||
),
|
||||
},
|
||||
(),
|
||||
metadata_headers,
|
||||
owned_names,
|
||||
)
|
||||
|
||||
def sign_request(
|
||||
self,
|
||||
headers: dict, # mutable-ok: provider interface
|
||||
optional_params: dict, # mutable-ok: provider interface
|
||||
request_data: dict, # mutable-ok: provider interface
|
||||
api_base: str,
|
||||
api_key: str | None = None,
|
||||
model: str | None = None,
|
||||
stream: bool | None = None,
|
||||
fake_stream: bool | None = None,
|
||||
) -> tuple[dict, bytes | None]: # mutable-ok: provider interface
|
||||
return self._sign_request(
|
||||
service_name="bedrock",
|
||||
headers=headers,
|
||||
optional_params=optional_params,
|
||||
request_data=request_data,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
stream=stream,
|
||||
fake_stream=fake_stream,
|
||||
)
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: provider interface
|
||||
) -> BedrockError:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
|
@ -34,6 +34,43 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
_ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"
|
||||
_BEDROCK_ROUTE_PREFIXES: Final[tuple[str, ...]] = (
|
||||
"invoke/",
|
||||
"claude_platform/",
|
||||
"converse_like/",
|
||||
"converse/",
|
||||
"agent/",
|
||||
"agentcore/",
|
||||
"async_invoke/",
|
||||
"openai/",
|
||||
"mantle/",
|
||||
)
|
||||
_BEDROCK_ROUTE_TYPES: Final[
|
||||
tuple[
|
||||
Literal[
|
||||
"invoke",
|
||||
"claude_platform",
|
||||
"converse_like",
|
||||
"converse",
|
||||
"agent",
|
||||
"agentcore",
|
||||
"async_invoke",
|
||||
"openai",
|
||||
"mantle",
|
||||
],
|
||||
...,
|
||||
]
|
||||
] = (
|
||||
"invoke",
|
||||
"claude_platform",
|
||||
"converse_like",
|
||||
"converse",
|
||||
"agent",
|
||||
"agentcore",
|
||||
"async_invoke",
|
||||
"openai",
|
||||
"mantle",
|
||||
)
|
||||
|
||||
|
||||
def error_response_text(response: httpx.Response) -> str:
|
||||
|
|
@ -1089,6 +1126,10 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
"""Wrapper for standalone function. See get_bedrock_base_model()."""
|
||||
return get_bedrock_base_model(model)
|
||||
|
||||
@staticmethod
|
||||
def has_explicit_route(model: str) -> bool:
|
||||
return any(BedrockModelInfo._model_has_route_prefix(model, prefix) for prefix in _BEDROCK_ROUTE_PREFIXES)
|
||||
|
||||
@staticmethod
|
||||
def _supported_cross_region_inference_region() -> list[str]:
|
||||
"""Wrapper for standalone function. See get_bedrock_cross_region_inference_regions()."""
|
||||
|
|
@ -1111,36 +1152,11 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
"""
|
||||
Get the bedrock route for the given model.
|
||||
"""
|
||||
route_mappings: dict[
|
||||
str,
|
||||
Literal[
|
||||
"invoke",
|
||||
"claude_platform",
|
||||
"converse_like",
|
||||
"converse",
|
||||
"agent",
|
||||
"agentcore",
|
||||
"async_invoke",
|
||||
"openai",
|
||||
"mantle",
|
||||
],
|
||||
] = {
|
||||
"invoke/": "invoke",
|
||||
"claude_platform/": "claude_platform",
|
||||
"converse_like/": "converse_like",
|
||||
"converse/": "converse",
|
||||
"agent/": "agent",
|
||||
"agentcore/": "agentcore",
|
||||
"async_invoke/": "async_invoke",
|
||||
"openai/": "openai",
|
||||
"mantle/": "mantle",
|
||||
}
|
||||
|
||||
# Check explicit routes first. Match each prefix only as a leading path
|
||||
# segment so the `bedrock_mantle/` provider prefix is never mistaken for
|
||||
# the `mantle/` invoke route (which would mangle
|
||||
# `bedrock_mantle/openai.gpt-5.5` into `bedrock_openai.gpt-5.5`).
|
||||
for prefix, route_type in route_mappings.items():
|
||||
for prefix, route_type in zip(_BEDROCK_ROUTE_PREFIXES, _BEDROCK_ROUTE_TYPES, strict=True):
|
||||
if BedrockModelInfo._model_has_route_prefix(model, prefix):
|
||||
return route_type
|
||||
|
||||
|
|
@ -1292,6 +1308,17 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
|
||||
return AmazonMantleMessagesConfig()
|
||||
|
||||
from litellm.utils import model_supports_native_endpoint
|
||||
|
||||
if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint(
|
||||
"/v1/messages", model, litellm.LlmProviders.BEDROCK
|
||||
):
|
||||
from litellm.llms.bedrock.messages.native_transformation import (
|
||||
AmazonBedrockNativeMessagesConfig,
|
||||
)
|
||||
|
||||
return AmazonBedrockNativeMessagesConfig()
|
||||
|
||||
#########################################################
|
||||
# This goes through litellm.AmazonAnthropicClaude3MessagesConfig()
|
||||
# Since bedrock Invoke supports Native Anthropic Messages API
|
||||
|
|
@ -1321,6 +1348,13 @@ def get_bedrock_chat_config(model: str):
|
|||
bedrock_invoke_provider: Final = BaseAWSLLM.get_bedrock_invoke_provider(model=model)
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
|
||||
from litellm.utils import model_supports_native_endpoint
|
||||
|
||||
if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint(
|
||||
"/v1/chat/completions", model, litellm.LlmProviders.BEDROCK
|
||||
):
|
||||
return litellm.AmazonBedrockOpenAIChatCompletionsConfig()
|
||||
|
||||
# Handle explicit routes first
|
||||
if bedrock_route == "claude_platform":
|
||||
return litellm.BedrockClaudePlatformConfig()
|
||||
|
|
|
|||
136
litellm/llms/bedrock/messages/native_transformation.py
Normal file
136
litellm/llms/bedrock/messages/native_transformation.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
from typing import Final, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
||||
DEFAULT_ANTHROPIC_API_VERSION,
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
|
||||
from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
||||
class MessagesURLBuilder(Protocol):
|
||||
def __call__(self, api_base: str | None, endpoint: str | None, region: str) -> str: ...
|
||||
|
||||
|
||||
class BearerTokenResolver(Protocol):
|
||||
def __call__(self, api_key: str | None) -> str | None: ...
|
||||
|
||||
|
||||
class AmazonBedrockNativeMessagesConfig(BaseAWSLLM, AnthropicMessagesConfig):
|
||||
def __init__(
|
||||
self,
|
||||
custom_llm_provider: str = "bedrock",
|
||||
url_builder: MessagesURLBuilder | None = None,
|
||||
bearer_token_resolver: BearerTokenResolver = bedrock_bearer_token,
|
||||
**kwargs: object, # kwargs-ok: provider interface
|
||||
) -> None:
|
||||
BaseAWSLLM.__init__(self, **kwargs)
|
||||
self._custom_llm_provider = custom_llm_provider
|
||||
self._url_builder = url_builder
|
||||
self._bearer_token_resolver = bearer_token_resolver
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> str:
|
||||
return self._custom_llm_provider
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
api_key: str | None,
|
||||
model: str,
|
||||
optional_params: dict, # mutable-ok: provider interface
|
||||
litellm_params: dict, # mutable-ok: provider interface
|
||||
stream: bool | None = None,
|
||||
) -> str:
|
||||
region: Final = self._get_aws_region_name(optional_params=optional_params, model=model)
|
||||
endpoint: Final = optional_params.get("aws_bedrock_runtime_endpoint")
|
||||
if self._url_builder is not None:
|
||||
return self._url_builder(api_base, endpoint, region)
|
||||
runtime_endpoint: Final = self.get_runtime_endpoint(
|
||||
api_base=api_base,
|
||||
aws_bedrock_runtime_endpoint=endpoint,
|
||||
aws_region_name=region,
|
||||
)[0]
|
||||
return f"{runtime_endpoint.rstrip('/')}/anthropic/v1/messages"
|
||||
|
||||
def validate_anthropic_messages_environment(
|
||||
self,
|
||||
headers: dict, # mutable-ok: provider interface
|
||||
model: str,
|
||||
messages: list[dict], # mutable-ok: provider interface
|
||||
optional_params: dict, # mutable-ok: provider interface
|
||||
litellm_params: dict, # mutable-ok: provider interface
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> tuple[dict, str | None]: # mutable-ok: provider interface
|
||||
content_type: Final = next(
|
||||
(value for key, value in headers.items() if key.lower() == "content-type"),
|
||||
"application/json",
|
||||
)
|
||||
return (
|
||||
{ # mutable-ok: provider interface
|
||||
**{key: value for key, value in headers.items() if key.lower() != "content-type"},
|
||||
"anthropic-version": headers.get("anthropic-version", DEFAULT_ANTHROPIC_API_VERSION),
|
||||
"Content-Type": content_type,
|
||||
},
|
||||
api_base,
|
||||
)
|
||||
|
||||
def transform_anthropic_messages_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[dict], # mutable-ok: provider interface
|
||||
anthropic_messages_optional_request_params: dict, # mutable-ok: provider interface
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict, # mutable-ok: provider interface
|
||||
) -> dict: # mutable-ok: provider interface
|
||||
return { # mutable-ok: provider interface
|
||||
**anthropic_messages_optional_request_params,
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
}
|
||||
|
||||
def sign_request(
|
||||
self,
|
||||
headers: dict, # mutable-ok: provider interface
|
||||
optional_params: dict, # mutable-ok: provider interface
|
||||
request_data: dict, # mutable-ok: provider interface
|
||||
api_base: str,
|
||||
api_key: str | None = None,
|
||||
model: str | None = None,
|
||||
stream: bool | None = None,
|
||||
fake_stream: bool | None = None,
|
||||
) -> tuple[dict, bytes | None]: # mutable-ok: provider interface
|
||||
return self._sign_request(
|
||||
service_name="bedrock",
|
||||
headers=headers,
|
||||
optional_params=optional_params,
|
||||
request_data=request_data,
|
||||
api_base=api_base,
|
||||
api_key=self._bearer_token_resolver(api_key),
|
||||
model=model,
|
||||
stream=stream,
|
||||
fake_stream=fake_stream,
|
||||
)
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: provider interface
|
||||
) -> BaseLLMException:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
|
||||
def mantle_native_messages_config() -> AmazonBedrockNativeMessagesConfig:
|
||||
from litellm.llms.bedrock_mantle.common_utils import resolve_mantle_bearer_token
|
||||
|
||||
return AmazonBedrockNativeMessagesConfig(
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
url_builder=build_mantle_messages_url,
|
||||
bearer_token_resolver=resolve_mantle_bearer_token,
|
||||
)
|
||||
91
litellm/llms/bedrock/responses/transformation.py
Normal file
91
litellm/llms/bedrock/responses/transformation.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.types.llms.openai import ResponseInputParam
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
||||
class AmazonBedrockResponsesAPIConfig(OpenAIResponsesAPIConfig, BaseAWSLLM):
|
||||
def __init__(self, **kwargs: object) -> None: # kwargs-ok: provider interface
|
||||
OpenAIResponsesAPIConfig.__init__(self)
|
||||
BaseAWSLLM.__init__(self, **kwargs)
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> LlmProviders:
|
||||
return LlmProviders.BEDROCK
|
||||
|
||||
def transform_responses_api_request(
|
||||
self,
|
||||
model: str,
|
||||
input: str | ResponseInputParam,
|
||||
response_api_optional_request_params: dict, # mutable-ok: provider interface
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict, # mutable-ok: provider interface
|
||||
) -> dict: # mutable-ok: provider interface
|
||||
request_params: Final = { # mutable-ok: provider interface
|
||||
key: value
|
||||
for key, value in response_api_optional_request_params.items()
|
||||
if key not in self.aws_authentication_params
|
||||
}
|
||||
return { # mutable-ok: provider interface
|
||||
**request_params,
|
||||
"model": model,
|
||||
"input": input,
|
||||
}
|
||||
|
||||
def get_complete_url(self, api_base: str | None, litellm_params: dict) -> str: # mutable-ok: provider interface
|
||||
region: Final = self._get_aws_region_name(optional_params=litellm_params, model="")
|
||||
endpoint_url: Final = self.get_runtime_endpoint(
|
||||
api_base=api_base,
|
||||
aws_bedrock_runtime_endpoint=litellm_params.get("aws_bedrock_runtime_endpoint"),
|
||||
aws_region_name=region,
|
||||
)[0]
|
||||
return f"{endpoint_url.rstrip('/')}/openai/v1/responses"
|
||||
|
||||
def validate_environment(
|
||||
self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None
|
||||
) -> dict: # mutable-ok: provider interface
|
||||
return {
|
||||
**{key: value for key, value in headers.items() if key.lower() != "content-type"},
|
||||
"Content-Type": next(
|
||||
(value for key, value in headers.items() if key.lower() == "content-type"),
|
||||
"application/json",
|
||||
),
|
||||
} # mutable-ok: provider interface
|
||||
|
||||
def sign_request(
|
||||
self,
|
||||
headers: dict, # mutable-ok: provider interface
|
||||
optional_params: dict, # mutable-ok: provider interface
|
||||
request_data: dict, # mutable-ok: provider interface
|
||||
api_base: str,
|
||||
api_key: str | None = None,
|
||||
model: str | None = None,
|
||||
stream: bool | None = None,
|
||||
fake_stream: bool | None = None,
|
||||
) -> tuple[dict, bytes | None]: # mutable-ok: provider interface
|
||||
return self._sign_request(
|
||||
service_name="bedrock",
|
||||
headers=headers,
|
||||
optional_params=optional_params,
|
||||
request_data=request_data,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
stream=stream,
|
||||
fake_stream=fake_stream,
|
||||
)
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: provider interface
|
||||
) -> BaseLLMException:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
|
@ -4062,6 +4062,10 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
|
|||
|
||||
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
|
||||
|
||||
from litellm.llms.bedrock.chat.openai_chat_completions_transformation import (
|
||||
AmazonBedrockOpenAIChatCompletionsConfig,
|
||||
)
|
||||
|
||||
if "aws_bedrock_client" in optional_params:
|
||||
verbose_logger.warning(
|
||||
"'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication."
|
||||
|
|
@ -4080,7 +4084,27 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
|
|||
optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name
|
||||
|
||||
bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model)
|
||||
if bedrock_route == "claude_platform":
|
||||
if isinstance(provider_config, AmazonBedrockOpenAIChatCompletionsConfig):
|
||||
return base_llm_http_handler.completion(
|
||||
model=model.removeprefix("bedrock/"),
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="bedrock",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=_get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
client=client,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
elif bedrock_route == "claude_platform":
|
||||
provider_config = ProviderConfigManager.get_provider_chat_config(
|
||||
model=model,
|
||||
provider=LlmProviders.BEDROCK,
|
||||
|
|
|
|||
|
|
@ -4550,7 +4550,17 @@ def get_optional_params(
|
|||
BedrockModelInfo: Final = getattr(sys.modules[__name__], "BedrockModelInfo")
|
||||
bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model)
|
||||
bedrock_base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
if bedrock_route == "converse" or bedrock_route == "converse_like":
|
||||
if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint(
|
||||
"/v1/chat/completions", model, LlmProviders.BEDROCK
|
||||
):
|
||||
native_provider_config: Final = provider_config or litellm.AmazonBedrockOpenAIChatCompletionsConfig()
|
||||
optional_params = native_provider_config.map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
drop_params=bool(drop_params),
|
||||
)
|
||||
elif bedrock_route == "converse" or bedrock_route == "converse_like":
|
||||
optional_params = litellm.AmazonConverseConfig().map_openai_params(
|
||||
model=model,
|
||||
non_default_params=non_default_params,
|
||||
|
|
@ -8148,7 +8158,7 @@ def _get_model_cost_entry_for_provider_config(
|
|||
model: str,
|
||||
provider: LlmProviders,
|
||||
) -> dict[str, Any]:
|
||||
candidate_keys: Final = (model, f"{provider.value}/{model}")
|
||||
candidate_keys: Final = (f"{provider.value}/{model}", model)
|
||||
for model_key in candidate_keys:
|
||||
model_info = litellm.model_cost.get(model_key)
|
||||
if model_info is not None:
|
||||
|
|
@ -8162,6 +8172,22 @@ def _get_model_cost_entry_for_provider_config(
|
|||
return {}
|
||||
|
||||
|
||||
def model_supports_native_endpoint(endpoint: str, model: str, provider: LlmProviders) -> bool:
|
||||
bedrock_model_info: Final = litellm.BedrockModelInfo
|
||||
base_model: Final = bedrock_model_info.get_base_model(model) if provider is LlmProviders.BEDROCK else model
|
||||
candidate_models: Final = (model, base_model) if base_model != model else (model,)
|
||||
return any(
|
||||
isinstance(
|
||||
supported_endpoints := _get_model_cost_entry_for_provider_config(candidate_model, provider).get(
|
||||
"supported_endpoints"
|
||||
),
|
||||
(list, tuple),
|
||||
)
|
||||
and endpoint in supported_endpoints
|
||||
for candidate_model in candidate_models
|
||||
)
|
||||
|
||||
|
||||
class ProviderConfigManager:
|
||||
# Dictionary mapping for O(1) provider lookup
|
||||
# Stores tuples of (factory_function, needs_model_parameter)
|
||||
|
|
@ -8620,6 +8646,22 @@ class ProviderConfigManager:
|
|||
model: str,
|
||||
provider: LlmProviders,
|
||||
) -> BaseAnthropicMessagesConfig | None:
|
||||
if (
|
||||
provider is LlmProviders.BEDROCK
|
||||
and not litellm.BedrockModelInfo.has_explicit_route(model)
|
||||
and model_supports_native_endpoint("/v1/messages", model, LlmProviders.BEDROCK)
|
||||
):
|
||||
from litellm.llms.bedrock.messages.native_transformation import (
|
||||
AmazonBedrockNativeMessagesConfig,
|
||||
)
|
||||
|
||||
return AmazonBedrockNativeMessagesConfig()
|
||||
if provider is LlmProviders.BEDROCK_MANTLE and model_supports_native_endpoint(
|
||||
"/v1/messages", model, LlmProviders.BEDROCK_MANTLE
|
||||
):
|
||||
from litellm.llms.bedrock.messages.native_transformation import mantle_native_messages_config
|
||||
|
||||
return mantle_native_messages_config()
|
||||
return ProviderConfigManager._get_provider_anthropic_messages_config_cached(model=model, provider=provider)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -8902,6 +8944,12 @@ class ProviderConfigManager:
|
|||
return litellm.BedrockMantleResponsesAPIConfig(
|
||||
use_openai_path=mantle_base_segment(model, litellm.model_cost) == "openai/v1"
|
||||
)
|
||||
elif litellm.LlmProviders.BEDROCK == provider:
|
||||
if model is None or not model_supports_native_endpoint("/v1/responses", model, LlmProviders.BEDROCK):
|
||||
return None
|
||||
from litellm.llms.bedrock.responses.transformation import AmazonBedrockResponsesAPIConfig
|
||||
|
||||
return AmazonBedrockResponsesAPIConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -0,0 +1,410 @@
|
|||
import json
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.bedrock.chat.openai_chat_completions_transformation import (
|
||||
AmazonBedrockOpenAIChatCompletionsConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import get_bedrock_chat_config
|
||||
from litellm.llms.bedrock.messages.native_transformation import (
|
||||
AmazonBedrockNativeMessagesConfig,
|
||||
mantle_native_messages_config,
|
||||
)
|
||||
from litellm.llms.bedrock.responses.transformation import AmazonBedrockResponsesAPIConfig
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager, model_supports_native_endpoint
|
||||
|
||||
|
||||
def _chat_response(url: str) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={ # mutable-ok: provider interface
|
||||
"id": "chat-test",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "native-model",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], # mutable-ok: provider interface
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, # mutable-ok: provider interface
|
||||
},
|
||||
request=httpx.Request("POST", url),
|
||||
)
|
||||
|
||||
|
||||
def _messages_response(url: str) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={ # mutable-ok: provider interface
|
||||
"id": "message-test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "native-model",
|
||||
"content": [{"type": "text", "text": "ok"}], # mutable-ok: provider interface
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1}, # mutable-ok: provider interface
|
||||
},
|
||||
request=httpx.Request("POST", url),
|
||||
)
|
||||
|
||||
|
||||
def _responses_response(url: str) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={ # mutable-ok: provider interface
|
||||
"id": "response-test",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"model": "native-model",
|
||||
"output": [], # mutable-ok: provider interface
|
||||
"status": "completed",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, # mutable-ok: provider interface
|
||||
},
|
||||
request=httpx.Request("POST", url),
|
||||
)
|
||||
|
||||
|
||||
def test_native_endpoint_gate_reads_provider_model_cost(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/native-model",
|
||||
{"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface
|
||||
)
|
||||
|
||||
assert model_supports_native_endpoint(
|
||||
"/v1/chat/completions", "native-model", LlmProviders.BEDROCK
|
||||
)
|
||||
assert not model_supports_native_endpoint("/v1/messages", "native-model", LlmProviders.BEDROCK)
|
||||
|
||||
|
||||
def test_native_endpoint_gate_uses_bedrock_base_model(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/anthropic.claude-3-7-sonnet",
|
||||
{"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface
|
||||
)
|
||||
|
||||
assert model_supports_native_endpoint(
|
||||
"/v1/messages", "us.anthropic.claude-3-7-sonnet", LlmProviders.BEDROCK
|
||||
)
|
||||
|
||||
|
||||
def test_bedrock_chat_route_prefers_explicit_converse(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/native-model",
|
||||
{"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface
|
||||
)
|
||||
|
||||
assert isinstance(get_bedrock_chat_config("native-model"), AmazonBedrockOpenAIChatCompletionsConfig)
|
||||
assert type(get_bedrock_chat_config("converse/native-model")).__name__ == "AmazonConverseConfig"
|
||||
|
||||
|
||||
def test_bedrock_converse_catalog_model_prefers_declared_native_chat(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/anthropic.claude-sonnet-4-6",
|
||||
{"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface
|
||||
)
|
||||
|
||||
assert isinstance(
|
||||
get_bedrock_chat_config("anthropic.claude-sonnet-4-6"),
|
||||
AmazonBedrockOpenAIChatCompletionsConfig,
|
||||
)
|
||||
assert type(get_bedrock_chat_config("converse/anthropic.claude-sonnet-4-6")).__name__ == "AmazonConverseConfig"
|
||||
|
||||
|
||||
def test_bedrock_messages_route_prefers_explicit_invoke(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/anthropic.claude-sonnet-4-6",
|
||||
{"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface
|
||||
)
|
||||
|
||||
assert isinstance(
|
||||
ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
model="anthropic.claude-sonnet-4-6",
|
||||
provider=LlmProviders.BEDROCK,
|
||||
),
|
||||
AmazonBedrockNativeMessagesConfig,
|
||||
)
|
||||
assert type(
|
||||
ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
model="invoke/anthropic.claude-sonnet-4-6",
|
||||
provider=LlmProviders.BEDROCK,
|
||||
)
|
||||
).__name__ == "AmazonAnthropicClaudeMessagesConfig"
|
||||
|
||||
|
||||
def test_native_chat_url_and_bearer_auth(monkeypatch):
|
||||
config = AmazonBedrockOpenAIChatCompletionsConfig() # rebind-ok: test capture
|
||||
|
||||
assert config.get_complete_url(
|
||||
None, None, "native-model", {"aws_region_name": "us-west-2"}, {} # mutable-ok: provider interface
|
||||
) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions"
|
||||
|
||||
monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token")
|
||||
validated_headers = config.validate_environment(
|
||||
{"content-type": "application/json"}, # mutable-ok: provider interface
|
||||
"native-model",
|
||||
[], # mutable-ok: provider interface
|
||||
{}, # mutable-ok: provider interface
|
||||
{}, # mutable-ok: provider interface
|
||||
)
|
||||
headers, body = config.sign_request(
|
||||
validated_headers,
|
||||
{"aws_region_name": "us-west-2"}, # mutable-ok: provider interface
|
||||
{"model": "native-model"}, # mutable-ok: provider interface
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions",
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer token"
|
||||
assert sum(key.lower() == "content-type" for key in headers) == 1
|
||||
assert body is not None
|
||||
|
||||
|
||||
def test_native_chat_sigv4_auth(monkeypatch):
|
||||
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
|
||||
config = AmazonBedrockOpenAIChatCompletionsConfig() # rebind-ok: test capture
|
||||
headers, _ = config.sign_request(
|
||||
{}, # mutable-ok: provider interface
|
||||
{ # mutable-ok: provider interface
|
||||
"aws_region_name": "us-west-2",
|
||||
"aws_access_key_id": "testing",
|
||||
"aws_secret_access_key": "testing",
|
||||
},
|
||||
{"model": "native-model"}, # mutable-ok: provider interface
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions",
|
||||
)
|
||||
|
||||
assert headers["Authorization"].startswith("AWS4-HMAC-SHA256")
|
||||
|
||||
|
||||
def test_native_messages_preserves_body_and_urls(monkeypatch):
|
||||
monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token")
|
||||
config = AmazonBedrockNativeMessagesConfig() # rebind-ok: test capture
|
||||
request = { # mutable-ok: provider interface # rebind-ok: test capture
|
||||
"model": "claude-native",
|
||||
"messages": [{"role": "user", "content": "hello"}], # mutable-ok: provider interface
|
||||
"stream": True,
|
||||
"cache_control": {"ttl": "5m"}, # mutable-ok: provider interface
|
||||
}
|
||||
body = config.transform_anthropic_messages_request( # rebind-ok: test capture
|
||||
model=request["model"],
|
||||
messages=request["messages"],
|
||||
anthropic_messages_optional_request_params={ # mutable-ok: provider interface
|
||||
"stream": request["stream"],
|
||||
"cache_control": request["cache_control"],
|
||||
},
|
||||
litellm_params={}, # mutable-ok: provider interface
|
||||
headers={}, # mutable-ok: provider interface
|
||||
)
|
||||
|
||||
assert body == request
|
||||
assert "anthropic_version" not in body
|
||||
assert config.get_complete_url(None, None, "claude-native", {"aws_region_name": "us-west-2"}, {}) == ( # mutable-ok: provider interface
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages"
|
||||
)
|
||||
assert mantle_native_messages_config().get_complete_url(
|
||||
None, None, "claude-native", {"aws_region_name": "us-west-2"}, {} # mutable-ok: provider interface
|
||||
) == "https://bedrock-mantle.us-west-2.api.aws/anthropic/v1/messages"
|
||||
headers, signed_body = config.sign_request(
|
||||
{}, # mutable-ok: provider interface
|
||||
{"aws_region_name": "us-west-2"}, # mutable-ok: provider interface
|
||||
body, # mutable-ok: provider interface
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages",
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer token"
|
||||
assert sum(key.lower() == "content-type" for key in headers) == 1
|
||||
assert signed_body is not None
|
||||
|
||||
|
||||
def test_native_messages_deduplicates_content_type_header(monkeypatch):
|
||||
monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token")
|
||||
config = AmazonBedrockNativeMessagesConfig() # rebind-ok: test capture
|
||||
headers, _ = config.validate_anthropic_messages_environment(
|
||||
headers={"content-type": "application/json"}, # mutable-ok: provider interface
|
||||
model="native-model",
|
||||
messages=[], # mutable-ok: provider interface
|
||||
optional_params={}, # mutable-ok: provider interface
|
||||
litellm_params={}, # mutable-ok: provider interface
|
||||
)
|
||||
|
||||
assert sum(key.lower() == "content-type" for key in headers) == 1
|
||||
assert headers["Content-Type"] == "application/json"
|
||||
|
||||
|
||||
def test_native_messages_selection_is_not_cached(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/native-model",
|
||||
{"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface
|
||||
)
|
||||
assert isinstance(
|
||||
ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
"native-model", LlmProviders.BEDROCK
|
||||
),
|
||||
AmazonBedrockNativeMessagesConfig,
|
||||
)
|
||||
|
||||
monkeypatch.setitem(litellm.model_cost, "bedrock/native-model", {}) # mutable-ok: provider interface
|
||||
assert type(
|
||||
ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
"claude-native", LlmProviders.BEDROCK
|
||||
)
|
||||
).__name__ == "AmazonAnthropicClaudeMessagesConfig"
|
||||
|
||||
|
||||
def test_native_responses_selection_and_url(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/native-model",
|
||||
{"supported_endpoints": ["/v1/responses"]}, # mutable-ok: provider interface
|
||||
)
|
||||
config = ProviderConfigManager.get_provider_responses_api_config( # rebind-ok: test capture
|
||||
LlmProviders.BEDROCK, "native-model"
|
||||
)
|
||||
|
||||
assert isinstance(config, AmazonBedrockResponsesAPIConfig)
|
||||
assert config.get_complete_url(None, {"aws_region_name": "us-west-2"}).endswith( # mutable-ok: provider interface
|
||||
"/openai/v1/responses"
|
||||
)
|
||||
validated_headers = config.validate_environment(
|
||||
{"content-type": "application/json"}, # mutable-ok: provider interface
|
||||
"native-model",
|
||||
{}, # mutable-ok: provider interface
|
||||
)
|
||||
assert sum(key.lower() == "content-type" for key in validated_headers) == 1
|
||||
|
||||
monkeypatch.setitem(litellm.model_cost, "bedrock/native-model", {}) # mutable-ok: provider interface
|
||||
assert ProviderConfigManager.get_provider_responses_api_config(
|
||||
LlmProviders.BEDROCK, "native-model"
|
||||
) is None
|
||||
|
||||
|
||||
def test_completion_forwards_native_chat_body(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/native-model",
|
||||
{"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface
|
||||
)
|
||||
requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture
|
||||
|
||||
def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility
|
||||
requests.append((url, json.loads(data)))
|
||||
return _chat_response(url)
|
||||
|
||||
with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary
|
||||
"litellm.llms.custom_httpx.http_handler.HTTPHandler.post", post
|
||||
):
|
||||
litellm.completion(
|
||||
model="bedrock/native-model",
|
||||
messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface
|
||||
api_base="https://bedrock-runtime.us-west-2.amazonaws.com",
|
||||
api_key="token",
|
||||
)
|
||||
|
||||
assert requests == [ # mutable-ok: provider interface
|
||||
(
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions",
|
||||
{"model": "native-model", "messages": [{"role": "user", "content": "hello"}]}, # mutable-ok: provider interface
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_completion_forwards_declared_native_chat_for_converse_catalog_model(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/anthropic.claude-sonnet-4-6",
|
||||
{"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface
|
||||
)
|
||||
requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture
|
||||
|
||||
def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility
|
||||
requests.append((url, json.loads(data)))
|
||||
return _chat_response(url)
|
||||
|
||||
with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary
|
||||
"litellm.llms.custom_httpx.http_handler.HTTPHandler.post", post
|
||||
):
|
||||
litellm.completion(
|
||||
model="bedrock/anthropic.claude-sonnet-4-6",
|
||||
messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface
|
||||
api_base="https://bedrock-runtime.us-west-2.amazonaws.com",
|
||||
api_key="token",
|
||||
)
|
||||
|
||||
assert requests[0][0].endswith("/openai/v1/chat/completions")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_forwards_native_body(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/native-model",
|
||||
{"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface
|
||||
)
|
||||
requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture
|
||||
|
||||
async def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility
|
||||
requests.append((url, json.loads(data)))
|
||||
return _messages_response(url)
|
||||
|
||||
with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary
|
||||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=post
|
||||
):
|
||||
await litellm.anthropic_messages(
|
||||
model="bedrock/native-model",
|
||||
messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface
|
||||
max_tokens=10,
|
||||
api_base="https://bedrock-runtime.us-west-2.amazonaws.com",
|
||||
api_key="token",
|
||||
)
|
||||
|
||||
assert requests == [ # mutable-ok: provider interface
|
||||
(
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages",
|
||||
{ # mutable-ok: provider interface
|
||||
"max_tokens": 10,
|
||||
"stream": False,
|
||||
"model": "native-model",
|
||||
"messages": [{"role": "user", "content": "hello"}], # mutable-ok: provider interface
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_responses_forwards_native_body(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/native-model",
|
||||
{"supported_endpoints": ["/v1/responses"]}, # mutable-ok: provider interface
|
||||
)
|
||||
requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture
|
||||
|
||||
def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility
|
||||
requests.append((url, json.loads(data)))
|
||||
return _responses_response(url)
|
||||
|
||||
with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary
|
||||
"litellm.llms.custom_httpx.http_handler.HTTPHandler.post", post
|
||||
):
|
||||
litellm.responses(
|
||||
model="bedrock/native-model",
|
||||
input="hello",
|
||||
instructions="be brief",
|
||||
api_base="https://bedrock-runtime.us-west-2.amazonaws.com",
|
||||
api_key="token",
|
||||
)
|
||||
|
||||
assert requests == [ # mutable-ok: provider interface
|
||||
(
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses",
|
||||
{ # mutable-ok: provider interface
|
||||
"model": "native-model",
|
||||
"input": "hello",
|
||||
"instructions": "be brief",
|
||||
},
|
||||
)
|
||||
]
|
||||
Loading…
Add table
Reference in a new issue