mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
Merge pull request #42049 from BerriAI/litellm_mantle_native_anthropic_messages
feat(bedrock_mantle): serve /v1/messages for Claude models on Mantle's native Anthropic Messages API
This commit is contained in:
Mateo Wang
GitHub
commit
2e35ae1065
20 changed files with 938 additions and 19 deletions
|
|
@ -1684,6 +1684,9 @@ if TYPE_CHECKING:
|
|||
from .llms.bedrock.messages.mantle_transformation import (
|
||||
AmazonMantleMessagesConfig as AmazonMantleMessagesConfig,
|
||||
)
|
||||
from .llms.bedrock_mantle.messages.transformation import (
|
||||
BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig,
|
||||
)
|
||||
from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
|
||||
from .llms.together_ai.chat.transformation import (
|
||||
TogetherAIChatConfig as TogetherAIChatConfig,
|
||||
|
|
|
|||
|
|
@ -176,6 +176,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"BedrockClaudePlatformMessagesConfig",
|
||||
"AmazonAnthropicClaudeMessagesConfig",
|
||||
"AmazonMantleMessagesConfig",
|
||||
"BedrockMantleAnthropicMessagesConfig",
|
||||
"TogetherAIConfig",
|
||||
"TogetherAIChatConfig",
|
||||
"NLPCloudConfig",
|
||||
|
|
@ -746,6 +747,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
".llms.bedrock.messages.mantle_transformation",
|
||||
"AmazonMantleMessagesConfig",
|
||||
),
|
||||
"BedrockMantleAnthropicMessagesConfig": (
|
||||
".llms.bedrock_mantle.messages.transformation",
|
||||
"BedrockMantleAnthropicMessagesConfig",
|
||||
),
|
||||
"TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"),
|
||||
"TogetherAIChatConfig": (
|
||||
".llms.together_ai.chat.transformation",
|
||||
|
|
|
|||
|
|
@ -131,6 +131,41 @@
|
|||
"web-fetch-2025-09-10": null,
|
||||
"web-search-2025-03-05": null
|
||||
},
|
||||
"bedrock_mantle": {
|
||||
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
|
||||
"advisor-tool-2026-03-01": null,
|
||||
"bash_20241022": null,
|
||||
"bash_20250124": null,
|
||||
"claude-code-20250219": "claude-code-20250219",
|
||||
"code-execution-2025-08-25": null,
|
||||
"compact-2026-01-12": "compact-2026-01-12",
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
"context-management-2025-06-27": "context-management-2025-06-27",
|
||||
"effort-2025-11-24": "effort-2025-11-24",
|
||||
"fast-mode-2026-02-01": null,
|
||||
"files-api-2025-04-14": null,
|
||||
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
|
||||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"mcp-client-2025-04-04": null,
|
||||
"mcp-client-2025-11-20": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"output-128k-2025-02-19": "output-128k-2025-02-19",
|
||||
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
|
||||
"prompt-caching-scope-2026-01-05": null,
|
||||
"skills-2025-10-02": null,
|
||||
"structured-output-2024-03-01": null,
|
||||
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
|
||||
"text_editor_20241022": null,
|
||||
"text_editor_20250124": null,
|
||||
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
|
||||
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
|
||||
"tool-examples-2025-10-29": "tool-examples-2025-10-29",
|
||||
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
|
||||
"web-fetch-2025-09-10": null,
|
||||
"web-search-2025-03-05": "web-search-2025-03-05"
|
||||
},
|
||||
"vertex_ai": {
|
||||
"advisor-tool-2026-03-01": null,
|
||||
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
|
||||
|
|
|
|||
|
|
@ -334,7 +334,7 @@ def update_headers_with_filtered_beta(
|
|||
Updated headers dict
|
||||
"""
|
||||
existing_beta: Final = headers.get("anthropic-beta")
|
||||
if not existing_beta:
|
||||
if existing_beta is None:
|
||||
return headers
|
||||
|
||||
# Parse existing beta headers
|
||||
|
|
|
|||
|
|
@ -651,6 +651,11 @@ def anthropic_messages_handler(
|
|||
"display": "summarized",
|
||||
}
|
||||
|
||||
resolved_api_base: Final = (
|
||||
dynamic_api_base
|
||||
if dynamic_api_base is not None and anthropic_messages_provider_config.uses_get_llm_provider_api_base()
|
||||
else api_base
|
||||
)
|
||||
return base_llm_http_handler.anthropic_messages_handler(
|
||||
model=model,
|
||||
messages=strip_provider_specific_fields_from_anthropic_messages(messages),
|
||||
|
|
@ -662,7 +667,7 @@ def anthropic_messages_handler(
|
|||
litellm_params=litellm_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
api_base=resolved_api_base,
|
||||
stream=stream,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -128,6 +128,9 @@ class BaseAnthropicMessagesConfig(ABC):
|
|||
"""
|
||||
return True
|
||||
|
||||
def uses_get_llm_provider_api_base(self) -> bool:
|
||||
return False
|
||||
|
||||
def get_async_streaming_response_iterator(
|
||||
self,
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
|
|
@ -445,13 +445,16 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
# Bedrock InvokeModel DOES support ``clear_tool_uses_20250919`` under the
|
||||
# ``context-management-2025-06-27`` beta. AWS docs:
|
||||
# https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-tool-use.md
|
||||
_BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: dict[str, str] = {
|
||||
"compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
|
||||
"clear_tool_uses_20250919": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
}
|
||||
_BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: Mapping[str, str] = MappingProxyType(
|
||||
{
|
||||
"compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
|
||||
"clear_tool_uses_20250919": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@classmethod
|
||||
def _filter_context_management_for_bedrock_invoke(
|
||||
cls,
|
||||
anthropic_messages_request: dict,
|
||||
beta_set: set,
|
||||
) -> None:
|
||||
|
|
@ -481,7 +484,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
anthropic_messages_request.pop("context_management", None)
|
||||
return
|
||||
|
||||
supported: Final = AmazonAnthropicClaudeMessagesConfig._BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS
|
||||
supported: Final = cls._BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS
|
||||
retained_edits: Final = [e for e in edits if isinstance(e, dict) and e.get("type") in supported]
|
||||
if not retained_edits:
|
||||
anthropic_messages_request.pop("context_management", None)
|
||||
|
|
@ -546,15 +549,16 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
if "tool-search-tool-2025-10-19" in beta_set:
|
||||
beta_set.add("tool-examples-2025-10-29")
|
||||
|
||||
beta_provider: Final = self.custom_llm_provider or "bedrock"
|
||||
filtered_betas: Final = sorted(
|
||||
filter_and_transform_beta_headers(
|
||||
beta_headers=list(beta_set),
|
||||
provider="bedrock",
|
||||
provider=beta_provider,
|
||||
)
|
||||
)
|
||||
|
||||
dropped_user_betas: Final = sorted(
|
||||
b for b in user_beta_set if not filter_and_transform_beta_headers([b], provider="bedrock")
|
||||
b for b in user_beta_set if not filter_and_transform_beta_headers([b], provider=beta_provider)
|
||||
)
|
||||
if dropped_user_betas:
|
||||
verbose_logger.warning(
|
||||
|
|
|
|||
0
litellm/llms/bedrock_mantle/messages/__init__.py
Normal file
0
litellm/llms/bedrock_mantle/messages/__init__.py
Normal file
127
litellm/llms/bedrock_mantle/messages/transformation.py
Normal file
127
litellm/llms/bedrock_mantle/messages/transformation.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
||||
DEFAULT_ANTHROPIC_API_VERSION,
|
||||
)
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import MANTLE_MESSAGES_PATH
|
||||
from litellm.llms.bedrock.messages.mantle_transformation import AmazonMantleMessagesConfig
|
||||
from litellm.llms.bedrock_mantle.common_utils import (
|
||||
MANTLE_HOST_RE,
|
||||
BedrockMantleAuthMixin,
|
||||
resolve_mantle_region,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
_BASE_SUFFIXES_TO_STRIP: Final = (
|
||||
MANTLE_MESSAGES_PATH,
|
||||
"/v1/messages",
|
||||
"/messages",
|
||||
"/anthropic/v1",
|
||||
"/openai/v1",
|
||||
"/v1",
|
||||
)
|
||||
_BODY_FIELDS_MANTLE_READS_FROM_HEADERS: Final = frozenset({"anthropic_version", "anthropic_beta"})
|
||||
_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...])
|
||||
_MANTLE_REQUEST: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def build_mantle_native_messages_url(api_base: str | None, litellm_params: Mapping[str, object]) -> str:
|
||||
region: Final = resolve_mantle_region(MappingProxyType({**litellm_params, "api_base": api_base}))
|
||||
configured: Final = (
|
||||
api_base or get_secret_str("BEDROCK_MANTLE_API_BASE") or f"https://bedrock-mantle.{region}.api.aws"
|
||||
).rstrip("/")
|
||||
stripped: Final = next(
|
||||
(configured[: -len(suffix)] for suffix in _BASE_SUFFIXES_TO_STRIP if configured.endswith(suffix)),
|
||||
configured,
|
||||
)
|
||||
host: Final = f"https://bedrock-mantle.{region}.api.aws" if MANTLE_HOST_RE.match(stripped) else stripped
|
||||
return f"{host}{MANTLE_MESSAGES_PATH}"
|
||||
|
||||
|
||||
class BedrockMantleAnthropicMessagesConfig(BedrockMantleAuthMixin, AmazonMantleMessagesConfig):
|
||||
_BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: Mapping[str, str] = MappingProxyType(
|
||||
{
|
||||
**AmazonMantleMessagesConfig._BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS,
|
||||
"clear_thinking_20251015": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None:
|
||||
AmazonMantleMessagesConfig.__init__(self)
|
||||
self._aws_signer = aws_signer or self
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> str | None:
|
||||
return "bedrock_mantle"
|
||||
|
||||
def uses_get_llm_provider_api_base(self) -> bool:
|
||||
return True
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
api_key: str | None,
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
stream: bool | None = None,
|
||||
) -> str:
|
||||
return build_mantle_native_messages_url(api_base=api_base, litellm_params=litellm_params)
|
||||
|
||||
def validate_anthropic_messages_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> tuple[dict, str | None]:
|
||||
merged_headers, resolved_api_base = super().validate_anthropic_messages_environment(
|
||||
headers=headers,
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
if any(name.lower() == "anthropic-version" for name in merged_headers):
|
||||
return merged_headers, resolved_api_base
|
||||
return { # mutable-ok: the base class contract returns a dict the handler signs into in place
|
||||
**merged_headers,
|
||||
"anthropic-version": DEFAULT_ANTHROPIC_API_VERSION,
|
||||
}, resolved_api_base
|
||||
|
||||
def transform_anthropic_messages_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
anthropic_messages_optional_request_params: dict,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
request: Final = _MANTLE_REQUEST.validate_python(
|
||||
super().transform_anthropic_messages_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
),
|
||||
)
|
||||
betas: Final = request.get("anthropic_beta")
|
||||
if betas is not None:
|
||||
header_betas: Final = ",".join(_ANTHROPIC_BETAS.validate_python(betas))
|
||||
headers["anthropic-beta"] = header_betas # rebind-ok: the handler signs and sends this same dict
|
||||
return { # mutable-ok: the base class contract returns the dict the handler serializes as the body
|
||||
key: value for key, value in request.items() if key not in _BODY_FIELDS_MANTLE_READS_FROM_HEADERS
|
||||
}
|
||||
|
|
@ -60564,6 +60564,34 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"bedrock_mantle/anthropic.claude-haiku-4-5": {
|
||||
"cache_creation_input_token_cost": 1.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-06,
|
||||
"source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"input_cost_per_token_batches": 5e-07,
|
||||
"output_cost_per_token_batches": 2.5e-06
|
||||
},
|
||||
"us.xai.grok-4.6": {
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
"output_cost_per_token": 6.6e-06,
|
||||
|
|
|
|||
|
|
@ -3418,7 +3418,12 @@ async def add_guardrails_from_policy_engine(
|
|||
|
||||
|
||||
_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
|
||||
(LlmProviders.ANTHROPIC.value, LlmProviders.BEDROCK.value, LlmProviders.VERTEX_AI.value)
|
||||
(
|
||||
LlmProviders.ANTHROPIC.value,
|
||||
LlmProviders.BEDROCK.value,
|
||||
LlmProviders.BEDROCK_MANTLE.value,
|
||||
LlmProviders.VERTEX_AI.value,
|
||||
)
|
||||
)
|
||||
_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = LlmProviders.ANTHROPIC.value
|
||||
|
||||
|
|
|
|||
|
|
@ -5624,6 +5624,12 @@ def _get_model_info_from_generalization(
|
|||
return None
|
||||
|
||||
|
||||
def _strip_mantle_region_prefix(model: str) -> str:
|
||||
from litellm.llms.bedrock_mantle.common_utils import split_mantle_region_prefix
|
||||
|
||||
return split_mantle_region_prefix(model)[1]
|
||||
|
||||
|
||||
def _get_potential_model_names(model: str, custom_llm_provider: str | None) -> PotentialModelNamesAndCustomLLMProvider:
|
||||
if custom_llm_provider is None:
|
||||
# Get custom_llm_provider
|
||||
|
|
@ -5656,20 +5662,30 @@ def _get_potential_model_names(model: str, custom_llm_provider: str | None) -> P
|
|||
|
||||
split_model = strip_bedrock_routing_prefix(split_model)
|
||||
|
||||
region_free_split_model: Final = (
|
||||
_strip_mantle_region_prefix(split_model) if custom_llm_provider == "bedrock_mantle" else split_model
|
||||
)
|
||||
region_free_combined_stripped_model_name: Final = (
|
||||
f"bedrock_mantle/{_strip_model_name(model=region_free_split_model, custom_llm_provider=custom_llm_provider)}"
|
||||
if custom_llm_provider == "bedrock_mantle"
|
||||
else combined_stripped_model_name
|
||||
)
|
||||
provider_model_info: Final = (
|
||||
ProviderConfigManager.get_provider_model_info(model=split_model, provider=LlmProviders(custom_llm_provider))
|
||||
ProviderConfigManager.get_provider_model_info(
|
||||
model=region_free_split_model, provider=LlmProviders(custom_llm_provider)
|
||||
)
|
||||
if custom_llm_provider in LlmProvidersSet
|
||||
else None
|
||||
)
|
||||
provider_cost_key: Final = (
|
||||
provider_model_info.get_model_cost_key(split_model) if provider_model_info is not None else None
|
||||
provider_model_info.get_model_cost_key(region_free_split_model) if provider_model_info is not None else None
|
||||
)
|
||||
|
||||
return PotentialModelNamesAndCustomLLMProvider(
|
||||
split_model=split_model,
|
||||
split_model=region_free_split_model,
|
||||
combined_model_name=combined_model_name,
|
||||
stripped_model_name=stripped_model_name,
|
||||
combined_stripped_model_name=combined_stripped_model_name,
|
||||
combined_stripped_model_name=region_free_combined_stripped_model_name,
|
||||
provider_prefixed_model_name=provider_cost_key or provider_prefixed_model_name,
|
||||
custom_llm_provider=cast(str, custom_llm_provider),
|
||||
)
|
||||
|
|
@ -8681,6 +8697,13 @@ class ProviderConfigManager:
|
|||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
|
||||
return BedrockModelInfo.get_bedrock_provider_config_for_messages_api(model)
|
||||
elif litellm.LlmProviders.BEDROCK_MANTLE == provider:
|
||||
if "claude" in model_lower:
|
||||
from litellm.llms.bedrock_mantle.messages.transformation import (
|
||||
BedrockMantleAnthropicMessagesConfig,
|
||||
)
|
||||
|
||||
return BedrockMantleAnthropicMessagesConfig()
|
||||
elif litellm.LlmProviders.VERTEX_AI == provider:
|
||||
if "claude" in model_lower:
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
|
||||
|
|
|
|||
|
|
@ -60564,6 +60564,34 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"bedrock_mantle/anthropic.claude-haiku-4-5": {
|
||||
"cache_creation_input_token_cost": 1.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-06,
|
||||
"source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"input_cost_per_token_batches": 5e-07,
|
||||
"output_cost_per_token_batches": 2.5e-06
|
||||
},
|
||||
"us.xai.grok-4.6": {
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
"output_cost_per_token": 6.6e-06,
|
||||
|
|
|
|||
|
|
@ -2003,7 +2003,7 @@ def test_provider_specific_header():
|
|||
)
|
||||
# Verify multi-provider support: anthropic headers work across multiple providers
|
||||
assert data["provider_specific_header"] == {
|
||||
"custom_llm_provider": "anthropic,bedrock,vertex_ai",
|
||||
"custom_llm_provider": "anthropic,bedrock,bedrock_mantle,vertex_ai",
|
||||
"extra_headers": {
|
||||
"anthropic-beta": "prompt-caching-2024-07-31",
|
||||
},
|
||||
|
|
@ -2075,7 +2075,7 @@ def test_provider_specific_header_multi_provider():
|
|||
assert "provider_specific_header" in data
|
||||
assert (
|
||||
data["provider_specific_header"]["custom_llm_provider"]
|
||||
== "anthropic,bedrock,vertex_ai"
|
||||
== "anthropic,bedrock,bedrock_mantle,vertex_ai"
|
||||
)
|
||||
assert data["provider_specific_header"]["extra_headers"] == {
|
||||
"anthropic-beta": "context-1m-2025-08-07",
|
||||
|
|
|
|||
|
|
@ -1440,6 +1440,46 @@ async def test_anthropic_messages_leaves_non_provider_failures_unmapped():
|
|||
assert "Traceback" not in str(excinfo.value)
|
||||
|
||||
|
||||
def _recording_client(seen_urls: list[str]) -> AsyncHTTPHandler:
|
||||
def record_and_answer(request: httpx.Request) -> httpx.Response:
|
||||
seen_urls.append(str(request.url))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "deepseek-chat",
|
||||
"content": [{"type": "text", "text": "pong"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
},
|
||||
)
|
||||
|
||||
upstream = AsyncHTTPHandler()
|
||||
upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(record_and_answer))
|
||||
return upstream
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_provider_messages_api_base_env_is_not_shadowed_by_the_chat_default(monkeypatch):
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages import handler
|
||||
|
||||
monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False)
|
||||
monkeypatch.setenv("DEEPSEEK_ANTHROPIC_API_BASE", "https://deepseek.internal.example/anthropic")
|
||||
seen_urls: list[str] = []
|
||||
|
||||
await handler.anthropic_messages(
|
||||
max_tokens=16,
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
model="deepseek/deepseek-chat",
|
||||
api_key="sk-test",
|
||||
client=_recording_client(seen_urls),
|
||||
)
|
||||
|
||||
assert seen_urls == ["https://deepseek.internal.example/anthropic/v1/messages"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthropic():
|
||||
"""Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21."""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,484 @@
|
|||
"""
|
||||
Unit tests for the bedrock_mantle native Anthropic Messages route.
|
||||
|
||||
Mantle serves its Claude models only on `/anthropic/v1/messages` (the OpenAI
|
||||
paths reject them), so `bedrock_mantle/anthropic.claude-*` requests on
|
||||
/v1/messages must hit that endpoint directly instead of the chat-completions
|
||||
bridge. These tests lock the dispatcher gate, the URL derivation from the
|
||||
OpenAI-surface base that get_llm_provider pre-fills, the version header, the
|
||||
Bearer/SigV4 auth chain, and the wire request through the public entrypoint.
|
||||
"""
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock_mantle.messages.transformation import (
|
||||
BedrockMantleAnthropicMessagesConfig,
|
||||
build_mantle_native_messages_url,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
MESSAGES_PATH = "/anthropic/v1/messages"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _httpx_transport_with_fresh_clients(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_ambient_mantle_env(monkeypatch):
|
||||
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
|
||||
monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False)
|
||||
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
|
||||
monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False)
|
||||
monkeypatch.delenv("AWS_REGION_NAME", raising=False)
|
||||
monkeypatch.delenv("AWS_REGION", raising=False)
|
||||
|
||||
|
||||
def _anthropic_response() -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "msg_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "anthropic.claude-sonnet-5",
|
||||
"content": [{"type": "text", "text": "pong"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_SSE_EVENTS = (
|
||||
(
|
||||
"message_start",
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_stream",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "anthropic.claude-sonnet-5",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
),
|
||||
("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),
|
||||
(
|
||||
"content_block_delta",
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}},
|
||||
),
|
||||
("content_block_stop", {"type": "content_block_stop", "index": 0}),
|
||||
("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}}),
|
||||
("message_stop", {"type": "message_stop"}),
|
||||
)
|
||||
|
||||
|
||||
def _sse_response() -> httpx.Response:
|
||||
body = "".join(f"event: {event}\ndata: {json.dumps(payload)}\n\n" for event, payload in _SSE_EVENTS).encode()
|
||||
return httpx.Response(status_code=200, content=body, headers={"content-type": "text/event-stream"})
|
||||
|
||||
|
||||
def _mantle_messages_route(region: str) -> respx.Route:
|
||||
return respx.post(f"https://bedrock-mantle.{region}.api.aws{MESSAGES_PATH}")
|
||||
|
||||
|
||||
def _sent_body(route: respx.Route) -> dict:
|
||||
return json.loads(route.calls.last.request.content)
|
||||
|
||||
|
||||
class TestDispatch:
|
||||
def test_claude_models_get_the_native_messages_config(self):
|
||||
config = ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
model="anthropic.claude-sonnet-5", provider=litellm.LlmProviders.BEDROCK_MANTLE
|
||||
)
|
||||
assert isinstance(config, BedrockMantleAnthropicMessagesConfig)
|
||||
assert config.custom_llm_provider == "bedrock_mantle"
|
||||
|
||||
@pytest.mark.parametrize("model", ["openai.gpt-5.6-sol", "openai.gpt-oss-120b-1:0", "google.gemma-4-31b"])
|
||||
def test_non_claude_models_keep_the_bridge(self, model):
|
||||
assert (
|
||||
ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
model=model, provider=litellm.LlmProviders.BEDROCK_MANTLE
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
class TestURL:
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
[
|
||||
"https://bedrock-mantle.us-east-1.api.aws/v1",
|
||||
"https://bedrock-mantle.us-east-1.api.aws/openai/v1",
|
||||
"https://bedrock-mantle.us-east-1.api.aws/openai/v1/",
|
||||
"https://bedrock-mantle.us-east-1.api.aws",
|
||||
"https://bedrock-mantle.us-east-1.api.aws/anthropic/v1/messages",
|
||||
],
|
||||
)
|
||||
def test_prefilled_openai_base_becomes_the_messages_endpoint(self, api_base):
|
||||
url = build_mantle_native_messages_url(api_base, {"aws_region_name": "us-east-1"})
|
||||
assert url == f"https://bedrock-mantle.us-east-1.api.aws{MESSAGES_PATH}"
|
||||
|
||||
def test_aws_region_name_wins_over_the_prefilled_host_region(self):
|
||||
url = build_mantle_native_messages_url(
|
||||
"https://bedrock-mantle.us-east-1.api.aws/v1", {"aws_region_name": "us-east-2"}
|
||||
)
|
||||
assert url == f"https://bedrock-mantle.us-east-2.api.aws{MESSAGES_PATH}"
|
||||
|
||||
def test_host_region_is_used_when_no_region_param(self):
|
||||
url = build_mantle_native_messages_url("https://bedrock-mantle.eu-west-1.api.aws/v1", {})
|
||||
assert url == f"https://bedrock-mantle.eu-west-1.api.aws{MESSAGES_PATH}"
|
||||
|
||||
def test_custom_host_is_preserved(self):
|
||||
url = build_mantle_native_messages_url("https://vpce-abc.bedrock-mantle.example.com/v1", {})
|
||||
assert url == f"https://vpce-abc.bedrock-mantle.example.com{MESSAGES_PATH}"
|
||||
|
||||
def test_env_base_is_used_without_api_base(self, monkeypatch):
|
||||
monkeypatch.setenv("BEDROCK_MANTLE_API_BASE", "https://mantle-proxy.internal/openai/v1")
|
||||
assert build_mantle_native_messages_url(None, {}) == f"https://mantle-proxy.internal{MESSAGES_PATH}"
|
||||
|
||||
def test_default_host_comes_from_mantle_region_env(self, monkeypatch):
|
||||
monkeypatch.setenv("BEDROCK_MANTLE_REGION", "ap-northeast-1")
|
||||
assert (
|
||||
build_mantle_native_messages_url(None, {})
|
||||
== f"https://bedrock-mantle.ap-northeast-1.api.aws{MESSAGES_PATH}"
|
||||
)
|
||||
|
||||
def test_config_get_complete_url_reads_litellm_params(self):
|
||||
config = BedrockMantleAnthropicMessagesConfig()
|
||||
url = config.get_complete_url(
|
||||
api_base="https://bedrock-mantle.us-east-1.api.aws/v1",
|
||||
api_key=None,
|
||||
model="anthropic.claude-sonnet-5",
|
||||
optional_params={},
|
||||
litellm_params={"aws_region_name": "us-west-2"},
|
||||
)
|
||||
assert url == f"https://bedrock-mantle.us-west-2.api.aws{MESSAGES_PATH}"
|
||||
|
||||
|
||||
class TestEnvironment:
|
||||
def _validate(self, headers: dict, litellm_params: dict) -> dict:
|
||||
config = BedrockMantleAnthropicMessagesConfig()
|
||||
merged, _ = config.validate_anthropic_messages_environment(
|
||||
headers=headers,
|
||||
model="anthropic.claude-sonnet-5",
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
return merged
|
||||
|
||||
def test_adds_the_anthropic_version_header(self):
|
||||
assert self._validate({}, {})["anthropic-version"] == "2023-06-01"
|
||||
|
||||
def test_keeps_a_caller_supplied_version_header(self):
|
||||
merged = self._validate({"Anthropic-Version": "2024-01-01"}, {})
|
||||
assert merged["Anthropic-Version"] == "2024-01-01"
|
||||
assert "anthropic-version" not in merged
|
||||
|
||||
def test_project_id_becomes_the_workspace_header(self):
|
||||
assert self._validate({}, {"aws_bedrock_project_id": "proj_123"})["anthropic-workspace"] == "proj_123"
|
||||
|
||||
|
||||
class TestRequestBody:
|
||||
def test_body_carries_model_and_stream_but_not_the_invoke_version(self):
|
||||
config = BedrockMantleAnthropicMessagesConfig()
|
||||
body = config.transform_anthropic_messages_request(
|
||||
model="anthropic.claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
anthropic_messages_optional_request_params={"max_tokens": 8, "stream": True},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert body["model"] == "anthropic.claude-sonnet-5"
|
||||
assert body["stream"] is True
|
||||
assert body["max_tokens"] == 8
|
||||
assert "anthropic_version" not in body
|
||||
|
||||
def test_body_omits_stream_when_not_streaming(self):
|
||||
config = BedrockMantleAnthropicMessagesConfig()
|
||||
body = config.transform_anthropic_messages_request(
|
||||
model="anthropic.claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
anthropic_messages_optional_request_params={"max_tokens": 8},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert "stream" not in body
|
||||
|
||||
|
||||
class TestAuth:
|
||||
def test_bearer_from_api_key_skips_aws_credentials(self):
|
||||
signer = BaseAWSLLM()
|
||||
signer.get_credentials = MagicMock(side_effect=AssertionError("must not resolve AWS credentials"))
|
||||
config = BedrockMantleAnthropicMessagesConfig(aws_signer=signer)
|
||||
headers, signed = config.sign_request(
|
||||
headers={"anthropic-version": "2023-06-01"},
|
||||
optional_params={},
|
||||
request_data={"model": "anthropic.claude-sonnet-5"},
|
||||
api_base=f"https://bedrock-mantle.us-east-1.api.aws{MESSAGES_PATH}",
|
||||
api_key="arg-bearer",
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer arg-bearer"
|
||||
assert headers["anthropic-version"] == "2023-06-01"
|
||||
assert signed == b'{"model": "anthropic.claude-sonnet-5"}'
|
||||
|
||||
def test_bearer_from_mantle_env_key(self, monkeypatch):
|
||||
monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-bearer")
|
||||
config = BedrockMantleAnthropicMessagesConfig()
|
||||
headers, _ = config.sign_request(
|
||||
headers={},
|
||||
optional_params={},
|
||||
request_data={},
|
||||
api_base=f"https://bedrock-mantle.us-east-1.api.aws{MESSAGES_PATH}",
|
||||
api_key=None,
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer env-bearer"
|
||||
|
||||
def test_sigv4_scope_is_pinned_to_the_url_host_region(self):
|
||||
config = BedrockMantleAnthropicMessagesConfig()
|
||||
headers, signed = config.sign_request(
|
||||
headers={"anthropic-version": "2023-06-01"},
|
||||
optional_params={
|
||||
"aws_access_key_id": "AKIAEXAMPLE",
|
||||
"aws_secret_access_key": "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0",
|
||||
"aws_region_name": "us-east-1",
|
||||
},
|
||||
request_data={"model": "anthropic.claude-sonnet-5"},
|
||||
api_base=f"https://bedrock-mantle.us-west-2.api.aws{MESSAGES_PATH}",
|
||||
api_key=None,
|
||||
)
|
||||
assert headers["Authorization"].startswith("AWS4-HMAC-SHA256")
|
||||
assert "/us-west-2/bedrock/aws4_request" in headers["Authorization"]
|
||||
assert signed == b'{"model": "anthropic.claude-sonnet-5"}'
|
||||
|
||||
|
||||
class TestWireRequest:
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_claude_request_hits_the_native_messages_endpoint(self):
|
||||
route = _mantle_messages_route("us-east-1").mock(return_value=_anthropic_response())
|
||||
|
||||
response = await litellm.anthropic_messages(
|
||||
model="bedrock_mantle/anthropic.claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
max_tokens=8,
|
||||
api_key="test-bearer",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
assert response["content"][0]["text"] == "pong"
|
||||
assert route.call_count == 1
|
||||
sent = route.calls.last.request
|
||||
assert sent.headers["authorization"] == "Bearer test-bearer"
|
||||
assert sent.headers["anthropic-version"] == "2023-06-01"
|
||||
assert "x-api-key" not in sent.headers
|
||||
body = _sent_body(route)
|
||||
assert body["model"] == "anthropic.claude-sonnet-5"
|
||||
assert body["messages"] == [{"role": "user", "content": "ping"}]
|
||||
assert "anthropic_version" not in body
|
||||
assert "stream" not in body
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_region_prefix_selects_the_host_and_is_not_sent_as_model(self):
|
||||
route = _mantle_messages_route("us-east-2").mock(return_value=_anthropic_response())
|
||||
|
||||
await litellm.anthropic_messages(
|
||||
model="bedrock_mantle/us-east-2/anthropic.claude-haiku-4-5",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
max_tokens=8,
|
||||
api_key="test-bearer",
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert _sent_body(route)["model"] == "anthropic.claude-haiku-4-5"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_streaming_sends_stream_and_passes_the_sse_through(self):
|
||||
route = _mantle_messages_route("us-east-1").mock(return_value=_sse_response())
|
||||
|
||||
response = await litellm.anthropic_messages(
|
||||
model="bedrock_mantle/anthropic.claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
max_tokens=8,
|
||||
stream=True,
|
||||
api_key="test-bearer",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
raw = b"".join([chunk async for chunk in response])
|
||||
|
||||
assert route.call_count == 1
|
||||
assert _sent_body(route)["stream"] is True
|
||||
text = raw.decode()
|
||||
assert "event: message_start" in text
|
||||
assert '"text": "pong"' in text
|
||||
assert "event: message_stop" in text
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_sigv4_request_signs_against_the_messages_url(self):
|
||||
route = _mantle_messages_route("us-east-1").mock(return_value=_anthropic_response())
|
||||
|
||||
await litellm.anthropic_messages(
|
||||
model="bedrock_mantle/anthropic.claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
max_tokens=8,
|
||||
aws_access_key_id="AKIAEXAMPLE",
|
||||
aws_secret_access_key="c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
authorization = route.calls.last.request.headers["authorization"]
|
||||
assert authorization.startswith("AWS4-HMAC-SHA256")
|
||||
assert "/us-east-1/bedrock/aws4_request" in authorization
|
||||
|
||||
|
||||
def _sent_betas(route: respx.Route) -> list[str]:
|
||||
return route.calls.last.request.headers["anthropic-beta"].split(",")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_beta_headers_config")
|
||||
class TestBetaHeadersOnTheWire:
|
||||
async def _send(self, **request_params) -> respx.Route:
|
||||
route = _mantle_messages_route("us-east-1").mock(return_value=_anthropic_response())
|
||||
await litellm.anthropic_messages(
|
||||
model="bedrock_mantle/anthropic.claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
max_tokens=8,
|
||||
api_key="test-bearer",
|
||||
aws_region_name="us-east-1",
|
||||
**request_params,
|
||||
)
|
||||
return route
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_betas_mantle_accepts_reach_it_in_the_header(self):
|
||||
route = await self._send(
|
||||
extra_headers={
|
||||
"anthropic-beta": "claude-code-20250219,interleaved-thinking-2025-05-14,context-management-2025-06-27"
|
||||
}
|
||||
)
|
||||
|
||||
assert _sent_betas(route) == [
|
||||
"claude-code-20250219",
|
||||
"context-management-2025-06-27",
|
||||
"interleaved-thinking-2025-05-14",
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_betas_a_proxy_client_sends_reach_mantle_filtered(self):
|
||||
from litellm.proxy.litellm_pre_call_utils import add_provider_specific_headers_to_request
|
||||
|
||||
proxy_request_data: dict = {}
|
||||
add_provider_specific_headers_to_request(
|
||||
data=proxy_request_data,
|
||||
headers={
|
||||
"anthropic-beta": "claude-code-20250219,fast-mode-2026-02-01,interleaved-thinking-2025-05-14",
|
||||
"anthropic-version": "2023-06-01",
|
||||
"user-agent": "claude-cli/2.1.239",
|
||||
},
|
||||
)
|
||||
|
||||
route = await self._send(**proxy_request_data)
|
||||
|
||||
assert _sent_betas(route) == ["claude-code-20250219", "interleaved-thinking-2025-05-14"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_betas_mantle_rejects_are_dropped_before_the_request(self):
|
||||
route = await self._send(
|
||||
extra_headers={"anthropic-beta": "code-execution-2025-08-25,context-1m-2025-08-07,files-api-2025-04-14"}
|
||||
)
|
||||
|
||||
assert _sent_betas(route) == ["context-1m-2025-08-07"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_no_beta_header_is_sent_when_every_value_is_rejected(self):
|
||||
route = await self._send(extra_headers={"anthropic-beta": "code-execution-2025-08-25"})
|
||||
|
||||
assert "anthropic-beta" not in route.calls.last.request.headers
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_advanced_tool_use_is_renamed_to_the_beta_mantle_knows(self):
|
||||
route = await self._send(extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"})
|
||||
|
||||
assert "tool-search-tool-2025-10-19" in _sent_betas(route)
|
||||
assert "advanced-tool-use-2025-11-20" not in _sent_betas(route)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_feature_beta_joins_the_callers_betas_in_the_header(self):
|
||||
route = await self._send(
|
||||
extra_headers={"anthropic-beta": "context-1m-2025-08-07"},
|
||||
context_management={"edits": [{"type": "clear_tool_uses_20250919"}]},
|
||||
)
|
||||
|
||||
assert _sent_betas(route) == ["context-1m-2025-08-07", "context-management-2025-06-27"]
|
||||
assert _sent_body(route)["context_management"] == {"edits": [{"type": "clear_tool_uses_20250919"}]}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_betas_and_version_never_travel_in_the_body(self):
|
||||
route = await self._send(
|
||||
extra_headers={"anthropic-beta": "context-1m-2025-08-07"},
|
||||
context_management={"edits": [{"type": "clear_tool_uses_20250919"}]},
|
||||
anthropic_version="bedrock-2023-05-31",
|
||||
)
|
||||
|
||||
body = _sent_body(route)
|
||||
assert "anthropic_beta" not in body
|
||||
assert "anthropic_version" not in body
|
||||
assert route.calls.last.request.headers["anthropic-version"] == "2023-06-01"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_clear_thinking_edit_is_forwarded_with_thinking_on(self):
|
||||
edits = [{"type": "clear_thinking_20251015", "keep": "all"}, {"type": "clear_tool_uses_20250919"}]
|
||||
route = await self._send(
|
||||
context_management={"edits": edits},
|
||||
thinking={"type": "adaptive"},
|
||||
)
|
||||
|
||||
body = _sent_body(route)
|
||||
assert body["context_management"] == {"edits": edits}
|
||||
assert body["thinking"] == {"type": "adaptive"}
|
||||
assert "context-management-2025-06-27" in _sent_betas(route)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_tools_reach_mantle_unchanged(self):
|
||||
tools = [
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Look up the weather",
|
||||
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
}
|
||||
]
|
||||
route = await self._send(tools=tools, tool_choice={"type": "auto"})
|
||||
|
||||
body = _sent_body(route)
|
||||
assert body["tools"] == tools
|
||||
assert body["tool_choice"] == {"type": "auto"}
|
||||
|
|
@ -7249,7 +7249,7 @@ CROSS_ACCOUNT_AUTHORIZATION = "Bearer deliberately-configured-pass-through-token
|
|||
|
||||
SIGV4_PREFIX = "AWS4-HMAC-SHA256"
|
||||
AUTHORIZATION_HEADER_CASINGS = ["authorization", "Authorization", "AUTHORIZATION"]
|
||||
LEAK_TARGET_PROVIDERS = ["bedrock", "bedrock_converse", "vertex_ai"]
|
||||
LEAK_TARGET_PROVIDERS = ["bedrock", "bedrock_converse", "bedrock_mantle", "vertex_ai"]
|
||||
|
||||
BEDROCK_ENDPOINT = (
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/model/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke"
|
||||
|
|
@ -7342,6 +7342,28 @@ def test_oauth_credential_entry_is_scoped_to_anthropic_alone():
|
|||
assert [entry["custom_llm_provider"] for entry in credential_entries] == ["anthropic"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom_llm_provider", ["anthropic", "bedrock", "bedrock_mantle", "vertex_ai"])
|
||||
def test_client_anthropic_api_headers_reach_every_anthropic_messages_provider(custom_llm_provider):
|
||||
client_headers = {
|
||||
"anthropic-beta": "claude-code-20250219,interleaved-thinking-2025-05-14",
|
||||
"anthropic-version": "2023-06-01",
|
||||
"user-agent": "claude-cli/2.1.239",
|
||||
}
|
||||
|
||||
forwarded = _headers_forwarded_to(client_headers, custom_llm_provider)
|
||||
|
||||
assert forwarded == {
|
||||
"anthropic-beta": "claude-code-20250219,interleaved-thinking-2025-05-14",
|
||||
"anthropic-version": "2023-06-01",
|
||||
}
|
||||
|
||||
|
||||
def test_client_anthropic_api_headers_stay_off_openai_compatible_providers():
|
||||
forwarded = _headers_forwarded_to({"anthropic-beta": "claude-code-20250219"}, "openai")
|
||||
|
||||
assert forwarded == {}
|
||||
|
||||
|
||||
def test_no_provider_specific_header_when_client_sends_nothing_anthropic():
|
||||
data: dict = {}
|
||||
add_provider_specific_headers_to_request(
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import pytest
|
|||
import litellm
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
filter_and_transform_beta_headers,
|
||||
update_headers_with_filtered_beta,
|
||||
update_request_with_filtered_beta,
|
||||
)
|
||||
|
||||
|
|
@ -511,3 +512,20 @@ class TestAnthropicBetaHeadersFiltering:
|
|||
assert (
|
||||
"unknown-header-123" not in filtered
|
||||
), f"Unknown header should not be in result for {provider}"
|
||||
|
||||
@pytest.mark.parametrize("provider", ["anthropic", "bedrock", "bedrock_mantle", "vertex_ai"])
|
||||
def test_blank_anthropic_beta_header_is_removed(self, provider):
|
||||
headers = {"anthropic-beta": "", "anthropic-version": "2023-06-01"}
|
||||
|
||||
assert update_headers_with_filtered_beta(headers, provider) == {"anthropic-version": "2023-06-01"}
|
||||
|
||||
@pytest.mark.parametrize("provider", ["anthropic", "bedrock", "bedrock_mantle", "vertex_ai"])
|
||||
def test_whitespace_only_anthropic_beta_header_is_removed(self, provider):
|
||||
headers = {"anthropic-beta": " , ", "anthropic-version": "2023-06-01"}
|
||||
|
||||
assert update_headers_with_filtered_beta(headers, provider) == {"anthropic-version": "2023-06-01"}
|
||||
|
||||
def test_absent_anthropic_beta_header_is_left_alone(self):
|
||||
headers = {"anthropic-version": "2023-06-01"}
|
||||
|
||||
assert update_headers_with_filtered_beta(headers, "bedrock_mantle") == {"anthropic-version": "2023-06-01"}
|
||||
|
|
|
|||
|
|
@ -3522,6 +3522,58 @@ def test_cost_per_token_region_name_applies_to_provider_prefixed_model(_local_mo
|
|||
)
|
||||
|
||||
|
||||
def test_completion_cost_mantle_native_messages_prices_claude_from_the_bedrock_row(_local_model_cost_map):
|
||||
"""Mantle's native Messages API answers with Anthropic's canonical model name and the proxy
|
||||
resolves a Mantle region for every call, so the first cost candidate is
|
||||
bedrock_mantle/<region>/claude-sonnet-5. That name has no row of its own and must fall through to
|
||||
the deployment's bare Bedrock row instead of stopping on an unpriced capability rule at $0."""
|
||||
|
||||
response = litellm.ModelResponse(
|
||||
id="msg_x",
|
||||
choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
model="claude-sonnet-5",
|
||||
usage={"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110},
|
||||
)
|
||||
row = litellm.model_cost["anthropic.claude-sonnet-5"]
|
||||
expected = 100 * row["input_cost_per_token"] + 10 * row["output_cost_per_token"]
|
||||
assert expected > 0
|
||||
|
||||
for region_name in ("us-east-1", None):
|
||||
assert litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="bedrock_mantle/anthropic.claude-sonnet-5",
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
region_name=region_name,
|
||||
) == pytest.approx(expected)
|
||||
|
||||
|
||||
def test_completion_cost_mantle_native_messages_prices_haiku_from_the_mantle_row(_local_model_cost_map):
|
||||
"""Mantle serves Anthropic's un-versioned haiku id, which has no bare Bedrock row (Bedrock's carries
|
||||
the -20251001-v1:0 suffix), and Claude Code sends every small-fast-model call to it. Both the plain
|
||||
and the region-prefixed deployment names must price from bedrock_mantle/anthropic.claude-haiku-4-5
|
||||
instead of billing $0."""
|
||||
|
||||
response = litellm.ModelResponse(
|
||||
id="msg_x",
|
||||
choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
model="claude-haiku-4-5",
|
||||
usage={"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110},
|
||||
)
|
||||
row = litellm.model_cost["bedrock_mantle/anthropic.claude-haiku-4-5"]
|
||||
expected = 100 * row["input_cost_per_token"] + 10 * row["output_cost_per_token"]
|
||||
assert expected > 0
|
||||
|
||||
for model in (
|
||||
"bedrock_mantle/anthropic.claude-haiku-4-5",
|
||||
"bedrock_mantle/us-east-2/anthropic.claude-haiku-4-5",
|
||||
):
|
||||
assert litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
) == pytest.approx(expected), model
|
||||
|
||||
|
||||
def test_select_model_name_keeps_base_model_free_of_region(_local_model_cost_map):
|
||||
"""An explicit base_model keeps pricing on that model's own key even when the request carries a
|
||||
region with different regional rates, so the private provider model never widens region pricing."""
|
||||
|
|
|
|||
|
|
@ -1163,6 +1163,21 @@ def test_get_model_info_bedrock_regional_inference_profile_pricing(local_model_c
|
|||
assert control["key"] == "au.anthropic.claude-opus-4-8"
|
||||
|
||||
|
||||
def test_get_model_info_bedrock_mantle_region_prefix_falls_back_to_the_mantle_row(local_model_cost_map):
|
||||
"""A Mantle deployment name may carry the region as a prefix (bedrock_mantle/us-east-2/<model>).
|
||||
That name has no cost row of its own, so pricing must fall through to the region-free
|
||||
bedrock_mantle/<model> row instead of raising, while a region that has its own row keeps it."""
|
||||
for model, expected_key in (
|
||||
("bedrock_mantle/us-east-2/anthropic.claude-haiku-4-5", "bedrock_mantle/anthropic.claude-haiku-4-5"),
|
||||
("bedrock_mantle/us-east-2/openai.gpt-5.6-sol", "bedrock_mantle/openai.gpt-5.6-sol"),
|
||||
("bedrock_mantle/us-gov-west-1/openai.gpt-5.4", "bedrock_mantle/us-gov-west-1/openai.gpt-5.4"),
|
||||
):
|
||||
info = litellm.get_model_info(model=model, custom_llm_provider="bedrock_mantle")
|
||||
assert info["key"] == expected_key, model
|
||||
assert info["input_cost_per_token"] == litellm.model_cost[expected_key]["input_cost_per_token"], model
|
||||
assert info["input_cost_per_token"] > 0, model
|
||||
|
||||
|
||||
def test_openai_models_in_model_info(monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
|
@ -3646,6 +3661,28 @@ class TestGetOptionalParamsTencent:
|
|||
assert isinstance(config, TencentAnthropicMessagesConfig)
|
||||
assert config.custom_llm_provider == "tencent"
|
||||
|
||||
def test_bedrock_mantle_claude_messages_config_routing(self):
|
||||
import litellm
|
||||
from litellm.llms.bedrock_mantle.messages.transformation import (
|
||||
BedrockMantleAnthropicMessagesConfig,
|
||||
)
|
||||
|
||||
config = ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
model="anthropic.claude-sonnet-5",
|
||||
provider=litellm.LlmProviders.BEDROCK_MANTLE,
|
||||
)
|
||||
assert isinstance(config, BedrockMantleAnthropicMessagesConfig)
|
||||
assert config.custom_llm_provider == "bedrock_mantle"
|
||||
|
||||
def test_bedrock_mantle_openai_models_keep_the_messages_bridge(self):
|
||||
import litellm
|
||||
|
||||
config = ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
model="openai.gpt-5.6-sol",
|
||||
provider=litellm.LlmProviders.BEDROCK_MANTLE,
|
||||
)
|
||||
assert config is None
|
||||
|
||||
|
||||
class TestValidateEnvironmentTencent:
|
||||
"""Tests that validate_environment resolves TENCENT_API_KEY for the tencent provider."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue