scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

fix(bedrock_mantle): send anthropic betas in the header Mantle reads on /v1/messages

Browse source
This commit is contained in:
mateo-berri 2026-09-19 18:08:41 -07:00
parent f2b6c0da81
commit 368a839640
4 changed files with 196 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
litellm/anthropic_beta_headers_config.json
View file

@ -131,6 +131,41 @@
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": null
},
"bedrock_mantle": {
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
"advisor-tool-2026-03-01": null,
"bash_20241022": null,
"bash_20250124": null,
"claude-code-20250219": "claude-code-20250219",
"code-execution-2025-08-25": null,
"compact-2026-01-12": "compact-2026-01-12",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-04-04": null,
"mcp-client-2025-11-20": null,
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": "output-128k-2025-02-19",
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-output-2024-03-01": null,
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
"tool-examples-2025-10-29": "tool-examples-2025-10-29",
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": "web-search-2025-03-05"
},
"vertex_ai": {
"advisor-tool-2026-03-01": null,
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",

22
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -1,4 +1,4 @@
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
@ -445,13 +445,16 @@ class AmazonAnthropicClaudeMessagesConfig(
# Bedrock InvokeModel DOES support ``clear_tool_uses_20250919`` under the
# ``context-management-2025-06-27`` beta. AWS docs:
# https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-tool-use.md
_BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: dict[str, str] = {
"compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
"clear_tool_uses_20250919": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
}
_BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: Mapping[str, str] = MappingProxyType(
{
"compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
"clear_tool_uses_20250919": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
}
)
@staticmethod
@classmethod
def _filter_context_management_for_bedrock_invoke(
cls,
anthropic_messages_request: dict,
beta_set: set,
) -> None:
@ -481,7 +484,7 @@ class AmazonAnthropicClaudeMessagesConfig(
anthropic_messages_request.pop("context_management", None)
return
supported: Final = AmazonAnthropicClaudeMessagesConfig._BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS
supported: Final = cls._BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS
retained_edits: Final = [e for e in edits if isinstance(e, dict) and e.get("type") in supported]
if not retained_edits:
anthropic_messages_request.pop("context_management", None)
@ -546,15 +549,16 @@ class AmazonAnthropicClaudeMessagesConfig(
if "tool-search-tool-2025-10-19" in beta_set:
beta_set.add("tool-examples-2025-10-29")
beta_provider: Final = self.custom_llm_provider or "bedrock"
filtered_betas: Final = sorted(
filter_and_transform_beta_headers(
beta_headers=list(beta_set),
provider="bedrock",
provider=beta_provider,
)
)
dropped_user_betas: Final = sorted(
b for b in user_beta_set if not filter_and_transform_beta_headers([b], provider="bedrock")
b for b in user_beta_set if not filter_and_transform_beta_headers([b], provider=beta_provider)
)
if dropped_user_betas:
verbose_logger.warning(

36
litellm/llms/bedrock_mantle/messages/transformation.py
View file

@ -1,6 +1,9 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from pydantic import TypeAdapter
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
DEFAULT_ANTHROPIC_API_VERSION,
)
@ -13,6 +16,7 @@ from litellm.llms.bedrock_mantle.common_utils import (
resolve_mantle_region,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.router import GenericLiteLLMParams
_BASE_SUFFIXES_TO_STRIP: Final = (
@ -23,6 +27,9 @@ _BASE_SUFFIXES_TO_STRIP: Final = (
"/openai/v1",
"/v1",
)
_BODY_FIELDS_MANTLE_READS_FROM_HEADERS: Final = frozenset({"anthropic_version", "anthropic_beta"})
_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...])
_MANTLE_REQUEST: Final = TypeAdapter(dict[str, object])
def build_mantle_native_messages_url(api_base: str | None, litellm_params: Mapping[str, object]) -> str:
@ -39,6 +46,13 @@ def build_mantle_native_messages_url(api_base: str | None, litellm_params: Mappi
class BedrockMantleAnthropicMessagesConfig(BedrockMantleAuthMixin, AmazonMantleMessagesConfig):
_BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: Mapping[str, str] = MappingProxyType(
{
**AmazonMantleMessagesConfig._BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS,
"clear_thinking_20251015": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
}
)
def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None:
AmazonMantleMessagesConfig.__init__(self)
self._aws_signer = aws_signer or self
@ -89,13 +103,17 @@ class BedrockMantleAnthropicMessagesConfig(BedrockMantleAuthMixin, AmazonMantleM
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> dict:
request: Final = super().transform_anthropic_messages_request(
model=model,
messages=messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
litellm_params=litellm_params,
headers=headers,
request: Final = _MANTLE_REQUEST.validate_python(
super().transform_anthropic_messages_request(
model=model,
messages=messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
litellm_params=litellm_params,
headers=headers,
),
)
if "anthropic_version" in anthropic_messages_optional_request_params:
return request
return {key: value for key, value in request.items() if key != "anthropic_version"}
betas: Final = request.get("anthropic_beta")
if betas is not None:
header_betas: Final = ",".join(_ANTHROPIC_BETAS.validate_python(betas))
headers["anthropic-beta"] = header_betas # rebind-ok: the handler signs and sends this same dict
return {key: value for key, value in request.items() if key not in _BODY_FIELDS_MANTLE_READS_FROM_HEADERS}

123
tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_messages_transformation.py
View file

@ -79,7 +79,10 @@ _SSE_EVENTS = (
},
),
("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),
("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}}),
(
"content_block_delta",
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}},
),
("content_block_stop", {"type": "content_block_stop", "index": 0}),
("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}}),
("message_stop", {"type": "message_stop"}),
@ -152,7 +155,10 @@ class TestURL:
def test_default_host_comes_from_mantle_region_env(self, monkeypatch):
monkeypatch.setenv("BEDROCK_MANTLE_REGION", "ap-northeast-1")
assert build_mantle_native_messages_url(None, {}) == f"https://bedrock-mantle.ap-northeast-1.api.aws{MESSAGES_PATH}"
assert (
build_mantle_native_messages_url(None, {})
== f"https://bedrock-mantle.ap-northeast-1.api.aws{MESSAGES_PATH}"
)
def test_config_get_complete_url_reads_litellm_params(self):
config = BedrockMantleAnthropicMessagesConfig()
@ -344,3 +350,116 @@ class TestWireRequest:
authorization = route.calls.last.request.headers["authorization"]
assert authorization.startswith("AWS4-HMAC-SHA256")
assert "/us-east-1/bedrock/aws4_request" in authorization
def _sent_betas(route: respx.Route) -> list[str]:
return route.calls.last.request.headers["anthropic-beta"].split(",")
@pytest.mark.usefixtures("local_beta_headers_config")
class TestBetaHeadersOnTheWire:
async def _send(self, **request_params) -> respx.Route:
route = _mantle_messages_route("us-east-1").mock(return_value=_anthropic_response())
await litellm.anthropic_messages(
model="bedrock_mantle/anthropic.claude-sonnet-5",
messages=[{"role": "user", "content": "ping"}],
max_tokens=8,
api_key="test-bearer",
aws_region_name="us-east-1",
**request_params,
)
return route
@pytest.mark.asyncio
@respx.mock
async def test_betas_mantle_accepts_reach_it_in_the_header(self):
route = await self._send(
extra_headers={
"anthropic-beta": "claude-code-20250219,interleaved-thinking-2025-05-14,context-management-2025-06-27"
}
)
assert _sent_betas(route) == [
"claude-code-20250219",
"context-management-2025-06-27",
"interleaved-thinking-2025-05-14",
]
@pytest.mark.asyncio
@respx.mock
async def test_betas_mantle_rejects_are_dropped_before_the_request(self):
route = await self._send(
extra_headers={"anthropic-beta": "code-execution-2025-08-25,context-1m-2025-08-07,files-api-2025-04-14"}
)
assert _sent_betas(route) == ["context-1m-2025-08-07"]
@pytest.mark.asyncio
@respx.mock
async def test_no_beta_header_is_sent_when_every_value_is_rejected(self):
route = await self._send(extra_headers={"anthropic-beta": "code-execution-2025-08-25"})
assert "anthropic-beta" not in route.calls.last.request.headers
@pytest.mark.asyncio
@respx.mock
async def test_advanced_tool_use_is_renamed_to_the_beta_mantle_knows(self):
route = await self._send(extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"})
assert "tool-search-tool-2025-10-19" in _sent_betas(route)
assert "advanced-tool-use-2025-11-20" not in _sent_betas(route)
@pytest.mark.asyncio
@respx.mock
async def test_a_feature_beta_joins_the_callers_betas_in_the_header(self):
route = await self._send(
extra_headers={"anthropic-beta": "context-1m-2025-08-07"},
context_management={"edits": [{"type": "clear_tool_uses_20250919"}]},
)
assert _sent_betas(route) == ["context-1m-2025-08-07", "context-management-2025-06-27"]
assert _sent_body(route)["context_management"] == {"edits": [{"type": "clear_tool_uses_20250919"}]}
@pytest.mark.asyncio
@respx.mock
async def test_betas_and_version_never_travel_in_the_body(self):
route = await self._send(
extra_headers={"anthropic-beta": "context-1m-2025-08-07"},
context_management={"edits": [{"type": "clear_tool_uses_20250919"}]},
anthropic_version="bedrock-2023-05-31",
)
body = _sent_body(route)
assert "anthropic_beta" not in body
assert "anthropic_version" not in body
assert route.calls.last.request.headers["anthropic-version"] == "2023-06-01"
@pytest.mark.asyncio
@respx.mock
async def test_clear_thinking_edit_is_forwarded_with_thinking_on(self):
edits = [{"type": "clear_thinking_20251015", "keep": "all"}, {"type": "clear_tool_uses_20250919"}]
route = await self._send(
context_management={"edits": edits},
thinking={"type": "adaptive"},
)
body = _sent_body(route)
assert body["context_management"] == {"edits": edits}
assert body["thinking"] == {"type": "adaptive"}
assert "context-management-2025-06-27" in _sent_betas(route)
@pytest.mark.asyncio
@respx.mock
async def test_tools_reach_mantle_unchanged(self):
tools = [
{
"name": "get_weather",
"description": "Look up the weather",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
}
]
route = await self._send(tools=tools, tool_choice={"type": "auto"})
body = _sent_body(route)
assert body["tools"] == tools
assert body["tool_choice"] == {"type": "auto"}