From 368a8396400bdf5f986f8379d84ac44c43b808c2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 19 Sep 2026 18:08:41 -0700 Subject: [PATCH] fix(bedrock_mantle): send anthropic betas in the header Mantle reads on /v1/messages --- litellm/anthropic_beta_headers_config.json | 35 +++++ .../anthropic_claude3_transformation.py | 22 ++-- .../bedrock_mantle/messages/transformation.py | 36 +++-- ..._bedrock_mantle_messages_transformation.py | 123 +++++++++++++++++- 4 files changed, 196 insertions(+), 20 deletions(-) diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index eb31cc17a15..1331de4c266 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -131,6 +131,41 @@ "web-fetch-2025-09-10": null, "web-search-2025-03-05": null }, + "bedrock_mantle": { + "advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19", + "advisor-tool-2026-03-01": null, + "bash_20241022": null, + "bash_20250124": null, + "claude-code-20250219": "claude-code-20250219", + "code-execution-2025-08-25": null, + "compact-2026-01-12": "compact-2026-01-12", + "computer-use-2025-01-24": "computer-use-2025-01-24", + "computer-use-2025-11-24": "computer-use-2025-11-24", + "context-1m-2025-08-07": "context-1m-2025-08-07", + "context-management-2025-06-27": "context-management-2025-06-27", + "effort-2025-11-24": "effort-2025-11-24", + "fast-mode-2026-02-01": null, + "files-api-2025-04-14": null, + "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14", + "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", + "mcp-client-2025-04-04": null, + "mcp-client-2025-11-20": null, + "mcp-servers-2025-12-04": null, + "output-128k-2025-02-19": "output-128k-2025-02-19", + "per-turn-control-2026-07-01": "per-turn-control-2026-07-01", + "prompt-caching-scope-2026-01-05": null, + "skills-2025-10-02": null, + "structured-output-2024-03-01": null, + "structured-outputs-2025-11-13": "structured-outputs-2025-11-13", + "text_editor_20241022": null, + "text_editor_20250124": null, + "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19", + "tool-examples-2025-10-29": "tool-examples-2025-10-29", + "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19", + "web-fetch-2025-09-10": null, + "web-search-2025-03-05": "web-search-2025-03-05" + }, "vertex_ai": { "advisor-tool-2026-03-01": null, "advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19", diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index d2be1ad9156..4b52a3bafe6 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast @@ -445,13 +445,16 @@ class AmazonAnthropicClaudeMessagesConfig( # Bedrock InvokeModel DOES support ``clear_tool_uses_20250919`` under the # ``context-management-2025-06-27`` beta. AWS docs: # https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-tool-use.md - _BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: dict[str, str] = { - "compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, - "clear_tool_uses_20250919": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, - } + _BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: Mapping[str, str] = MappingProxyType( + { + "compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + "clear_tool_uses_20250919": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + } + ) - @staticmethod + @classmethod def _filter_context_management_for_bedrock_invoke( + cls, anthropic_messages_request: dict, beta_set: set, ) -> None: @@ -481,7 +484,7 @@ class AmazonAnthropicClaudeMessagesConfig( anthropic_messages_request.pop("context_management", None) return - supported: Final = AmazonAnthropicClaudeMessagesConfig._BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS + supported: Final = cls._BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS retained_edits: Final = [e for e in edits if isinstance(e, dict) and e.get("type") in supported] if not retained_edits: anthropic_messages_request.pop("context_management", None) @@ -546,15 +549,16 @@ class AmazonAnthropicClaudeMessagesConfig( if "tool-search-tool-2025-10-19" in beta_set: beta_set.add("tool-examples-2025-10-29") + beta_provider: Final = self.custom_llm_provider or "bedrock" filtered_betas: Final = sorted( filter_and_transform_beta_headers( beta_headers=list(beta_set), - provider="bedrock", + provider=beta_provider, ) ) dropped_user_betas: Final = sorted( - b for b in user_beta_set if not filter_and_transform_beta_headers([b], provider="bedrock") + b for b in user_beta_set if not filter_and_transform_beta_headers([b], provider=beta_provider) ) if dropped_user_betas: verbose_logger.warning( diff --git a/litellm/llms/bedrock_mantle/messages/transformation.py b/litellm/llms/bedrock_mantle/messages/transformation.py index a4365cfa49b..480c09a0476 100644 --- a/litellm/llms/bedrock_mantle/messages/transformation.py +++ b/litellm/llms/bedrock_mantle/messages/transformation.py @@ -1,6 +1,9 @@ from collections.abc import Mapping +from types import MappingProxyType from typing import Final +from pydantic import TypeAdapter + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( DEFAULT_ANTHROPIC_API_VERSION, ) @@ -13,6 +16,7 @@ from litellm.llms.bedrock_mantle.common_utils import ( resolve_mantle_region, ) from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES from litellm.types.router import GenericLiteLLMParams _BASE_SUFFIXES_TO_STRIP: Final = ( @@ -23,6 +27,9 @@ _BASE_SUFFIXES_TO_STRIP: Final = ( "/openai/v1", "/v1", ) +_BODY_FIELDS_MANTLE_READS_FROM_HEADERS: Final = frozenset({"anthropic_version", "anthropic_beta"}) +_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...]) +_MANTLE_REQUEST: Final = TypeAdapter(dict[str, object]) def build_mantle_native_messages_url(api_base: str | None, litellm_params: Mapping[str, object]) -> str: @@ -39,6 +46,13 @@ def build_mantle_native_messages_url(api_base: str | None, litellm_params: Mappi class BedrockMantleAnthropicMessagesConfig(BedrockMantleAuthMixin, AmazonMantleMessagesConfig): + _BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: Mapping[str, str] = MappingProxyType( + { + **AmazonMantleMessagesConfig._BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS, + "clear_thinking_20251015": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + } + ) + def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None: AmazonMantleMessagesConfig.__init__(self) self._aws_signer = aws_signer or self @@ -89,13 +103,17 @@ class BedrockMantleAnthropicMessagesConfig(BedrockMantleAuthMixin, AmazonMantleM litellm_params: GenericLiteLLMParams, headers: dict, ) -> dict: - request: Final = super().transform_anthropic_messages_request( - model=model, - messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, - litellm_params=litellm_params, - headers=headers, + request: Final = _MANTLE_REQUEST.validate_python( + super().transform_anthropic_messages_request( + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ), ) - if "anthropic_version" in anthropic_messages_optional_request_params: - return request - return {key: value for key, value in request.items() if key != "anthropic_version"} + betas: Final = request.get("anthropic_beta") + if betas is not None: + header_betas: Final = ",".join(_ANTHROPIC_BETAS.validate_python(betas)) + headers["anthropic-beta"] = header_betas # rebind-ok: the handler signs and sends this same dict + return {key: value for key, value in request.items() if key not in _BODY_FIELDS_MANTLE_READS_FROM_HEADERS} diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_messages_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_messages_transformation.py index 2961eee925c..3544262996c 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_messages_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_messages_transformation.py @@ -79,7 +79,10 @@ _SSE_EVENTS = ( }, ), ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), - ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}}), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}}, + ), ("content_block_stop", {"type": "content_block_stop", "index": 0}), ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}}), ("message_stop", {"type": "message_stop"}), @@ -152,7 +155,10 @@ class TestURL: def test_default_host_comes_from_mantle_region_env(self, monkeypatch): monkeypatch.setenv("BEDROCK_MANTLE_REGION", "ap-northeast-1") - assert build_mantle_native_messages_url(None, {}) == f"https://bedrock-mantle.ap-northeast-1.api.aws{MESSAGES_PATH}" + assert ( + build_mantle_native_messages_url(None, {}) + == f"https://bedrock-mantle.ap-northeast-1.api.aws{MESSAGES_PATH}" + ) def test_config_get_complete_url_reads_litellm_params(self): config = BedrockMantleAnthropicMessagesConfig() @@ -344,3 +350,116 @@ class TestWireRequest: authorization = route.calls.last.request.headers["authorization"] assert authorization.startswith("AWS4-HMAC-SHA256") assert "/us-east-1/bedrock/aws4_request" in authorization + + +def _sent_betas(route: respx.Route) -> list[str]: + return route.calls.last.request.headers["anthropic-beta"].split(",") + + +@pytest.mark.usefixtures("local_beta_headers_config") +class TestBetaHeadersOnTheWire: + async def _send(self, **request_params) -> respx.Route: + route = _mantle_messages_route("us-east-1").mock(return_value=_anthropic_response()) + await litellm.anthropic_messages( + model="bedrock_mantle/anthropic.claude-sonnet-5", + messages=[{"role": "user", "content": "ping"}], + max_tokens=8, + api_key="test-bearer", + aws_region_name="us-east-1", + **request_params, + ) + return route + + @pytest.mark.asyncio + @respx.mock + async def test_betas_mantle_accepts_reach_it_in_the_header(self): + route = await self._send( + extra_headers={ + "anthropic-beta": "claude-code-20250219,interleaved-thinking-2025-05-14,context-management-2025-06-27" + } + ) + + assert _sent_betas(route) == [ + "claude-code-20250219", + "context-management-2025-06-27", + "interleaved-thinking-2025-05-14", + ] + + @pytest.mark.asyncio + @respx.mock + async def test_betas_mantle_rejects_are_dropped_before_the_request(self): + route = await self._send( + extra_headers={"anthropic-beta": "code-execution-2025-08-25,context-1m-2025-08-07,files-api-2025-04-14"} + ) + + assert _sent_betas(route) == ["context-1m-2025-08-07"] + + @pytest.mark.asyncio + @respx.mock + async def test_no_beta_header_is_sent_when_every_value_is_rejected(self): + route = await self._send(extra_headers={"anthropic-beta": "code-execution-2025-08-25"}) + + assert "anthropic-beta" not in route.calls.last.request.headers + + @pytest.mark.asyncio + @respx.mock + async def test_advanced_tool_use_is_renamed_to_the_beta_mantle_knows(self): + route = await self._send(extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}) + + assert "tool-search-tool-2025-10-19" in _sent_betas(route) + assert "advanced-tool-use-2025-11-20" not in _sent_betas(route) + + @pytest.mark.asyncio + @respx.mock + async def test_a_feature_beta_joins_the_callers_betas_in_the_header(self): + route = await self._send( + extra_headers={"anthropic-beta": "context-1m-2025-08-07"}, + context_management={"edits": [{"type": "clear_tool_uses_20250919"}]}, + ) + + assert _sent_betas(route) == ["context-1m-2025-08-07", "context-management-2025-06-27"] + assert _sent_body(route)["context_management"] == {"edits": [{"type": "clear_tool_uses_20250919"}]} + + @pytest.mark.asyncio + @respx.mock + async def test_betas_and_version_never_travel_in_the_body(self): + route = await self._send( + extra_headers={"anthropic-beta": "context-1m-2025-08-07"}, + context_management={"edits": [{"type": "clear_tool_uses_20250919"}]}, + anthropic_version="bedrock-2023-05-31", + ) + + body = _sent_body(route) + assert "anthropic_beta" not in body + assert "anthropic_version" not in body + assert route.calls.last.request.headers["anthropic-version"] == "2023-06-01" + + @pytest.mark.asyncio + @respx.mock + async def test_clear_thinking_edit_is_forwarded_with_thinking_on(self): + edits = [{"type": "clear_thinking_20251015", "keep": "all"}, {"type": "clear_tool_uses_20250919"}] + route = await self._send( + context_management={"edits": edits}, + thinking={"type": "adaptive"}, + ) + + body = _sent_body(route) + assert body["context_management"] == {"edits": edits} + assert body["thinking"] == {"type": "adaptive"} + assert "context-management-2025-06-27" in _sent_betas(route) + + @pytest.mark.asyncio + @respx.mock + async def test_tools_reach_mantle_unchanged(self): + tools = [ + { + "name": "get_weather", + "description": "Look up the weather", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + ] + route = await self._send(tools=tools, tool_choice={"type": "auto"}) + + body = _sent_body(route) + assert body["tools"] == tools + assert body["tool_choice"] == {"type": "auto"}