From a4950a1fcf891b92d8c6e75c2bda4b4477e37966 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 27 Oct 2026 20:55:26 +0000 Subject: [PATCH] fix(bedrock): strip beta-gated fields against the final body anthropic_beta list Converse counts the body anthropic_beta list as active, Invoke strips after its auto-injected betas are known and injects thinking-binding-controls for thinking.block_binding, test helpers are fully typed Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/anthropic_beta_gated_fields.py | 20 +++++++++++--- .../messages/native/transformation.py | 17 ++++++++++-- .../bedrock/chat/converse_transformation.py | 8 ++++-- .../anthropic_claude_transformation.py | 21 +++++++++++++++ .../chat/test_anthropic_chat_handler.py | 22 +++++++++++----- .../test_anthropic_messages_transformation.py | 6 ++--- .../chat/test_converse_transformation.py | 26 +++++++++++++++---- .../test_anthropic_claude_transformation.py | 25 ++++++++++++++++++ 8 files changed, 123 insertions(+), 22 deletions(-) diff --git a/litellm/anthropic_beta_gated_fields.py b/litellm/anthropic_beta_gated_fields.py index d16e15860f7..ed5647e58b7 100644 --- a/litellm/anthropic_beta_gated_fields.py +++ b/litellm/anthropic_beta_gated_fields.py @@ -239,11 +239,25 @@ class StrippedNativeRequest: stripped: tuple[str, ...] +def unstripped_native_request(optional_params: object, messages: object) -> StrippedNativeRequest: + """The identity result for legs that strip later, once their final ``anthropic_beta`` list exists.""" + params: Final = _as_mapping(optional_params) or MappingProxyType({}) + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + return StrippedNativeRequest( + thinking=params.get("thinking"), + thinking_stripped=False, + messages=message_sequence if message_sequence is not None else (), + messages_stripped=False, + stripped=(), + ) + + def strip_beta_gated_native_request( - headers: object, optional_params: object, messages: object + headers: object, optional_params: object, messages: object, body_betas: object = () ) -> StrippedNativeRequest: - """The ``/v1/messages`` seam, fed the final (already filtered) headers plus the body parts it may rewrite.""" - active_betas: Final = active_beta_values(headers) + """The ``/v1/messages`` seam, fed the final (already filtered) headers, the final body ``anthropic_beta`` + list on legs that carry betas in the body, plus the body parts it may rewrite.""" + active_betas: Final = active_beta_values(headers, body_betas) params: Final = _as_mapping(optional_params) or MappingProxyType({}) thinking: Final = strip_beta_gated_thinking(params.get("thinking"), active_betas) message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index 43bb4abb02b..39fec46d5ad 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -6,7 +6,11 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger -from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_native_request +from litellm.anthropic_beta_gated_fields import ( + log_beta_gated_strips, + strip_beta_gated_native_request, + unstripped_native_request, +) from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params @@ -610,6 +614,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if thinking_enabled or effort_enabled: optional_params.pop("temperature", None) + def strips_beta_gated_fields_from_final_headers(self) -> bool: + """False on legs whose final ``anthropic_beta`` list is only known after this transform (Bedrock Invoke), + which strip beta-gated fields themselves once that list exists.""" + return True + def transform_anthropic_messages_request( self, model: str, @@ -710,7 +719,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if not _has_advisor: messages = strip_advisor_blocks_from_messages(messages) - gated: Final = strip_beta_gated_native_request(headers, anthropic_messages_optional_request_params, messages) # pyright: ignore[reportUnknownArgumentType] # the base signature types these as bare dicts + gated: Final = ( + strip_beta_gated_native_request(headers, anthropic_messages_optional_request_params, messages) # pyright: ignore[reportUnknownArgumentType] # the base signature types these as bare dicts + if self.strips_beta_gated_fields_from_final_headers() + else unstripped_native_request(anthropic_messages_optional_request_params, messages) # pyright: ignore[reportUnknownArgumentType] # the base signature types these as bare dicts + ) log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( messages=strip_encrypted_reasoning_blocks_from_anthropic_messages( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 50a3aa7e9b1..04059570347 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -17,7 +17,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger -from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_thinking +from litellm.anthropic_beta_gated_fields import active_beta_values, log_beta_gated_strips, strip_beta_gated_thinking from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, @@ -2033,7 +2033,11 @@ class AmazonConverseConfig(BaseConfig): original_tools, model, headers, additional_request_params, litellm_params, deployment_model ) requested_thinking: Final = additional_request_params.get("thinking") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # _prepare_request_params hands back a bare dict - gated_thinking: Final = strip_beta_gated_thinking(requested_thinking, frozenset(anthropic_beta_list)) # pyright: ignore[reportUnknownArgumentType] # _process_tools_and_beta hands back a bare list + body_betas: Final = additional_request_params.get("anthropic_beta") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # same bare dict + gated_thinking: Final = strip_beta_gated_thinking( + requested_thinking, # pyright: ignore[reportUnknownArgumentType] # same bare dict + active_beta_values({}, body_betas) | frozenset(anthropic_beta_list), # pyright: ignore[reportUnknownArgumentType] # _process_tools_and_beta hands back a bare list + ) if gated_thinking.stripped: log_beta_gated_strips(gated_thinking.stripped, model=model, provider="bedrock_converse") additional_request_params["thinking"] = gated_thinking.thinking diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index e1e15861a90..b6d5323f0f6 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -7,6 +7,11 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger +from litellm.anthropic_beta_gated_fields import ( + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER, + log_beta_gated_strips, + strip_beta_gated_native_request, +) from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, @@ -546,6 +551,9 @@ class AmazonAnthropicClaudeMessagesConfig( "edits": retained_edits, } + def strips_beta_gated_fields_from_final_headers(self) -> bool: + return False + def _get_bedrock_invoke_anthropic_beta_headers( self, model: str, @@ -590,6 +598,9 @@ class AmazonAnthropicClaudeMessagesConfig( ) beta_set.update(auto_betas) + if "block_binding" in (anthropic_messages_request.get("thinking") or {}): # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the base signature types the request as a bare dict + beta_set.add(ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER) + if injected_thinking_for_clear_thinking: beta_set.add("interleaved-thinking-2025-05-14") @@ -822,6 +833,16 @@ class AmazonAnthropicClaudeMessagesConfig( if filtered_betas: anthropic_messages_request["anthropic_beta"] = filtered_betas + final_request: Final = OBJECT_MAPPING_ADAPTER.validate_python(anthropic_messages_request) # pyright: ignore[reportUnknownArgumentType] # built from the base signature's bare dict + gated: Final = strip_beta_gated_native_request( + {}, final_request, final_request.get("messages"), body_betas=filtered_betas + ) + log_beta_gated_strips(gated.stripped, model=model, provider="bedrock_invoke") + if gated.thinking_stripped: + anthropic_messages_request["thinking"] = gated.thinking + if gated.messages_stripped: + anthropic_messages_request["messages"] = list(gated.messages) + outbound_tools: Final = tools_without_eager_input_streaming(anthropic_messages_request) if outbound_tools is not None: anthropic_messages_request["tools"] = outbound_tools diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 333351bfffd..ae73fbe7d2f 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -494,7 +494,11 @@ def test_streamed_thinking_blocks_accumulate_in_the_openai_sdk_and_replay_withou {"type": "content_block_delta", "index": 2, "delta": {"type": "thinking_delta", "thinking": "Step 3."}}, {"type": "content_block_delta", "index": 2, "delta": {"type": "signature_delta", "signature": "sig-b"}}, {"type": "content_block_stop", "index": 2}, - {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 9}}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 9}, + }, {"type": "message_stop"}, ) @@ -515,7 +519,11 @@ def test_streamed_thinking_blocks_accumulate_in_the_openai_sdk_and_replay_withou "thinking_blocks": message["thinking_blocks"], } replayed: Final = anthropic_messages_pt( - messages=[{"role": "user", "content": "Capital of France?"}, next_turn, {"role": "user", "content": "And Spain?"}], + messages=[ + {"role": "user", "content": "Capital of France?"}, + next_turn, + {"role": "user", "content": "And Spain?"}, + ], model="claude-haiku-4-5", llm_provider="anthropic", ) @@ -3609,8 +3617,8 @@ def test_streamed_end_turn_carries_no_stop_details(): assert "stop_reason" not in provider_fields -def _anthropic_chat_body_for_bound_thinking(extra_headers: dict | None) -> dict: - captured_requests: list[httpx.Request] = [] +def _anthropic_chat_body_for_bound_thinking(extra_headers: dict[str, str] | None) -> dict[str, object]: + captured_requests: Final[list[httpx.Request]] = [] def respond(request: httpx.Request) -> httpx.Response: captured_requests.append(request) @@ -3629,7 +3637,7 @@ def _anthropic_chat_body_for_bound_thinking(extra_headers: dict | None) -> dict: ) thinking: Final = {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} - client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) try: litellm.completion( model="anthropic/claude-opus-4-8", @@ -3647,11 +3655,11 @@ def _anthropic_chat_body_for_bound_thinking(extra_headers: dict | None) -> dict: def test_anthropic_chat_keeps_block_binding_when_the_final_header_carries_the_binding_beta(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") - body = _anthropic_chat_body_for_bound_thinking({"anthropic-beta": "thinking-binding-controls-2026-08-01"}) + body: Final = _anthropic_chat_body_for_bound_thinking({"anthropic-beta": "thinking-binding-controls-2026-08-01"}) assert body["thinking"] == {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} def test_anthropic_chat_drops_block_binding_when_no_header_carries_the_binding_beta(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") - body = _anthropic_chat_body_for_bound_thinking(None) + body: Final = _anthropic_chat_body_for_bound_thinking(None) assert body["thinking"] == {"type": "adaptive"} diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index c12be4b4d18..9d8e0b49774 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -422,7 +422,7 @@ def test_drop_params_forwards_a_non_object_message_item_as_sent_on_the_native_re BINDING_AND_DISPLAY_BETAS: Final = "thinking-binding-controls-2026-08-01,thinking-display-updates-2026-08-18" -def _bound_adaptive_request(headers: dict) -> dict: +def _bound_adaptive_request(headers: dict[str, str]) -> dict[str, object]: thinking: Final = { "type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}, @@ -445,7 +445,7 @@ def _bound_adaptive_request(headers: dict) -> dict: def test_messages_request_keeps_beta_gated_thinking_fields_when_the_final_header_carries_the_betas(): - request = _bound_adaptive_request({"anthropic-beta": BINDING_AND_DISPLAY_BETAS}) + request: Final = _bound_adaptive_request({"anthropic-beta": BINDING_AND_DISPLAY_BETAS}) assert request["thinking"] == { "type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}, @@ -464,7 +464,7 @@ def test_messages_request_drops_beta_gated_thinking_fields_the_final_header_does def test_messages_request_drops_clear_at_from_a_mid_conversation_system_turn_without_its_beta(): system_turn: Final = {"role": "system", "content": "be terse", "clear_at": "end_turn"} - request = AnthropicMessagesConfig().transform_anthropic_messages_request( + request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( model="claude-opus-4-8", messages=[{"role": "user", "content": "hi"}, system_turn, {"role": "user", "content": "go"}], anthropic_messages_optional_request_params={"max_tokens": 64}, diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index c50d5fbc7d2..7d93b8e8be0 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -5349,7 +5349,6 @@ def test_recursive_output_config_format_follows_the_profile_native_key( assert ("outputConfig" in request) is forwarded - def test_sonnet_4_6_json_schema_still_uses_native_output_config(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) @@ -10666,7 +10665,10 @@ def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_infere _CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT: Final = { - "textFormat": {"type": "json_schema", "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}} + "textFormat": { + "type": "json_schema", + "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}, + } } _BEDROCK_SCHEMA_TYPE_ERROR: Final = "outputConfig.textFormat.structure.jsonSchema.schema must be a string" @@ -10792,16 +10794,18 @@ def test_a_deployment_with_a_batch_control_plane_endpoint_keeps_it_out_of_chat_r assert "aws_bedrock_control_plane_endpoint" not in route.calls[0].request.content.decode() -def _converse_bound_thinking_request(headers: dict | None) -> dict: +def _converse_bound_thinking_request( + headers: dict[str, str] | None, body_betas: tuple[str, ...] = () +) -> dict[str, object]: thinking: Final = { "type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}, "display": "updates", } - data = AmazonConverseConfig()._transform_request_helper( + data: Final = AmazonConverseConfig()._transform_request_helper( model="anthropic.claude-opus-4-8-v1:0", system_content_blocks=[], - optional_params={"thinking": thinking}, + optional_params={"thinking": thinking, **({"anthropic_beta": list(body_betas)} if body_betas else {})}, messages=[{"role": "user", "content": "hi"}], headers=headers, ) @@ -10834,3 +10838,15 @@ def test_converse_drops_beta_gated_thinking_fields_when_the_request_carries_no_b "thinking.block_binding" in record.getMessage() and "provider=bedrock_converse" in record.getMessage() for record in caplog.records ) + + +def test_converse_keeps_beta_gated_thinking_fields_the_body_anthropic_beta_carries(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + fields: Final = _converse_bound_thinking_request( + None, body_betas=("thinking-binding-controls-2026-08-01", "thinking-display-updates-2026-08-18") + ) + assert fields["thinking"] == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 3c0bbadc133..5fdb5278b09 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -4198,3 +4198,28 @@ def test_bedrock_messages_without_guardrail_config_sends_no_guardrail_headers() headers={}, model="anthropic.claude-sonnet-5-v1:0", messages=[], optional_params={}, litellm_params={} ) assert not any(name.startswith("X-Amzn-Bedrock-Guardrail") for name in headers) + + +def test_bedrock_messages_keeps_mid_conversation_output_config_its_auto_added_beta_gates(): + """Invoke learns its body betas after the shared transform, so the gated field it supports must survive.""" + from unittest.mock import patch + + from litellm.types.router import GenericLiteLLMParams + + messages: Final = [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Hi"}]}, + {"role": "system", "content": "Answer briefly", "output_config": {"effort": "high"}}, + {"role": "user", "content": [{"type": "text", "text": "Go on"}]}, + ] + with patch("litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=True): + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base signature returns a bare dict + model="anthropic.claude-opus-4-8-v1:0", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 4096}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert "mid-conversation-output-config-2026-07-01" in result["anthropic_beta"] + assert {"role": "system", "content": "Answer briefly", "output_config": {"effort": "high"}} in result["messages"]