diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index c03f7cda990..fd72bf7d2e7 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -427,6 +427,7 @@ def update_headers_with_filtered_beta( provider: str, api_base: str | None = None, unlisted: UnlistedBetas = "drop", + keep: frozenset[str] = frozenset(), ) -> dict: """ Update headers dict by filtering and transforming anthropic-beta header values. @@ -437,6 +438,7 @@ def update_headers_with_filtered_beta( provider: Provider name api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``) unlisted: Whether a beta missing from the row is dropped or forwarded as sent + keep: Betas the routed model accepts, forwarded as sent whatever the row says Returns: Updated headers dict @@ -446,13 +448,20 @@ def update_headers_with_filtered_beta( return headers # Parse existing beta headers - beta_values: Final = [b.strip() for b in existing_beta.split(",") if b.strip()] + beta_values: Final[tuple[str, ...]] = tuple(b.strip() for b in existing_beta.split(",") if b.strip()) # Filter and transform based on provider - filtered_beta_values: Final = filter_and_transform_beta_headers( - beta_headers=beta_values, - provider=resolve_beta_provider(provider, api_base), - unlisted=unlisted, + filtered_beta_values: Final = tuple( + dict.fromkeys( + [ + *filter_and_transform_beta_headers( + beta_headers=[b for b in beta_values if b not in keep], + provider=resolve_beta_provider(provider, api_base), + unlisted=unlisted, + ), + *(b for b in beta_values if b in keep), + ] + ) ) # Update or remove the header diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index d234ac40c3d..487e7aca663 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -139,6 +139,9 @@ class BaseAnthropicMessagesConfig(ABC): """ return True + def model_specific_anthropic_betas(self, model: str, litellm_params: GenericLiteLLMParams) -> frozenset[str]: + return frozenset() + def supports_safeguards(self) -> bool: return True diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 52b27da4def..cc81266af9e 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1708,6 +1708,15 @@ def bedrock_supports_tool_search(model: str) -> bool: return AnthropicModelInfo.supports_model_capability(model, "supports_tool_search", "bedrock") +def bedrock_supports_per_turn_control(model: str, model_info: Mapping[str, object] | None) -> bool: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + deployment_flag: Final = (model_info or {}).get("supports_per_turn_control") + if isinstance(deployment_flag, bool): + return deployment_flag + return AnthropicModelInfo.supports_model_capability(model, "supports_per_turn_control", "bedrock") + + def supports_bedrock_prompt_cache_ttl(model: str) -> bool: """ Whether Bedrock accepts a ``ttl`` ("5m"/"1h") on prompt ``cache_control`` diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index 51f37364245..cf48fcf5c2f 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -42,6 +42,7 @@ from litellm.llms.bedrock.common_utils import ( apply_bedrock_invoke_structured_output, bedrock_native_output_format, bedrock_native_output_retry_requested, + bedrock_supports_per_turn_control, bedrock_supports_tool_search, ensure_bedrock_anthropic_messages_tool_names, for_bedrock_json_tool_retry, @@ -113,6 +114,13 @@ class AmazonAnthropicClaudeMessagesConfig( def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool: return True + def model_specific_anthropic_betas(self, model: str, litellm_params: GenericLiteLLMParams) -> frozenset[str]: + if bedrock_supports_per_turn_control( + model, _MODEL_INFO.validate_python(getattr(litellm_params, "model_info", None)) + ): + return frozenset({ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value}) + return frozenset() + BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys()) def get_error_class( @@ -574,6 +582,7 @@ class AmazonAnthropicClaudeMessagesConfig( headers: dict, anthropic_messages_request: dict, injected_thinking_for_clear_thinking: bool, + litellm_params: GenericLiteLLMParams, ) -> list[str]: anthropic_model_info: Final = AnthropicModelInfo() tools: Final = anthropic_messages_optional_request_params.get("tools") @@ -642,16 +651,22 @@ class AmazonAnthropicClaudeMessagesConfig( if "tool-search-tool-2025-10-19" in beta_set: beta_set.add("tool-examples-2025-10-29") + model_supported_betas: Final = self.model_specific_anthropic_betas(model, litellm_params) & user_beta_set beta_provider: Final = self.beta_headers_provider filtered_betas: Final = sorted( - filter_and_transform_beta_headers( - beta_headers=list(beta_set), - provider=beta_provider, - ) + { + *filter_and_transform_beta_headers( + beta_headers=list(beta_set), + provider=beta_provider, + ), + *model_supported_betas, + } ) dropped_user_betas: Final = sorted( - b for b in user_beta_set if not filter_and_transform_beta_headers([b], provider=beta_provider) + b + for b in user_beta_set - model_supported_betas + if not filter_and_transform_beta_headers([b], provider=beta_provider) ) if dropped_user_betas: verbose_logger.warning( @@ -837,6 +852,7 @@ class AmazonAnthropicClaudeMessagesConfig( headers=headers, anthropic_messages_request=anthropic_messages_request, injected_thinking_for_clear_thinking=injected_thinking_for_clear_thinking, + litellm_params=litellm_params, ) if filtered_betas: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 4a10db31606..169c21cf635 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2642,6 +2642,7 @@ class BaseLLMHTTPHandler: provider=custom_llm_provider, api_base=affinity_api_base, unlisted="drop" if provider_filters_betas else "forward", + keep=anthropic_messages_provider_config.model_specific_anthropic_betas(model, litellm_params), ) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 868dcdad74d..95adfda8c49 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -64871,6 +64871,15 @@ "supports_tool_search": true } }, + { + "name": "claude-per-turn-control", + "pattern": "claude-[a-z]+-(?:4[-._](?:[5-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "fill_missing_for_providers": ["bedrock", "bedrock_converse"], + "description": "Claude at version 4.5 or higher, with the same id shapes and version boundaries as claude-tool-search. On 2026-11-05 Bedrock InvokeModel answered 200 with the per-turn-control-2026-07-01 beta on every Claude 4.5 and newer model a QA account reached across 13 regions, and 400 invalid beta flag on Claude 3 Haiku in ca-central-1. The jp., au. and in. profiles answered 200 too, AWS lists no apac. profile for Claude 4.5 or newer, and GovCloud was not tested. Claude 3.5, 3.7, Sonnet 4, Opus 4 and Opus 4.1 were unreachable, so they keep the beta dropped as before. The flag means Bedrock accepts the beta name on the model, as supports_tool_search does for the tool search beta, and Bedrock Invoke forwards the beta only where it is set. It does not promise per-turn output_config works, since on 2026-11-05 Opus 4.8, Opus 5, Sonnet 5 and Fable 5 rejected it with or without the beta. A deployment turns the forward off with model_info supports_per_turn_control false.", + "model_info": { + "supports_per_turn_control": true + } + }, { "name": "wandb-reasoning-baseline", "pattern": "^wandb/", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 7c137b72c3e..5a14dc95e7c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -226,6 +226,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): thinking_always_on: ReadOnly[bool | None] thinking_on_by_default: ReadOnly[bool | None] supports_tool_search: bool | None + supports_per_turn_control: ReadOnly[bool | None] supports_mid_conversation_system: bool | None supports_url_context: bool | None supports_continuation: ReadOnly[bool | None] @@ -4302,6 +4303,7 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( "supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output", + "supports_per_turn_control", "supports_pdf_input", "supports_continuation", "reasoning_modes", diff --git a/litellm/utils.py b/litellm/utils.py index 53b02ebddac..8944ecb5648 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7600,6 +7600,7 @@ def get_model_info_helper( thinking_always_on=_model_info.get("thinking_always_on", None), thinking_on_by_default=_model_info.get("thinking_on_by_default", None), supports_tool_search=_model_info.get("supports_tool_search", None), + supports_per_turn_control=_model_info.get("supports_per_turn_control", None), supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None), supports_anthropic_compaction=_model_info.get("supports_anthropic_compaction", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 868dcdad74d..95adfda8c49 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -64871,6 +64871,15 @@ "supports_tool_search": true } }, + { + "name": "claude-per-turn-control", + "pattern": "claude-[a-z]+-(?:4[-._](?:[5-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "fill_missing_for_providers": ["bedrock", "bedrock_converse"], + "description": "Claude at version 4.5 or higher, with the same id shapes and version boundaries as claude-tool-search. On 2026-11-05 Bedrock InvokeModel answered 200 with the per-turn-control-2026-07-01 beta on every Claude 4.5 and newer model a QA account reached across 13 regions, and 400 invalid beta flag on Claude 3 Haiku in ca-central-1. The jp., au. and in. profiles answered 200 too, AWS lists no apac. profile for Claude 4.5 or newer, and GovCloud was not tested. Claude 3.5, 3.7, Sonnet 4, Opus 4 and Opus 4.1 were unreachable, so they keep the beta dropped as before. The flag means Bedrock accepts the beta name on the model, as supports_tool_search does for the tool search beta, and Bedrock Invoke forwards the beta only where it is set. It does not promise per-turn output_config works, since on 2026-11-05 Opus 4.8, Opus 5, Sonnet 5 and Fable 5 rejected it with or without the beta. A deployment turns the forward off with model_info supports_per_turn_control false.", + "model_info": { + "supports_per_turn_control": true + } + }, { "name": "wandb-reasoning-baseline", "pattern": "^wandb/", diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index ff4e3650464..c4614dd636f 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -2370,6 +2370,32 @@ def test_bedrock_invoke_transform_hoists_only_leading_system_run(local_model_cos ] +def test_bedrock_invoke_transform_forwards_per_turn_control_beta_with_message_output_config( + local_model_cost_map: None, local_beta_headers_config: None +) -> None: + """A live InvokeModel call on 2026-11-05 answered 200 with per-turn-control-2026-07-01 next to a message + ``output_config`` on Opus 5.5, so the client's beta and that message must both reach Bedrock.""" + from litellm.types.router import GenericLiteLLMParams + + messages: Final = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "system", "content": [], "output_config": {"effort": "low"}}, + {"role": "user", "content": "continue"}, + ] + + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model="us.anthropic.claude-opus-5-5", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False}, + litellm_params=GenericLiteLLMParams(), + headers={"anthropic-beta": "per-turn-control-2026-07-01"}, + ) + + assert result["messages"] == messages + assert result["anthropic_beta"].count("per-turn-control-2026-07-01") == 1 + + def test_bedrock_invoke_transform_converts_mid_conversation_system_for_older_claude(local_model_cost_map): """Invoke rejects ``role: "system"`` in every position on Opus 4.7, Sonnet 4.6, Haiku 4.5, etc. ("role 'system' is not supported on this model"), but @@ -4093,6 +4119,96 @@ async def _send_bedrock_messages( return bedrock, None +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize( + ("model", "headers", "forwarded"), + [ + ("anthropic.claude-3-haiku-20240307-v1:0", {"anthropic-beta": "per-turn-control-2026-07-01"}, False), + ("us.anthropic.claude-opus-4-1-20250805-v1:0", {"anthropic-beta": "per-turn-control-2026-07-01"}, False), + ("us.anthropic.claude-sonnet-4-5-20250929-v1:0", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("jp.anthropic.claude-opus-4-6-v1", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("au.anthropic.claude-opus-4-8", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("eu.anthropic.claude-opus-5-5", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("global.anthropic.claude-sonnet-5-5", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("us.anthropic.claude-newfamily-6", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("us.anthropic.claude-sonnet-5-5", {}, False), + ], +) +async def test_bedrock_invoke_messages_forward_per_turn_control_beta_only_where_the_model_map_flags_it( + model: str, headers: dict[str, str], forwarded: bool +) -> None: + """On 2026-11-05 Bedrock InvokeModel answered 400 ``invalid beta flag`` to per-turn-control-2026-07-01 on Claude 3 + Haiku and 200 on every Claude 4.5 and newer model a QA account reached in 13 regions and in the jp., au. and in. + profiles.""" + bedrock: Final = ScriptedBedrockInvoke(replies=((200, _ANTHROPIC_MESSAGE),)) + + await litellm.anthropic_messages( + model=f"bedrock/invoke/{model}", + messages=[{"role": "user", "content": "hi"}], + max_tokens=16, + extra_headers=headers, + aws_access_key_id="test-access-key", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)), + ) + + (sent,) = bedrock.bodies + sent_betas: Final = TypeAdapter(tuple[str, ...]).validate_python(sent.get("anthropic_beta", ())) + assert ("per-turn-control-2026-07-01" in sent_betas) is forwarded + + +@pytest.mark.parametrize( + ("model", "flag", "expected"), + [ + ("us.anthropic.claude-opus-5-5", False, (False, True)), + ("anthropic.claude-3-haiku-20240307-v1:0", True, (True, False)), + ], + ids=["opt_out", "opt_in"], +) +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +async def test_bedrock_invoke_messages_per_turn_control_flag_stays_on_its_own_deployment( + respx_mock: respx.Router, + monkeypatch: pytest.MonkeyPatch, + model: str, + flag: bool, + expected: tuple[bool, bool], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + bedrock: Final = ScriptedBedrockInvoke(replies=((200, _ANTHROPIC_MESSAGE), (200, _ANTHROPIC_MESSAGE))) + respx_mock.post(url__regex=r"https://bedrock-runtime\..*").mock(side_effect=bedrock) + router: Final = litellm.Router( + model_list=[ + { + "model_name": model_name, + "litellm_params": { + "model": f"bedrock/invoke/{model}", + "aws_access_key_id": "test-access-key", + "aws_secret_access_key": "test-secret-key", + "aws_region_name": "us-east-1", + }, + "model_info": model_info, + } + for model_name, model_info in (("flagged", {"supports_per_turn_control": flag}), ("default", {})) + ], + num_retries=0, + ) + + for model_name in ("flagged", "default"): + await router.aanthropic_messages( + model=model_name, + messages=[{"role": "user", "content": "hi"}], + max_tokens=16, + extra_headers={"anthropic-beta": "per-turn-control-2026-07-01"}, + ) + + forwarded: Final = tuple( + "per-turn-control-2026-07-01" in TypeAdapter(tuple[str, ...]).validate_python(sent.get("anthropic_beta", ())) + for sent in bedrock.bodies + ) + assert forwarded == expected + + _OPT_IN_DEPLOYMENTS: Final = pytest.mark.parametrize( ("model", "model_info"), [