From 04fa760bf2a0a2d7be05302f2bb079a65593e478 Mon Sep 17 00:00:00 2001 From: shrey-berri Date: Wed, 30 Sep 2026 00:50:09 -0700 Subject: [PATCH] fix(bedrock): add beta header for output config in message (#43778) --- litellm/anthropic_beta_headers_config.json | 6 ++- litellm/llms/anthropic/common_utils.py | 17 ++++++ .../anthropic_claude3_transformation.py | 2 + .../anthropic_claude3_transformation.py | 9 +++- litellm/types/llms/anthropic.py | 2 + .../anthropic/test_anthropic_common_utils.py | 31 +++++++++++ .../test_anthropic_claude3_transformation.py | 53 +++++++++++++++++++ 7 files changed, 117 insertions(+), 3 deletions(-) diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 3a938007633..0e0eeff83fe 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -33,7 +33,8 @@ "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19", "web-fetch-2025-09-10": "web-fetch-2025-09-10", - "web-search-2025-03-05": "web-search-2025-03-05" + "web-search-2025-03-05": "web-search-2025-03-05", + "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01" }, "azure_ai": { "advisor-tool-2026-03-01": null, @@ -134,7 +135,8 @@ "token-efficient-tools-2025-02-19": null, "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19", "web-fetch-2025-09-10": null, - "web-search-2025-03-05": null + "web-search-2025-03-05": null, + "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01" }, "bedrock_mantle": { "advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19", diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index bf2d588dd3a..3e61a0caa90 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -31,6 +31,7 @@ from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic import ( ANTHROPIC_HOSTED_TOOLS, + ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, ANTHROPIC_OAUTH_BETA_HEADER, ANTHROPIC_OAUTH_TOKEN_PREFIX, AllAnthropicToolsValues, @@ -326,6 +327,12 @@ class AnthropicModelInfo(BaseLLMModelInfo): file_ids: Final = get_file_ids_from_messages(messages) return len(file_ids) > 0 + def is_mid_conversation_output_config_used(self, messages: list[AllMessageValues]) -> bool: + """ + Return if "output_config" is in a message + """ + return any("output_config" in message for message in messages) + def is_mcp_server_used(self, mcp_servers: list[AnthropicMcpServerTool] | None) -> bool: if mcp_servers is None: return False @@ -851,6 +858,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): mcp_server_used: bool = False, *, custom_llm_provider: str, + is_mid_conversation_output_config_used: bool = False, ) -> list[str]: """ Get list of common beta headers based on the features that are active. @@ -883,6 +891,9 @@ class AnthropicModelInfo(BaseLLMModelInfo): if mcp_server_used: betas.append("mcp-client-2025-04-04") + if is_mid_conversation_output_config_used: + betas.append(ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER) + return list(set(betas)) @staticmethod @@ -915,6 +926,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): container_with_skills_used: bool = False, api_base: str | None = None, use_bearer_for_custom_base: bool = False, + is_mid_conversation_output_config_used: bool = False, ) -> dict: betas: Final = set() # Anthropic no longer requires the prompt-caching beta header @@ -950,6 +962,9 @@ class AnthropicModelInfo(BaseLLMModelInfo): if container_with_skills_used: betas.add("skills-2025-10-02") + if is_mid_conversation_output_config_used: + betas.add(ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER) + _is_oauth: Final = api_key and api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX) headers: Final = { "anthropic-version": anthropic_version or "2023-06-01", @@ -1015,6 +1030,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): mcp_server_used: Final = self.is_mcp_server_used(mcp_servers=optional_params.get("mcp_servers")) pdf_used: Final = self.is_pdf_used(messages=messages) file_id_used: Final = self.is_file_id_used(messages=messages) + is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages=messages) web_search_tool_used: Final = self.is_web_search_tool_used(tools=tools) tool_search_used: Final = self.is_tool_search_used(tools=tools) programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools=tools) @@ -1032,6 +1048,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): api_key=api_key, auth_token=auth_token, file_id_used=file_id_used, + is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, web_search_tool_used=web_search_tool_used, is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index c5abb5e9a1c..4d758640d72 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -255,6 +255,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): tool_search_used: Final = self.is_tool_search_used(tools) programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools) input_examples_used: Final = self.is_input_examples_used(tools) + is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages) user_beta_set: Final = set(get_anthropic_beta_from_headers(headers)) beta_set: Final = set(user_beta_set) @@ -266,6 +267,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): file_id_used=self.is_file_id_used(messages), mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), custom_llm_provider="bedrock", + is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, ) beta_set.update(auto_betas) diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 94eb0c92e40..f018077ebcd 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -515,7 +515,13 @@ class AmazonAnthropicClaudeMessagesConfig( tool_search_used: Final = anthropic_model_info.is_tool_search_used(tools) programmatic_tool_calling_used: Final = anthropic_model_info.is_programmatic_tool_calling_used(tools) input_examples_used: Final = anthropic_model_info.is_input_examples_used(tools) - + outgoing_messages_typed: Final = cast( + list[AllMessageValues], + anthropic_messages_request["messages"], + ) + is_mid_conversation_output_config_used: Final = anthropic_model_info.is_mid_conversation_output_config_used( + outgoing_messages_typed + ) user_beta_set: Final = set(get_anthropic_beta_from_headers(headers)) beta_set: Final = set(user_beta_set) auto_betas: Final = anthropic_model_info.get_anthropic_beta_list( @@ -528,6 +534,7 @@ class AmazonAnthropicClaudeMessagesConfig( anthropic_messages_optional_request_params.get("mcp_servers") ), custom_llm_provider="bedrock", + is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, ) beta_set.update(auto_betas) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index a818daf554d..60d5450a1f2 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -774,6 +774,8 @@ ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: Final = frozenset( # Effort beta header constant ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" +ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01" + ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER: Final = "fine-grained-tool-streaming-2025-05-14" # OAuth constants diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 52b53769457..b80129a55bf 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -2345,3 +2345,34 @@ class TestMalformedContentListItems: api_key=FAKE_REGULAR_KEY, max_tokens=5, ) + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize("nested_output_config", [False, True]) +@pytest.mark.parametrize("explicit_beta", [False, True]) +@pytest.mark.parametrize("output_config", [{}, {"effort": "high"}, {"format": {"type": "text"}}]) +def test_validate_environment_adds_mid_conversation_output_config_beta( + nested_output_config: bool, explicit_beta: bool, output_config: dict[str, object] +) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + from litellm.types.llms.anthropic import ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER + + beta: Final = ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER + + messages: Final = [ + {"role": "user", "content": "Hello"}, + *([{"role": "system", "content": [], "output_config": output_config}] if nested_output_config else []), + {"role": "user", "content": "Reply with OK"}, + ] + + headers: Final = AnthropicModelInfo().validate_environment( + headers={"anthropic-beta": beta} if explicit_beta else {}, + model="claude-fable-5-1", + messages=messages, + optional_params={"output_config": {"effort": "high"}}, + litellm_params={}, + api_key=FAKE_REGULAR_KEY, + ) + + assert headers.get("anthropic-beta", "").split(",").count(beta) == int(nested_output_config or explicit_beta) + assert headers["x-api-key"] == FAKE_REGULAR_KEY diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 79207ece259..f92de7370bd 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -3494,3 +3494,56 @@ async def test_get_async_streaming_response_iterator_yields_small_frame_before_u remaining: Final = tuple([chunk async for chunk in iterator]) assert any(chunk.startswith(b"event: message_stop\n") for chunk in remaining), remaining await iterator.aclose() + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize("nested_output_config", [False, True]) +@pytest.mark.parametrize("explicit_beta", [False, True]) +@pytest.mark.parametrize("output_config", [{}, {"effort": "high"}, {"format": {"type": "text"}}]) +def test_bedrock_messages_mid_conversation_output_config_beta( + nested_output_config: bool, explicit_beta: bool, output_config: dict[str, object] +) -> None: + from litellm.types.llms.anthropic import ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER + from litellm.types.router import GenericLiteLLMParams + + beta: Final = ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER + + messages: Final = [ + {"role": "user", "content": "Hello"}, + *([{"role": "system", "content": [], "output_config": output_config}] if nested_output_config else []), + {"role": "user", "content": "Reply with OK"}, + ] + + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model="global.anthropic.claude-fable-5-1", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 1024, "output_config": {"effort": "high"}}, + litellm_params=GenericLiteLLMParams(), + headers={"anthropic-beta": beta} if explicit_beta else {}, + ) + + assert result.get("anthropic_beta", []).count(beta) == int(nested_output_config or explicit_beta) + assert result["messages"] == messages + assert result["output_config"] == {"effort": "high"} + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize("explicit_beta", [False, True]) +def test_bedrock_messages_removed_output_config_does_not_add_beta(explicit_beta: bool) -> None: + from litellm.types.llms.anthropic import ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER + from litellm.types.router import GenericLiteLLMParams + + beta: Final = ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model="global.anthropic.claude-fable-5-1", + messages=[ + {"role": "system", "content": "Answer briefly", "output_config": {"effort": "high"}}, + {"role": "user", "content": "Reply with OK"}, + ], + anthropic_messages_optional_request_params={"max_tokens": 1024}, + litellm_params=GenericLiteLLMParams(), + headers={"anthropic-beta": beta} if explicit_beta else {}, + ) + + assert result["messages"] == [{"role": "user", "content": "Reply with OK"}] + assert result.get("anthropic_beta", []).count(beta) == int(explicit_beta)