scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-01 02:02:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 92

fix(bedrock): add beta header for output config in message (#43778)

Browse source
This commit is contained in:
shrey-berri 2026-09-30 00:50:09 -07:00 • committed by GitHub
parent 314ff111e5
commit 04fa760bf2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 117 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/anthropic_beta_headers_config.json
View file

@ -33,7 +33,8 @@
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
"web-search-2025-03-05": "web-search-2025-03-05"
"web-search-2025-03-05": "web-search-2025-03-05",
"mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01"
},
"azure_ai": {
"advisor-tool-2026-03-01": null,
@ -134,7 +135,8 @@
"token-efficient-tools-2025-02-19": null,
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": null
"web-search-2025-03-05": null,
"mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01"
},
"bedrock_mantle": {
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",

17
litellm/llms/anthropic/common_utils.py
View file

@ -31,6 +31,7 @@ from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
ANTHROPIC_OAUTH_BETA_HEADER,
ANTHROPIC_OAUTH_TOKEN_PREFIX,
AllAnthropicToolsValues,
@ -326,6 +327,12 @@ class AnthropicModelInfo(BaseLLMModelInfo):
file_ids: Final = get_file_ids_from_messages(messages)
return len(file_ids) > 0
def is_mid_conversation_output_config_used(self, messages: list[AllMessageValues]) -> bool:
"""
Return if "output_config" is in a message
"""
return any("output_config" in message for message in messages)
def is_mcp_server_used(self, mcp_servers: list[AnthropicMcpServerTool] | None) -> bool:
if mcp_servers is None:
return False
@ -851,6 +858,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
mcp_server_used: bool = False,
*,
custom_llm_provider: str,
is_mid_conversation_output_config_used: bool = False,
) -> list[str]:
"""
Get list of common beta headers based on the features that are active.
@ -883,6 +891,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if mcp_server_used:
betas.append("mcp-client-2025-04-04")
if is_mid_conversation_output_config_used:
betas.append(ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER)
return list(set(betas))
@staticmethod
@ -915,6 +926,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
container_with_skills_used: bool = False,
api_base: str | None = None,
use_bearer_for_custom_base: bool = False,
is_mid_conversation_output_config_used: bool = False,
) -> dict:
betas: Final = set()
# Anthropic no longer requires the prompt-caching beta header
@ -950,6 +962,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if container_with_skills_used:
betas.add("skills-2025-10-02")
if is_mid_conversation_output_config_used:
betas.add(ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER)
_is_oauth: Final = api_key and api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)
headers: Final = {
"anthropic-version": anthropic_version or "2023-06-01",
@ -1015,6 +1030,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
mcp_server_used: Final = self.is_mcp_server_used(mcp_servers=optional_params.get("mcp_servers"))
pdf_used: Final = self.is_pdf_used(messages=messages)
file_id_used: Final = self.is_file_id_used(messages=messages)
is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages=messages)
web_search_tool_used: Final = self.is_web_search_tool_used(tools=tools)
tool_search_used: Final = self.is_tool_search_used(tools=tools)
programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools=tools)
@ -1032,6 +1048,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
api_key=api_key,
auth_token=auth_token,
file_id_used=file_id_used,
is_mid_conversation_output_config_used=is_mid_conversation_output_config_used,
web_search_tool_used=web_search_tool_used,
is_vertex_request=optional_params.get("is_vertex_request", False),
user_anthropic_beta_headers=user_anthropic_beta_headers,

2
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -255,6 +255,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
tool_search_used: Final = self.is_tool_search_used(tools)
programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools)
input_examples_used: Final = self.is_input_examples_used(tools)
is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages)
user_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
beta_set: Final = set(user_beta_set)
@ -266,6 +267,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
file_id_used=self.is_file_id_used(messages),
mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")),
custom_llm_provider="bedrock",
is_mid_conversation_output_config_used=is_mid_conversation_output_config_used,
)
beta_set.update(auto_betas)

9
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -515,7 +515,13 @@ class AmazonAnthropicClaudeMessagesConfig(
tool_search_used: Final = anthropic_model_info.is_tool_search_used(tools)
programmatic_tool_calling_used: Final = anthropic_model_info.is_programmatic_tool_calling_used(tools)
input_examples_used: Final = anthropic_model_info.is_input_examples_used(tools)
outgoing_messages_typed: Final = cast(
list[AllMessageValues],
anthropic_messages_request["messages"],
)
is_mid_conversation_output_config_used: Final = anthropic_model_info.is_mid_conversation_output_config_used(
outgoing_messages_typed
)
user_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
beta_set: Final = set(user_beta_set)
auto_betas: Final = anthropic_model_info.get_anthropic_beta_list(
@ -528,6 +534,7 @@ class AmazonAnthropicClaudeMessagesConfig(
anthropic_messages_optional_request_params.get("mcp_servers")
),
custom_llm_provider="bedrock",
is_mid_conversation_output_config_used=is_mid_conversation_output_config_used,
)
beta_set.update(auto_betas)

2
litellm/types/llms/anthropic.py
View file

@ -774,6 +774,8 @@ ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: Final = frozenset(
# Effort beta header constant
ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24"
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01"
ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER: Final = "fine-grained-tool-streaming-2025-05-14"
# OAuth constants

31
tests/unit/llms/anthropic/test_anthropic_common_utils.py
View file

@ -2345,3 +2345,34 @@ class TestMalformedContentListItems:
api_key=FAKE_REGULAR_KEY,
max_tokens=5,
)
@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
@pytest.mark.parametrize("nested_output_config", [False, True])
@pytest.mark.parametrize("explicit_beta", [False, True])
@pytest.mark.parametrize("output_config", [{}, {"effort": "high"}, {"format": {"type": "text"}}])
def test_validate_environment_adds_mid_conversation_output_config_beta(
nested_output_config: bool, explicit_beta: bool, output_config: dict[str, object]
) -> None:
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.types.llms.anthropic import ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER
beta: Final = ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER
messages: Final = [
{"role": "user", "content": "Hello"},
*([{"role": "system", "content": [], "output_config": output_config}] if nested_output_config else []),
{"role": "user", "content": "Reply with OK"},
]
headers: Final = AnthropicModelInfo().validate_environment(
headers={"anthropic-beta": beta} if explicit_beta else {},
model="claude-fable-5-1",
messages=messages,
optional_params={"output_config": {"effort": "high"}},
litellm_params={},
api_key=FAKE_REGULAR_KEY,
)
assert headers.get("anthropic-beta", "").split(",").count(beta) == int(nested_output_config or explicit_beta)
assert headers["x-api-key"] == FAKE_REGULAR_KEY

53
tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py
View file

@ -3494,3 +3494,56 @@ async def test_get_async_streaming_response_iterator_yields_small_frame_before_u
remaining: Final = tuple([chunk async for chunk in iterator])
assert any(chunk.startswith(b"event: message_stop\n") for chunk in remaining), remaining
await iterator.aclose()
@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
@pytest.mark.parametrize("nested_output_config", [False, True])
@pytest.mark.parametrize("explicit_beta", [False, True])
@pytest.mark.parametrize("output_config", [{}, {"effort": "high"}, {"format": {"type": "text"}}])
def test_bedrock_messages_mid_conversation_output_config_beta(
nested_output_config: bool, explicit_beta: bool, output_config: dict[str, object]
) -> None:
from litellm.types.llms.anthropic import ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER
from litellm.types.router import GenericLiteLLMParams
beta: Final = ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER
messages: Final = [
{"role": "user", "content": "Hello"},
*([{"role": "system", "content": [], "output_config": output_config}] if nested_output_config else []),
{"role": "user", "content": "Reply with OK"},
]
result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
model="global.anthropic.claude-fable-5-1",
messages=messages,
anthropic_messages_optional_request_params={"max_tokens": 1024, "output_config": {"effort": "high"}},
litellm_params=GenericLiteLLMParams(),
headers={"anthropic-beta": beta} if explicit_beta else {},
)
assert result.get("anthropic_beta", []).count(beta) == int(nested_output_config or explicit_beta)
assert result["messages"] == messages
assert result["output_config"] == {"effort": "high"}
@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
@pytest.mark.parametrize("explicit_beta", [False, True])
def test_bedrock_messages_removed_output_config_does_not_add_beta(explicit_beta: bool) -> None:
from litellm.types.llms.anthropic import ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER
from litellm.types.router import GenericLiteLLMParams
beta: Final = ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER
result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
model="global.anthropic.claude-fable-5-1",
messages=[
{"role": "system", "content": "Answer briefly", "output_config": {"effort": "high"}},
{"role": "user", "content": "Reply with OK"},
],
anthropic_messages_optional_request_params={"max_tokens": 1024},
litellm_params=GenericLiteLLMParams(),
headers={"anthropic-beta": beta} if explicit_beta else {},
)
assert result["messages"] == [{"role": "user", "content": "Reply with OK"}]
assert result.get("anthropic_beta", []).count(beta) == int(explicit_beta)