scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

fix(bedrock): add beta for mid-conversation tool changes (#43833)

Browse source
This commit is contained in:
shrey-berri 2026-10-01 16:42:46 -07:00 • committed by GitHub
parent aa601ce4e8
commit 339f5b6a4d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 169 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
litellm/llms/anthropic/common_utils.py
View file

@ -27,11 +27,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.factory import (
THOUGHT_SIGNATURE_SEPARATOR,
)
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
ANTHROPIC_OAUTH_BETA_HEADER,
ANTHROPIC_OAUTH_TOKEN_PREFIX,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
@ -344,6 +346,18 @@ class AnthropicModelInfo(BaseLLMModelInfo):
return False
return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates"
def is_mid_conversation_tool_change_used(self, messages: Sequence[object]) -> bool:
for message in messages:
if message_field(message, "role") != "system":
continue
for block in parts_of(message_field(message, "content")):
if (
message_field(block, "type") in ("tool_addition", "tool_removal")
and message_field(message_field(block, "tool"), "type") == "tool_reference"
):
return True
return False
def is_mid_conversation_output_config_used(self, messages: list[AllMessageValues]) -> bool:
"""
Return if "output_config" is in a message
@ -881,6 +895,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
custom_llm_provider: str,
is_mid_conversation_output_config_used: bool = False,
is_thinking_display_updates_used: bool = False,
is_mid_conversation_tool_change_used: bool = False,
) -> list[str]:
"""
Get list of common beta headers based on the features that are active.
@ -919,7 +934,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
thinking_display_betas: Final = (
(ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else ()
)
return list(set(betas).union(thinking_display_betas))
tool_change_betas: Final = (
(ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else ()
)
return list(set(betas).union(thinking_display_betas, tool_change_betas))
@staticmethod
def _make_api_key_auth_header(api_key: str, api_base: str | None, use_bearer_for_custom_base: bool = False) -> dict:
@ -953,6 +971,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
use_bearer_for_custom_base: bool = False,
is_mid_conversation_output_config_used: bool = False,
is_thinking_display_updates_used: bool = False,
is_mid_conversation_tool_change_used: bool = False,
) -> dict:
betas: Final = set()
# Anthropic no longer requires the prompt-caching beta header
@ -1010,7 +1029,8 @@ class AnthropicModelInfo(BaseLLMModelInfo):
betas.update(user_anthropic_beta_headers)
all_betas: Final = betas.union(
(ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else ()
(ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else (),
(ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else (),
)
# Don't send any beta headers to Vertex, except web search which is required
@ -1080,6 +1100,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
file_id_used=file_id_used,
is_mid_conversation_output_config_used=is_mid_conversation_output_config_used,
is_thinking_display_updates_used=self.is_thinking_display_updates_used(optional_params.get("thinking")),
is_mid_conversation_tool_change_used=self.is_mid_conversation_tool_change_used(messages),
web_search_tool_used=web_search_tool_used,
is_vertex_request=optional_params.get("is_vertex_request", False),
user_anthropic_beta_headers=user_anthropic_beta_headers,

8
litellm/llms/anthropic/pass_through/messages/transformation.py
View file

@ -12,6 +12,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import (
from litellm.types.llms.anthropic import (
ANTHROPIC_ADVISOR_TOOL_TYPE,
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
AnthropicMessagesRequest,
)
@ -694,7 +695,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if AnthropicModelInfo().is_thinking_display_updates_used(optional_params.get("thinking"))
else ()
)
all_beta_values: Final = beta_values.union(thinking_display_betas)
tool_change_betas: Final = (
(ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,)
if AnthropicModelInfo().is_mid_conversation_tool_change_used(messages)
else ()
)
all_beta_values: Final = beta_values.union(thinking_display_betas, tool_change_betas)
if not all_beta_values:
return headers

3
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -549,6 +549,9 @@ class AmazonAnthropicClaudeMessagesConfig(
is_thinking_display_updates_used=anthropic_model_info.is_thinking_display_updates_used(
anthropic_messages_request.get("thinking")
),
is_mid_conversation_tool_change_used=anthropic_model_info.is_mid_conversation_tool_change_used(
outgoing_messages_typed
),
)
beta_set.update(auto_betas)

1
litellm/types/llms/anthropic.py
View file

@ -777,6 +777,7 @@ ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24"
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01"
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: Final = "thinking-display-updates-2026-08-18"
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: Final = "mid-conversation-tool-changes-2026-07-01"
ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER: Final = "fine-grained-tool-streaming-2025-05-14"

30
tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_per_turn_control.py
View file

@ -150,3 +150,33 @@ def test_native_messages_thinking_display_updates_beta(display: str | None, expl
)
assert headers.get("anthropic-beta", "").split(",").count(beta) == int(display == "updates" or explicit_beta)
@pytest.mark.parametrize("action", (None, "tool_addition", "tool_removal"))
@pytest.mark.parametrize("explicit_beta", (False, True))
def test_native_messages_tool_changes_beta(action: str | None, explicit_beta: bool) -> None:
from typing import Final
from litellm.types.llms.anthropic import (
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
)
beta: Final = ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER
content: Final = (
[{"type": action, "tool": {"type": "tool_reference", "name": "mcp__test__ping"}}]
if action
else "Answer briefly"
)
headers, _ = AnthropicMessagesConfig().validate_anthropic_messages_environment(
headers={"anthropic-beta": beta} if explicit_beta else {},
model="claude-fable-5-1",
messages=["not a message dict", {"role": "user", "content": "Hello"}, {"role": "system", "content": content}],
optional_params={"thinking": {"type": "adaptive", "display": "updates"}},
litellm_params={},
api_key="sk-ant-test",
)
assert headers.get("anthropic-beta", "").split(",").count(beta) == int(action is not None or explicit_beta)
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in headers.get("anthropic-beta", "").split(",")

54
tests/unit/llms/anthropic/test_anthropic_common_utils.py
View file

@ -2450,3 +2450,57 @@ def test_shared_legacy_thinking_translation_preserves_supported_display(
)
assert optional_params["thinking"] == expected_thinking
@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
@pytest.mark.parametrize("action", (None, "tool_addition", "tool_removal"))
@pytest.mark.parametrize("explicit_beta", (False, True))
def test_validate_environment_adds_tool_changes_beta(action: str | None, explicit_beta: bool) -> None:
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.types.llms.anthropic import ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER
beta: Final = ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER
content: Final = (
[{"type": action, "tool": {"type": "tool_reference", "name": "mcp__test__ping"}}]
if action
else "Answer briefly"
)
headers: Final = AnthropicModelInfo().validate_environment(
headers={"anthropic-beta": beta} if explicit_beta else {},
model="claude-fable-5-1",
messages=[{"role": "user", "content": "Hello"}, {"role": "system", "content": content}],
optional_params={},
litellm_params={},
api_key=FAKE_REGULAR_KEY,
)
assert headers.get("anthropic-beta", "").split(",").count(beta) == int(action is not None or explicit_beta)
assert headers["x-api-key"] == FAKE_REGULAR_KEY
@pytest.mark.parametrize(
("role", "content"),
(
("user", [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]),
("assistant", [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]),
("system", "tool_addition"),
("system", None),
("system", ["tool_addition"]),
("system", [{"type": "tool_reference", "name": "ping"}]),
("system", [{"type": "tool_addition", "tool": {"type": "tool_definition", "definition": {"name": "ping"}}}]),
),
)
def test_tool_changes_beta_requires_system_tool_reference(role: str, content: object) -> None:
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.types.llms.anthropic import ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER
headers: Final = AnthropicModelInfo().validate_environment(
headers={},
model="claude-fable-5-1",
messages=[{"role": role, "content": content}],
optional_params={},
litellm_params={},
api_key=FAKE_REGULAR_KEY,
)
assert ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",")

51
tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py
View file

@ -3527,3 +3527,54 @@ def test_bedrock_clear_thinking_preserves_display_updates() -> None:
assert result.get("thinking") == {"type": "adaptive", "display": "updates"}
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in result.get("anthropic_beta", [])
@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
@pytest.mark.parametrize("action", (None, "tool_addition", "tool_removal"))
@pytest.mark.parametrize("explicit_beta", (False, True))
def test_bedrock_messages_tool_changes_beta(action: str | None, explicit_beta: bool) -> None:
from litellm.types.llms.anthropic import ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER
from litellm.types.router import GenericLiteLLMParams
beta: Final = ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER
content: Final = (
[{"type": action, "tool": {"type": "tool_reference", "name": "mcp__test__ping"}}]
if action
else "Answer briefly"
)
messages: Final = [{"role": "user", "content": "Hello"}, {"role": "system", "content": content}]
result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
model="global.anthropic.claude-fable-5-1",
messages=messages,
anthropic_messages_optional_request_params={"max_tokens": 512},
litellm_params=GenericLiteLLMParams(),
headers={"anthropic-beta": beta} if explicit_beta else {},
)
assert result.get("anthropic_beta", []).count(beta) == int(action is not None or explicit_beta)
assert result["messages"] == messages
@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
@pytest.mark.parametrize("explicit_beta", (False, True))
def test_bedrock_removed_tool_change_does_not_add_beta(explicit_beta: bool) -> None:
from litellm.types.llms.anthropic import ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER
from litellm.types.router import GenericLiteLLMParams
beta: Final = ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER
result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
model="global.anthropic.claude-fable-5-1",
messages=[
{
"role": "system",
"content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}],
},
{"role": "user", "content": "Reply with OK"},
],
anthropic_messages_optional_request_params={"max_tokens": 512},
litellm_params=GenericLiteLLMParams(),
headers={"anthropic-beta": beta} if explicit_beta else {},
)
assert result["messages"] == [{"role": "user", "content": "Reply with OK"}]
assert result.get("anthropic_beta", []).count(beta) == int(explicit_beta)