From 989f5e06f5858aaec59f039e4f206b46a5d3ca5a Mon Sep 17 00:00:00 2001 From: Kent Date: Fri, 26 Jun 2026 12:34:06 +0800 Subject: [PATCH] feat(bedrock): add opt-out flag for orphaned-tool-block neutralization Address Greptile P1 review feedback on PR #31400. P1 (backward compat): gate neutralization behind a new litellm.bedrock_neutralize_orphaned_tool_blocks flag (default True, so both bugs stay fixed). Setting it False restores the legacy contract: raise UnsupportedParamsError, or inject a dummy tool under modify_params. A new _handle_orphaned_tool_blocks dispatcher selects the path; the pure _neutralize_orphaned_tool_blocks helper is unchanged. Adds tests for flag-off-raises, flag-off-with-modify_params-injects, and default-on. P1 (CI credentials): the live-network test_parallel_function_call_anthropic_error_msg no longer needs Bedrock creds for the Converse case. That case is dropped (its no-raise behavior is already covered offline in test_converse_transformation.py); the now-unused expect_unsupported_params_error param and dead pytest.raises branch are removed. --- litellm/__init__.py | 4 ++ .../bedrock/chat/converse_transformation.py | 30 +++++++- tests/local_testing/test_function_calling.py | 65 +++++------------ .../chat/test_converse_transformation.py | 71 +++++++++++++++++-- 4 files changed, 118 insertions(+), 52 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 5ae5942f32f..4b10130326e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -242,6 +242,10 @@ telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) +bedrock_neutralize_orphaned_tool_blocks = ( + os.getenv("LITELLM_BEDROCK_NEUTRALIZE_ORPHANED_TOOL_BLOCKS", "true").lower() + == "true" +) use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) ) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 5781b97ce00..4bdba492ee5 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -66,7 +66,9 @@ from litellm.types.utils import ( Usage, ) from litellm.utils import ( + add_dummy_tool, any_assistant_message_has_thinking_blocks, + has_tool_call_blocks, last_assistant_with_tool_calls_has_no_thinking_blocks, supports_reasoning, token_counter, @@ -247,6 +249,30 @@ class AmazonConverseConfig(BaseConfig): ) return [_rewrite(message) for message in messages] + @staticmethod + def _handle_orphaned_tool_blocks( + messages: list[AllMessageValues], optional_params: dict + ) -> list[AllMessageValues]: + if litellm.bedrock_neutralize_orphaned_tool_blocks: + return AmazonConverseConfig._neutralize_orphaned_tool_blocks( + messages, optional_params + ) + + if "tools" in optional_params or not has_tool_call_blocks(messages): + return messages + + if litellm.modify_params: + optional_params["tools"] = add_dummy_tool( + custom_llm_provider="bedrock_converse" + ) + return messages + + raise litellm.utils.UnsupportedParamsError( + message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.", + model="", + llm_provider="bedrock", + ) + @classmethod def get_config(cls): return { @@ -1742,7 +1768,7 @@ class AmazonConverseConfig(BaseConfig): messages, model=model ) - messages = self._neutralize_orphaned_tool_blocks(messages, optional_params) + messages = self._handle_orphaned_tool_blocks(messages, optional_params) # Convert last user message to guarded_text if guardrailConfig is present messages = self._convert_consecutive_user_messages_to_guarded_text( @@ -1803,7 +1829,7 @@ class AmazonConverseConfig(BaseConfig): messages, model=model ) - messages = self._neutralize_orphaned_tool_blocks(messages, optional_params) + messages = self._handle_orphaned_tool_blocks(messages, optional_params) # Convert last user message to guarded_text if guardrailConfig is present messages = self._convert_consecutive_user_messages_to_guarded_text( diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index e0984cd4311..3d32e2e24f9 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -267,7 +267,6 @@ def test_aaparallel_function_call_with_anthropic_thinking(model): from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message - _PARALLEL_TOOL_HISTORY_MESSAGES = [ { "role": "user", @@ -299,20 +298,11 @@ _PARALLEL_TOOL_HISTORY_MESSAGES = [ @pytest.mark.parametrize( - "model, messages, expect_unsupported_params_error", + "model, messages", [ - # Bedrock Converse neutralizes orphaned tool blocks to text; no error. - ( - "anthropic.claude-3-sonnet-20240229-v1:0", - _PARALLEL_TOOL_HISTORY_MESSAGES, - False, - ), - # Anthropic Messages API: dummy tool is injected without modify_params. - ( - "claude-haiku-4-5-20251001", - _PARALLEL_TOOL_HISTORY_MESSAGES, - False, - ), + # Anthropic Messages API: a dummy tool is injected without modify_params, + # so tool history with no tools= completes instead of raising. + ("claude-haiku-4-5-20251001", _PARALLEL_TOOL_HISTORY_MESSAGES), ( "anthropic.claude-3-sonnet-20240229-v1:0", [ @@ -321,7 +311,6 @@ _PARALLEL_TOOL_HISTORY_MESSAGES = [ "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", } ], - False, ), ( "claude-haiku-4-5-20251001", @@ -331,22 +320,18 @@ _PARALLEL_TOOL_HISTORY_MESSAGES = [ "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", } ], - False, ), ], ) -def test_parallel_function_call_anthropic_error_msg( - model, messages, expect_unsupported_params_error -): +def test_parallel_function_call_anthropic_error_msg(model, messages): """ - Tool history without an explicit ``tools`` param: + Tool history without an explicit ``tools`` param must complete, not raise. - - Bedrock **Converse** neutralizes the orphaned tool blocks into plain text - and sends no ``toolConfig`` (see #24158, #27138). It no longer raises. - - **Anthropic** (and Bedrock Invoke via ``AnthropicConfig.transform_request``) - always get a dummy tool so CLIs work with ``modify_params`` left off. - - Reference Issue: https://github.com/BerriAI/litellm/issues/24158, https://github.com/BerriAI/litellm/issues/27138 + Anthropic (and Bedrock Invoke via ``AnthropicConfig.transform_request``) + inject a dummy tool so CLIs work with ``modify_params`` left off. Bedrock + Converse's no-raise behavior is covered offline in + ``tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py`` + (see #24158, #27138), which needs no live credentials. """ # Force modify_params off as a clean baseline: it exercises the Anthropic # dummy-tool path, which injects regardless of modify_params @@ -355,26 +340,14 @@ def test_parallel_function_call_anthropic_error_msg( litellm.modify_params = False try: litellm.set_verbose = True - - if expect_unsupported_params_error: - with pytest.raises(litellm.UnsupportedParamsError) as e: - second_response = litellm.completion( - model=model, - messages=messages, - temperature=0.2, - seed=22, - drop_params=True, - ) # get a new response from the model where it can see the function response - print("second response\n", second_response) - else: - second_response = litellm.completion( - model=model, - messages=messages, - temperature=0.2, - seed=22, - drop_params=True, - ) # get a new response from the model where it can see the function response - print("second response\n", second_response) + second_response = litellm.completion( + model=model, + messages=messages, + temperature=0.2, + seed=22, + drop_params=True, + ) # get a new response from the model where it can see the function response + print("second response\n", second_response) except litellm.InternalServerError as e: print(e) except litellm.RateLimitError as e: diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 8750d587dda..e608c2c73b6 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -5523,14 +5523,22 @@ def test_neutralize_orphaned_tool_blocks_non_text_result_marked_not_empty(): "role": "assistant", "content": None, "tool_calls": [ - {"id": "c1", "type": "function", - "function": {"name": "render", "arguments": "{}"}} + { + "id": "c1", + "type": "function", + "function": {"name": "render", "arguments": "{}"}, + } ], }, { "role": "tool", "tool_call_id": "c1", - "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}], + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,AAAA"}, + } + ], }, ] @@ -5538,7 +5546,9 @@ def test_neutralize_orphaned_tool_blocks_non_text_result_marked_not_empty(): messages, optional_params={} ) - rewritten = next(m for m in result if m.get("role") == "user" and m is not messages[0]) + rewritten = next( + m for m in result if m.get("role") == "user" and m is not messages[0] + ) text = rewritten["content"] assert text.strip() # never empty assert "non-text tool result omitted" in text @@ -5756,3 +5766,56 @@ def test_transform_request_with_tools_still_builds_toolconfig(monkeypatch): ) assert "toolConfig" in result + + +def test_transform_request_flag_off_restores_raise(monkeypatch): + """Opt-out: with bedrock_neutralize_orphaned_tool_blocks=False and + modify_params=False, the legacy UnsupportedParamsError contract is restored.""" + monkeypatch.setattr(litellm, "bedrock_neutralize_orphaned_tool_blocks", False) + monkeypatch.setattr(litellm, "modify_params", False) + config = AmazonConverseConfig() + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="without `tools="): + config.transform_request( + model="us.anthropic.claude-opus-4-5-20251101-v1:0", + messages=_orphaned_tool_history_messages(), + optional_params={}, + litellm_params={}, + headers={}, + ) + + +def test_transform_request_flag_off_with_modify_params_restores_dummy_tool(monkeypatch): + """Opt-out: with the flag off and modify_params=True, the legacy dummy-tool + injection is restored (a toolConfig is produced, not neutralized text).""" + monkeypatch.setattr(litellm, "bedrock_neutralize_orphaned_tool_blocks", False) + monkeypatch.setattr(litellm, "modify_params", True) + config = AmazonConverseConfig() + + result = config.transform_request( + model="us.anthropic.claude-opus-4-5-20251101-v1:0", + messages=_orphaned_tool_history_messages(), + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert "toolConfig" in result + assert "dummy_tool" in json.dumps(result) + + +def test_transform_request_flag_on_is_default(monkeypatch): + """Default-on: without touching the flag, neutralization is the behavior.""" + monkeypatch.setattr(litellm, "modify_params", False) + config = AmazonConverseConfig() + + assert litellm.bedrock_neutralize_orphaned_tool_blocks is True + result = config.transform_request( + model="us.anthropic.claude-opus-4-5-20251101-v1:0", + messages=_orphaned_tool_history_messages(), + optional_params={}, + litellm_params={}, + headers={}, + ) + + _assert_no_structured_tool_blocks(result)