From f4e976e22570d23d9aed896d85a8bbcbd6c9b2db Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 22 Apr 2026 09:45:21 +0530 Subject: [PATCH] fix(anthropic): handle response_format tool alongside user tools in non-streaming Non-streaming path required len(tool_calls)==1 to unwrap json_tool_call, so mixed user tools leaked the internal tool. Align with Bedrock converse handling: strip internal tools, merge structured JSON into content. Made-with: Cursor --- litellm/llms/anthropic/chat/transformation.py | 79 ++++++++++++++----- .../test_anthropic_chat_transformation.py | 34 ++++++++ 2 files changed, 93 insertions(+), 20 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index cd5bb731717..4a6743b74be 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1549,25 +1549,56 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) data["output_config"] = output_config + def _resolve_json_mode_non_streaming( + self, + json_mode: Optional[bool], + tool_calls: List[ChatCompletionToolCallChunk], + ) -> Tuple[ + Optional[LitellmMessage], + List[ChatCompletionToolCallChunk], + Optional[str], + ]: + """Strip internal response_format tool calls; merge payload into content when mixed with user tools.""" + if json_mode is not True or not tool_calls: + return None, tool_calls, None + + json_indices = [ + i + for i, t in enumerate(tool_calls) + if t.get("function", {}).get("name") == RESPONSE_FORMAT_TOOL_NAME + ] + if not json_indices: + return None, tool_calls, None + + if len(json_indices) == len(tool_calls): + json_tool = tool_calls[json_indices[0]] + if json_tool.get("function", {}).get("arguments") is None: + return None, tool_calls, None + _message = AnthropicConfig._convert_tool_response_to_message( + tool_calls=[json_tool] + ) + return _message, [], None + + first_json = tool_calls[json_indices[0]] + json_msg = AnthropicConfig._convert_tool_response_to_message([first_json]) + extra_content: Optional[str] = ( + json_msg.content if json_msg is not None else None + ) + filtered_tools = [ + t for i, t in enumerate(tool_calls) if i not in json_indices + ] + return None, filtered_tools, extra_content + def _transform_response_for_json_mode( self, json_mode: Optional[bool], tool_calls: List[ChatCompletionToolCallChunk], ) -> Optional[LitellmMessage]: - _message: Optional[LitellmMessage] = None - if json_mode is True and len(tool_calls) == 1: - # check if tool name is the default tool name - json_mode_content_str: Optional[str] = None - if ( - "name" in tool_calls[0]["function"] - and tool_calls[0]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME - ): - json_mode_content_str = tool_calls[0]["function"].get("arguments") - if json_mode_content_str is not None: - _message = AnthropicConfig._convert_tool_response_to_message( - tool_calls=tool_calls, - ) - return _message + replacement, _, _ = self._resolve_json_mode_non_streaming( + json_mode=json_mode, + tool_calls=tool_calls, + ) + return replacement def extract_response_content(self, completion_response: dict) -> Tuple[ str, @@ -1927,19 +1958,27 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_calls, ) + json_mode_message, tool_calls_for_message, json_extra_content = ( + self._resolve_json_mode_non_streaming( + json_mode=json_mode, + tool_calls=tool_calls, + ) + ) + merged_text = text_content or "" + if json_extra_content: + merged_text = ( + merged_text + json_extra_content if merged_text else json_extra_content + ) + _message = litellm.Message( - tool_calls=tool_calls, - content=text_content or None, + tool_calls=tool_calls_for_message, + content=merged_text or None, provider_specific_fields=provider_specific_fields, thinking_blocks=thinking_blocks, reasoning_content=reasoning_content, ) _message.provider_specific_fields = provider_specific_fields - json_mode_message = self._transform_response_for_json_mode( - json_mode=json_mode, - tool_calls=tool_calls, - ) if json_mode_message is not None: completion_response["stop_reason"] = "stop" _message = json_mode_message diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index a7f5f92ab05..84a985b275f 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -8,6 +8,7 @@ sys.path.insert( ) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -38,6 +39,39 @@ def test_response_format_transformation_unit_test(): print(result) +def test_anthropic_json_mode_non_streaming_mixed_internal_and_user_tools(): + """Non-streaming + response_format: internal json tool must not require len(tool_calls)==1.""" + config = AnthropicConfig() + tool_calls = [ + { + "id": "toolu_json", + "type": "function", + "function": { + "name": RESPONSE_FORMAT_TOOL_NAME, + "arguments": '{"values": {"answer": 42}}', + }, + "index": 0, + }, + { + "id": "toolu_user", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NY"}', + }, + "index": 1, + }, + ] + replacement, filtered, extra = config._resolve_json_mode_non_streaming( + json_mode=True, + tool_calls=tool_calls, + ) + assert replacement is None + assert len(filtered) == 1 + assert filtered[0]["function"]["name"] == "get_weather" + assert extra == '{"answer": 42}' + + def test_calculate_usage(): """ Do not include cache_creation_input_tokens in the prompt_tokens