diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fe8387476ee..4d3ceca2b6e 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2235,6 +2235,45 @@ def _is_orphaned_tool_result( return False + + +def _merge_consecutive_assistant_messages(messages: List[AllMessageValues]) -> List[AllMessageValues]: + if not messages: + return [] + + merged: List[AllMessageValues] = [] + current_msg: Optional[AllMessageValues] = None + + for msg in messages: + if current_msg is None: + current_msg = copy.deepcopy(msg) + continue + + if msg.get("role") == "assistant" and current_msg.get("role") == "assistant": + # Merge content + if isinstance(current_msg.get("content"), str) and isinstance(msg.get("content"), str): + current_msg["content"] += "\n" + msg["content"] + elif isinstance(current_msg.get("content"), list) and isinstance(msg.get("content"), list): + current_msg["content"].extend(msg["content"]) + elif isinstance(current_msg.get("content"), str) and isinstance(msg.get("content"), list): + current_msg["content"] = [{"type": "text", "text": current_msg["content"]}] + msg["content"] + elif isinstance(current_msg.get("content"), list) and isinstance(msg.get("content"), str): + current_msg["content"].append({"type": "text", "text": msg["content"]}) + + # Merge tool_calls + if "tool_calls" in msg: + if "tool_calls" not in current_msg: + current_msg["tool_calls"] = [] + current_msg["tool_calls"].extend(msg["tool_calls"]) + else: + merged.append(current_msg) + current_msg = copy.deepcopy(msg) + + if current_msg: + merged.append(current_msg) + + return merged + def sanitize_messages_for_tool_calling( messages: List[AllMessageValues], ) -> List[AllMessageValues]: @@ -2263,6 +2302,8 @@ def sanitize_messages_for_tool_calling( if not litellm.modify_params: return messages + messages = _merge_consecutive_assistant_messages(messages) + sanitized_messages: List[AllMessageValues] = [] i = 0 @@ -2340,6 +2381,160 @@ def sanitize_messages_for_tool_calling( return sanitized_messages +def sanitize_anthropic_native_messages_for_tool_calling( + messages: List[Dict], +) -> List[Dict]: + """ + Sanitize Anthropic-native format messages for tool calling issues. + + This handles the case where messages use Anthropic's native format + (content is a list containing tool_use/tool_result blocks) rather than + OpenAI format (tool_calls field). This is needed for /v1/messages pass-through + requests where modify_params=True is set. + + Case A: Orphaned tool_use blocks in assistant messages + - If an assistant message has tool_use blocks in its content list, but the + next user message does not have corresponding tool_result blocks, add + dummy tool_result blocks to the next user message (or insert a new user + message if none exists). + + Case B: Orphaned tool_result blocks in user messages + - If a user message has tool_result blocks that reference tool_use ids + not found in the previous assistant message, remove those tool_result blocks. + + Only applies when litellm.modify_params is True. + """ + if not litellm.modify_params: + return messages + + if not messages: + return messages + + sanitized: List[Dict] = [] + i = 0 + + while i < len(messages): + current_msg = messages[i] + role = current_msg.get("role") + + if role == "assistant": + content = current_msg.get("content") + if isinstance(content, list): + # Case C: Reorder blocks so all text blocks precede tool_use blocks. + # Anthropic rejects content where a text block appears after a tool_use block + # (e.g. the pattern [text, tool_use, text, tool_use] produced by context + # compaction merging two assistant turns into one). + has_text_after_tool_use = False + seen_tool_use = False + for block in content: + if isinstance(block, dict): + btype = block.get("type") + if btype == "tool_use": + seen_tool_use = True + elif btype == "text" and seen_tool_use: + has_text_after_tool_use = True + break + + if has_text_after_tool_use: + import copy as _copy + text_blocks = [b for b in content if isinstance(b, dict) and b.get("type") == "text"] + non_text_blocks = [b for b in content if not (isinstance(b, dict) and b.get("type") == "text")] + reordered_msg = _copy.deepcopy(current_msg) + reordered_msg["content"] = text_blocks + non_text_blocks + current_msg = reordered_msg + verbose_logger.debug( + "sanitize_anthropic_native_messages_for_tool_calling: " + "Reordered assistant content blocks to put text before tool_use." + ) + content = reordered_msg["content"] + + # Collect tool_use ids from this assistant message (Anthropic native format) + tool_use_ids: List[str] = [] + tool_use_names: Dict[str, str] = {} + if isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + tool_id = block.get("id") + if tool_id: + tool_use_ids.append(tool_id) + tool_use_names[tool_id] = block.get("name", "unknown_tool") + + if tool_use_ids: + # Look ahead: find tool_result blocks in the next user message + found_tool_result_ids: set = set() + next_user_msg: Optional[Dict] = None + next_user_idx: Optional[int] = None + + if i + 1 < len(messages): + candidate = messages[i + 1] + if candidate.get("role") == "user": + next_user_msg = candidate + next_user_idx = i + 1 + candidate_content = candidate.get("content", []) + if isinstance(candidate_content, list): + for block in candidate_content: + if ( + isinstance(block, dict) + and block.get("type") == "tool_result" + ): + tid = block.get("tool_use_id") + if tid: + found_tool_result_ids.add(tid) + + missing_ids = [ + tid for tid in tool_use_ids if tid not in found_tool_result_ids + ] + + if missing_ids: + verbose_logger.debug( + f"sanitize_anthropic_native_messages_for_tool_calling: " + f"Found {len(missing_ids)} orphaned tool_use blocks. Adding dummy tool_results." + ) + sanitized.append(current_msg) + + dummy_tool_results = [ + { + "type": "tool_result", + "tool_use_id": tid, + "content": ( + f"[System: Tool execution skipped/interrupted. " + f"No result provided for tool '{tool_use_names.get(tid, 'unknown_tool')}'.]" + ), + } + for tid in missing_ids + ] + + if next_user_msg is not None and next_user_idx is not None: + # Prepend dummy results to existing user message content + existing_content = next_user_msg.get("content", []) + if isinstance(existing_content, list): + merged_content = dummy_tool_results + existing_content + else: + # existing_content is a string; wrap it + merged_content = dummy_tool_results + [ + {"type": "text", "text": existing_content} + ] + import copy + + patched_user_msg = copy.deepcopy(next_user_msg) + patched_user_msg["content"] = merged_content + sanitized.append(patched_user_msg) + i = next_user_idx + 1 + else: + # No following user message; insert one + sanitized.append( + {"role": "user", "content": dummy_tool_results} + ) + i += 1 + continue + + # Default: keep message as-is + sanitized.append(current_msg) + i += 1 + + return sanitized + + def anthropic_messages_pt( # noqa: PLR0915 messages: List[AllMessageValues], model: str, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 0c59e812e0b..d44b3027a1a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -336,6 +336,12 @@ def anthropic_messages_handler( """ from litellm.types.utils import LlmProviders + from litellm.litellm_core_utils.prompt_templates.factory import ( + sanitize_anthropic_native_messages_for_tool_calling, + ) + + messages = sanitize_anthropic_native_messages_for_tool_calling(messages) + metadata = validate_anthropic_api_metadata(metadata) local_vars = locals() diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_native_message_sanitization.py b/tests/test_litellm/llms/anthropic/test_anthropic_native_message_sanitization.py new file mode 100644 index 00000000000..e5ce00e61cc --- /dev/null +++ b/tests/test_litellm/llms/anthropic/test_anthropic_native_message_sanitization.py @@ -0,0 +1,299 @@ +""" +Tests for sanitize_anthropic_native_messages_for_tool_calling(). + +Covers the /v1/messages pass-through path where messages arrive in +Anthropic-native format (content is a list with tool_use/tool_result blocks) +rather than OpenAI format (tool_calls field). + +Bug scenario: + An Anthropic API client (e.g. an agentic coding tool) sends /v1/messages + requests through LiteLLM proxy. After context compaction, two separate + assistant turns may be merged into a single message whose content list + looks like: + + [text_A, tool_use_A, text_B, tool_use_B] + + Anthropic rejects this with: + "messages.N: `tool_use` ids were found without `tool_result` blocks + immediately after: " + + because text blocks must not appear *after* tool_use blocks in the same + assistant message. + + The fix reorders content so all text blocks precede all tool_use blocks. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../.."))) + +import litellm +from litellm.litellm_core_utils.prompt_templates.factory import ( + sanitize_anthropic_native_messages_for_tool_calling, +) + + +class TestSanitizeAnthropicNativeMessages: + def setup_method(self): + self._orig = litellm.modify_params + litellm.modify_params = True + + def teardown_method(self): + litellm.modify_params = self._orig + + # ------------------------------------------------------------------ + # Case C – text block appears after tool_use block (the primary bug) + # ------------------------------------------------------------------ + + def test_text_after_tool_use_reordered(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "do work"}]}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Thinking from first turn."}, + {"type": "tool_use", "id": "toolu_aaa", "name": "tool_a", "input": {}}, + {"type": "text", "text": "Thinking from second turn."}, + {"type": "tool_use", "id": "toolu_bbb", "name": "tool_b", "input": {}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_aaa", "content": "result_a"}, + {"type": "tool_result", "tool_use_id": "toolu_bbb", "content": "result_b"}, + ], + }, + ] + + result = sanitize_anthropic_native_messages_for_tool_calling(messages) + + assert len(result) == 3 + content = result[1]["content"] + types = [b["type"] for b in content] + assert types == ["text", "text", "tool_use", "tool_use"], ( + f"Expected all text blocks before tool_use blocks, got: {types}" + ) + assert content[2]["id"] == "toolu_aaa" + assert content[3]["id"] == "toolu_bbb" + + def test_valid_text_before_tool_use_unchanged(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "go"}]}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "I will call two tools."}, + {"type": "tool_use", "id": "toolu_aaa", "name": "tool_a", "input": {}}, + {"type": "tool_use", "id": "toolu_bbb", "name": "tool_b", "input": {}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_aaa", "content": "r1"}, + {"type": "tool_result", "tool_use_id": "toolu_bbb", "content": "r2"}, + ], + }, + ] + + result = sanitize_anthropic_native_messages_for_tool_calling(messages) + + assert result == messages + + def test_no_text_blocks_unchanged(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "go"}]}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_aaa", "name": "tool_a", "input": {}}, + {"type": "tool_use", "id": "toolu_bbb", "name": "tool_b", "input": {}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_aaa", "content": "r1"}, + {"type": "tool_result", "tool_use_id": "toolu_bbb", "content": "r2"}, + ], + }, + ] + + result = sanitize_anthropic_native_messages_for_tool_calling(messages) + + assert result == messages + + # ------------------------------------------------------------------ + # Case A – orphaned tool_use (no matching tool_result) + # ------------------------------------------------------------------ + + def test_orphaned_tool_use_injects_dummy_user_message(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "do something"}]}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Sure."}, + {"type": "tool_use", "id": "toolu_orphan", "name": "my_tool", "input": {}}, + ], + }, + ] + + result = sanitize_anthropic_native_messages_for_tool_calling(messages) + + assert len(result) == 3 + injected = result[2] + assert injected["role"] == "user" + assert isinstance(injected["content"], list) + tool_results = [b for b in injected["content"] if b.get("type") == "tool_result"] + assert len(tool_results) == 1 + assert tool_results[0]["tool_use_id"] == "toolu_orphan" + assert "my_tool" in tool_results[0]["content"] + + def test_orphaned_tool_use_prepends_to_existing_user_message(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "start"}]}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_x", "name": "tool_x", "input": {}} + ], + }, + { + "role": "user", + "content": [{"type": "text", "text": "never mind"}], + }, + ] + + result = sanitize_anthropic_native_messages_for_tool_calling(messages) + + assert len(result) == 3 + content = result[2]["content"] + assert isinstance(content, list) + tool_results = [b for b in content if b.get("type") == "tool_result"] + text_blocks = [b for b in content if b.get("type") == "text"] + assert len(tool_results) == 1 + assert tool_results[0]["tool_use_id"] == "toolu_x" + assert len(text_blocks) == 1 + + def test_orphaned_tool_use_string_user_message_wrapped(self): + messages = [ + {"role": "user", "content": "start"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_y", "name": "tool_y", "input": {}} + ], + }, + {"role": "user", "content": "follow-up as plain string"}, + ] + + result = sanitize_anthropic_native_messages_for_tool_calling(messages) + + assert len(result) == 3 + content = result[2]["content"] + assert isinstance(content, list) + types = {b.get("type") for b in content} + assert "tool_result" in types + assert "text" in types + + def test_partial_tool_results_get_dummies_for_missing_ids(self): + messages = [ + {"role": "user", "content": "run two tools"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "tid_1", "name": "tool_a", "input": {}}, + {"type": "tool_use", "id": "tid_2", "name": "tool_b", "input": {}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "tid_1", "content": "result_a"} + ], + }, + ] + + result = sanitize_anthropic_native_messages_for_tool_calling(messages) + + assert len(result) == 3 + user_content = result[2]["content"] + result_ids = {b["tool_use_id"] for b in user_content if b.get("type") == "tool_result"} + assert "tid_1" in result_ids + assert "tid_2" in result_ids + + def test_complete_tool_use_result_pair_passes_through_unchanged(self): + messages = [ + {"role": "user", "content": "ping"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_z", "name": "ping_tool", "input": {}} + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_z", "content": "pong"} + ], + }, + ] + + result = sanitize_anthropic_native_messages_for_tool_calling(messages) + + assert result == messages + + # ------------------------------------------------------------------ + # modify_params gate + # ------------------------------------------------------------------ + + def test_modify_params_false_skips_all_sanitization(self): + litellm.modify_params = False + + messages = [ + {"role": "user", "content": "go"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "A"}, + {"type": "tool_use", "id": "toolu_a", "name": "t", "input": {}}, + {"type": "text", "text": "B"}, + ], + }, + ] + + result = sanitize_anthropic_native_messages_for_tool_calling(messages) + + assert result == messages + + # ------------------------------------------------------------------ + # Edge cases + # ------------------------------------------------------------------ + + def test_empty_messages_returns_empty(self): + assert sanitize_anthropic_native_messages_for_tool_calling([]) == [] + + def test_no_tool_use_messages_unchanged(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "hello"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "hi"}]}, + ] + + result = sanitize_anthropic_native_messages_for_tool_calling(messages) + + assert result == messages + + def test_string_content_messages_unchanged(self): + messages = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "hi there"}, + ] + + result = sanitize_anthropic_native_messages_for_tool_calling(messages) + + assert result == messages