mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(anthropic): sanitize /v1/messages pass-through for tool_use content ordering
When an Anthropic API client uses the /v1/messages endpoint with context compaction, two assistant turns can be merged into one message with content: [text_A, tool_use_A, text_B, tool_use_B] Anthropic rejects this: "tool_use ids were found without tool_result blocks immediately after". The fix reorders content so all text blocks precede tool_use blocks, and injects dummy tool_results for any orphaned tool_use. Only active when modify_params=True. Adds sanitize_anthropic_native_messages_for_tool_calling() in factory.py and calls it from anthropic_messages_handler() in handler.py. Tests: tests/test_litellm/llms/anthropic/test_anthropic_native_message_sanitization.py
This commit is contained in:
xy.kong
xykong
parent
3e1479c052
commit
11c931540e
3 changed files with 500 additions and 0 deletions
|
|
@ -2235,6 +2235,45 @@ def _is_orphaned_tool_result(
|
|||
return False
|
||||
|
||||
|
||||
|
||||
|
||||
def _merge_consecutive_assistant_messages(messages: List[AllMessageValues]) -> List[AllMessageValues]:
|
||||
if not messages:
|
||||
return []
|
||||
|
||||
merged: List[AllMessageValues] = []
|
||||
current_msg: Optional[AllMessageValues] = None
|
||||
|
||||
for msg in messages:
|
||||
if current_msg is None:
|
||||
current_msg = copy.deepcopy(msg)
|
||||
continue
|
||||
|
||||
if msg.get("role") == "assistant" and current_msg.get("role") == "assistant":
|
||||
# Merge content
|
||||
if isinstance(current_msg.get("content"), str) and isinstance(msg.get("content"), str):
|
||||
current_msg["content"] += "\n" + msg["content"]
|
||||
elif isinstance(current_msg.get("content"), list) and isinstance(msg.get("content"), list):
|
||||
current_msg["content"].extend(msg["content"])
|
||||
elif isinstance(current_msg.get("content"), str) and isinstance(msg.get("content"), list):
|
||||
current_msg["content"] = [{"type": "text", "text": current_msg["content"]}] + msg["content"]
|
||||
elif isinstance(current_msg.get("content"), list) and isinstance(msg.get("content"), str):
|
||||
current_msg["content"].append({"type": "text", "text": msg["content"]})
|
||||
|
||||
# Merge tool_calls
|
||||
if "tool_calls" in msg:
|
||||
if "tool_calls" not in current_msg:
|
||||
current_msg["tool_calls"] = []
|
||||
current_msg["tool_calls"].extend(msg["tool_calls"])
|
||||
else:
|
||||
merged.append(current_msg)
|
||||
current_msg = copy.deepcopy(msg)
|
||||
|
||||
if current_msg:
|
||||
merged.append(current_msg)
|
||||
|
||||
return merged
|
||||
|
||||
def sanitize_messages_for_tool_calling(
|
||||
messages: List[AllMessageValues],
|
||||
) -> List[AllMessageValues]:
|
||||
|
|
@ -2263,6 +2302,8 @@ def sanitize_messages_for_tool_calling(
|
|||
if not litellm.modify_params:
|
||||
return messages
|
||||
|
||||
messages = _merge_consecutive_assistant_messages(messages)
|
||||
|
||||
sanitized_messages: List[AllMessageValues] = []
|
||||
i = 0
|
||||
|
||||
|
|
@ -2340,6 +2381,160 @@ def sanitize_messages_for_tool_calling(
|
|||
return sanitized_messages
|
||||
|
||||
|
||||
def sanitize_anthropic_native_messages_for_tool_calling(
|
||||
messages: List[Dict],
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Sanitize Anthropic-native format messages for tool calling issues.
|
||||
|
||||
This handles the case where messages use Anthropic's native format
|
||||
(content is a list containing tool_use/tool_result blocks) rather than
|
||||
OpenAI format (tool_calls field). This is needed for /v1/messages pass-through
|
||||
requests where modify_params=True is set.
|
||||
|
||||
Case A: Orphaned tool_use blocks in assistant messages
|
||||
- If an assistant message has tool_use blocks in its content list, but the
|
||||
next user message does not have corresponding tool_result blocks, add
|
||||
dummy tool_result blocks to the next user message (or insert a new user
|
||||
message if none exists).
|
||||
|
||||
Case B: Orphaned tool_result blocks in user messages
|
||||
- If a user message has tool_result blocks that reference tool_use ids
|
||||
not found in the previous assistant message, remove those tool_result blocks.
|
||||
|
||||
Only applies when litellm.modify_params is True.
|
||||
"""
|
||||
if not litellm.modify_params:
|
||||
return messages
|
||||
|
||||
if not messages:
|
||||
return messages
|
||||
|
||||
sanitized: List[Dict] = []
|
||||
i = 0
|
||||
|
||||
while i < len(messages):
|
||||
current_msg = messages[i]
|
||||
role = current_msg.get("role")
|
||||
|
||||
if role == "assistant":
|
||||
content = current_msg.get("content")
|
||||
if isinstance(content, list):
|
||||
# Case C: Reorder blocks so all text blocks precede tool_use blocks.
|
||||
# Anthropic rejects content where a text block appears after a tool_use block
|
||||
# (e.g. the pattern [text, tool_use, text, tool_use] produced by context
|
||||
# compaction merging two assistant turns into one).
|
||||
has_text_after_tool_use = False
|
||||
seen_tool_use = False
|
||||
for block in content:
|
||||
if isinstance(block, dict):
|
||||
btype = block.get("type")
|
||||
if btype == "tool_use":
|
||||
seen_tool_use = True
|
||||
elif btype == "text" and seen_tool_use:
|
||||
has_text_after_tool_use = True
|
||||
break
|
||||
|
||||
if has_text_after_tool_use:
|
||||
import copy as _copy
|
||||
text_blocks = [b for b in content if isinstance(b, dict) and b.get("type") == "text"]
|
||||
non_text_blocks = [b for b in content if not (isinstance(b, dict) and b.get("type") == "text")]
|
||||
reordered_msg = _copy.deepcopy(current_msg)
|
||||
reordered_msg["content"] = text_blocks + non_text_blocks
|
||||
current_msg = reordered_msg
|
||||
verbose_logger.debug(
|
||||
"sanitize_anthropic_native_messages_for_tool_calling: "
|
||||
"Reordered assistant content blocks to put text before tool_use."
|
||||
)
|
||||
content = reordered_msg["content"]
|
||||
|
||||
# Collect tool_use ids from this assistant message (Anthropic native format)
|
||||
tool_use_ids: List[str] = []
|
||||
tool_use_names: Dict[str, str] = {}
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "tool_use":
|
||||
tool_id = block.get("id")
|
||||
if tool_id:
|
||||
tool_use_ids.append(tool_id)
|
||||
tool_use_names[tool_id] = block.get("name", "unknown_tool")
|
||||
|
||||
if tool_use_ids:
|
||||
# Look ahead: find tool_result blocks in the next user message
|
||||
found_tool_result_ids: set = set()
|
||||
next_user_msg: Optional[Dict] = None
|
||||
next_user_idx: Optional[int] = None
|
||||
|
||||
if i + 1 < len(messages):
|
||||
candidate = messages[i + 1]
|
||||
if candidate.get("role") == "user":
|
||||
next_user_msg = candidate
|
||||
next_user_idx = i + 1
|
||||
candidate_content = candidate.get("content", [])
|
||||
if isinstance(candidate_content, list):
|
||||
for block in candidate_content:
|
||||
if (
|
||||
isinstance(block, dict)
|
||||
and block.get("type") == "tool_result"
|
||||
):
|
||||
tid = block.get("tool_use_id")
|
||||
if tid:
|
||||
found_tool_result_ids.add(tid)
|
||||
|
||||
missing_ids = [
|
||||
tid for tid in tool_use_ids if tid not in found_tool_result_ids
|
||||
]
|
||||
|
||||
if missing_ids:
|
||||
verbose_logger.debug(
|
||||
f"sanitize_anthropic_native_messages_for_tool_calling: "
|
||||
f"Found {len(missing_ids)} orphaned tool_use blocks. Adding dummy tool_results."
|
||||
)
|
||||
sanitized.append(current_msg)
|
||||
|
||||
dummy_tool_results = [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": tid,
|
||||
"content": (
|
||||
f"[System: Tool execution skipped/interrupted. "
|
||||
f"No result provided for tool '{tool_use_names.get(tid, 'unknown_tool')}'.]"
|
||||
),
|
||||
}
|
||||
for tid in missing_ids
|
||||
]
|
||||
|
||||
if next_user_msg is not None and next_user_idx is not None:
|
||||
# Prepend dummy results to existing user message content
|
||||
existing_content = next_user_msg.get("content", [])
|
||||
if isinstance(existing_content, list):
|
||||
merged_content = dummy_tool_results + existing_content
|
||||
else:
|
||||
# existing_content is a string; wrap it
|
||||
merged_content = dummy_tool_results + [
|
||||
{"type": "text", "text": existing_content}
|
||||
]
|
||||
import copy
|
||||
|
||||
patched_user_msg = copy.deepcopy(next_user_msg)
|
||||
patched_user_msg["content"] = merged_content
|
||||
sanitized.append(patched_user_msg)
|
||||
i = next_user_idx + 1
|
||||
else:
|
||||
# No following user message; insert one
|
||||
sanitized.append(
|
||||
{"role": "user", "content": dummy_tool_results}
|
||||
)
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# Default: keep message as-is
|
||||
sanitized.append(current_msg)
|
||||
i += 1
|
||||
|
||||
return sanitized
|
||||
|
||||
|
||||
def anthropic_messages_pt( # noqa: PLR0915
|
||||
messages: List[AllMessageValues],
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -336,6 +336,12 @@ def anthropic_messages_handler(
|
|||
"""
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
sanitize_anthropic_native_messages_for_tool_calling,
|
||||
)
|
||||
|
||||
messages = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
metadata = validate_anthropic_api_metadata(metadata)
|
||||
|
||||
local_vars = locals()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,299 @@
|
|||
"""
|
||||
Tests for sanitize_anthropic_native_messages_for_tool_calling().
|
||||
|
||||
Covers the /v1/messages pass-through path where messages arrive in
|
||||
Anthropic-native format (content is a list with tool_use/tool_result blocks)
|
||||
rather than OpenAI format (tool_calls field).
|
||||
|
||||
Bug scenario:
|
||||
An Anthropic API client (e.g. an agentic coding tool) sends /v1/messages
|
||||
requests through LiteLLM proxy. After context compaction, two separate
|
||||
assistant turns may be merged into a single message whose content list
|
||||
looks like:
|
||||
|
||||
[text_A, tool_use_A, text_B, tool_use_B]
|
||||
|
||||
Anthropic rejects this with:
|
||||
"messages.N: `tool_use` ids were found without `tool_result` blocks
|
||||
immediately after: <id>"
|
||||
|
||||
because text blocks must not appear *after* tool_use blocks in the same
|
||||
assistant message.
|
||||
|
||||
The fix reorders content so all text blocks precede all tool_use blocks.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")))
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
sanitize_anthropic_native_messages_for_tool_calling,
|
||||
)
|
||||
|
||||
|
||||
class TestSanitizeAnthropicNativeMessages:
|
||||
def setup_method(self):
|
||||
self._orig = litellm.modify_params
|
||||
litellm.modify_params = True
|
||||
|
||||
def teardown_method(self):
|
||||
litellm.modify_params = self._orig
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Case C – text block appears after tool_use block (the primary bug)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_text_after_tool_use_reordered(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "do work"}]},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Thinking from first turn."},
|
||||
{"type": "tool_use", "id": "toolu_aaa", "name": "tool_a", "input": {}},
|
||||
{"type": "text", "text": "Thinking from second turn."},
|
||||
{"type": "tool_use", "id": "toolu_bbb", "name": "tool_b", "input": {}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_aaa", "content": "result_a"},
|
||||
{"type": "tool_result", "tool_use_id": "toolu_bbb", "content": "result_b"},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
assert len(result) == 3
|
||||
content = result[1]["content"]
|
||||
types = [b["type"] for b in content]
|
||||
assert types == ["text", "text", "tool_use", "tool_use"], (
|
||||
f"Expected all text blocks before tool_use blocks, got: {types}"
|
||||
)
|
||||
assert content[2]["id"] == "toolu_aaa"
|
||||
assert content[3]["id"] == "toolu_bbb"
|
||||
|
||||
def test_valid_text_before_tool_use_unchanged(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "go"}]},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "I will call two tools."},
|
||||
{"type": "tool_use", "id": "toolu_aaa", "name": "tool_a", "input": {}},
|
||||
{"type": "tool_use", "id": "toolu_bbb", "name": "tool_b", "input": {}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_aaa", "content": "r1"},
|
||||
{"type": "tool_result", "tool_use_id": "toolu_bbb", "content": "r2"},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
assert result == messages
|
||||
|
||||
def test_no_text_blocks_unchanged(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "go"}]},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "tool_use", "id": "toolu_aaa", "name": "tool_a", "input": {}},
|
||||
{"type": "tool_use", "id": "toolu_bbb", "name": "tool_b", "input": {}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_aaa", "content": "r1"},
|
||||
{"type": "tool_result", "tool_use_id": "toolu_bbb", "content": "r2"},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
assert result == messages
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Case A – orphaned tool_use (no matching tool_result)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_orphaned_tool_use_injects_dummy_user_message(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "do something"}]},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Sure."},
|
||||
{"type": "tool_use", "id": "toolu_orphan", "name": "my_tool", "input": {}},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
assert len(result) == 3
|
||||
injected = result[2]
|
||||
assert injected["role"] == "user"
|
||||
assert isinstance(injected["content"], list)
|
||||
tool_results = [b for b in injected["content"] if b.get("type") == "tool_result"]
|
||||
assert len(tool_results) == 1
|
||||
assert tool_results[0]["tool_use_id"] == "toolu_orphan"
|
||||
assert "my_tool" in tool_results[0]["content"]
|
||||
|
||||
def test_orphaned_tool_use_prepends_to_existing_user_message(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "start"}]},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "tool_use", "id": "toolu_x", "name": "tool_x", "input": {}}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "never mind"}],
|
||||
},
|
||||
]
|
||||
|
||||
result = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
assert len(result) == 3
|
||||
content = result[2]["content"]
|
||||
assert isinstance(content, list)
|
||||
tool_results = [b for b in content if b.get("type") == "tool_result"]
|
||||
text_blocks = [b for b in content if b.get("type") == "text"]
|
||||
assert len(tool_results) == 1
|
||||
assert tool_results[0]["tool_use_id"] == "toolu_x"
|
||||
assert len(text_blocks) == 1
|
||||
|
||||
def test_orphaned_tool_use_string_user_message_wrapped(self):
|
||||
messages = [
|
||||
{"role": "user", "content": "start"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "tool_use", "id": "toolu_y", "name": "tool_y", "input": {}}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "follow-up as plain string"},
|
||||
]
|
||||
|
||||
result = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
assert len(result) == 3
|
||||
content = result[2]["content"]
|
||||
assert isinstance(content, list)
|
||||
types = {b.get("type") for b in content}
|
||||
assert "tool_result" in types
|
||||
assert "text" in types
|
||||
|
||||
def test_partial_tool_results_get_dummies_for_missing_ids(self):
|
||||
messages = [
|
||||
{"role": "user", "content": "run two tools"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "tool_use", "id": "tid_1", "name": "tool_a", "input": {}},
|
||||
{"type": "tool_use", "id": "tid_2", "name": "tool_b", "input": {}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "tid_1", "content": "result_a"}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
assert len(result) == 3
|
||||
user_content = result[2]["content"]
|
||||
result_ids = {b["tool_use_id"] for b in user_content if b.get("type") == "tool_result"}
|
||||
assert "tid_1" in result_ids
|
||||
assert "tid_2" in result_ids
|
||||
|
||||
def test_complete_tool_use_result_pair_passes_through_unchanged(self):
|
||||
messages = [
|
||||
{"role": "user", "content": "ping"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "tool_use", "id": "toolu_z", "name": "ping_tool", "input": {}}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_z", "content": "pong"}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
assert result == messages
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# modify_params gate
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_modify_params_false_skips_all_sanitization(self):
|
||||
litellm.modify_params = False
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "go"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "A"},
|
||||
{"type": "tool_use", "id": "toolu_a", "name": "t", "input": {}},
|
||||
{"type": "text", "text": "B"},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
assert result == messages
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Edge cases
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_empty_messages_returns_empty(self):
|
||||
assert sanitize_anthropic_native_messages_for_tool_calling([]) == []
|
||||
|
||||
def test_no_tool_use_messages_unchanged(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "hello"}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "hi"}]},
|
||||
]
|
||||
|
||||
result = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
assert result == messages
|
||||
|
||||
def test_string_content_messages_unchanged(self):
|
||||
messages = [
|
||||
{"role": "user", "content": "hello"},
|
||||
{"role": "assistant", "content": "hi there"},
|
||||
]
|
||||
|
||||
result = sanitize_anthropic_native_messages_for_tool_calling(messages)
|
||||
|
||||
assert result == messages
|
||||
Loading…
Add table
Reference in a new issue