mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-04 02:31:27 +00:00
fix(anthropic): skip tool-id rewrite only for anthropic pass-through
Host-only skip turned sanitization off for azure_ai and github_copilot, which still speak Anthropic's id charset (#32214). Skip only when custom_llm_provider is anthropic and api_base hostname is not api.anthropic.com. Keep rewriting for azure_ai, github_copilot, bedrock, vertex_ai, and empty api_base. Curl to localhost:4000: merge base forwarded functions_Bash_0; this branch forwards functions.Bash:0.
This commit is contained in:
Chaitanya Laxman
parent
45169b10a9
commit
87d542f079
4 changed files with 109 additions and 49 deletions
|
|
@ -8,6 +8,7 @@ from collections.abc import Mapping, MutableMapping, Sequence
|
|||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, Literal
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
|
@ -1240,37 +1241,42 @@ def _sanitize_tool_use_id_content_block(block: object) -> object:
|
|||
return block
|
||||
|
||||
|
||||
_ANTHROPIC_TOOL_ID_CHARSET_HOST_MARKERS: Final = frozenset(
|
||||
(
|
||||
"api.anthropic.com",
|
||||
"amazonaws.com",
|
||||
"googleapis.com",
|
||||
"cloud.google.com",
|
||||
)
|
||||
)
|
||||
_ANTHROPIC_TOOL_ID_CHARSET_HOSTNAME: Final = "api.anthropic.com"
|
||||
|
||||
|
||||
def _upstream_enforces_anthropic_tool_id_charset(api_base: str | None) -> bool:
|
||||
def _should_sanitize_anthropic_tool_use_ids(
|
||||
*,
|
||||
api_base: str | None,
|
||||
custom_llm_provider: str | None,
|
||||
) -> bool:
|
||||
if custom_llm_provider is not None and custom_llm_provider.casefold() != "anthropic":
|
||||
return True
|
||||
if api_base is None or not api_base.strip():
|
||||
return True
|
||||
host: Final = api_base.casefold()
|
||||
return any(marker in host for marker in _ANTHROPIC_TOOL_ID_CHARSET_HOST_MARKERS)
|
||||
hostname: Final = urlparse(api_base).hostname
|
||||
if hostname is None:
|
||||
return True
|
||||
return hostname.casefold() == _ANTHROPIC_TOOL_ID_CHARSET_HOSTNAME
|
||||
|
||||
|
||||
def sanitize_tool_use_ids_in_anthropic_messages(
|
||||
messages: list[Any],
|
||||
*,
|
||||
api_base: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Rewrite ``tool_use`` / ``server_tool_use`` ``id`` and ``tool_result``
|
||||
``tool_use_id`` values to Anthropic's ``^[a-zA-Z0-9_-]+$`` pattern.
|
||||
|
||||
No-op when ``api_base`` is a host that is not Anthropic, Bedrock, or Vertex.
|
||||
Those upstreams (vLLM, Kimi, SGLang) echo the original ids; rewriting them
|
||||
breaks the next tool_result turn. See #32214.
|
||||
No-op when ``custom_llm_provider`` is ``anthropic`` and ``api_base`` is a
|
||||
non-Anthropic host. vLLM/Kimi echo the original ids; rewriting them breaks
|
||||
the next tool_result turn. See #32214.
|
||||
"""
|
||||
if not _upstream_enforces_anthropic_tool_id_charset(api_base):
|
||||
if not _should_sanitize_anthropic_tool_use_ids(
|
||||
api_base=api_base,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
):
|
||||
return messages
|
||||
out: Final[list[Any]] = []
|
||||
for m in messages:
|
||||
|
|
|
|||
|
|
@ -263,7 +263,9 @@ async def anthropic_messages(
|
|||
messages = strip_empty_content_blocks_from_anthropic_messages(messages)
|
||||
# Replay of cross-provider tool history (e.g. kimi -> Anthropic) may carry
|
||||
# ids like ``functions.Bash:0`` that violate Anthropic's id pattern.
|
||||
messages = sanitize_tool_use_ids_in_anthropic_messages(messages, api_base=api_base)
|
||||
messages = sanitize_tool_use_ids_in_anthropic_messages(
|
||||
messages, api_base=api_base, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages)
|
||||
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
|
|
@ -460,7 +462,9 @@ def anthropic_messages_handler(
|
|||
# full-messages scan. Pop it so it never leaks into provider params.
|
||||
if not kwargs.pop("_litellm_messages_presanitized", False):
|
||||
messages = strip_empty_content_blocks_from_anthropic_messages(messages)
|
||||
messages = sanitize_tool_use_ids_in_anthropic_messages(messages, api_base=api_base)
|
||||
messages = sanitize_tool_use_ids_in_anthropic_messages(
|
||||
messages, api_base=api_base, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages)
|
||||
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
|
|
|
|||
|
|
@ -220,6 +220,49 @@ async def test_anthropic_messages_sanitizes_tool_use_ids_before_dispatch():
|
|||
assert msgs[0]["content"][0]["id"] == "functions.Bash:0"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_keeps_tool_use_ids_for_non_anthropic_api_base():
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages import handler
|
||||
|
||||
msgs = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "functions.Bash:0",
|
||||
"name": "Bash",
|
||||
"input": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
captured = {}
|
||||
|
||||
def fake_handler(*args, **kwargs):
|
||||
captured["messages"] = kwargs.get("messages")
|
||||
return "stub"
|
||||
|
||||
fake_loop = MagicMock()
|
||||
fake_loop.run_in_executor = lambda _e, func: _async_return(func())
|
||||
|
||||
with (
|
||||
patch.object(handler, "anthropic_messages_handler", side_effect=fake_handler),
|
||||
patch("asyncio.get_event_loop", return_value=fake_loop),
|
||||
):
|
||||
await handler.anthropic_messages(
|
||||
max_tokens=100,
|
||||
messages=msgs,
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
custom_llm_provider="anthropic",
|
||||
api_key="k",
|
||||
api_base="http://127.0.0.1:8000/v1",
|
||||
)
|
||||
|
||||
assert captured["messages"][0]["content"][0]["id"] == "functions.Bash:0"
|
||||
assert msgs[0]["content"][0]["id"] == "functions.Bash:0"
|
||||
|
||||
|
||||
async def _async_return(value):
|
||||
return value
|
||||
|
||||
|
|
@ -702,27 +745,6 @@ def test_handler_strips_when_no_presanitized_flag():
|
|||
assert result is not None
|
||||
|
||||
|
||||
def test_handler_forwards_api_base_to_tool_id_sanitize():
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages import handler
|
||||
|
||||
with patch.object(
|
||||
handler,
|
||||
"sanitize_tool_use_ids_in_anthropic_messages",
|
||||
wraps=handler.sanitize_tool_use_ids_in_anthropic_messages,
|
||||
) as spy:
|
||||
result = handler.anthropic_messages_handler(
|
||||
max_tokens=10,
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
model="anthropic/claude-3-5-sonnet-20241022",
|
||||
custom_llm_provider="anthropic",
|
||||
api_base="http://127.0.0.1:8000/v1",
|
||||
mock_response="hi there",
|
||||
)
|
||||
assert result is not None
|
||||
assert spy.call_count == 1
|
||||
assert spy.call_args.kwargs["api_base"] == "http://127.0.0.1:8000/v1"
|
||||
|
||||
|
||||
def test_handler_skips_strip_when_presanitized():
|
||||
"""Async wrapper already sanitized -> handler must NOT rescan."""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages import handler
|
||||
|
|
|
|||
|
|
@ -1823,12 +1823,38 @@ class TestAnthropicThinkingSignatureSelfHeal:
|
|||
},
|
||||
]
|
||||
out = sanitize_tool_use_ids_in_anthropic_messages(
|
||||
msgs, api_base="http://127.0.0.1:8000/v1"
|
||||
msgs, api_base="http://127.0.0.1:8000/v1", custom_llm_provider="anthropic"
|
||||
)
|
||||
assert out is msgs
|
||||
assert out[0]["content"][0]["id"] == "functions.Bash:0"
|
||||
assert out[1]["content"][0]["tool_use_id"] == "functions.Bash:0"
|
||||
|
||||
def test_sanitize_tool_use_ids_uses_url_hostname_not_query_string(self):
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
sanitize_tool_use_ids_in_anthropic_messages,
|
||||
)
|
||||
|
||||
msgs = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "functions.Bash:0",
|
||||
"name": "Bash",
|
||||
"input": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
out = sanitize_tool_use_ids_in_anthropic_messages(
|
||||
msgs,
|
||||
api_base="http://vllm:8000/v1?x=api.anthropic.com",
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
assert out is msgs
|
||||
assert out[0]["content"][0]["id"] == "functions.Bash:0"
|
||||
|
||||
def test_sanitize_tool_use_ids_still_runs_for_anthropic_hosts(self):
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
sanitize_tool_use_ids_in_anthropic_messages,
|
||||
|
|
@ -1847,17 +1873,19 @@ class TestAnthropicThinkingSignatureSelfHeal:
|
|||
],
|
||||
}
|
||||
]
|
||||
anthropic_hosts = (
|
||||
"",
|
||||
"https://api.anthropic.com",
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
"https://us-east5-aiplatform.googleapis.com",
|
||||
"https://aiplatform.googleapis.com",
|
||||
"https://cloud.google.com/vertex-ai",
|
||||
still_sanitize = (
|
||||
("anthropic", ""),
|
||||
("anthropic", "https://api.anthropic.com"),
|
||||
("azure_ai", "https://myres.services.ai.azure.com/anthropic"),
|
||||
("github_copilot", "https://api.githubcopilot.com"),
|
||||
("bedrock", "https://bedrock-runtime.us-east-1.amazonaws.com"),
|
||||
("vertex_ai", "https://us-east5-aiplatform.googleapis.com"),
|
||||
)
|
||||
for api_base in anthropic_hosts:
|
||||
out = sanitize_tool_use_ids_in_anthropic_messages(msgs, api_base=api_base)
|
||||
assert out[0]["content"][0]["id"] == "functions_Bash_0", api_base
|
||||
for custom_llm_provider, api_base in still_sanitize:
|
||||
out = sanitize_tool_use_ids_in_anthropic_messages(
|
||||
msgs, api_base=api_base, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
assert out[0]["content"][0]["id"] == "functions_Bash_0", (custom_llm_provider, api_base)
|
||||
assert msgs[0]["content"][0]["id"] == "functions.Bash:0"
|
||||
|
||||
def test_normalize_anthropic_tool_use_id_strips_thought_signature(self):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue