From 45169b10a98f0a675167e7033d81450066642d61 Mon Sep 17 00:00:00 2001 From: Chaitanya Laxman Date: Mon, 7 Sep 2026 18:16:04 +0400 Subject: [PATCH] fix(anthropic): keep vLLM tool ids intact on /v1/messages Native /v1/messages always rewrote tool ids (functions.Bash:0 -> functions_Bash_0). vLLM/Kimi echoes the original ids, so the next tool_result turn breaks (#32214). Skip the rewrite when api_base is not Anthropic, Bedrock, or Vertex. Empty api_base still sanitizes. Tests: skip for 127.0.0.1, still rewrite anthropic hosts, handler forwards api_base. Revert of production files: skip test TypeError, handler test KeyError. --- litellm/llms/anthropic/common_utils.py | 39 ++++++++--- .../messages/handler.py | 4 +- ...erimental_pass_through_messages_handler.py | 21 ++++++ .../anthropic/test_anthropic_common_utils.py | 66 +++++++++++++++++++ 4 files changed, 119 insertions(+), 11 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index d9424d6a243..34b2015f9eb 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1240,17 +1240,38 @@ def _sanitize_tool_use_id_content_block(block: object) -> object: return block -def sanitize_tool_use_ids_in_anthropic_messages(messages: list[Any]) -> list[Any]: - """ - Return a new message list with ``tool_use`` / ``server_tool_use`` ``id`` and - ``tool_result`` ``tool_use_id`` values rewritten to satisfy Anthropic's - ``^[a-zA-Z0-9_-]+$`` requirement. +_ANTHROPIC_TOOL_ID_CHARSET_HOST_MARKERS: Final = frozenset( + ( + "api.anthropic.com", + "amazonaws.com", + "googleapis.com", + "cloud.google.com", + ) +) - Cross-provider clients (e.g. Claude Code routed through kimi) may replay - conversation history containing ids like ``functions.Bash:0`` with ``.`` - and ``:`` — valid on the upstream provider but rejected by Anthropic when - the session is switched to a native Anthropic deployment. + +def _upstream_enforces_anthropic_tool_id_charset(api_base: str | None) -> bool: + if api_base is None or not api_base.strip(): + return True + host: Final = api_base.casefold() + return any(marker in host for marker in _ANTHROPIC_TOOL_ID_CHARSET_HOST_MARKERS) + + +def sanitize_tool_use_ids_in_anthropic_messages( + messages: list[Any], + *, + api_base: str | None = None, +) -> list[Any]: """ + Rewrite ``tool_use`` / ``server_tool_use`` ``id`` and ``tool_result`` + ``tool_use_id`` values to Anthropic's ``^[a-zA-Z0-9_-]+$`` pattern. + + No-op when ``api_base`` is a host that is not Anthropic, Bedrock, or Vertex. + Those upstreams (vLLM, Kimi, SGLang) echo the original ids; rewriting them + breaks the next tool_result turn. See #32214. + """ + if not _upstream_enforces_anthropic_tool_id_charset(api_base): + return messages out: Final[list[Any]] = [] for m in messages: if not isinstance(m, dict) or not isinstance(m.get("content"), list): diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 9d1e921cce4..bd9ec052e2d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -263,7 +263,7 @@ async def anthropic_messages( messages = strip_empty_content_blocks_from_anthropic_messages(messages) # Replay of cross-provider tool history (e.g. kimi -> Anthropic) may carry # ids like ``functions.Bash:0`` that violate Anthropic's id pattern. - messages = sanitize_tool_use_ids_in_anthropic_messages(messages) + messages = sanitize_tool_use_ids_in_anthropic_messages(messages, api_base=api_base) messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages) from litellm.integrations.anthropic_cache_control_hook import ( @@ -460,7 +460,7 @@ def anthropic_messages_handler( # full-messages scan. Pop it so it never leaks into provider params. if not kwargs.pop("_litellm_messages_presanitized", False): messages = strip_empty_content_blocks_from_anthropic_messages(messages) - messages = sanitize_tool_use_ids_in_anthropic_messages(messages) + messages = sanitize_tool_use_ids_in_anthropic_messages(messages, api_base=api_base) messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages) from litellm.integrations.anthropic_cache_control_hook import ( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 01f7a2fb7ab..dbe96b322a5 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -702,6 +702,27 @@ def test_handler_strips_when_no_presanitized_flag(): assert result is not None +def test_handler_forwards_api_base_to_tool_id_sanitize(): + from litellm.llms.anthropic.experimental_pass_through.messages import handler + + with patch.object( + handler, + "sanitize_tool_use_ids_in_anthropic_messages", + wraps=handler.sanitize_tool_use_ids_in_anthropic_messages, + ) as spy: + result = handler.anthropic_messages_handler( + max_tokens=10, + messages=[{"role": "user", "content": "Hello"}], + model="anthropic/claude-3-5-sonnet-20241022", + custom_llm_provider="anthropic", + api_base="http://127.0.0.1:8000/v1", + mock_response="hi there", + ) + assert result is not None + assert spy.call_count == 1 + assert spy.call_args.kwargs["api_base"] == "http://127.0.0.1:8000/v1" + + def test_handler_skips_strip_when_presanitized(): """Async wrapper already sanitized -> handler must NOT rescan.""" from litellm.llms.anthropic.experimental_pass_through.messages import handler diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 794613942a1..d0dddea0ff0 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1794,6 +1794,72 @@ class TestAnthropicThinkingSignatureSelfHeal: assert out[1]["content"][0]["tool_use_id"] == "functions_Bash_0" assert msgs[0]["content"][0]["id"] == "functions.Bash:0" + def test_sanitize_tool_use_ids_skips_non_anthropic_api_base(self): + from litellm.llms.anthropic.common_utils import ( + sanitize_tool_use_ids_in_anthropic_messages, + ) + + msgs = [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "functions.Bash:0", + "name": "Bash", + "input": {}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "functions.Bash:0", + "content": "ok", + } + ], + }, + ] + out = sanitize_tool_use_ids_in_anthropic_messages( + msgs, api_base="http://127.0.0.1:8000/v1" + ) + assert out is msgs + assert out[0]["content"][0]["id"] == "functions.Bash:0" + assert out[1]["content"][0]["tool_use_id"] == "functions.Bash:0" + + def test_sanitize_tool_use_ids_still_runs_for_anthropic_hosts(self): + from litellm.llms.anthropic.common_utils import ( + sanitize_tool_use_ids_in_anthropic_messages, + ) + + msgs = [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "functions.Bash:0", + "name": "Bash", + "input": {}, + } + ], + } + ] + anthropic_hosts = ( + "", + "https://api.anthropic.com", + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://us-east5-aiplatform.googleapis.com", + "https://aiplatform.googleapis.com", + "https://cloud.google.com/vertex-ai", + ) + for api_base in anthropic_hosts: + out = sanitize_tool_use_ids_in_anthropic_messages(msgs, api_base=api_base) + assert out[0]["content"][0]["id"] == "functions_Bash_0", api_base + assert msgs[0]["content"][0]["id"] == "functions.Bash:0" + def test_normalize_anthropic_tool_use_id_strips_thought_signature(self): from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR,