From 0119e9cd60f3d51184fe550b953e52de44ff4b5c Mon Sep 17 00:00:00 2001 From: SynapShift <127070675+SynapShift@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:39:41 +0800 Subject: [PATCH] fix(anthropic): preserve mid-conversation tools across adapters --- .../mid_conversation_tools.py | 81 +++++++++++ .../chat/guardrail_translation/handler.py | 20 ++- .../pass_through/adapters/transformation.py | 25 ++-- .../responses_adapters/transformation.py | 34 ++--- .../test_anthropic_guardrail_handler.py | 120 ++++++++++++++++ ...al_pass_through_adapters_transformation.py | 132 ++++++++++++++++++ .../test_responses_adapters_transformation.py | 61 ++++++++ .../proxy/test_tools_allowlist_enforcement.py | 46 ++++++ 8 files changed, 486 insertions(+), 33 deletions(-) create mode 100644 litellm/litellm_core_utils/prompt_templates/mid_conversation_tools.py diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_tools.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_tools.py new file mode 100644 index 00000000000..7c2f9626827 --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_tools.py @@ -0,0 +1,81 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from itertools import chain +from typing import Final, TypeVar, cast + +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of + +_Request: Final = TypeVar("_Request", bound=Mapping[str, object]) + + +@dataclass(frozen=True, slots=True) +class _ToolChange: + name: str + remove: bool + definition: Mapping[str, object] | None + + +def _tool_change(block: object) -> _ToolChange | None: + action: Final = message_field(block, "type") + if action not in ("tool_addition", "tool_removal"): + return None + tool: Final = message_field(block, "tool") + tool_type: Final = message_field(tool, "type") + definition: Final = ( + as_str_mapping(message_field(tool, "definition")) + if action == "tool_addition" and tool_type == "tool_definition" + else None + ) + if definition is None and tool_type != "tool_reference": + return None + name: Final = message_field(definition if definition is not None else tool, "name") + return _ToolChange(name, action == "tool_removal", definition) if isinstance(name, str) and name else None + + +def _message_tool_changes(message: object) -> tuple[_ToolChange, ...]: + if message_field(message, "role") != "system": + return () + return tuple( + change for block in parts_of(message_field(message, "content")) if (change := _tool_change(block)) is not None + ) + + +def _without_tool_changes(message: object) -> object | None: + mapping: Final = as_str_mapping(message) + if mapping is None or not _message_tool_changes(mapping): + return message + content: Final = [block for block in parts_of(mapping.get("content")) if _tool_change(block) is None] + return {**mapping, "content": content} if content else None + + +def _tool_name(tool: object) -> str | None: + definition: Final = message_field(tool, "function") if message_field(tool, "type") == "function" else tool + name: Final = message_field(definition, "name") + return name if isinstance(name, str) else None + + +def resolve_midturn_tool_changes(request: _Request) -> _Request: + """Resolve tool availability once so authorization and translation agree.""" + messages: Final = parts_of(request.get("messages")) + changes: Final = tuple(chain.from_iterable(_message_tool_changes(message) for message in messages)) + if not changes: + return request + tools: Final = parts_of(request.get("tools")) + existing_names: Final = frozenset(_tool_name(tool) for tool in tools) + last_change: Final = {change.name: change for change in changes} + definitions: Final = { + change.name: change.definition for change in reversed(changes) if change.definition is not None + } + added: Final = tuple( + definitions[name] for name in last_change if name in definitions and name not in existing_names + ) + removed: Final = frozenset(change.name for change in last_change.values() if change.remove) + return cast( # cast-ok: preserve request fields, replacing only its tools and messages payloads + _Request, + { + **request, + "tools": [tool for tool in (*tools, *added) if _tool_name(tool) not in removed], + "messages": [cleaned for message in messages if (cleaned := _without_tool_changes(message)) is not None], + }, + ) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 4c6bded68ea..d8e77b00007 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -25,6 +25,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.prompt_templates.mid_conversation_tools import resolve_midturn_tool_changes from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.pass_through.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, @@ -603,6 +604,10 @@ class AnthropicMessagesHandler(BaseTranslation): messages: Final = data.get("messages") if messages is None: return data + request_view: Final = cast( # cast-ok: JSON request keys are strings; values are inspected as objects + Mapping[str, object], data + ) + resolved_request: Final = resolve_midturn_tool_changes(request_view) skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply) skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply) @@ -685,9 +690,8 @@ class AnthropicMessagesHandler(BaseTranslation): model: Final = data.get("model") if model: inputs["model"] = model - original_tool_names: Final = _original_tool_names( - cast(Mapping[str, object], data) # cast-ok: Anthropic request body - ) + original_tool_names: Final = _original_tool_names(resolved_request) + original_tools: Final = deepcopy(tools_to_check) guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, request_data=data, @@ -697,7 +701,8 @@ class AnthropicMessagesHandler(BaseTranslation): guardrailed_texts: Final = guardrailed_inputs.get("texts", []) guardrailed_tools: Final = guardrailed_inputs.get("tools") - if guardrailed_tools is not None: + tools_changed: Final = guardrailed_tools is not None and guardrailed_tools != original_tools + if guardrailed_tools is not None and (resolved_request is data or tools_changed): # Convert tools back from OpenAI format to Anthropic format anthropic_config: Final = AnthropicConfig() anthropic_tools: Final[list[AllAnthropicToolsValues]] = [] @@ -755,6 +760,10 @@ class AnthropicMessagesHandler(BaseTranslation): responses=guardrailed_texts, scanned=scanned, ) + if tools_changed and resolved_request is not data: + data["messages"] = ( # rebind-ok: guardrails rewrite the caller's request + resolve_midturn_tool_changes(request_view)["messages"] + ) verbose_proxy_logger.debug("Anthropic Messages: Processed input messages: %s", messages) @@ -1001,7 +1010,8 @@ class AnthropicMessagesHandler(BaseTranslation): def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]: """Extract every tool name in an Anthropic messages request: tools[].name, plus tools[].function.name for OpenAI-format tools the bridge forwards verbatim.""" - return [name for tool in list_field(data, "tools") for name in anthropic_tool_names(tool)] + request: Final = resolve_midturn_tool_changes(data) + return [name for tool in list_field(request, "tools") for name in anthropic_tool_names(tool)] @classmethod def _extract_input_text_and_images( diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index 9b1297dbd75..f4bd446185a 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -10,6 +10,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.hidden_params import get_hidden_params +from litellm.litellm_core_utils.prompt_templates.mid_conversation_tools import resolve_midturn_tool_changes from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_part from litellm.llms.anthropic.pass_through.utils import ( is_reasoning_auto_summary_enabled, @@ -1246,63 +1247,63 @@ class LiteLLMAnthropicMessagesAdapter: - tool_name_mapping maps truncated tool names back to original names for tools that exceeded OpenAI's 64-char limit """ - # Debug: Processing Anthropic message request + request: Final = resolve_midturn_tool_changes(anthropic_message_request) new_messages: list[AllMessageValues] = [] tool_name_mapping: dict[str, str] = {} ## CONVERT ANTHROPIC MESSAGES TO OPENAI messages_list: Final[list[AllAnthropicPassThroughMessageValues]] = cast( list[AllAnthropicPassThroughMessageValues], - anthropic_message_request["messages"], + request["messages"], ) new_messages = self.translate_anthropic_messages_to_openai( messages=messages_list, - model=anthropic_message_request.get("model"), + model=request.get("model"), custom_llm_provider=custom_llm_provider, preserve_midturn_system=preserve_midturn_system, ) ## ADD SYSTEM MESSAGE TO MESSAGES - self._add_system_message_to_messages(new_messages, anthropic_message_request) + self._add_system_message_to_messages(new_messages, request) new_kwargs: Final[ChatCompletionRequest] = { - "model": anthropic_message_request["model"], + "model": request["model"], "messages": new_messages, } ## CONVERT METADATA (user_id + litellm metadata) self._translate_metadata_to_openai( - anthropic_message_request=anthropic_message_request, + anthropic_message_request=request, new_kwargs=new_kwargs, custom_llm_provider=custom_llm_provider, ) ## CONVERT TOOL CHOICE self._translate_tool_choice_to_openai( - anthropic_message_request=anthropic_message_request, + anthropic_message_request=request, new_kwargs=new_kwargs, ) ## CONVERT TOOLS tool_name_mapping = self._translate_tools_to_openai( - anthropic_message_request=anthropic_message_request, + anthropic_message_request=request, new_kwargs=new_kwargs, ) ## CONVERT THINKING self._translate_thinking_to_openai( - anthropic_message_request=anthropic_message_request, + anthropic_message_request=request, new_kwargs=new_kwargs, custom_llm_provider=custom_llm_provider, drop_params=drop_params, ) ## CONVERT STOP_SEQUENCES self._translate_stop_sequences_to_openai( - anthropic_message_request=anthropic_message_request, + anthropic_message_request=request, new_kwargs=new_kwargs, ) ## CONVERT OUTPUT_FORMAT to RESPONSE_FORMAT self._translate_output_format_to_openai( - anthropic_message_request=anthropic_message_request, + anthropic_message_request=request, new_kwargs=new_kwargs, ) self._copy_untranslated_anthropic_params( - anthropic_message_request=anthropic_message_request, + anthropic_message_request=request, new_kwargs=new_kwargs, ) diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index bd0f57c3ed9..68e44f46c72 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, ) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_tools import resolve_midturn_tool_changes from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, ) @@ -605,14 +606,15 @@ class LiteLLMAnthropicToResponsesAPIAdapter: Responses API rejects ``include``. ``auto_summary_level`` is the ``reasoning.summary`` the opt-in auto summary injects when the client sent none. """ - model: Final[str] = anthropic_request["model"] + request: Final = resolve_midturn_tool_changes(anthropic_request) + model: Final[str] = request["model"] messages_list: Final = cast( list[AllAnthropicPassThroughMessageValues], - anthropic_request["messages"], + request["messages"], ) input_items: Final = self.translate_messages_to_responses_input(messages_list) - system: Final = anthropic_request.get("system") + system: Final = request.get("system") developer_parts: Final = ( self._translate_midturn_system_content_to_responses(system) if isinstance(system, list) @@ -643,34 +645,34 @@ class LiteLLMAnthropicToResponsesAPIAdapter: responses_kwargs["instructions"] = self.system_blocks_to_instructions(system) # max_tokens -> max_output_tokens - max_tokens: Final = anthropic_request.get("max_tokens") + max_tokens: Final = request.get("max_tokens") if max_tokens: responses_kwargs["max_output_tokens"] = max_tokens # temperature / top_p passed through - if "temperature" in anthropic_request: - responses_kwargs["temperature"] = anthropic_request["temperature"] - if "top_p" in anthropic_request: - responses_kwargs["top_p"] = anthropic_request["top_p"] + if "temperature" in request: + responses_kwargs["temperature"] = request["temperature"] + if "top_p" in request: + responses_kwargs["top_p"] = request["top_p"] # tools - tools: Final = anthropic_request.get("tools") + tools: Final = request.get("tools") if tools: responses_kwargs["tools"] = self.translate_tools_to_responses_api( cast(list[AllAnthropicToolsValues], tools) ) # tool_choice - tool_choice: Final = anthropic_request.get("tool_choice") + tool_choice: Final = request.get("tool_choice") if tool_choice: responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api( cast(AnthropicMessagesToolChoice, tool_choice) ) # thinking -> reasoning - thinking: Final = anthropic_request.get("thinking") + thinking: Final = request.get("thinking") if isinstance(thinking, dict): - output_config = anthropic_request.get("output_config") + output_config = request.get("output_config") reasoning: Final = self.translate_thinking_to_reasoning( thinking, output_config=cast(dict[str, Any] | None, output_config), @@ -683,8 +685,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: # output_format / output_config.format -> text format # output_format: {"type": "json_schema", "schema": {...}} # output_config: {"format": {"type": "json_schema", "schema": {...}}} - output_format: object = anthropic_request.get("output_format") - output_config = anthropic_request.get("output_config") + output_format: object = request.get("output_format") + output_config = request.get("output_config") if not isinstance(output_format, dict) and isinstance(output_config, dict): output_format = output_config.get("format") if isinstance(output_format, dict) and output_format.get("type") == "json_schema": @@ -700,14 +702,14 @@ class LiteLLMAnthropicToResponsesAPIAdapter: } # context_management: Anthropic dict -> OpenAI array - context_management: Final = anthropic_request.get("context_management") + context_management: Final = request.get("context_management") if isinstance(context_management, dict): openai_cm: Final = self.translate_context_management_to_responses_api(context_management) if openai_cm is not None: responses_kwargs["context_management"] = openai_cm # metadata user_id -> user and prompt_cache_key - metadata: Final = anthropic_request.get("metadata") + metadata: Final = request.get("metadata") if isinstance(metadata, dict) and "user_id" in metadata: responses_kwargs["user"] = str(metadata["user_id"])[:64] prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"]) diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 8218c120153..ad7ab41535c 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -6,6 +6,7 @@ with guardrail transformations, specifically testing edge cases with empty choic """ import json +from copy import deepcopy from typing import Any, Literal, Optional, Final from unittest.mock import MagicMock, patch @@ -19,9 +20,58 @@ from litellm.llms.anthropic.chat.guardrail_translation.handler import ( AnthropicMessagesHandler, ) from litellm.llms.anthropic.pass_through.adapters.transformation import truncate_tool_name +from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.types.utils import ChatCompletionMessageToolCall, GenericGuardrailAPIInputs +class DropToolDefinitionsGuardrail(CustomGuardrail): + def __init__(self, *, remove_in_place: bool) -> None: + super().__init__(guardrail_name="drop-tools") + self.remove_in_place = remove_in_place + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: object | None = None, + ) -> GenericGuardrailAPIInputs: + if self.remove_in_place: + inputs["tools"].clear() + return inputs + return {**inputs, "tools": []} + + +@pytest.mark.parametrize("remove_in_place", [False, True]) +@pytest.mark.parametrize("with_text", [False, True]) +@pytest.mark.asyncio +async def test_guardrail_removed_midturn_tools_are_not_reintroduced(with_text: bool, remove_in_place: bool) -> None: + user: Final = {"role": "user", "content": "Check the weather"} + text: Final = [{"type": "text", "text": "A tool is available"}] if with_text else [] + data: Final = { + "model": "test-model", + "max_tokens": 128, + "messages": [ + user, + {"role": "system", "content": [*text, { + "type": "tool_addition", + "tool": { + "type": "tool_definition", + "definition": {"name": "get_weather", "input_schema": {"type": "object"}}, + }, + }]}, + ], + } + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=DropToolDefinitionsGuardrail(remove_in_place=remove_in_place) + ) + translated, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(data) + + assert translated.get("tools", []) == [] + assert data["messages"] == [user, *([{"role": "system", "content": text}] if with_text else [])] + + class MockPassThroughGuardrail(CustomGuardrail): """Mock guardrail that passes through without blocking - for testing streaming fallback behavior""" @@ -36,6 +86,32 @@ class MockPassThroughGuardrail(CustomGuardrail): return inputs +@pytest.mark.asyncio +async def test_unchanged_guardrail_keeps_midturn_tool_history() -> None: + data: Final = { + "model": "test-model", + "max_tokens": 128, + "messages": [ + {"role": "user", "content": "Check the weather"}, + {"role": "system", "content": [{ + "type": "tool_addition", + "tool": { + "type": "tool_definition", + "definition": {"name": "get_weather", "input_schema": {"type": "object"}}, + }, + }]}, + ], + } + original: Final = deepcopy(data) + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockPassThroughGuardrail(guardrail_name="pass-through") + ) + + assert data["messages"] == original["messages"] + assert data.get("tools") == original.get("tools") + + class MockDynamicGuardrail(CustomGuardrail): """Mock guardrail that records dynamic params from request metadata.""" @@ -1633,6 +1709,50 @@ class TestAnthropicMessagesHandlerToolInjection: """A tool a guardrail injects in OpenAI format must survive the write-back to Anthropic format alongside the request's original tools.""" + @pytest.mark.asyncio + async def test_injected_tool_keeps_inline_long_tool_name_and_schema(self) -> None: + name: Final = "mcp__workspace_resources__get_accessible_projects_with_detailed_permissions" + tool: Final = { + "name": name, + "description": "List accessible projects", + "input_schema": { + "type": "object", + "properties": {"workspace_id": {"type": "string"}}, + "required": ["workspace_id"], + }, + } + user: Final = {"role": "user", "content": "List my projects"} + data: Final = { + "model": "test-model", + "max_tokens": 128, + "tool_choice": {"type": "tool", "name": name}, + "messages": [ + user, + {"role": "system", "content": [ + {"type": "tool_addition", "tool": {"type": "tool_definition", "definition": tool}}, + ]}, + ], + } + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=ToolAppendingGuardrail(guardrail_name="append-tool") + ) + translated, mapping = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(data) + + assert data["tools"] == [ + {"type": "custom", **tool}, + { + "type": "custom", + "name": "injected_tool", + "description": "injected by guardrail", + "input_schema": {"type": "object", "properties": {}}, + }, + ] + assert data["messages"] == [user] + assert data["tool_choice"] == {"type": "tool", "name": name} + assert mapping == {truncate_tool_name(name): name} + assert translated["tool_choice"] == {"type": "function", "function": {"name": truncate_tool_name(name)}} + @pytest.mark.asyncio async def test_injected_tool_survives_when_request_already_has_tools(self): handler = AnthropicMessagesHandler() diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 658aa009683..1e65f56f445 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1,4 +1,5 @@ import base64 +import copy import functools import json from collections.abc import Mapping @@ -5419,3 +5420,134 @@ async def test_chat_bridge_tool_result_text_block_without_text_reaches_the_model ], } ] + + +@pytest.mark.parametrize("preserve_midturn_system", [False, True]) +@pytest.mark.parametrize("with_text", [False, True]) +def test_midturn_tool_definition_reaches_chat_completions(preserve_midturn_system: bool, with_text: bool) -> None: + tool: Final = { + "name": "get_weather", + "description": "Get the weather for a city", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + text: Final = [{"type": "text", "text": "A new tool is available."}] if with_text else [] + request: Final = { + "model": "hosted_vllm/test-model", + "max_tokens": 128, + "messages": [ + {"role": "user", "content": "What is the weather in Dallas?"}, + { + "role": "system", + "content": [*text, {"type": "tool_addition", "tool": {"type": "tool_definition", "definition": tool}}], + }, + ], + } + original: Final = copy.deepcopy(request) + + translated, mapping = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + request, preserve_midturn_system=preserve_midturn_system + ) + + assert translated.get("tools") == [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": tool["description"], + "parameters": tool["input_schema"], + }, + } + ] + expected_note: Final = text if preserve_midturn_system else [{"type": "text", "text": CONVERTED_SYSTEM_NOTE}, *text] + assert translated["messages"] == [ + request["messages"][0], + *([{"role": "system" if preserve_midturn_system else "user", "content": expected_note}] if with_text else []), + ] + assert mapping == {} + assert request == original + + +@pytest.mark.parametrize( + "actions,available", + [ + (("tool_addition",), True), + (("tool_addition", "tool_addition"), True), + (("tool_addition", "tool_removal"), False), + (("tool_addition", "tool_removal", "tool_addition"), True), + (("tool_removal",), False), + ], +) +def test_midturn_tool_changes_update_existing_tools_in_message_order(actions: tuple[str, ...], available: bool) -> None: + tool: Final = {"name": "get_weather", "input_schema": {"type": "object", "properties": {}}} + request: Final = { + "model": "hosted_vllm/test-model", + "max_tokens": 128, + "tools": [tool], + "messages": [ + {"role": "user", "content": "Check the weather."}, + *( + { + "role": "system", + "content": [{"type": action, "tool": {"type": "tool_reference", "name": "get_weather"}}], + } + for action in actions + ), + ], + } + translated, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(request) + assert translated.get("tools", []) == ( + [{"type": "function", "function": {"name": "get_weather", "parameters": tool["input_schema"]}}] + if available + else [] + ) + assert translated["messages"] == [request["messages"][0]] + + +@pytest.mark.parametrize("already_declared", [False, True]) +@pytest.mark.parametrize("reactivate", [False, True]) +def test_inline_tool_changes_keep_definitions_and_historical_calls(already_declared: bool, reactivate: bool) -> None: + tool_name: Final = "mcp__weather__" + "long_tool_name_" * 5 + tool: Final = {"name": tool_name, "description": "Original", "input_schema": {"type": "object", "properties": {}}} + addition: Final = {"type": "tool_addition", "tool": {"type": "tool_definition", "definition": tool}} + reference: Final = {"type": "tool_reference", "name": tool_name} + request: Final = { + "model": "hosted_vllm/test-model", + "max_tokens": 128, + "tools": [tool] if already_declared else [], + "messages": [ + {"role": "user", "content": "Check the weather."}, + {"role": "system", "content": [addition, addition]}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "call_weather", "name": tool_name, "input": {}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_weather", "content": "Sunny"}]}, + {"role": "system", "content": [{"type": "tool_removal", "tool": reference}]}, + *([{"role": "system", "content": [{"type": "tool_addition", "tool": reference}]}] if reactivate else []), + ], + } + original: Final = copy.deepcopy(request) + translated, mapping = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(request) + short_name: Final = truncate_tool_name(tool_name) + + assert translated.get("tools", []) == ( + [ + { + "type": "function", + "function": {"name": short_name, "description": "Original", "parameters": tool["input_schema"]}, + } + ] + if reactivate + else [] + ) + assert translated["messages"][1]["tool_calls"] == [ + {"id": "call_weather", "type": "function", "function": {"name": short_name, "arguments": "{}"}} + ] + assert translated["messages"][2] == {"role": "tool", "tool_call_id": "call_weather", "content": "Sunny"} + assert len(translated["messages"]) == 3 + assert mapping == ({short_name: tool_name} if reactivate else {}) + assert request == original diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py index 829000c0c5f..28b65a039c3 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -3,6 +3,7 @@ Tests for LiteLLMAnthropicToResponsesAPIAdapter (litellm/llms/anthropic/pass_through/responses_adapters/transformation.py) """ +import copy import json import os from typing import Any, Dict, Final, List @@ -2424,3 +2425,63 @@ class TestPromptCacheBreakpointToResponses: extra_kwargs={"prompt_cache_options": {"mode": "explicit"}}, ) assert kwargs["prompt_cache_options"] == {"mode": "explicit"} + + +@pytest.mark.parametrize("remove", [False, True]) +def test_midturn_tool_changes_reach_responses_api(remove: bool) -> None: + tool: Final = { + "name": "get_weather", + "description": "Get the weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, + } + request: Final = { + "model": "test-model", + "max_tokens": 128, + "messages": [ + {"role": "user", "content": "What is the weather in Dallas?"}, + { + "role": "system", + "content": [ + {"type": "text", "text": "A new tool is available."}, + {"type": "tool_addition", "tool": {"type": "tool_definition", "definition": tool}}, + ], + }, + *( + [ + { + "role": "system", + "content": [ + {"type": "tool_removal", "tool": {"type": "tool_reference", "name": "get_weather"}} + ], + } + ] + if remove + else [] + ), + ], + } + original: Final = copy.deepcopy(request) + translated: Final = _ADAPTER.translate_request(request) + + assert translated.get("tools", []) == ( + [] + if remove + else [ + { + "type": "function", + "name": "get_weather", + "description": tool["description"], + "parameters": tool["input_schema"], + "strict": False, + } + ] + ) + assert translated["input"] == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "What is the weather in Dallas?"}]}, + { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": "A new tool is available."}], + }, + ] + assert request == original diff --git a/tests/unit/proxy/test_tools_allowlist_enforcement.py b/tests/unit/proxy/test_tools_allowlist_enforcement.py index e306f7be4f8..ba59ef6afff 100644 --- a/tests/unit/proxy/test_tools_allowlist_enforcement.py +++ b/tests/unit/proxy/test_tools_allowlist_enforcement.py @@ -7,6 +7,7 @@ Covers: """ from unittest.mock import AsyncMock, MagicMock, patch +from typing import Final import pytest @@ -18,6 +19,51 @@ from litellm.proxy.guardrails.tool_name_extraction import ( ) +@pytest.mark.parametrize("already_declared", [False, True]) +@pytest.mark.asyncio +async def test_midturn_tool_additions_cannot_bypass_key_allowlist(already_declared: bool) -> None: + data: Final = { + "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}] if already_declared else [], + "messages": [ + {"role": "user", "content": "Use the available tool"}, + { + "role": "system", + "content": [{ + "type": "tool_addition", + "tool": { + "type": "tool_definition", + "definition": {"name": "run_sql", "input_schema": {"type": "object"}}, + }, + }], + }, + ], + } + token: Final = UserAPIKeyAuth(api_key="test-key", metadata={"allowed_tools": ["get_weather"]}) + + with pytest.raises(ProxyException, match="run_sql"): + await check_tools_allowlist(request_body=data, valid_token=token, team_object=None, route="/v1/messages") + + +@pytest.mark.asyncio +async def test_allowed_midturn_tool_addition_reaches_authorization() -> None: + data: Final = { + "messages": [{ + "role": "system", + "content": [{ + "type": "tool_addition", + "tool": { + "type": "tool_definition", + "definition": {"name": "get_weather", "input_schema": {"type": "object"}}, + }, + }], + }], + } + token: Final = UserAPIKeyAuth(api_key="test-key", metadata={"allowed_tools": ["get_weather"]}) + + assert extract_request_tool_names("/v1/messages", data) == ["get_weather"] + await check_tools_allowlist(request_body=data, valid_token=token, team_object=None, route="/v1/messages") + + def _token(metadata=None, team_metadata=None): return UserAPIKeyAuth( api_key="test-key",