From ba49b2026d0916fdf4adf266ab4096e0fceef268 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:24:41 -0700 Subject: [PATCH 1/2] fix(responses): drop malformed tool flags under drop_params and keep stored search calls off non-Anthropic follow-ups With drop_params on, a non-boolean defer_loading or a non-list allowed_callers on a Responses API tool (or a namespace member) is dropped before the bridge instead of answering 400, so only a request without drop_params gets the 400. A custom tool now carries defer_loading to the chat tool like a function tool does A previous_response_id follow-up routed to a provider that does not build an Anthropic request no longer replays the stored Claude search call (srvtoolu_ id) as a client tool call. Mistral answered that history 400 "Unexpected tool call id". An assistant turn that held only the search call is dropped there. The Anthropic route keeps the call, since its factory pairs it with the stored result The tool_search_call output model is renamed OutputToolSearchCall so the proxy's OpenAPI schema keeps the OpenAI SDK's ResponseToolSearchCall name. answered_tool_call_ids reads the history by index, so the lookup is linear in the history length. The streaming done event reads the response it is given, which is the same object it used to read from the iterator --- .../prompt_templates/factory.py | 5 +- litellm/proxy/_lazy_openapi_snapshot.json | 244 +++++++++--------- .../custom_tools.py | 53 +++- .../streaming_iterator.py | 4 +- .../tool_search.py | 40 ++- .../transformation.py | 92 ++++--- litellm/types/llms/openai.py | 4 +- litellm/types/responses/main.py | 2 +- ...esponses_bridge_tool_search_replay_wire.py | 51 ++++ ...nses_bridge_tool_search_undeferred_wire.py | 71 ++++- .../test_responses_bridge_tool_search_wire.py | 19 ++ .../test_anthropic_responses_bridge.py | 85 +++++- .../test_litellm_completion_responses.py | 92 ++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 114 ++++---- 14 files changed, 636 insertions(+), 240 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d0cf1d10d92..eebf6b73b44 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1828,9 +1828,8 @@ def _tool_result_ids(answers: Iterable[AllMessageValues]) -> Iterator[str]: def answered_tool_call_ids(messages: Sequence[AllMessageValues], assistant_index: int) -> frozenset[str]: - following: Final = itertools.dropwhile( - lambda message: message["role"] == "assistant", itertools.islice(messages, assistant_index, None) - ) + from_assistant: Final = (messages[index] for index in range(assistant_index, len(messages))) + following: Final = itertools.dropwhile(lambda message: message["role"] == "assistant", from_assistant) answers: Final = itertools.takewhile(lambda message: message["role"] in ("tool", "function"), following) return frozenset(_tool_result_ids(answers)) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 44b59f175c8..9d3e140eadd 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -24116,6 +24116,60 @@ "title": "OutputTokensDetails", "type": "object" }, + "OutputToolSearchCall": { + "additionalProperties": true, + "properties": { + "arguments": { + "title": "Arguments" + }, + "call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Call Id" + }, + "execution": { + "enum": [ + "server", + "client" + ], + "title": "Execution", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "status": { + "enum": [ + "in_progress", + "completed", + "incomplete" + ], + "title": "Status", + "type": "string" + }, + "type": { + "const": "tool_search_call", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "arguments", + "execution", + "status", + "type" + ], + "title": "OutputToolSearchCall", + "type": "object" + }, "PendingSafetyCheck": { "additionalProperties": true, "description": "A pending safety check for the computer call.", @@ -25748,7 +25802,7 @@ "$ref": "#/components/schemas/ResponseFunctionToolCallOutputItem" }, { - "$ref": "#/components/schemas/openai__types__responses__response_tool_search_call__ResponseToolSearchCall" + "$ref": "#/components/schemas/ResponseToolSearchCall" }, { "$ref": "#/components/schemas/ResponseToolSearchOutputItem" @@ -26102,6 +26156,71 @@ "title": "ResponseTextConfigParam", "type": "object" }, + "ResponseToolSearchCall": { + "additionalProperties": true, + "properties": { + "arguments": { + "title": "Arguments" + }, + "call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Call Id" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "execution": { + "enum": [ + "server", + "client" + ], + "title": "Execution", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "status": { + "enum": [ + "in_progress", + "completed", + "incomplete" + ], + "title": "Status", + "type": "string" + }, + "type": { + "const": "tool_search_call", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "arguments", + "execution", + "status", + "type" + ], + "title": "ResponseToolSearchCall", + "type": "object" + }, "ResponseToolSearchOutputItem": { "additionalProperties": true, "properties": { @@ -26336,7 +26455,7 @@ "$ref": "#/components/schemas/ResponseReasoningItem" }, { - "$ref": "#/components/schemas/openai__types__responses__response_tool_search_call__ResponseToolSearchCall" + "$ref": "#/components/schemas/ResponseToolSearchCall" }, { "$ref": "#/components/schemas/ResponseToolSearchOutputItem" @@ -26419,7 +26538,7 @@ "$ref": "#/components/schemas/CustomToolCallOutputItem" }, { - "$ref": "#/components/schemas/litellm__types__responses__main__ResponseToolSearchCall" + "$ref": "#/components/schemas/OutputToolSearchCall" } ] }, @@ -27358,125 +27477,6 @@ "title": "WebSearchTool", "type": "object" }, - "litellm__types__responses__main__ResponseToolSearchCall": { - "additionalProperties": true, - "properties": { - "arguments": { - "title": "Arguments" - }, - "call_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Call Id" - }, - "execution": { - "enum": [ - "server", - "client" - ], - "title": "Execution", - "type": "string" - }, - "id": { - "title": "Id", - "type": "string" - }, - "status": { - "enum": [ - "in_progress", - "completed", - "incomplete" - ], - "title": "Status", - "type": "string" - }, - "type": { - "const": "tool_search_call", - "title": "Type", - "type": "string" - } - }, - "required": [ - "id", - "arguments", - "execution", - "status", - "type" - ], - "title": "ResponseToolSearchCall", - "type": "object" - }, - "openai__types__responses__response_tool_search_call__ResponseToolSearchCall": { - "additionalProperties": true, - "properties": { - "arguments": { - "title": "Arguments" - }, - "call_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Call Id" - }, - "created_by": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Created By" - }, - "execution": { - "enum": [ - "server", - "client" - ], - "title": "Execution", - "type": "string" - }, - "id": { - "title": "Id", - "type": "string" - }, - "status": { - "enum": [ - "in_progress", - "completed", - "incomplete" - ], - "title": "Status", - "type": "string" - }, - "type": { - "const": "tool_search_call", - "title": "Type", - "type": "string" - } - }, - "required": [ - "id", - "arguments", - "execution", - "status", - "type" - ], - "title": "ResponseToolSearchCall", - "type": "object" - }, "openai__types__responses__web_search_preview_tool__UserLocation": { "additionalProperties": true, "description": "The user's location.", diff --git a/litellm/responses/litellm_completion_transformation/custom_tools.py b/litellm/responses/litellm_completion_transformation/custom_tools.py index 51942fa5fff..3492bc99af6 100644 --- a/litellm/responses/litellm_completion_transformation/custom_tools.py +++ b/litellm/responses/litellm_completion_transformation/custom_tools.py @@ -180,6 +180,50 @@ def validated_defer_loading(value: object) -> bool: raise BadRequestError(message="defer_loading must be a boolean", model="", llm_provider="") from exc +_TOOL_FLAG_ADAPTERS: Final = MappingProxyType( + {"defer_loading": _DEFER_LOADING_ADAPTER, "allowed_callers": _ALLOWED_CALLERS_ADAPTER} +) + + +def _is_well_formed_tool_flag(key: str, value: object) -> bool: + adapter: Final = _TOOL_FLAG_ADAPTERS.get(key) + if adapter is None: + return True + try: + adapter.validate_python(value, strict=True) + except ValidationError: + return False + return True + + +_NAMESPACE_MEMBERS_ADAPTER: Final = TypeAdapter[list[dict[str, object]]](list[dict[str, object]]) + + +def without_malformed_tool_flags(tool: Mapping[str, object]) -> Mapping[str, object]: + kept: Final = {key: value for key, value in tool.items() if _is_well_formed_tool_flag(key, value)} + if tool.get("type") != "namespace": + return kept + try: + members: Final = _NAMESPACE_MEMBERS_ADAPTER.validate_python(tool.get("tools")) + except ValidationError: + return kept + return {**kept, "tools": [without_malformed_tool_flags(member) for member in members]} + + +def function_chat_tool( + function: ChatCompletionToolParamFunctionChunk, allowed_callers: Sequence[str] | None, deferred: bool +) -> ChatCompletionToolParam: + if allowed_callers is None and not deferred: + return ChatCompletionToolParam(type="function", function=function) + if allowed_callers is None: + return ChatCompletionToolParam(type="function", function=function, defer_loading=True) + if not deferred: + return ChatCompletionToolParam(type="function", function=function, allowed_callers=list(allowed_callers)) + return ChatCompletionToolParam( + type="function", function=function, allowed_callers=list(allowed_callers), defer_loading=True + ) + + def custom_tool_grammar_suffix(fmt: object) -> str: try: parsed: Final = _CustomToolFormat.model_validate(fmt) @@ -196,8 +240,8 @@ def convert_custom_tool_to_function_tool(tool: Mapping[str, object]) -> ChatComp The grammar definition is embedded in the description so the model can produce correctly-formatted output. Returns ``None`` if the tool is not a - custom tool. Raises ``ValueError`` if ``allowed_callers`` is not a list of - strings. + custom tool. Raises ``BadRequestError`` if ``allowed_callers`` is not a list + of strings or ``defer_loading`` is not a boolean. """ if tool.get("type") != "custom": return None @@ -208,6 +252,7 @@ def convert_custom_tool_to_function_tool(tool: Mapping[str, object]) -> ChatComp tool.get("format") ) allowed_callers: Final = validated_allowed_callers(tool.get("allowed_callers")) + deferred: Final = validated_defer_loading(tool.get("defer_loading")) function_chunk: Final = ChatCompletionToolParamFunctionChunk( name=name, description=description, @@ -222,6 +267,4 @@ def convert_custom_tool_to_function_tool(tool: Mapping[str, object]) -> ChatComp "required": ["content"], }, ) - if allowed_callers is None: - return ChatCompletionToolParam(type="function", function=function_chunk) - return ChatCompletionToolParam(type="function", function=function_chunk, allowed_callers=allowed_callers) + return function_chat_tool(function_chunk, allowed_callers, deferred) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index bdc4796b2ea..418bdb3d6db 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -858,8 +858,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self._cached_item_id is None: self._cached_item_id = f"msg_{uuid.uuid4()}" - text: Final = self.litellm_model_response.choices[0].message.content or "" - annotations = getattr(self.litellm_model_response.choices[0].message, "annotations", None) + text: Final = litellm_complete_object.choices[0].message.content or "" + annotations = getattr(litellm_complete_object.choices[0].message, "annotations", None) response_annotations: Final = ( LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations( diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index bab28279176..dd1048eea3e 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -6,22 +6,25 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError from litellm.litellm_core_utils.prompt_templates.factory import is_anthropic_tool_search_use from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AnthropicToolSearchToolRegex from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.responses.main import ResponseToolSearchCall -from litellm.types.utils import LlmProviders +from litellm.types.responses.main import OutputToolSearchCall +from litellm.types.utils import ChatCompletionMessageToolCall, LlmProviders, Message from .custom_tools import openai_shaped_tool_call_item_id __all__ = [ "ANTHROPIC_TOOL_SEARCH_TOOL_NAME", "ANTHROPIC_TOOL_SEARCH_TOOL_TYPE", + "CompletionRoute", "ToolSearchCallStatus", "anthropic_tool_search_chat_tool", "build_tool_search_call", "has_server_tool_search", + "holds_tool_search_use", "is_anthropic_tool_search_chat_tool", "is_anthropic_tool_search_use", "is_server_tool_search_tool", "provider_builds_anthropic_request", + "without_tool_search_uses", ] ANTHROPIC_TOOL_SEARCH_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" @@ -72,6 +75,35 @@ def provider_builds_anthropic_request(model: str, custom_llm_provider: str | Non return isinstance(chat_config, AnthropicConfig) +class CompletionRoute(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model: str + custom_llm_provider: str | None = None + + +def _is_tool_search_call(tool_call: object) -> bool: + return isinstance(tool_call, ChatCompletionMessageToolCall) and is_anthropic_tool_search_use( + tool_call.id or "", tool_call.function.name or "" + ) + + +def holds_tool_search_use(message: object) -> bool: + return isinstance(message, Message) and any( + _is_tool_search_call(tool_call) for tool_call in message.tool_calls or () + ) + + +def without_tool_search_uses(message: Message) -> Message | None: + tool_calls: Final = message.tool_calls or [] + kept: Final = tuple(tool_call for tool_call in tool_calls if not _is_tool_search_call(tool_call)) + if len(kept) == len(tool_calls): + return message + if not kept and not message.content: + return None + return message.model_copy(update={"tool_calls": list(kept) or None}) + + def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None: try: return _ARGUMENTS_ADAPTER.validate_json(arguments_json or "{}") @@ -79,9 +111,9 @@ def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None: return None -def build_tool_search_call(tool_id: str, arguments_json: str, status: ToolSearchCallStatus) -> ResponseToolSearchCall: +def build_tool_search_call(tool_id: str, arguments_json: str, status: ToolSearchCallStatus) -> OutputToolSearchCall: arguments: Final = _parsed_arguments(arguments_json) - return ResponseToolSearchCall( + return OutputToolSearchCall( id=openai_shaped_tool_call_item_id("tool_search_call", tool_id), call_id=tool_id, arguments={} if arguments is None else arguments, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 9250c95e9af..7cdcc963191 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -33,9 +33,11 @@ from openai.types.responses.tool_param import FunctionToolParam from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict +import litellm from litellm._logging import verbose_logger from litellm.caching import InMemoryCache from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) @@ -91,7 +93,7 @@ from litellm.types.responses.main import ( OutputFunctionToolCall, OutputImageGenerationCall, OutputText, - ResponseToolSearchCall, + OutputToolSearchCall, ) from litellm.types.utils import ( ChatCompletionAnnotation, @@ -106,22 +108,27 @@ from litellm.types.utils import ( from .custom_tools import ( convert_custom_tool_to_function_tool, extract_custom_tool_names, + function_chat_tool, is_custom_tool_call, openai_shaped_tool_call_item_id, serialize_tool_call_arguments, unwrap_custom_tool_arguments, validated_allowed_callers, validated_defer_loading, + without_malformed_tool_flags, ) from .reasoning_items import decode_thinking_blocks, encode_thinking_blocks, mint_reasoning_item_id from .tool_search import ( + CompletionRoute, anthropic_tool_search_chat_tool, build_tool_search_call, has_server_tool_search, + holds_tool_search_use, is_anthropic_tool_search_chat_tool, is_anthropic_tool_search_use, is_server_tool_search_tool, provider_builds_anthropic_request, + without_tool_search_uses, ) NamespaceNameMap: TypeAlias = Mapping[str, tuple[str, str]] @@ -242,14 +249,21 @@ def _input_item_role(input_item: Mapping[str, object]) -> str: return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated +SessionMessage: TypeAlias = ( + AllMessageValues + | GenericChatCompletionMessage + | ChatCompletionMessageToolCall + | ChatCompletionResponseMessage + | Message +) + + +def _without_stored_tool_search_uses(message: SessionMessage) -> SessionMessage | None: + return without_tool_search_uses(message) if isinstance(message, Message) else message + + class ChatCompletionSession(TypedDict, total=False): - messages: list[ - AllMessageValues - | GenericChatCompletionMessage - | ChatCompletionMessageToolCall - | ChatCompletionResponseMessage - | Message - ] + messages: list[SessionMessage] litellm_session_id: str | None instructions: ReadOnly[str | None] @@ -382,6 +396,12 @@ class LiteLLMCompletionResponsesConfig: ) return supported_params is not None and "web_search_options" not in supported_params + @staticmethod + def _without_malformed_tool_flags_when_dropping_params(tools: ResponseTools, drop_params: object) -> ResponseTools: + if litellm.drop_params is not True and normalize_drop_params(drop_params) is not True: + return tools + return [without_malformed_tool_flags(tool) for tool in tools or ()] + @staticmethod def _without_unsupported_tool_search( tools: ResponseTools, model: str, custom_llm_provider: str | None @@ -498,13 +518,18 @@ class LiteLLMCompletionResponsesConfig: custom_llm_provider: str | None = None, stream: bool | None = None, extra_headers: Mapping[str, object] | None = None, + drop_params: object = None, **kwargs, ) -> dict: """ Transform a Responses API request into a Chat Completion request """ request_tools: Final = LiteLLMCompletionResponsesConfig._without_unsupported_tool_search( - responses_api_request.get("tools") or [], model=model, custom_llm_provider=custom_llm_provider + LiteLLMCompletionResponsesConfig._without_malformed_tool_flags_when_dropping_params( + responses_api_request.get("tools") or [], drop_params + ), + model=model, + custom_llm_provider=custom_llm_provider, ) ( converted_tools, @@ -627,6 +652,14 @@ class LiteLLMCompletionResponsesConfig: return messages + @staticmethod + def _without_tool_search_uses_off_anthropic( + messages: Sequence[SessionMessage], route: CompletionRoute + ) -> tuple[SessionMessage, ...]: + if provider_builds_anthropic_request(route.model, route.custom_llm_provider): + return tuple(messages) + return tuple(kept for message in messages if (kept := _without_stored_tool_search_uses(message)) is not None) + @staticmethod async def async_responses_api_session_handler( previous_response_id: str, @@ -644,7 +677,14 @@ class LiteLLMCompletionResponsesConfig: ) ) _messages: Final = litellm_completion_request.get("messages") or [] - session_messages: Final = chat_completion_session.get("messages") or [] + stored_messages: Final = chat_completion_session.get("messages") or [] + session_messages: Final = ( + LiteLLMCompletionResponsesConfig._without_tool_search_uses_off_anthropic( + stored_messages, CompletionRoute.model_validate(litellm_completion_request) + ) + if any(holds_tool_search_use(message) for message in stored_messages) + else stored_messages + ) instructions_end: Final = 1 if instructions else 0 carried_instructions: Final = None if instructions else chat_completion_session.get("instructions") leading_system_messages: Final = ( @@ -658,7 +698,7 @@ class LiteLLMCompletionResponsesConfig: # Store original _messages before combining for safety check original_new_messages: Final = _messages.copy() if _messages else [] - combined_messages = leading_system_messages + session_messages + _messages[instructions_end:] + combined_messages = [*leading_system_messages, *session_messages, *_messages[instructions_end:]] # Fix: Ensure tool_results have corresponding tool_calls in previous assistant message # Pass tools parameter to help reconstruct tool_calls if not in cache @@ -680,7 +720,7 @@ class LiteLLMCompletionResponsesConfig: elif session_messages: # If we had session messages but they got filtered out, # restore them - combined_messages = session_messages + combined_messages = list(session_messages) else: # Both are empty - this likely means function_call_output had empty/invalid call_id # Provide a helpful error message @@ -2001,16 +2041,10 @@ class LiteLLMCompletionResponsesConfig: parameters=dict(normalized_parameters), strict=bool(namespace_tool.get("strict", False)), ) - allowed_callers: Final = validated_allowed_callers(namespace_tool.get("allowed_callers")) - deferred: Final = validated_defer_loading(namespace_tool.get("defer_loading")) - if allowed_callers is None and not deferred: - return ChatCompletionToolParam(type="function", function=function) - if allowed_callers is None: - return ChatCompletionToolParam(type="function", function=function, defer_loading=True) - if not deferred: - return ChatCompletionToolParam(type="function", function=function, allowed_callers=allowed_callers) - return ChatCompletionToolParam( - type="function", function=function, allowed_callers=allowed_callers, defer_loading=True + return function_chat_tool( + function, + validated_allowed_callers(namespace_tool.get("allowed_callers")), + validated_defer_loading(namespace_tool.get("defer_loading")), ) @staticmethod @@ -2230,7 +2264,7 @@ class LiteLLMCompletionResponsesConfig: def transform_chat_completion_tools_to_responses_tools( chat_completion_response: ModelResponse, responses_api_request: ResponsesAPIOptionalRequestParams | None = None, - ) -> list[ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem | ResponseToolSearchCall]: + ) -> list[ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem | OutputToolSearchCall]: """ Transform a Chat Completion tools into a Responses API tools. @@ -2238,7 +2272,7 @@ class LiteLLMCompletionResponsesConfig: with ``type="custom_tool_call"``. For regular function tools, returns ``ResponseFunctionToolCall`` with ``type="function_call"``. A provider-side tool search call made for a request's ``tool_search`` tool returns - ``ResponseToolSearchCall`` with ``type="tool_search_call"``. + ``OutputToolSearchCall`` with ``type="tool_search_call"``. """ all_chat_completion_tools: Final[list[ChatCompletionMessageToolCall]] = [] for choice in chat_completion_response.choices: @@ -2260,9 +2294,7 @@ class LiteLLMCompletionResponsesConfig: chat_completion_response ) responses_tools: Final[ - list[ - ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem | ResponseToolSearchCall - ] + list[ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem | OutputToolSearchCall] ] = [] for tool in all_chat_completion_tools: if tool.type == "function": @@ -2339,7 +2371,7 @@ class LiteLLMCompletionResponsesConfig: tool_arguments: str, web_search_calls: Mapping[str, ResponseFunctionWebSearch], tool_search_requested: bool, - ) -> ResponseFunctionWebSearch | ResponseToolSearchCall | None: + ) -> ResponseFunctionWebSearch | OutputToolSearchCall | None: web_search_call: Final = web_search_calls.get(tool_id) if web_search_call is not None: return web_search_call @@ -2614,7 +2646,7 @@ class LiteLLMCompletionResponsesConfig: | ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem - | ResponseToolSearchCall + | OutputToolSearchCall ]: responses_output: list[ GenericResponseOutputItem @@ -2624,7 +2656,7 @@ class LiteLLMCompletionResponsesConfig: | ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem - | ResponseToolSearchCall + | OutputToolSearchCall ] = [] responses_output.extend( diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 1acb448511e..33986d80df3 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -91,7 +91,7 @@ from litellm.types.responses.main import ( OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, - ResponseToolSearchCall, + OutputToolSearchCall, ) from .base import CachedTokensDetails @@ -1445,7 +1445,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): | ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem - | ResponseToolSearchCall + | OutputToolSearchCall ] ) parallel_tool_calls: bool | None = None diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 49b27c9494b..5588527c141 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -133,7 +133,7 @@ class CustomToolCallOutputItem(BaseLiteLLMOpenAIResponseObject): status: Literal["in_progress", "completed", "incomplete"] | None = None -class ResponseToolSearchCall(BaseLiteLLMOpenAIResponseObject): +class OutputToolSearchCall(BaseLiteLLMOpenAIResponseObject): model_config = ConfigDict(extra="allow", frozen=True, protected_namespaces=()) id: str diff --git a/tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py index 50ac31c6aa0..eb5ac2765fc 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py @@ -29,6 +29,7 @@ _CACHE_BUST: Final[Mapping[str, JsonValue]] = MappingProxyType({"cache": {"no-ca _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]]) _EMPTY_TURN: Final[Mapping[str, JsonValue]] = MappingProxyType({"role": "user", "content": ""}) +_EARLIER_PAIRS: Final = 200 _WEATHER_PARAMETERS: Final[dict[str, JsonValue]] = { "type": "object", "properties": {"city": {"type": "string"}}, @@ -283,6 +284,56 @@ def test_a_replayed_search_call_is_dropped_only_when_nothing_answers_it( _assert_wire(call, request, body) +def _earlier_pairs(marker: str) -> Iterator[JsonValue]: + for index in range(_EARLIER_PAIRS): + call_id = f"toolu_{marker}_{index}" + call: JsonValue = { + "id": call_id, + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + yield {"role": "assistant", "content": f"step {index}", "tool_calls": [call]} + yield _tool_message(call_id, "22C") + + +def _long_history(marker: str, answers: Sequence[Mapping[str, JsonValue]]) -> tuple[JsonValue, ...]: + first, *rest = _history(marker, answers) + return (first, *_earlier_pairs(marker), *rest) + + +def _last_assistant_tool_uses(messages: Sequence[Mapping[str, JsonValue]]) -> tuple[tuple[str, str], ...]: + last: Final = [turn for turn in messages if turn["role"] == "assistant"][-1] + blocks: Final = _ITEMS.validate_python(last["content"]) + return tuple(sorted((str(block["id"]), str(block["name"])) for block in blocks if block["type"] == "tool_use")) + + +@pytest.mark.parametrize("answer", ["unanswered", "answered"]) +def test_a_search_call_after_two_hundred_answered_tool_turns_is_paired_only_by_its_own_answers( + gateway: Gateway, answer: Answer +) -> None: + call: Final = _Call("anthropic", answer, False, uuid.uuid4().hex) + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url) + history: Final = _long_history(call.marker, _answers(call.marker, answer)) + response: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, call.marker, history, False)) + _assert_served(response, call.marker, False) + request, body = _only_request(wire) + assert request.target == "/v1/messages", request.target + _assert_tools_forwarded("anthropic", body) + messages: Final = _ITEMS.validate_python(body["messages"]) + earlier: Final = [f"toolu_{call.marker}_{index}" for index in range(_EARLIER_PAIRS)] + weather: Final = (f"toolu_{call.marker}", "get_weather") + search: Final = (f"srvtoolu_{call.marker}", _REGEX_TOOL_NAME) + match answer: + case "unanswered": + assert "srvtoolu_" not in json.dumps(messages), messages[-4:] + assert _last_assistant_tool_uses(messages) == (weather,), messages[-4:] + assert sorted(_tool_results(messages)) == sorted([*earlier, weather[0]]), messages[-4:] + case "answered": + assert _last_assistant_tool_uses(messages) == (search, weather), messages[-4:] + assert sorted(_tool_results(messages)) == sorted([*earlier, search[0], weather[0]]), messages[-4:] + + def test_a_function_role_answer_keeps_the_replayed_search_call(gateway: Gateway) -> None: call: Final = _Call("anthropic", "answered", False, uuid.uuid4().hex) answer: Final[dict[str, JsonValue]] = { diff --git a/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py index bc95861e467..af8765f4b7f 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py @@ -1,10 +1,11 @@ import asyncio +import itertools import json import uuid from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Final, Literal +from typing import Final, Literal, TypeAlias from urllib.parse import unquote import httpx @@ -23,6 +24,8 @@ _GEMINI: Final = "gemini-2.5-flash" _MISTRAL: Final = "mistral-large-latest" _REGEX_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" _REGEX_TOOL_NAME: Final = "tool_search_tool_regex" +_SEARCH_TOOL_TYPES: Final = frozenset({_REGEX_TOOL_TYPE, "tool_search_tool_bm25_20251119"}) +_AZURE_MESSAGES_PATH: Final = "/anthropic/v1/messages" _ALL_DEFERRED_ERROR: Final = "At least one tool must have defer_loading=false" _DEFERRED_NAMES: Final = ("get_weather", "finance__get_stock_price") _CACHE_BUST: Final[Mapping[str, JsonValue]] = MappingProxyType({"cache": {"no-cache": True}}) @@ -44,8 +47,9 @@ _EMAIL_PARAMETERS: Final[dict[str, JsonValue]] = { "required": ["to"], } -Route = Literal["gemini", "bedrock_converse", "mistral", "unlisted_claude", "anthropic"] +Route = Literal["gemini", "bedrock_converse", "mistral", "unlisted_claude", "anthropic", "azure_ai"] Search = Literal["absent", "dropped"] +NativeSearch: TypeAlias = Literal["regex", "bm25"] @dataclass(frozen=True, slots=True) @@ -70,6 +74,10 @@ def _tool_search(**fields: JsonValue) -> dict[str, JsonValue]: return {"type": "tool_search", **fields} +def _native_search_tool(kind: NativeSearch) -> Mapping[str, JsonValue]: + return MappingProxyType({"type": f"tool_search_tool_{kind}_20251119", "name": f"tool_search_tool_{kind}"}) + + def _function_tool(defer_loading: JsonValue = True) -> dict[str, JsonValue]: return { "type": "function", @@ -249,9 +257,9 @@ def _claude(request: Request) -> Reply: marker: Final = rv.newest_marker(request.body.decode()) or "health" body: Final = _JSON_OBJECT.validate_json(request.body) tools: Final = _request_tools(body) - regex_tools: Final = [tool for tool in tools if tool.get("type") == _REGEX_TOOL_TYPE] - function_tools: Final = [tool for tool in tools if tool.get("type") != _REGEX_TOOL_TYPE] - if not regex_tools and function_tools and all(tool.get("defer_loading") is True for tool in function_tools): + search_tools: Final = [tool for tool in tools if tool.get("type") in _SEARCH_TOOL_TYPES] + function_tools: Final = [tool for tool in tools if tool.get("type") not in _SEARCH_TOOL_TYPES] + if not search_tools and function_tools and all(tool.get("defer_loading") is True for tool in function_tools): return _anthropic_error(400, f"tools: {_ALL_DEFERRED_ERROR} marker-{marker}") if body.get("stream") is True: return Reply(content_type="text/event-stream", chunks=sse_chunks(_claude_events(marker))) @@ -266,7 +274,7 @@ def _vendor(route: Route) -> Callable[[Request], Reply]: return _converse case "mistral": return _mistral - case "unlisted_claude" | "anthropic": + case "unlisted_claude" | "anthropic" | "azure_ai": return _claude @@ -290,6 +298,10 @@ def _deployment(scenario: Scenario, route: Route, wire_url: str) -> str: ) case "anthropic": return scenario.model(model=f"anthropic/{_CLAUDE}", api_base=wire_url, api_key=cc.ANTHROPIC_API_KEY) + case "azure_ai": + return scenario.model( + model=f"azure_ai/{_CLAUDE}", api_base=f"{wire_url}{_AZURE_MESSAGES_PATH}", api_key="synthetic-azure-key" + ) def _responses_body( @@ -343,6 +355,9 @@ def _assert_target(route: Route, stream: bool, request: Request, body: Mapping[s case "unlisted_claude" | "anthropic": assert request.target == "/v1/messages", request.target assert body.get("stream", False) is stream, body + case "azure_ai": + assert request.target == _AZURE_MESSAGES_PATH, request.target + assert body.get("stream", False) is stream, body def _wire_tool_names(route: Route, body: Mapping[str, JsonValue]) -> list[str]: @@ -360,7 +375,7 @@ def _wire_tool_names(route: Route, body: Mapping[str, JsonValue]) -> list[str]: str(_JSON_OBJECT.validate_python(tool["function"])["name"]) for tool in _ITEMS.validate_python(body["tools"]) ] - case "unlisted_claude" | "anthropic": + case "unlisted_claude" | "anthropic" | "azure_ai": return [str(tool["name"]) for tool in _ITEMS.validate_python(body["tools"])] @@ -434,6 +449,48 @@ def test_the_supported_anthropic_route_keeps_each_tools_own_defer_loading_with_t assert "defer_loading" not in json.dumps(sent[3]), sent +def _native_cells() -> tuple[ParameterSet, ...]: + routes: Final[tuple[tuple[Route, NativeSearch], ...]] = ( + ("anthropic", "regex"), + ("anthropic", "bm25"), + ("azure_ai", "regex"), + ) + return tuple( + pytest.param(route, kind, stream, id=f"{route}-{kind}-{'stream' if stream else 'sync'}") + for (route, kind), stream in itertools.product(routes, (False, True)) + ) + + +@pytest.mark.parametrize(("route", "kind", "stream"), _native_cells()) +def test_a_native_anthropic_search_tool_reaches_the_wire_untouched_and_keeps_every_deferral( + gateway: Gateway, route: Route, kind: NativeSearch, stream: bool +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_claude) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, route, wire.url) + tools: Final = (_native_search_tool(kind), _function_tool(), _namespace_tool()) + response: Final = _raw(gateway, _responses_body(model, marker, tools, stream)) + _assert_served(response, marker, stream) + request, body = _only_request(wire) + _assert_target(route, stream, request, body) + sent: Final = _ITEMS.validate_python(body["tools"]) + assert sent[0] == dict(_native_search_tool(kind)), sent + assert [(tool["name"], tool.get("defer_loading")) for tool in sent[1:]] == [ + (name, True) for name in _DEFERRED_NAMES + ], sent + + +def test_bedrock_converse_skips_a_native_search_tool_and_serves_the_deferred_tools(gateway: Gateway) -> None: + call: Final = _Call("bedrock_converse", False, uuid.uuid4().hex) + with wire_server(_converse) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "bedrock_converse", wire.url) + tools: Final = (_native_search_tool("regex"), _function_tool(), _namespace_tool()) + response: Final = _raw(gateway, _responses_body(model, call.marker, tools, False)) + _assert_served(response, call.marker, False) + request, body = _only_request(wire) + _assert_undeferred_wire(call, request, body) + + @pytest.mark.parametrize("route", ["gemini", "mistral"]) def test_a_non_boolean_defer_loading_is_refused_with_400_on_a_route_without_tool_search( gateway: Gateway, route: Route diff --git a/tests/integration/providers/test_responses_bridge_tool_search_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_wire.py index 26fe899714b..054c2e136b9 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_wire.py @@ -35,6 +35,7 @@ _BEDROCK_REGEX_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( ) _REGEX_TOOL_TYPES: Final = frozenset({_REGEX_TOOL_TYPE, _REGEX_TOOL_NAME}) _TOOL_SEARCH_BETA_PREFIXES: Final = ("advanced-tool-use", "tool-search-tool") +_TOOL_SEARCH_CALL_KEYS: Final = frozenset({"id", "type", "call_id", "arguments", "execution", "status"}) _SEARCH_ARGUMENTS: Final = '{"pattern": "weather|stock"}' _TRUNCATED_ARGUMENTS: Final = '{"pattern": ' _ALL_DEFERRED_ERROR: Final = "At least one tool must have defer_loading=false" @@ -587,6 +588,24 @@ def test_bedrock_invoke_and_vertex_claude_keep_the_tool_search_tool( _assert_spend_rows(model, (str(answer.payload["id"]),)) +@pytest.mark.parametrize("provider", ["anthropic", "bedrock_invoke"]) +def test_a_non_streaming_tool_search_call_item_carries_only_its_own_keys(gateway: Gateway, provider: Provider) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, provider, wire.url, gateway.upstream_url) + tools: Final = (_tool_search(), _function_tool(), _namespace_tool()) + response: Final = _raw( + gateway, "/v1/responses", {**_responses_body(model, marker, tools, False), **_CACHE_BUST} + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + _assert_search_call_output(payload, marker) + (search,) = [item for item in _tool_items(payload) if item["type"] == "tool_search_call"] + assert frozenset(search) == _TOOL_SEARCH_CALL_KEYS, search + request, body = _only_request(wire) + _assert_claude_wire(provider, False, request, body, _DEFERRED_NAMES) + + def test_search_arguments_that_never_parse_stream_an_incomplete_tool_search_call(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(_Vendor(search_arguments=_TRUNCATED_ARGUMENTS).respond) as wire, gateway.scenario() as scenario: diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index 5563c2826bb..45d3f8996bf 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -17,7 +17,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.openai import ResponsesAPIStreamEvents -from litellm.types.utils import ModelResponse +from litellm.types.utils import Message, ModelResponse def test_response_api_handler_merges_metadata_and_service_tier_without_error(): @@ -304,6 +304,89 @@ async def test_previous_response_id_without_instructions_carries_the_latest_ones assert [block.text for block in anthropic.request.system] == [expected_system] +_SEARCH_CALL: Final = { + "id": "srvtoolu_search", + "type": "function", + "function": {"name": "tool_search_tool_regex", "arguments": '{"pattern": "weather"}'}, +} +_WEATHER_CALL: Final = { + "id": "toolu_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Tokyo"}'}, +} + + +def _search_turn(tool_calls: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + return { + "request_id": "chatcmpl-search-turn", + "call_type": "aresponses", + "session_id": "session-search", + "proxy_server_request": {"model": _MODEL, "input": "What is the weather in Tokyo?"}, + "response": { + "id": "chatcmpl-search-turn", + "object": "chat.completion", + "created": 0, + "model": _MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": {"role": "assistant", "content": None, "tool_calls": list(tool_calls)}, + } + ], + }, + } + + +async def _follow_up_messages( + tool_calls: Sequence[Mapping[str, object]], model: str, custom_llm_provider: str +) -> Sequence[object]: + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_search_turn(tool_calls)])): + request: Final = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( + previous_response_id="chatcmpl-search-turn", + litellm_completion_request={ + "model": model, + "custom_llm_provider": custom_llm_provider, + "messages": [{"role": "user", "content": "And in Osaka?"}], + }, + instructions=None, + ) + return request["messages"] + + +def _replayed_tool_call_ids(messages: Sequence[object]) -> list[str]: + assistant_turns: Final = [message for message in messages if isinstance(message, Message)] + return [tool_call.id for turn in assistant_turns for tool_call in turn.tool_calls or ()] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected_tool_call_ids"), + [ + ("mistral/mistral-small-latest", "mistral", ["toolu_weather"]), + ("gemini/gemini-2.5-flash", "gemini", ["toolu_weather"]), + (_MODEL, "anthropic", ["srvtoolu_search", "toolu_weather"]), + ], +) +async def test_previous_response_id_keeps_a_stored_claude_search_call_only_on_an_anthropic_route( + model: str, custom_llm_provider: str, expected_tool_call_ids: list[str] +) -> None: + """ + A follow-up that routes a Claude turn's history elsewhere must not replay Anthropic's server-side + search call as a client tool call, which Mistral answers 400 "Unexpected tool call id" + """ + messages: Final = await _follow_up_messages([_SEARCH_CALL, _WEATHER_CALL], model, custom_llm_provider) + + assert _replayed_tool_call_ids(messages) == expected_tool_call_ids + + +@pytest.mark.asyncio +async def test_previous_response_id_drops_a_claude_turn_that_held_only_a_search_call_off_anthropic() -> None: + messages: Final = await _follow_up_messages([_SEARCH_CALL], "mistral/mistral-small-latest", "mistral") + + assert [message["role"] for message in messages] == ["user", "user"] + + def _anthropic_sse(*events: Mapping[str, object]) -> bytes: return b"".join( f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 01c2638efcf..b81da5cba64 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -20,7 +20,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.base import HiddenParams -from litellm.types.responses.main import ResponseToolSearchCall, build_web_search_call +from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, Choices, @@ -1929,6 +1929,88 @@ class TestToolTransformation: assert [tool["function"]["name"] for tool in not_deferred] == ["finance__get_stock_price", "get_weather"] assert all("defer_loading" not in tool for tool in not_deferred) + @pytest.mark.parametrize("global_drop_params", [False, True]) + def test_malformed_tool_flags_answer_400_unless_drop_params_drops_them( + self, monkeypatch: pytest.MonkeyPatch, global_drop_params: bool + ) -> None: + monkeypatch.setattr(litellm, "drop_params", global_drop_params) + transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request + tools: Final = [ + {"type": "tool_search", "execution": "server"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}, "defer_loading": "false"}, + { + "type": "namespace", + "name": "finance", + "tools": [ + { + "type": "function", + "name": "get_stock_price", + "parameters": {"type": "object"}, + "defer_loading": 1, + "allowed_callers": "direct", + }, + { + "type": "function", + "name": "get_fx_rate", + "parameters": {"type": "object"}, + "defer_loading": True, + }, + ], + }, + {"type": "custom", "name": "apply_patch", "defer_loading": "yes"}, + ] + + if not global_drop_params: + with pytest.raises(litellm.BadRequestError): + transform( + model="anthropic/claude-sonnet-5-5", + input="What is the weather in Paris right now?", + responses_api_request={"tools": tools}, + custom_llm_provider="anthropic", + ) + result: Final = transform( + model="anthropic/claude-sonnet-5-5", + input="What is the weather in Paris right now?", + responses_api_request={"tools": tools}, + custom_llm_provider="anthropic", + drop_params=None if global_drop_params else True, + ) + + function_tools: Final = [tool for tool in result["tools"] if tool["type"] == "function"] + assert {tool["function"]["name"]: tool.get("defer_loading", False) for tool in function_tools} == { + "get_weather": False, + "finance__get_stock_price": False, + "finance__get_fx_rate": True, + "apply_patch": False, + } + assert all("allowed_callers" not in tool for tool in function_tools) + + def test_custom_tools_stay_deferred_when_tool_search_goes_out(self) -> None: + result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input="Patch the README", + responses_api_request={ + "tools": [ + {"type": "tool_search", "execution": "server"}, + {"type": "custom", "name": "apply_patch", "defer_loading": True}, + { + "type": "namespace", + "name": "editor", + "tools": [{"type": "custom", "name": "insert", "defer_loading": True}], + }, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + custom_llm_provider="anthropic", + ) + + function_tools: Final = [tool for tool in result["tools"] if tool["type"] == "function"] + assert {tool["function"]["name"]: tool.get("defer_loading", False) for tool in function_tools} == { + "apply_patch": True, + "insert": True, + "get_weather": False, + } + def test_anthropic_tool_search_call_becomes_tool_search_call_item(self) -> None: response: Final = _chat_response_with_tool_search_call('{"pattern": "weather|forecast"}') with_tool_search: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools( @@ -1944,7 +2026,7 @@ class TestToolTransformation: assert [item.type for item in with_tool_search] == ["tool_search_call", "function_call"] search_call: Final = with_tool_search[0] - assert isinstance(search_call, ResponseToolSearchCall) + assert isinstance(search_call, OutputToolSearchCall) assert search_call.id == "tsc_srvtoolu_01Search" assert search_call.call_id == "srvtoolu_01Search" assert search_call.arguments == {"pattern": "weather|forecast"} @@ -1969,7 +2051,7 @@ class TestToolTransformation: ) search_call: Final = items[0] - assert isinstance(search_call, ResponseToolSearchCall) + assert isinstance(search_call, OutputToolSearchCall) assert search_call.status == "incomplete" assert search_call.arguments == {} @@ -2130,9 +2212,7 @@ class TestToolTransformation: tools = [custom_tool] with pytest.raises(litellm.BadRequestError, match="allowed_callers must be a list of strings"): - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(tools=tools) def test_transform_web_search_tools_to_web_search_options(self): """Test that web_search tools are converted to web_search_options""" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 58126b12400..62a9d3943cf 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -39598,6 +39598,32 @@ export interface components { } & { [key: string]: unknown; }; + /** OutputToolSearchCall */ + OutputToolSearchCall: { + /** Arguments */ + arguments: unknown; + /** Call Id */ + call_id?: string | null; + /** + * Execution + * @enum {string} + */ + execution: "server" | "client"; + /** Id */ + id: string; + /** + * Status + * @enum {string} + */ + status: "in_progress" | "completed" | "incomplete"; + /** + * Type + * @constant + */ + type: "tool_search_call"; + } & { + [key: string]: unknown; + }; /** * PageLinks * @description Hypermedia for a paginated list. No `first`/`last`: without a total count the last page is unknown. @@ -43077,7 +43103,7 @@ export interface components { */ ResponseItemList: { /** Data */ - data: (components["schemas"]["ResponseInputMessageItem"] | components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseFunctionToolCallItem"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["openai__types__responses__response_tool_search_call__ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["McpCall"] | components["schemas"]["ResponseCustomToolCallItem"] | components["schemas"]["ResponseCustomToolCallOutputItem"])[]; + data: (components["schemas"]["ResponseInputMessageItem"] | components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseFunctionToolCallItem"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["McpCall"] | components["schemas"]["ResponseCustomToolCallItem"] | components["schemas"]["ResponseCustomToolCallOutputItem"])[]; /** First Id */ first_id: string; /** Has More */ @@ -43299,6 +43325,34 @@ export interface components { } & { [key: string]: unknown; }; + /** ResponseToolSearchCall */ + ResponseToolSearchCall: { + /** Arguments */ + arguments: unknown; + /** Call Id */ + call_id?: string | null; + /** Created By */ + created_by?: string | null; + /** + * Execution + * @enum {string} + */ + execution: "server" | "client"; + /** Id */ + id: string; + /** + * Status + * @enum {string} + */ + status: "in_progress" | "completed" | "incomplete"; + /** + * Type + * @constant + */ + type: "tool_search_call"; + } & { + [key: string]: unknown; + }; /** ResponseToolSearchOutputItem */ ResponseToolSearchOutputItem: { /** Call Id */ @@ -43533,9 +43587,9 @@ export interface components { /** Object */ object?: string | null; /** Output */ - output: (components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["openai__types__responses__response_tool_search_call__ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpCall"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["ResponseCustomToolCall"] | components["schemas"]["ResponseCustomToolCallOutputItem"] | { + output: (components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpCall"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["ResponseCustomToolCall"] | components["schemas"]["ResponseCustomToolCallOutputItem"] | { [key: string]: unknown; - })[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["litellm__types__responses__main__ResponseToolSearchCall"])[]; + })[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["OutputToolSearchCall"])[]; /** Parallel Tool Calls */ parallel_tool_calls?: boolean | null; /** Previous Response Id */ @@ -49368,32 +49422,6 @@ export interface components { /** Type */ type: "computer_use_preview" | string; }; - /** ResponseToolSearchCall */ - litellm__types__responses__main__ResponseToolSearchCall: { - /** Arguments */ - arguments: unknown; - /** Call Id */ - call_id?: string | null; - /** - * Execution - * @enum {string} - */ - execution: "server" | "client"; - /** Id */ - id: string; - /** - * Status - * @enum {string} - */ - status: "in_progress" | "completed" | "incomplete"; - /** - * Type - * @constant - */ - type: "tool_search_call"; - } & { - [key: string]: unknown; - }; /** ModelInfo */ litellm__types__router__ModelInfo: { /** Access Windows */ @@ -49476,34 +49504,6 @@ export interface components { */ type: "computer"; }; - /** ResponseToolSearchCall */ - openai__types__responses__response_tool_search_call__ResponseToolSearchCall: { - /** Arguments */ - arguments: unknown; - /** Call Id */ - call_id?: string | null; - /** Created By */ - created_by?: string | null; - /** - * Execution - * @enum {string} - */ - execution: "server" | "client"; - /** Id */ - id: string; - /** - * Status - * @enum {string} - */ - status: "in_progress" | "completed" | "incomplete"; - /** - * Type - * @constant - */ - type: "tool_search_call"; - } & { - [key: string]: unknown; - }; /** * UserLocation * @description The user's location. From 67fe21b3a24ef0f3cd686ea3976c25e05ffe887e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:26:57 -0700 Subject: [PATCH 2/2] fix(responses): validate function allowed_callers, send one search tool, and keep replays clean off Anthropic A function tool whose allowed_callers is not a list of strings now answers 400 instead of 500. The guardrail scan reads tools without malformed flags, so the route decides between a 400 and a drop under drop_params. When the caller sends a native Anthropic search tool beside tool_search, only the native one goes out. Databricks Claude no longer counts as an Anthropic request route. A previous_response_id follow-up off Anthropic also drops a search call the client replayed as a function_call with a srvtoolu_ id, and its output. The answered tool id scan in the Anthropic message factory runs once per assistant run instead of once per message. A null created_by on tool_search_call and tool_search_output output items is left out of the serialized response, so a cache hit returns the same keys as the first call. --- .../prompt_templates/factory.py | 5 +- .../guardrail_translation/handler.py | 4 +- .../custom_tools.py | 18 +- .../tool_search.py | 81 +++- .../transformation.py | 37 +- litellm/types/llms/openai.py | 19 +- ...test_responses_bridge_tool_search_chaos.py | 383 +++++++++++++++++- ...nses_bridge_tool_search_undeferred_wire.py | 132 ++++++ .../test_responses_bridge_tool_search_wire.py | 180 +++++++- ...llm_core_utils_prompt_templates_factory.py | 22 + ...test_openai_responses_guardrail_handler.py | 19 + .../test_anthropic_responses_bridge.py | 93 ++++- .../test_litellm_completion_responses.py | 40 +- .../unit/types/llms/test_types_llms_openai.py | 18 + 14 files changed, 1020 insertions(+), 31 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index eebf6b73b44..560d756ab9a 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2617,6 +2617,7 @@ def anthropic_messages_pt( unique_tool_ids: set[str] = set() assistant_content: list[AnthropicMessagesAssistantMessageValues] = [] + answered_tool_ids = answered_tool_call_ids(messages, msg_i) ## MERGE CONSECUTIVE ASSISTANT CONTENT ## while msg_i < len(messages) and messages[msg_i]["role"] == "assistant": assistant_content_block: ChatCompletionAssistantMessage = messages[msg_i] @@ -2670,7 +2671,7 @@ def anthropic_messages_pt( assistant_tool_calls, web_search_results=_web_search_results_tc, tool_results=_tool_results_tc, - answered_tool_ids=answered_tool_call_ids(messages, msg_i), + answered_tool_ids=answered_tool_ids, ) # Group tool invoke results into (server_tool_use, result) pairs @@ -2854,7 +2855,7 @@ def anthropic_messages_pt( assistant_tool_calls, web_search_results=_web_search_results, tool_results=_tool_results, - answered_tool_ids=answered_tool_call_ids(messages, msg_i), + answered_tool_ids=answered_tool_ids, ) # Prevent "tool_use ids must be unique" errors by filtering duplicates diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 61e0127f5b1..e5cfbd4d28b 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -63,6 +63,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( unappliable_request_rewrite, ) from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools +from litellm.responses.litellm_completion_transformation.custom_tools import without_malformed_tool_flags from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -567,8 +568,9 @@ class OpenAIResponsesHandler(BaseTranslation): original_tools: Final[tuple[Mapping[str, object], ...]] = ( tuple(raw_tools) if isinstance(raw_tools, list) else () ) + scanned_tools: Final = tuple(without_malformed_tool_flags(tool) for tool in original_tools) flattened_tool_groups: Final = tuple( - form.chat_tools for form in LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(original_tools) + form.chat_tools for form in LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(scanned_tools) ) extracted: Final = self._extract_guardrail_inputs( data, input_data, flattened_tool_groups, skip_system=skip_system diff --git a/litellm/responses/litellm_completion_transformation/custom_tools.py b/litellm/responses/litellm_completion_transformation/custom_tools.py index 3492bc99af6..a69e3d28b56 100644 --- a/litellm/responses/litellm_completion_transformation/custom_tools.py +++ b/litellm/responses/litellm_completion_transformation/custom_tools.py @@ -196,18 +196,30 @@ def _is_well_formed_tool_flag(key: str, value: object) -> bool: return True -_NAMESPACE_MEMBERS_ADAPTER: Final = TypeAdapter[list[dict[str, object]]](list[dict[str, object]]) +_NAMESPACE_MEMBERS_ADAPTER: Final = TypeAdapter[list[object]](list[object]) +_TOOL_ADAPTER: Final = TypeAdapter[dict[str, object]](dict[str, object]) + + +def _with_well_formed_flags(tool: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in tool.items() if _is_well_formed_tool_flag(key, value)} + + +def _member_with_well_formed_flags(member: object) -> object: + try: + return _with_well_formed_flags(_TOOL_ADAPTER.validate_python(member)) + except ValidationError: + return member def without_malformed_tool_flags(tool: Mapping[str, object]) -> Mapping[str, object]: - kept: Final = {key: value for key, value in tool.items() if _is_well_formed_tool_flag(key, value)} + kept: Final = _with_well_formed_flags(tool) if tool.get("type") != "namespace": return kept try: members: Final = _NAMESPACE_MEMBERS_ADAPTER.validate_python(tool.get("tools")) except ValidationError: return kept - return {**kept, "tools": [without_malformed_tool_flags(member) for member in members]} + return {**kept, "tools": [_member_with_well_formed_flags(member) for member in members]} def function_chat_tool( diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index dd1048eea3e..e17aec939fc 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence, Set from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, TypeAdapter, ValidationError @@ -7,7 +7,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import is_anthropic_too from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AnthropicToolSearchToolRegex from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.responses.main import OutputToolSearchCall -from litellm.types.utils import ChatCompletionMessageToolCall, LlmProviders, Message +from litellm.types.utils import LlmProviders, Message from .custom_tools import openai_shaped_tool_call_item_id @@ -16,6 +16,7 @@ __all__ = [ "ANTHROPIC_TOOL_SEARCH_TOOL_TYPE", "CompletionRoute", "ToolSearchCallStatus", + "answers_tool_search", "anthropic_tool_search_chat_tool", "build_tool_search_call", "has_server_tool_search", @@ -24,6 +25,8 @@ __all__ = [ "is_anthropic_tool_search_use", "is_server_tool_search_tool", "provider_builds_anthropic_request", + "tool_search_call_ids", + "without_tool_search_calls", "without_tool_search_uses", ] @@ -64,6 +67,7 @@ def is_anthropic_tool_search_chat_tool(tool: Mapping[str, object]) -> bool: def provider_builds_anthropic_request(model: str, custom_llm_provider: str | None) -> bool: from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.anthropic.chat.transformation import AnthropicConfig + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.utils import ProviderConfigManager _, provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) @@ -72,7 +76,7 @@ def provider_builds_anthropic_request(model: str, custom_llm_provider: str | Non except ValueError: return False chat_config: Final = ProviderConfigManager.get_provider_chat_config(model=model, provider=llm_provider) - return isinstance(chat_config, AnthropicConfig) + return isinstance(chat_config, AnthropicConfig) and not isinstance(chat_config, OpenAIGPTConfig) class CompletionRoute(LiteLLMBaseModel): @@ -82,16 +86,75 @@ class CompletionRoute(LiteLLMBaseModel): custom_llm_provider: str | None = None +class _ToolCallFunctionFields(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + name: str | None = None + + +class _ToolCallFields(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + id: str | None = None + function: _ToolCallFunctionFields | None = None + + +class _SessionMessageFields(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + role: str | None = None + tool_calls: tuple[object, ...] | None = None + tool_call_id: str | None = None + + +def _session_message_fields(message: object) -> _SessionMessageFields: + try: + return _SessionMessageFields.model_validate(message) + except ValidationError: + return _SessionMessageFields() + + +def _tool_search_call_id(tool_call: object) -> str | None: + try: + parsed: Final = _ToolCallFields.model_validate(tool_call) + except ValidationError: + return None + tool_id: Final = parsed.id or "" + tool_name: Final = (parsed.function.name if parsed.function is not None else None) or "" + return tool_id if is_anthropic_tool_search_use(tool_id, tool_name) else None + + def _is_tool_search_call(tool_call: object) -> bool: - return isinstance(tool_call, ChatCompletionMessageToolCall) and is_anthropic_tool_search_use( - tool_call.id or "", tool_call.function.name or "" - ) + return _tool_search_call_id(tool_call) is not None + + +def _assistant_tool_calls(message: object) -> tuple[object, ...]: + fields: Final = _session_message_fields(message) + return (fields.tool_calls or ()) if fields.role == "assistant" else () def holds_tool_search_use(message: object) -> bool: - return isinstance(message, Message) and any( - _is_tool_search_call(tool_call) for tool_call in message.tool_calls or () - ) + return any(_is_tool_search_call(tool_call) for tool_call in _assistant_tool_calls(message)) + + +def _tool_search_call_ids_of(messages: Iterable[object]) -> Iterator[str]: + for message in messages: + for tool_call in _assistant_tool_calls(message): + if (tool_id := _tool_search_call_id(tool_call)) is not None: + yield tool_id + + +def tool_search_call_ids(messages: Iterable[object]) -> frozenset[str]: + return frozenset(_tool_search_call_ids_of(messages)) + + +def answers_tool_search(message: object, search_call_ids: Set[str]) -> bool: + fields: Final = _session_message_fields(message) + return fields.role == "tool" and fields.tool_call_id in search_call_ids + + +def without_tool_search_calls(message: object) -> tuple[object, ...]: + return tuple(tool_call for tool_call in _assistant_tool_calls(message) if not _is_tool_search_call(tool_call)) def without_tool_search_uses(message: Message) -> Message | None: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 7cdcc963191..ff14649ab15 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -120,6 +120,7 @@ from .custom_tools import ( from .reasoning_items import decode_thinking_blocks, encode_thinking_blocks, mint_reasoning_item_id from .tool_search import ( CompletionRoute, + answers_tool_search, anthropic_tool_search_chat_tool, build_tool_search_call, has_server_tool_search, @@ -128,6 +129,8 @@ from .tool_search import ( is_anthropic_tool_search_use, is_server_tool_search_tool, provider_builds_anthropic_request, + tool_search_call_ids, + without_tool_search_calls, without_tool_search_uses, ) @@ -258,8 +261,24 @@ SessionMessage: TypeAlias = ( ) -def _without_stored_tool_search_uses(message: SessionMessage) -> SessionMessage | None: - return without_tool_search_uses(message) if isinstance(message, Message) else message +def _replayed_assistant_without_tool_search_uses(message: Mapping[str, object]) -> SessionMessage | None: + kept_calls: Final = without_tool_search_calls(message) + if not kept_calls and not message.get("content"): + return None + without_calls: Final = {key: value for key, value in message.items() if key != "tool_calls"} + return cast( # cast-ok: the replayed assistant message minus its search calls keeps every other field as stored + SessionMessage, {**without_calls, "tool_calls": list(kept_calls)} if kept_calls else without_calls + ) + + +def _without_stored_tool_search_uses(message: SessionMessage, search_call_ids: frozenset[str]) -> SessionMessage | None: + if isinstance(message, Message): + return without_tool_search_uses(message) + if answers_tool_search(message, search_call_ids): + return None + if isinstance(message, Mapping) and holds_tool_search_use(message): + return _replayed_assistant_without_tool_search_uses(message) + return message class ChatCompletionSession(TypedDict, total=False): @@ -410,6 +429,8 @@ class LiteLLMCompletionResponsesConfig: if not has_server_tool_search(tools): return tools + if any(is_anthropic_tool_search_chat_tool(tool) for tool in tools or ()): + return [tool for tool in tools or () if not is_server_tool_search_tool(tool)] route_runs_tool_search: Final = supports_tool_search( model=model, custom_llm_provider=custom_llm_provider ) and provider_builds_anthropic_request(model, custom_llm_provider) @@ -658,7 +679,12 @@ class LiteLLMCompletionResponsesConfig: ) -> tuple[SessionMessage, ...]: if provider_builds_anthropic_request(route.model, route.custom_llm_provider): return tuple(messages) - return tuple(kept for message in messages if (kept := _without_stored_tool_search_uses(message)) is not None) + search_call_ids: Final = tool_search_call_ids(messages) + return tuple( + kept + for message in messages + if (kept := _without_stored_tool_search_uses(message, search_call_ids)) is not None + ) @staticmethod async def async_responses_api_session_handler( @@ -2130,8 +2156,9 @@ class LiteLLMCompletionResponsesConfig: chat_completion_tool["cache_control"] = tool.get("cache_control") if validated_defer_loading(tool.get("defer_loading")): chat_completion_tool["defer_loading"] = True - if tool.get("allowed_callers"): - chat_completion_tool["allowed_callers"] = tool.get("allowed_callers") + function_allowed_callers: Final = validated_allowed_callers(tool.get("allowed_callers")) + if function_allowed_callers: + chat_completion_tool["allowed_callers"] = function_allowed_callers if tool.get("input_examples"): chat_completion_tool["input_examples"] = tool.get("input_examples") if tool.get("eager_input_streaming") is not None: diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 33986d80df3..493ce762cd6 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1426,6 +1426,17 @@ One of: completed, failed, in_progress, cancelled, queued, or incomplete. """ +_NULL_DROPPED_OUTPUT_ITEM_KEYS: Final = frozenset( + ( + ("reasoning", "status"), + ("reasoning", "content"), + ("reasoning", "encrypted_content"), + ("tool_search_call", "created_by"), + ("tool_search_output", "created_by"), + ) +) + + class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): id: str created_at: int @@ -1515,8 +1526,12 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): return serialized return [ ( - {k: v for k, v in item.items() if v is not None or k not in ("status", "content", "encrypted_content")} - if isinstance(item, dict) and item.get("type") == "reasoning" + { + k: v + for k, v in item.items() + if v is not None or (item.get("type"), k) not in _NULL_DROPPED_OUTPUT_ITEM_KEYS + } + if isinstance(item, dict) else item ) for item in serialized diff --git a/tests/integration/providers/test_responses_bridge_tool_search_chaos.py b/tests/integration/providers/test_responses_bridge_tool_search_chaos.py index 6ca8c9f0f75..2a3b98c3659 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_chaos.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_chaos.py @@ -1,25 +1,27 @@ import asyncio +import itertools import json import re import socket import threading import uuid -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from queue import SimpleQueue -from types import MappingProxyType -from typing import Final, Literal +from types import EllipsisType, MappingProxyType +from typing import Final, Literal, TypeAlias from urllib.parse import unquote, urlsplit import httpx import psutil import pytest import yaml +from _pytest.mark.structures import ParameterSet from integration._support import claude_code as cc from integration._support import responses_vendor as rv -from integration._support.anthropic_thinking import EVENT_STREAM, Event, aws_chunks, sse_chunks -from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.anthropic_thinking import BEDROCK_MODEL, EVENT_STREAM, Event, aws_chunks, sse_chunks +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment from integration._support.database import read_rows from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process from integration._support.wire import Reply, Request, Wire, wire_server @@ -27,6 +29,7 @@ from pydantic import JsonValue _CLAUDE: Final = cc.OPUS _BEDROCK_CLAUDE: Final = "anthropic.claude-sonnet-5-5" +_MISTRAL: Final = "mistral-large-latest" _CONFIG_MODEL: Final = "responses-bridge-tool-search-chaos" _ANTHROPIC_BASE: Final = "http://api.anthropic.com" _REGEX_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" @@ -43,7 +46,15 @@ _WEATHER_PARAMETERS: Final[dict[str, JsonValue]] = { "required": ["city"], } +_ORDER_PARAMETERS: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"order_id": {"type": "string"}}, + "required": ["order_id"], +} + Endpoint = Literal["responses", "chat", "messages"] +StoredShape: TypeAlias = Literal["tool_use", "search_only", "search_and_text"] +OffAnthropicRoute: TypeAlias = Literal["mistral", "bedrock_converse"] @dataclass(frozen=True, slots=True) @@ -635,3 +646,365 @@ def test_bedrock_invoke_pass_through_relays_the_search_frames_untouched(owned_ri seconds=70, ) assert str(rows[0]["status"]) == "success", rows + + +def _server_tool_use(marker: str) -> Event: + return { + "type": "server_tool_use", + "id": f"srvtoolu_{marker}", + "name": _REGEX_TOOL_NAME, + "input": json.loads(_SEARCH_ARGUMENTS), + } + + +def _stored_turn(marker: str, shape: StoredShape) -> dict[str, JsonValue]: + match shape: + case "tool_use": + return _message(marker) + case "search_only": + return { + **_message(marker), + "content": [_server_tool_use(marker), _search_result(marker)], + "stop_reason": "pause_turn", + } + case "search_and_text": + return { + **_message(marker), + "content": [ + _server_tool_use(marker), + _search_result(marker), + {"type": "text", "text": f"found marker-{marker}"}, + ], + "stop_reason": "end_turn", + } + + +def _stored_turn_vendor(shape: StoredShape) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method != "POST": + return Reply() + marker: Final = rv.newest_marker(request.body.decode()) or "health" + return Reply(body=json.dumps(_stored_turn(marker, shape)).encode()) + + return respond + + +def _tool_call_ids(message: Mapping[str, JsonValue]) -> frozenset[str]: + return frozenset(str(call["id"]) for call in rv.ITEMS.validate_python(message.get("tool_calls") or [])) + + +def _answer_ids_after(messages: Sequence[Mapping[str, JsonValue]], index: int) -> frozenset[str]: + answers: Final = itertools.takewhile(lambda message: message.get("role") == "tool", messages[index + 1 :]) + return frozenset(str(message.get("tool_call_id")) for message in answers) + + +def _opening_call_ids(messages: Sequence[Mapping[str, JsonValue]], index: int) -> frozenset[str]: + opener: Final = next((message for message in reversed(messages[:index]) if message.get("role") != "tool"), None) + return frozenset() if opener is None else _tool_call_ids(opener) + + +def _unmatched_tool_ids(messages: Sequence[Mapping[str, JsonValue]]) -> Iterator[str]: + for index, message in enumerate(messages): + if message.get("role") == "assistant": + yield from sorted(_tool_call_ids(message) - _answer_ids_after(messages, index)) + if message.get("role") == "tool" and str(message.get("tool_call_id")) not in _opening_call_ids(messages, index): + yield str(message.get("tool_call_id")) + + +def _chat_answer(marker: str, stream: bool) -> Reply: + head: Final[Mapping[str, JsonValue]] = {"id": f"cmpl-{marker}", "created": 1, "model": _MISTRAL} + if not stream: + message: Final = {"role": "assistant", "content": f"answered marker-{marker}"} + completion: Final = { + **head, + "object": "chat.completion", + "choices": [{"index": 0, "message": message, "finish_reason": "stop"}], + "usage": rv.CHAT_USAGE, + } + return Reply(body=json.dumps(completion).encode()) + delta: Final[dict[str, JsonValue]] = {"role": "assistant", "content": f"answered marker-{marker}"} + opened: Final[dict[str, JsonValue]] = {"index": 0, "delta": delta, "finish_reason": None} + return Reply( + content_type="text/event-stream", + chunks=( + rv.chat_sse({**head, "object": "chat.completion.chunk", "choices": [opened]}), + rv.chat_sse( + { + **head, + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": rv.CHAT_USAGE, + } + ), + b"data: [DONE]\n\n", + ), + ) + + +def _strict_mistral(request: Request) -> Reply: + if request.method != "POST": + return Reply() + marker: Final = rv.newest_marker(request.body.decode()) or "health" + if request.target != "/v1/chat/completions": + return rv.error(404, f"unexpected target {request.target}", "not_found") + if "defer_loading" in request.body.decode(): + return rv.error(422, f"tools: defer_loading: Extra inputs are not permitted marker-{marker}", "extra_forbidden") + body: Final = rv.JSON_OBJECT.validate_json(request.body) + unmatched: Final = tuple(_unmatched_tool_ids(rv.ITEMS.validate_python(body.get("messages") or []))) + if unmatched: + return rv.error( + 400, + f"Not the same number of function calls and responses: {', '.join(unmatched)} marker-{marker}", + "invalid_request_message_order", + ) + return _chat_answer(marker, body.get("stream") is True) + + +def _converse_ids(message: Mapping[str, JsonValue], block_key: str) -> frozenset[str]: + return frozenset( + str(rv.JSON_OBJECT.validate_python(block[block_key])["toolUseId"]) + for block in rv.ITEMS.validate_python(message.get("content") or []) + if block_key in block + ) + + +def _unpaired_converse_turns(messages: Sequence[Mapping[str, JsonValue]]) -> tuple[int, ...]: + return tuple( + index + 1 + for index, (previous, current) in enumerate(zip(messages, messages[1:])) + if _converse_ids(previous, "toolUse") != _converse_ids(current, "toolResult") + ) + + +def _strict_converse(request: Request) -> Reply: + if request.method != "POST": + return Reply() + marker: Final = rv.newest_marker(request.body.decode()) or "health" + if unquote(request.target) != f"/model/{BEDROCK_MODEL}/converse": + return rv.error(404, f"unexpected target {request.target}", "not_found") + body: Final = rv.JSON_OBJECT.validate_json(request.body) + unpaired: Final = _unpaired_converse_turns(rv.ITEMS.validate_python(body.get("messages") or [])) + if unpaired: + refusal: Final = ( + f"Expected toolResult blocks at messages.{unpaired[0]}.content for the toolUse ids of the previous turn " + f"marker-{marker}" + ) + return Reply(status=400, body=json.dumps({"message": refusal}).encode()) + answer: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": f"answered marker-{marker}"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + } + return Reply(body=json.dumps(answer).encode()) + + +def _strict_vendor(route: OffAnthropicRoute) -> Callable[[Request], Reply]: + match route: + case "mistral": + return _strict_mistral + case "bedrock_converse": + return _strict_converse + + +def _off_anthropic_deployment(scenario: Scenario, route: OffAnthropicRoute, wire_url: str) -> str: + match route: + case "mistral": + return scenario.model(model=f"mistral/{_MISTRAL}", api_base=wire_url, api_key="synthetic-mistral-key") + case "bedrock_converse": + return scenario.model( + model=f"bedrock/converse/{BEDROCK_MODEL}", + aws_access_key_id="AKIASCRIPTEDPROVIDER", + aws_secret_access_key="scripted-secret", + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire_url, + ) + + +def _stored_first_turn(proxy: Gateway, model: str, marker: str) -> str: + first: Final = proxy.request("POST", "/v1/responses", _body(model, _Call("responses", False, marker))) + assert first.status_code == 200, first.text + payload: Final = rv.JSON_OBJECT.validate_json(first.text) + types: Final = [item["type"] for item in rv.ITEMS.validate_python(payload["output"])] + assert "tool_search_call" in types, types + response_id: Final = str(payload["id"]) + eventually( + lambda: read_rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda rows: any(rv.same_response(str(row["request_id"]), response_id) for row in rows), + seconds=70, + ) + return response_id + + +def _follow_up_input(marker: str, shape: StoredShape) -> list[JsonValue]: + if shape == "tool_use": + return [{"type": "function_call_output", "call_id": f"toolu_{marker}", "output": f"22C marker-{marker}"}] + return [{"role": "user", "content": f"continue marker-{marker}"}] + + +def _chat_turn(message: Mapping[str, JsonValue]) -> tuple[str, tuple[str, ...]]: + if message.get("role") == "tool": + return "tool", (str(message.get("tool_call_id")),) + return str(message["role"]), tuple(sorted(_tool_call_ids(message))) + + +def _expected_chat_turns(marker: str, shape: StoredShape) -> tuple[tuple[str, tuple[str, ...]], ...]: + match shape: + case "tool_use": + return (("user", ()), ("assistant", (f"toolu_{marker}",)), ("tool", (f"toolu_{marker}",))) + case "search_only": + return (("user", ()), ("user", ())) + case "search_and_text": + return (("user", ()), ("assistant", ()), ("user", ())) + + +def _assert_converse_follow_up(messages: Sequence[Mapping[str, JsonValue]], marker: str) -> None: + assert [message["role"] for message in messages] == ["user", "assistant", "user"], messages + assert _converse_ids(messages[1], "toolUse") == {f"toolu_{marker}"}, messages + assert _converse_ids(messages[2], "toolResult") == {f"toolu_{marker}"}, messages + + +def _off_anthropic_cells() -> tuple[ParameterSet, ...]: + cases: Final[tuple[tuple[OffAnthropicRoute, StoredShape, bool], ...]] = ( + ("mistral", "search_only", False), + ("mistral", "search_and_text", False), + ("mistral", "search_and_text", True), + ("bedrock_converse", "tool_use", False), + ) + return tuple( + pytest.param(route, shape, stream, id=f"{route}-{shape}-{'stream' if stream else 'sync'}") + for route, shape, stream in cases + ) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@pytest.mark.parametrize(("route", "shape", "stream"), _off_anthropic_cells()) +def test_a_follow_up_off_anthropic_drops_the_stored_search_call_and_is_served( + owned_rig: _OwnedRig, route: OffAnthropicRoute, shape: StoredShape, stream: bool +) -> None: + marker: Final = uuid.uuid4().hex + proxy: Final = owned_rig.proxy.gateway + with ( + wire_server(_stored_turn_vendor(shape)) as claude_wire, + wire_server(_strict_vendor(route)) as follow_up_wire, + proxy.scenario() as scenario, + ): + claude: Final = scenario.model( + model=f"anthropic/{_CLAUDE}", api_base=claude_wire.url, api_key=cc.ANTHROPIC_API_KEY + ) + target: Final = _off_anthropic_deployment(scenario, route, follow_up_wire.url) + response_id: Final = _stored_first_turn(proxy, claude, marker) + follow_up: Final = proxy.request( + "POST", + "/v1/responses", + { + **_body(target, _Call("responses", stream, marker)), + "previous_response_id": response_id, + "input": _follow_up_input(marker, shape), + }, + ) + assert follow_up.status_code == 200, follow_up.text + assert f"answered marker-{marker}" in follow_up.text, follow_up.text + (request,) = _marked_posts(follow_up_wire.drain(), marker) + sent: Final = request.body.decode() + assert "srvtoolu_" not in sent and "tool_search" not in sent and "defer_loading" not in sent, sent + body: Final = rv.JSON_OBJECT.validate_json(request.body) + messages: Final = rv.ITEMS.validate_python(body["messages"]) + if route == "bedrock_converse": + _assert_converse_follow_up(messages, marker) + return + assert tuple(_chat_turn(message) for message in messages) == _expected_chat_turns(marker, shape), messages + tools: Final = rv.ITEMS.validate_python(body["tools"]) + assert [rv.JSON_OBJECT.validate_python(tool["function"])["name"] for tool in tools] == ["get_weather"], tools + if shape == "search_and_text": + assert f"found marker-{marker}" in json.dumps(messages[1]), messages + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_a_follow_up_on_anthropic_that_answers_the_stored_search_call_replays_it_with_its_answer( + owned_rig: _OwnedRig, +) -> None: + marker: Final = uuid.uuid4().hex + proxy: Final = owned_rig.proxy.gateway + with wire_server(_stored_turn_vendor("tool_use")) as claude_wire, proxy.scenario() as scenario: + claude: Final = scenario.model( + model=f"anthropic/{_CLAUDE}", api_base=claude_wire.url, api_key=cc.ANTHROPIC_API_KEY + ) + response_id: Final = _stored_first_turn(proxy, claude, marker) + answers: Final[list[JsonValue]] = [ + {"type": "function_call_output", "call_id": f"srvtoolu_{marker}", "output": f"get_weather marker-{marker}"}, + {"type": "function_call_output", "call_id": f"toolu_{marker}", "output": f"22C marker-{marker}"}, + ] + follow_up: Final = proxy.request( + "POST", + "/v1/responses", + {**_body(claude, _Call("responses", False, marker)), "previous_response_id": response_id, "input": answers}, + ) + assert follow_up.status_code == 200, follow_up.text + opening, replay = _marked_posts(claude_wire.drain(), marker) + _assert_forwarded_with_tool_search(opening) + _assert_forwarded_with_tool_search(replay) + messages: Final = rv.ITEMS.validate_python(rv.JSON_OBJECT.validate_json(replay.body)["messages"]) + stored_turn: Final = next(turn for turn in messages if turn["role"] == "assistant") + blocks: Final = rv.ITEMS.validate_python(stored_turn["content"]) + assert [(block.get("id"), block.get("name")) for block in blocks if block["type"] != "text"] == [ + (f"srvtoolu_{marker}", _REGEX_TOOL_NAME), + (f"toolu_{marker}", "get_weather"), + ], blocks + assert sorted(str(block["tool_use_id"]) for block in _tool_results(messages)) == [ + f"srvtoolu_{marker}", + f"toolu_{marker}", + ], messages + + +@pytest.fixture(scope="module") +def drop_params_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]: + directory: Final = tmp_path_factory.mktemp("tool-search-drop-params") + config: Final = rv.JSON_OBJECT.validate_python( + yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + ) + settings: Final = rv.JSON_OBJECT.validate_python(config["litellm_settings"]) + path: Final = directory / "responses-bridge-tool-search-drop-params.yaml" + path.write_text(yaml.safe_dump({**config, "litellm_settings": {**settings, "drop_params": True}})) + with ( + gateway_from_environment() as gateway, + owned_proxy_process(gateway, directory, {}, config=path, workers=2) as owned, + ): + yield owned + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@pytest.mark.parametrize("request_drop_params", [..., False], ids=["absent", "false"]) +def test_a_global_drop_params_drops_a_malformed_defer_loading_and_serves_the_search_call( + drop_params_proxy: OwnedProxy, request_drop_params: bool | EllipsisType +) -> None: + marker: Final = uuid.uuid4().hex + proxy: Final = drop_params_proxy.gateway + with wire_server(_Vendor().respond) as wire, proxy.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_CLAUDE}", api_base=wire.url, api_key=cc.ANTHROPIC_API_KEY) + opening: Final = _body(model, _Call("responses", False, marker)) + lookup: Final[dict[str, JsonValue]] = { + "type": "function", + "name": "lookup_order", + "description": "Look up an order", + "parameters": _ORDER_PARAMETERS, + "defer_loading": "true", + } + flag: Final[Mapping[str, JsonValue]] = ( + {} if isinstance(request_drop_params, EllipsisType) else {"drop_params": request_drop_params} + ) + response: Final = proxy.request( + "POST", + "/v1/responses", + {**opening, "tools": [*rv.ITEMS.validate_python(opening["tools"]), lookup], **flag}, + ) + _assert_carries_the_search_call( + _Served(_Call("responses", False, marker), response.status_code, response.text, "") + ) + (request,) = _marked_posts(wire.drain(), marker) + tools: Final = rv.ITEMS.validate_python(rv.JSON_OBJECT.validate_json(request.body)["tools"]) + assert tools[0] == dict(_REGEX_TOOL), tools + assert [(tool["name"], tool.get("defer_loading")) for tool in tools[1:]] == [ + ("get_weather", True), + ("lookup_order", None), + ], tools diff --git a/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py index af8765f4b7f..c43194100cc 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py @@ -92,6 +92,10 @@ def _loaded_tool() -> dict[str, JsonValue]: return {"type": "function", "name": "send_email", "description": "Send an email", "parameters": _EMAIL_PARAMETERS} +def _custom_tool() -> dict[str, JsonValue]: + return {"type": "custom", "name": "run_sql", "description": "Run a SQL query", "defer_loading": True} + + def _namespace_tool() -> dict[str, JsonValue]: return { "type": "namespace", @@ -480,6 +484,32 @@ def test_a_native_anthropic_search_tool_reaches_the_wire_untouched_and_keeps_eve ], sent +@pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"]) +@pytest.mark.parametrize("route", ["anthropic", "mistral"]) +def test_a_deferred_custom_tool_keeps_its_deferral_only_while_a_search_tool_goes_out( + gateway: Gateway, route: Route, stream: bool +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_vendor(route)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, route, wire.url) + tools: Final = (_tool_search(), _custom_tool(), _loaded_tool()) + response: Final = _raw(gateway, _responses_body(model, marker, tools, stream)) + _assert_served(response, marker, stream) + request, body = _only_request(wire) + _assert_target(route, stream, request, body) + if route == "mistral": + assert _wire_tool_names(route, body) == ["run_sql", "send_email"], body + assert "defer_loading" not in request.body.decode(), body + assert "tool_search" not in request.body.decode(), body + return + sent: Final = _ITEMS.validate_python(body["tools"]) + assert sent[0] == {"type": _REGEX_TOOL_TYPE, "name": _REGEX_TOOL_NAME}, sent + assert [(tool["name"], tool.get("defer_loading")) for tool in sent[1:]] == [ + ("run_sql", True), + ("send_email", None), + ], sent + + def test_bedrock_converse_skips_a_native_search_tool_and_serves_the_deferred_tools(gateway: Gateway) -> None: call: Final = _Call("bedrock_converse", False, uuid.uuid4().hex) with wire_server(_converse) as wire, gateway.scenario() as scenario: @@ -512,6 +542,108 @@ def test_a_non_boolean_defer_loading_is_refused_with_400_on_a_route_without_tool _assert_undeferred_wire(follow_up, request, body) +def _text_parts(marker: str) -> tuple[str, ...]: + return ("part one ", "part two ", f"marker-{marker}") + + +def _split_claude_events(marker: str) -> tuple[Event, ...]: + opened: Final = {**_claude_message(marker), "content": [], "stop_reason": None} + deltas: Final[tuple[Event, ...]] = tuple( + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": part}} + for part in _text_parts(marker) + ) + return ( + {"type": "message_start", "message": opened}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + *deltas, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 7}, + }, + {"type": "message_stop"}, + ) + + +def _split_chat_chunks(marker: str) -> tuple[bytes, ...]: + head: Final[Mapping[str, JsonValue]] = { + "id": f"cmpl-{marker}", + "object": "chat.completion.chunk", + "created": 1, + "model": _MISTRAL, + } + parts: Final = tuple( + rv.chat_sse( + { + **head, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": part}, "finish_reason": None}], + } + ) + for part in _text_parts(marker) + ) + return ( + *parts, + rv.chat_sse( + { + **head, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + } + ), + b"data: [DONE]\n\n", + ) + + +def _split_vendor(route: Route) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method != "POST": + return Reply() + marker: Final = rv.newest_marker(request.body.decode()) or "health" + if route == "mistral": + return Reply(content_type="text/event-stream", chunks=_split_chat_chunks(marker)) + return Reply(content_type="text/event-stream", chunks=sse_chunks(_split_claude_events(marker))) + + return respond + + +def _frames(text: str) -> tuple[dict[str, JsonValue], ...]: + return tuple(_JSON_OBJECT.validate_json(line[6:]) for line in text.splitlines() if line.startswith("data: {")) + + +def _message_texts(items: Sequence[Mapping[str, JsonValue]]) -> tuple[str, ...]: + return tuple( + "".join(str(part["text"]) for part in _ITEMS.validate_python(item["content"])) + for item in items + if item["type"] == "message" + ) + + +@pytest.mark.parametrize("route", ["anthropic", "mistral"]) +def test_a_streamed_answer_closes_its_message_item_with_every_streamed_delta(gateway: Gateway, route: Route) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_split_vendor(route)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, route, wire.url) + tools: Final = (_tool_search(), _function_tool(), _loaded_tool()) + response: Final = _raw(gateway, _responses_body(model, marker, tools, True)) + assert response.status_code == 200, response.text + frames: Final = _frames(response.text) + expected: Final = "".join(_text_parts(marker)) + deltas: Final = [str(frame["delta"]) for frame in frames if frame.get("type") == "response.output_text.delta"] + assert "".join(deltas) == expected, frames + done: Final = [ + _JSON_OBJECT.validate_python(frame["item"]) + for frame in frames + if frame.get("type") == "response.output_item.done" + ] + assert _message_texts(done) == (expected,), done + (completed,) = [frame for frame in frames if frame.get("type") == "response.completed"] + output: Final = _ITEMS.validate_python(_JSON_OBJECT.validate_python(completed["response"])["output"]) + assert _message_texts(output) == (expected,), output + request, body = _only_request(wire) + _assert_target(route, True, request, body) + + async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served: body: Final = _responses_body(model, call.marker, (_function_tool(), _namespace_tool()), call.stream) async with client.stream( diff --git a/tests/integration/providers/test_responses_bridge_tool_search_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_wire.py index 054c2e136b9..dbae068e861 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_wire.py @@ -1,26 +1,29 @@ import asyncio +import itertools import json import os import uuid from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import EllipsisType, MappingProxyType -from typing import Final, Literal +from typing import Final, Literal, TypeAlias from urllib.parse import unquote import anthropic import httpx import openai import pytest +from _pytest.mark.structures import ParameterSet from anthropic.types.beta import BetaToolUnionParam from integration._support import claude_code as cc from integration._support import responses_vendor as rv from integration._support.anthropic_thinking import BEDROCK_MODEL, EVENT_STREAM, MODEL, Event, aws_chunks, sse_chunks -from integration._support.client import Gateway, Scenario, eventually +from integration._support.client import Gateway, Scenario, eventually, object_value from integration._support.database import read_rows from integration._support.vertex import service_account_json from integration._support.wire import Reply, Request, Wire, wire_server from integration.providers.test_bedrock_auth_wire import RESPONSE as CONVERSE_RESPONSE +from openai.types.responses import ResponseToolSearchCall from pydantic import JsonValue, TypeAdapter from redis import Redis @@ -65,8 +68,26 @@ _GEMINI_REPLY: Final[dict[str, JsonValue]] = { "modelVersion": _GEMINI, } +_ORDER_PARAMETERS: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"order_id": {"type": "string"}}, + "required": ["order_id"], +} +_MALFORMED_FLAGS: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"defer_loading": "true", "allowed_callers": "direct"} +) +_WELL_FORMED_FLAGS: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"defer_loading": True, "allowed_callers": ["direct"]} +) +_FLAG_REFUSALS: Final[Mapping[str, str]] = MappingProxyType( + {"defer_loading": "defer_loading must be a boolean", "allowed_callers": "allowed_callers must be a list of strings"} +) + _Client = Literal["sdk", "async_sdk", "raw"] Provider = Literal["anthropic", "bedrock_invoke", "vertex_ai"] +FlaggedKind: TypeAlias = Literal["function", "custom", "namespace_member"] +ToolFlag: TypeAlias = Literal["defer_loading", "allowed_callers"] +DropSource: TypeAlias = Literal["off", "request", "deployment", "null", "request_string"] @dataclass(frozen=True, slots=True) @@ -606,6 +627,35 @@ def test_a_non_streaming_tool_search_call_item_carries_only_its_own_keys(gateway _assert_claude_wire(provider, False, request, body, _DEFERRED_NAMES) +@pytest.mark.parametrize("provider", ["anthropic", "bedrock_invoke"]) +def test_a_streamed_tool_search_call_item_carries_only_its_own_keys(gateway: Gateway, provider: Provider) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, provider, wire.url, gateway.upstream_url) + tools: Final = (_tool_search(), _function_tool(), _namespace_tool()) + answer: Final = _raw_answer(gateway, _responses_body(model, marker, tools, True)) + _assert_search_call_output(answer.payload, marker) + _assert_search_call_events(answer.events, marker, "completed") + streamed: Final = [_item_of(event) for event in _item_events(answer.events, "tool_search_call")] + (completed,) = [item for item in _tool_items(answer.payload) if item["type"] == "tool_search_call"] + assert [frozenset(item) for item in (*streamed, completed)] == [_TOOL_SEARCH_CALL_KEYS] * 3, streamed + request, body = _only_request(wire) + _assert_claude_wire(provider, True, request, body, _DEFERRED_NAMES) + + +def _schema_properties(schemas: Mapping[str, JsonValue], name: str) -> frozenset[str]: + return frozenset(object_value(object_value(schemas[name])["properties"])) + + +def test_the_openapi_schema_keeps_the_sdk_tool_search_call_beside_the_proxy_output_item(gateway: Gateway) -> None: + schemas: Final = object_value(object_value(gateway.get("/openapi.json")["components"])["schemas"]) + proxy_item: Final = _schema_properties(schemas, "OutputToolSearchCall") + sdk_item: Final = _schema_properties(schemas, "ResponseToolSearchCall") + assert proxy_item == _TOOL_SEARCH_CALL_KEYS, schemas.get("OutputToolSearchCall") + assert sdk_item == frozenset(ResponseToolSearchCall.model_fields), schemas.get("ResponseToolSearchCall") + assert [name for name in schemas if name.endswith("ToolSearchCall") and "__" in name] == [], sorted(schemas) + + def test_search_arguments_that_never_parse_stream_an_incomplete_tool_search_call(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(_Vendor(search_arguments=_TRUNCATED_ARGUMENTS).respond) as wire, gateway.scenario() as scenario: @@ -700,6 +750,24 @@ def test_a_cache_hit_still_carries_the_tool_search_call(gateway: Gateway) -> Non assert sorted(str(row["cache_hit"]) == "True" for row in rows) == [False, True], rows +def test_a_cache_hit_tool_search_call_item_carries_only_its_own_keys(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url, gateway.upstream_url) + body: Final = _responses_body(model, marker, (_tool_search(), _function_tool()), False) + first: Final = _raw(gateway, "/v1/responses", body) + assert first.status_code == 200, first.text + assert len(wire.drain()) == 1 + _await_cached(f"srvtoolu_{marker}") + hit: Final = _raw(gateway, "/v1/responses", body) + assert hit.status_code == 200, hit.text + assert wire.drain() == (), hit.text + payload: Final = _JSON_OBJECT.validate_json(hit.content) + _assert_search_call_output(payload, marker) + (search,) = [item for item in _tool_items(payload) if item["type"] == "tool_search_call"] + assert frozenset(search) == _TOOL_SEARCH_CALL_KEYS, search + + def test_model_info_reports_tool_search_support_for_the_claude_deployment(gateway: Gateway) -> None: with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, "anthropic", wire.url, gateway.upstream_url) @@ -964,6 +1032,114 @@ def test_a_malformed_allowed_callers_on_a_namespace_member_is_refused_with_400( assert wire.drain() == (), refused.text +def _tool_flags(flag: ToolFlag) -> Mapping[str, JsonValue]: + return {key: _MALFORMED_FLAGS[key] if key == flag else value for key, value in _WELL_FORMED_FLAGS.items()} + + +def _flagged_tool(kind: FlaggedKind, flag: ToolFlag) -> dict[str, JsonValue]: + match kind: + case "function": + return { + "type": "function", + "name": "lookup_order", + "description": "Look up an order", + "parameters": _ORDER_PARAMETERS, + **_tool_flags(flag), + } + case "custom": + return {"type": "custom", "name": "run_sql", "description": "Run a SQL query", **_tool_flags(flag)} + case "namespace_member": + return _namespace_tool(**_tool_flags(flag)) + + +def _flagged_wire_name(kind: FlaggedKind) -> str: + match kind: + case "function": + return "lookup_order" + case "custom": + return "run_sql" + case "namespace_member": + return "finance__get_stock_price" + + +def _request_drop_params(source: DropSource) -> Mapping[str, JsonValue]: + match source: + case "off" | "deployment": + return {} + case "request": + return {"drop_params": True} + case "null": + return {"drop_params": None} + case "request_string": + return {"drop_params": "true"} + + +def _drop_params_deployment(scenario: Scenario, source: DropSource, wire_url: str) -> str: + if source == "deployment": + return scenario.model( + model=f"anthropic/{_CLAUDE}", api_base=wire_url, api_key=cc.ANTHROPIC_API_KEY, drop_params=True + ) + return scenario.model(model=f"anthropic/{_CLAUDE}", api_base=wire_url, api_key=cc.ANTHROPIC_API_KEY) + + +def _drop_params_cells() -> tuple[ParameterSet, ...]: + kinds: Final[tuple[FlaggedKind, ...]] = ("function", "custom", "namespace_member") + flags: Final[tuple[ToolFlag, ...]] = ("defer_loading", "allowed_callers") + sources: Final[tuple[DropSource, ...]] = ("off", "request", "deployment") + grid: Final = tuple( + pytest.param(kind, flag, source, id=f"{kind}-{flag}-{source}") + for kind, flag, source in itertools.product(kinds, flags, sources) + ) + edges: Final = tuple( + pytest.param("function", "defer_loading", source, id=f"function-defer_loading-{source}") + for source in ("null", "request_string") + ) + return (*grid, *edges) + + +def _assert_flag_dropped_on_wire(body: Mapping[str, JsonValue], kind: FlaggedKind, flag: ToolFlag) -> None: + tools: Final = _ITEMS.validate_python(body["tools"]) + assert tools[0] == dict(_REGEX_TOOL), tools + (flagged,) = [tool for tool in tools if tool.get("name") == _flagged_wire_name(kind)] + assert flag not in flagged, flagged + kept: Final = "allowed_callers" if flag == "defer_loading" else "defer_loading" + assert flagged[kept] == _WELL_FORMED_FLAGS[kept], flagged + (weather,) = [tool for tool in tools if tool.get("name") == "get_weather"] + assert weather.get("defer_loading") is True, weather + + +@pytest.mark.parametrize(("kind", "flag", "source"), _drop_params_cells()) +def test_drop_params_drops_a_malformed_tool_flag_and_without_it_the_request_is_refused_with_400( + gateway: Gateway, kind: FlaggedKind, flag: ToolFlag, source: DropSource +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: + model: Final = _drop_params_deployment(scenario, source, wire.url) + tools: Final = (_tool_search(), _function_tool(), _flagged_tool(kind, flag)) + response: Final = _raw( + gateway, + "/v1/responses", + {**_responses_body(model, marker, tools, False), **_CACHE_BUST, **_request_drop_params(source)}, + ) + if source in ("off", "null"): + assert response.status_code == 400, response.text + assert "litellm.BadRequestError" in response.text, response.text + assert _FLAG_REFUSALS[flag] in response.text, response.text + assert wire.drain() == (), response.text + follow_up: Final = uuid.uuid4().hex + served: Final = _raw_answer( + gateway, _responses_body(model, follow_up, (_tool_search(), _function_tool()), False) + ) + _assert_search_call_output(served.payload, follow_up) + assert len(wire.drain()) == 1 + return + assert response.status_code == 200, response.text + _assert_search_call_output(_JSON_OBJECT.validate_json(response.content), marker) + request, body = _only_request(wire) + assert request.target == "/v1/messages", request.target + _assert_flag_dropped_on_wire(body, kind, flag) + + def test_duplicate_tool_search_tools_reach_claude_and_its_refusal_reaches_the_caller(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 90dbad52036..14fea46516e 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -3,6 +3,7 @@ import json import logging import os import re +import time from typing import Final, List from unittest.mock import MagicMock, patch @@ -7176,3 +7177,24 @@ def test_anthropic_messages_pt_keeps_a_replayed_tool_search_call_the_caller_answ "srvtoolu_01Search", "toolu_01Weather", ] + + +def _merged_assistant_run_cost(run_length: int) -> float: + messages: Final = [ + {"role": "user", "content": "Plan the trip"}, + *({"role": "assistant", "content": "Working on it", "tool_calls": []} for _ in range(run_length)), + {"role": "user", "content": "Go on"}, + ] + + def once() -> float: + started: Final = time.perf_counter() + anthropic_messages_pt(messages=messages, model="claude-sonnet-5-5", llm_provider="anthropic") + return time.perf_counter() - started + + return min(once() for _ in range(3)) + + +def test_anthropic_messages_pt_merges_a_long_assistant_run_in_linear_time(): + growth: Final = _merged_assistant_run_cost(4000) / _merged_assistant_run_cost(1000) + + assert growth < 8.0, f"cost grew {growth:.1f}x for 4x the assistant run" diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 8bd242308f7..d7d4c1b3aae 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -3197,6 +3197,25 @@ class TestOpenAIResponsesHandlerNamespaceTools: assert result["tools"] == expected_tools + @pytest.mark.asyncio + async def test_scanning_tools_with_malformed_flags_leaves_them_for_the_route_to_judge(self): + data: Final = _codex_request("hi") + data["tools"][0]["defer_loading"] = "false" + data["tools"][1]["tools"][0]["allowed_callers"] = "direct" + data["tools"][2]["defer_loading"] = 1 + expected_tools: Final = copy.deepcopy(data["tools"]) + guardrail: Final = MockRecordingGuardrail(guardrail_name="test") + + result: Final = await OpenAIResponsesHandler().process_input_messages(data, guardrail) + + assert result["tools"] == expected_tools + assert [tool["function"]["name"] for tool in guardrail.seen_inputs[0]["tools"]] == [ + "get_weather", + "mcp__confluence__confluence_get_page", + "mcp__confluence__confluence_search", + "apply_patch", + ] + @pytest.mark.asyncio async def test_appending_guardrail_keeps_namespace_and_adds_tool(self): data = _codex_request("hi") diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index 45d3f8996bf..6086a3f6013 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -1,5 +1,6 @@ import json from collections.abc import Mapping, Sequence +from itertools import chain from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -341,7 +342,13 @@ def _search_turn(tool_calls: Sequence[Mapping[str, object]]) -> Mapping[str, obj async def _follow_up_messages( tool_calls: Sequence[Mapping[str, object]], model: str, custom_llm_provider: str ) -> Sequence[object]: - with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_search_turn(tool_calls)])): + return await _follow_up_messages_after(_search_turn(tool_calls), model, custom_llm_provider) + + +async def _follow_up_messages_after( + spend_log: Mapping[str, object], model: str, custom_llm_provider: str +) -> Sequence[object]: + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([spend_log])): request: Final = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( previous_response_id="chatcmpl-search-turn", litellm_completion_request={ @@ -365,6 +372,7 @@ def _replayed_tool_call_ids(messages: Sequence[object]) -> list[str]: [ ("mistral/mistral-small-latest", "mistral", ["toolu_weather"]), ("gemini/gemini-2.5-flash", "gemini", ["toolu_weather"]), + ("databricks/databricks-claude-sonnet-4-5", "databricks", ["toolu_weather"]), (_MODEL, "anthropic", ["srvtoolu_search", "toolu_weather"]), ], ) @@ -387,6 +395,89 @@ async def test_previous_response_id_drops_a_claude_turn_that_held_only_a_search_ assert [message["role"] for message in messages] == ["user", "user"] +def _replayed_search_turn(search_answered: bool) -> Mapping[str, object]: + search_result: Final = ( + [{"type": "function_call_output", "call_id": "srvtoolu_search", "output": "get_weather"}] + if search_answered + else [] + ) + return { + **_search_turn([]), + "proxy_server_request": { + "model": _MODEL, + "input": [ + {"role": "user", "content": "What is the weather in Tokyo?"}, + { + "type": "function_call", + "call_id": "srvtoolu_search", + "name": "tool_search_tool_regex", + "arguments": '{"pattern": "weather"}', + }, + *search_result, + { + "type": "function_call", + "call_id": "toolu_weather", + "name": "get_weather", + "arguments": '{"city": "Tokyo"}', + }, + {"type": "function_call_output", "call_id": "toolu_weather", "output": '{"temp_c": 18}'}, + ], + }, + "response": { + "id": "chatcmpl-search-turn", + "object": "chat.completion", + "created": 0, + "model": _MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "It is 18C in Tokyo."}, + } + ], + }, + } + + +def _tool_call_ids_of(message: object) -> Sequence[str]: + if isinstance(message, Message): + return [tool_call.id or "" for tool_call in message.tool_calls or ()] + if isinstance(message, dict): + return [tool_call["id"] for tool_call in message.get("tool_calls") or ()] + return [] + + +def _history_tool_ids(messages: Sequence[object]) -> tuple[Sequence[str], Sequence[str]]: + called: Final = list(chain.from_iterable(map(_tool_call_ids_of, messages))) + answered: Final = [ + message["tool_call_id"] for message in messages if isinstance(message, dict) and message["role"] == "tool" + ] + return called, answered + + +@pytest.mark.asyncio +@pytest.mark.parametrize("search_answered", [False, True]) +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expect_search"), + [("mistral/mistral-small-latest", "mistral", False), (_MODEL, "anthropic", True)], +) +async def test_previous_response_id_drops_a_client_replayed_search_call_only_off_anthropic( + model: str, custom_llm_provider: str, expect_search: bool, search_answered: bool +) -> None: + """ + A client can replay Claude's search call as a ``function_call`` input item with its ``srvtoolu_`` id. A later + turn sent elsewhere must not carry it, which OpenAI-compatible providers answer 400 "did not have response messages" + """ + messages: Final = await _follow_up_messages_after( + _replayed_search_turn(search_answered), model, custom_llm_provider + ) + + called, answered = _history_tool_ids(messages) + search_ids: Final = ["srvtoolu_search"] if expect_search else [] + assert called == [*search_ids, "toolu_weather"] + assert answered == [*(search_ids if search_answered else []), "toolu_weather"] + + def _anthropic_sse(*events: Mapping[str, object]) -> bytes: return b"".join( f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index b81da5cba64..d0d75c6a33c 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1,4 +1,5 @@ import json +from collections.abc import Mapping from copy import deepcopy from typing import Final, Literal @@ -1853,6 +1854,27 @@ class TestToolTransformation: assert chat_tools[0] == {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} assert [tool["defer_loading"] for tool in chat_tools[1:]] == [True, True] + @pytest.mark.parametrize( + "native_search_tool", + [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}, + ], + ) + def test_server_tool_search_beside_a_native_search_tool_sends_only_the_native_one( + self, native_search_tool: Mapping[str, str] + ) -> None: + result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input="What is the weather in Paris right now?", + responses_api_request={"tools": [native_search_tool, *_deferred_tools_with_server_tool_search()]}, + custom_llm_provider="anthropic", + ) + + chat_tools: Final = result["tools"] + assert [tool for tool in chat_tools if tool["type"] != "function"] == [native_search_tool] + assert [tool["defer_loading"] for tool in chat_tools if tool["type"] == "function"] == [True, True] + def test_native_search_tool_passes_through_routes_without_tool_search(self) -> None: model: Final = "azure_ai/claude-sonnet-5-5" assert litellm.supports_tool_search(model, "azure_ai") is False @@ -1937,11 +1959,18 @@ class TestToolTransformation: transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request tools: Final = [ {"type": "tool_search", "execution": "server"}, - {"type": "function", "name": "get_weather", "parameters": {"type": "object"}, "defer_loading": "false"}, + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object"}, + "defer_loading": "false", + "allowed_callers": "direct", + }, { "type": "namespace", "name": "finance", "tools": [ + "not a tool", { "type": "function", "name": "get_stock_price", @@ -1985,6 +2014,15 @@ class TestToolTransformation: } assert all("allowed_callers" not in tool for tool in function_tools) + def test_top_level_function_allowed_callers_must_be_a_list_of_strings(self) -> None: + transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools + weather_tool: Final = {"type": "function", "name": "get_weather", "parameters": {"type": "object"}} + + with pytest.raises(litellm.BadRequestError, match="allowed_callers must be a list of strings"): + transform(tools=[{**weather_tool, "allowed_callers": "direct"}]) + converted, _ = transform(tools=[{**weather_tool, "allowed_callers": ["direct"]}]) + assert converted[0]["allowed_callers"] == ["direct"] + def test_custom_tools_stay_deferred_when_tool_search_goes_out(self) -> None: result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model="anthropic/claude-sonnet-5-5", diff --git a/tests/unit/types/llms/test_types_llms_openai.py b/tests/unit/types/llms/test_types_llms_openai.py index 871e4cc8cfb..e90d48b64af 100644 --- a/tests/unit/types/llms/test_types_llms_openai.py +++ b/tests/unit/types/llms/test_types_llms_openai.py @@ -668,3 +668,21 @@ def test_openai_video_object_hidden_params_public_accessor_preserves_identity() video.hidden_params = replacement assert video._hidden_params is replacement assert video.hidden_params["public_key"] == "visible" + + +@pytest.mark.parametrize("extra_keys", ({}, {"created_by": "server"}), ids=("bridge-item", "openai-item")) +def test_rebuilding_a_dumped_responses_answer_keeps_the_tool_search_call_keys(extra_keys: dict[str, str]) -> None: + search_call: Final = { + "id": "tsc_1", + "arguments": {"query": "weather"}, + "call_id": None, + "execution": "server", + "status": "completed", + "type": "tool_search_call", + **extra_keys, + } + answer: Final = ResponsesAPIResponse(id="resp_1", created_at=1, output=[search_call]) + + rebuilt: Final = ResponsesAPIResponse(**answer.model_dump()) + + assert rebuilt.model_dump()["output"] == [search_call]