From ba49b2026d0916fdf4adf266ab4096e0fceef268 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:24:41 -0700 Subject: [PATCH] fix(responses): drop malformed tool flags under drop_params and keep stored search calls off non-Anthropic follow-ups With drop_params on, a non-boolean defer_loading or a non-list allowed_callers on a Responses API tool (or a namespace member) is dropped before the bridge instead of answering 400, so only a request without drop_params gets the 400. A custom tool now carries defer_loading to the chat tool like a function tool does A previous_response_id follow-up routed to a provider that does not build an Anthropic request no longer replays the stored Claude search call (srvtoolu_ id) as a client tool call. Mistral answered that history 400 "Unexpected tool call id". An assistant turn that held only the search call is dropped there. The Anthropic route keeps the call, since its factory pairs it with the stored result The tool_search_call output model is renamed OutputToolSearchCall so the proxy's OpenAPI schema keeps the OpenAI SDK's ResponseToolSearchCall name. answered_tool_call_ids reads the history by index, so the lookup is linear in the history length. The streaming done event reads the response it is given, which is the same object it used to read from the iterator --- .../prompt_templates/factory.py | 5 +- litellm/proxy/_lazy_openapi_snapshot.json | 244 +++++++++--------- .../custom_tools.py | 53 +++- .../streaming_iterator.py | 4 +- .../tool_search.py | 40 ++- .../transformation.py | 92 ++++--- litellm/types/llms/openai.py | 4 +- litellm/types/responses/main.py | 2 +- ...esponses_bridge_tool_search_replay_wire.py | 51 ++++ ...nses_bridge_tool_search_undeferred_wire.py | 71 ++++- .../test_responses_bridge_tool_search_wire.py | 19 ++ .../test_anthropic_responses_bridge.py | 85 +++++- .../test_litellm_completion_responses.py | 92 ++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 114 ++++---- 14 files changed, 636 insertions(+), 240 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d0cf1d10d92..eebf6b73b44 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1828,9 +1828,8 @@ def _tool_result_ids(answers: Iterable[AllMessageValues]) -> Iterator[str]: def answered_tool_call_ids(messages: Sequence[AllMessageValues], assistant_index: int) -> frozenset[str]: - following: Final = itertools.dropwhile( - lambda message: message["role"] == "assistant", itertools.islice(messages, assistant_index, None) - ) + from_assistant: Final = (messages[index] for index in range(assistant_index, len(messages))) + following: Final = itertools.dropwhile(lambda message: message["role"] == "assistant", from_assistant) answers: Final = itertools.takewhile(lambda message: message["role"] in ("tool", "function"), following) return frozenset(_tool_result_ids(answers)) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 44b59f175c8..9d3e140eadd 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -24116,6 +24116,60 @@ "title": "OutputTokensDetails", "type": "object" }, + "OutputToolSearchCall": { + "additionalProperties": true, + "properties": { + "arguments": { + "title": "Arguments" + }, + "call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Call Id" + }, + "execution": { + "enum": [ + "server", + "client" + ], + "title": "Execution", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "status": { + "enum": [ + "in_progress", + "completed", + "incomplete" + ], + "title": "Status", + "type": "string" + }, + "type": { + "const": "tool_search_call", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "arguments", + "execution", + "status", + "type" + ], + "title": "OutputToolSearchCall", + "type": "object" + }, "PendingSafetyCheck": { "additionalProperties": true, "description": "A pending safety check for the computer call.", @@ -25748,7 +25802,7 @@ "$ref": "#/components/schemas/ResponseFunctionToolCallOutputItem" }, { - "$ref": "#/components/schemas/openai__types__responses__response_tool_search_call__ResponseToolSearchCall" + "$ref": "#/components/schemas/ResponseToolSearchCall" }, { "$ref": "#/components/schemas/ResponseToolSearchOutputItem" @@ -26102,6 +26156,71 @@ "title": "ResponseTextConfigParam", "type": "object" }, + "ResponseToolSearchCall": { + "additionalProperties": true, + "properties": { + "arguments": { + "title": "Arguments" + }, + "call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Call Id" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "execution": { + "enum": [ + "server", + "client" + ], + "title": "Execution", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "status": { + "enum": [ + "in_progress", + "completed", + "incomplete" + ], + "title": "Status", + "type": "string" + }, + "type": { + "const": "tool_search_call", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "arguments", + "execution", + "status", + "type" + ], + "title": "ResponseToolSearchCall", + "type": "object" + }, "ResponseToolSearchOutputItem": { "additionalProperties": true, "properties": { @@ -26336,7 +26455,7 @@ "$ref": "#/components/schemas/ResponseReasoningItem" }, { - "$ref": "#/components/schemas/openai__types__responses__response_tool_search_call__ResponseToolSearchCall" + "$ref": "#/components/schemas/ResponseToolSearchCall" }, { "$ref": "#/components/schemas/ResponseToolSearchOutputItem" @@ -26419,7 +26538,7 @@ "$ref": "#/components/schemas/CustomToolCallOutputItem" }, { - "$ref": "#/components/schemas/litellm__types__responses__main__ResponseToolSearchCall" + "$ref": "#/components/schemas/OutputToolSearchCall" } ] }, @@ -27358,125 +27477,6 @@ "title": "WebSearchTool", "type": "object" }, - "litellm__types__responses__main__ResponseToolSearchCall": { - "additionalProperties": true, - "properties": { - "arguments": { - "title": "Arguments" - }, - "call_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Call Id" - }, - "execution": { - "enum": [ - "server", - "client" - ], - "title": "Execution", - "type": "string" - }, - "id": { - "title": "Id", - "type": "string" - }, - "status": { - "enum": [ - "in_progress", - "completed", - "incomplete" - ], - "title": "Status", - "type": "string" - }, - "type": { - "const": "tool_search_call", - "title": "Type", - "type": "string" - } - }, - "required": [ - "id", - "arguments", - "execution", - "status", - "type" - ], - "title": "ResponseToolSearchCall", - "type": "object" - }, - "openai__types__responses__response_tool_search_call__ResponseToolSearchCall": { - "additionalProperties": true, - "properties": { - "arguments": { - "title": "Arguments" - }, - "call_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Call Id" - }, - "created_by": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Created By" - }, - "execution": { - "enum": [ - "server", - "client" - ], - "title": "Execution", - "type": "string" - }, - "id": { - "title": "Id", - "type": "string" - }, - "status": { - "enum": [ - "in_progress", - "completed", - "incomplete" - ], - "title": "Status", - "type": "string" - }, - "type": { - "const": "tool_search_call", - "title": "Type", - "type": "string" - } - }, - "required": [ - "id", - "arguments", - "execution", - "status", - "type" - ], - "title": "ResponseToolSearchCall", - "type": "object" - }, "openai__types__responses__web_search_preview_tool__UserLocation": { "additionalProperties": true, "description": "The user's location.", diff --git a/litellm/responses/litellm_completion_transformation/custom_tools.py b/litellm/responses/litellm_completion_transformation/custom_tools.py index 51942fa5fff..3492bc99af6 100644 --- a/litellm/responses/litellm_completion_transformation/custom_tools.py +++ b/litellm/responses/litellm_completion_transformation/custom_tools.py @@ -180,6 +180,50 @@ def validated_defer_loading(value: object) -> bool: raise BadRequestError(message="defer_loading must be a boolean", model="", llm_provider="") from exc +_TOOL_FLAG_ADAPTERS: Final = MappingProxyType( + {"defer_loading": _DEFER_LOADING_ADAPTER, "allowed_callers": _ALLOWED_CALLERS_ADAPTER} +) + + +def _is_well_formed_tool_flag(key: str, value: object) -> bool: + adapter: Final = _TOOL_FLAG_ADAPTERS.get(key) + if adapter is None: + return True + try: + adapter.validate_python(value, strict=True) + except ValidationError: + return False + return True + + +_NAMESPACE_MEMBERS_ADAPTER: Final = TypeAdapter[list[dict[str, object]]](list[dict[str, object]]) + + +def without_malformed_tool_flags(tool: Mapping[str, object]) -> Mapping[str, object]: + kept: Final = {key: value for key, value in tool.items() if _is_well_formed_tool_flag(key, value)} + if tool.get("type") != "namespace": + return kept + try: + members: Final = _NAMESPACE_MEMBERS_ADAPTER.validate_python(tool.get("tools")) + except ValidationError: + return kept + return {**kept, "tools": [without_malformed_tool_flags(member) for member in members]} + + +def function_chat_tool( + function: ChatCompletionToolParamFunctionChunk, allowed_callers: Sequence[str] | None, deferred: bool +) -> ChatCompletionToolParam: + if allowed_callers is None and not deferred: + return ChatCompletionToolParam(type="function", function=function) + if allowed_callers is None: + return ChatCompletionToolParam(type="function", function=function, defer_loading=True) + if not deferred: + return ChatCompletionToolParam(type="function", function=function, allowed_callers=list(allowed_callers)) + return ChatCompletionToolParam( + type="function", function=function, allowed_callers=list(allowed_callers), defer_loading=True + ) + + def custom_tool_grammar_suffix(fmt: object) -> str: try: parsed: Final = _CustomToolFormat.model_validate(fmt) @@ -196,8 +240,8 @@ def convert_custom_tool_to_function_tool(tool: Mapping[str, object]) -> ChatComp The grammar definition is embedded in the description so the model can produce correctly-formatted output. Returns ``None`` if the tool is not a - custom tool. Raises ``ValueError`` if ``allowed_callers`` is not a list of - strings. + custom tool. Raises ``BadRequestError`` if ``allowed_callers`` is not a list + of strings or ``defer_loading`` is not a boolean. """ if tool.get("type") != "custom": return None @@ -208,6 +252,7 @@ def convert_custom_tool_to_function_tool(tool: Mapping[str, object]) -> ChatComp tool.get("format") ) allowed_callers: Final = validated_allowed_callers(tool.get("allowed_callers")) + deferred: Final = validated_defer_loading(tool.get("defer_loading")) function_chunk: Final = ChatCompletionToolParamFunctionChunk( name=name, description=description, @@ -222,6 +267,4 @@ def convert_custom_tool_to_function_tool(tool: Mapping[str, object]) -> ChatComp "required": ["content"], }, ) - if allowed_callers is None: - return ChatCompletionToolParam(type="function", function=function_chunk) - return ChatCompletionToolParam(type="function", function=function_chunk, allowed_callers=allowed_callers) + return function_chat_tool(function_chunk, allowed_callers, deferred) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index bdc4796b2ea..418bdb3d6db 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -858,8 +858,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self._cached_item_id is None: self._cached_item_id = f"msg_{uuid.uuid4()}" - text: Final = self.litellm_model_response.choices[0].message.content or "" - annotations = getattr(self.litellm_model_response.choices[0].message, "annotations", None) + text: Final = litellm_complete_object.choices[0].message.content or "" + annotations = getattr(litellm_complete_object.choices[0].message, "annotations", None) response_annotations: Final = ( LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations( diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index bab28279176..dd1048eea3e 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -6,22 +6,25 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError from litellm.litellm_core_utils.prompt_templates.factory import is_anthropic_tool_search_use from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AnthropicToolSearchToolRegex from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.responses.main import ResponseToolSearchCall -from litellm.types.utils import LlmProviders +from litellm.types.responses.main import OutputToolSearchCall +from litellm.types.utils import ChatCompletionMessageToolCall, LlmProviders, Message from .custom_tools import openai_shaped_tool_call_item_id __all__ = [ "ANTHROPIC_TOOL_SEARCH_TOOL_NAME", "ANTHROPIC_TOOL_SEARCH_TOOL_TYPE", + "CompletionRoute", "ToolSearchCallStatus", "anthropic_tool_search_chat_tool", "build_tool_search_call", "has_server_tool_search", + "holds_tool_search_use", "is_anthropic_tool_search_chat_tool", "is_anthropic_tool_search_use", "is_server_tool_search_tool", "provider_builds_anthropic_request", + "without_tool_search_uses", ] ANTHROPIC_TOOL_SEARCH_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" @@ -72,6 +75,35 @@ def provider_builds_anthropic_request(model: str, custom_llm_provider: str | Non return isinstance(chat_config, AnthropicConfig) +class CompletionRoute(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model: str + custom_llm_provider: str | None = None + + +def _is_tool_search_call(tool_call: object) -> bool: + return isinstance(tool_call, ChatCompletionMessageToolCall) and is_anthropic_tool_search_use( + tool_call.id or "", tool_call.function.name or "" + ) + + +def holds_tool_search_use(message: object) -> bool: + return isinstance(message, Message) and any( + _is_tool_search_call(tool_call) for tool_call in message.tool_calls or () + ) + + +def without_tool_search_uses(message: Message) -> Message | None: + tool_calls: Final = message.tool_calls or [] + kept: Final = tuple(tool_call for tool_call in tool_calls if not _is_tool_search_call(tool_call)) + if len(kept) == len(tool_calls): + return message + if not kept and not message.content: + return None + return message.model_copy(update={"tool_calls": list(kept) or None}) + + def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None: try: return _ARGUMENTS_ADAPTER.validate_json(arguments_json or "{}") @@ -79,9 +111,9 @@ def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None: return None -def build_tool_search_call(tool_id: str, arguments_json: str, status: ToolSearchCallStatus) -> ResponseToolSearchCall: +def build_tool_search_call(tool_id: str, arguments_json: str, status: ToolSearchCallStatus) -> OutputToolSearchCall: arguments: Final = _parsed_arguments(arguments_json) - return ResponseToolSearchCall( + return OutputToolSearchCall( id=openai_shaped_tool_call_item_id("tool_search_call", tool_id), call_id=tool_id, arguments={} if arguments is None else arguments, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 9250c95e9af..7cdcc963191 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -33,9 +33,11 @@ from openai.types.responses.tool_param import FunctionToolParam from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict +import litellm from litellm._logging import verbose_logger from litellm.caching import InMemoryCache from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) @@ -91,7 +93,7 @@ from litellm.types.responses.main import ( OutputFunctionToolCall, OutputImageGenerationCall, OutputText, - ResponseToolSearchCall, + OutputToolSearchCall, ) from litellm.types.utils import ( ChatCompletionAnnotation, @@ -106,22 +108,27 @@ from litellm.types.utils import ( from .custom_tools import ( convert_custom_tool_to_function_tool, extract_custom_tool_names, + function_chat_tool, is_custom_tool_call, openai_shaped_tool_call_item_id, serialize_tool_call_arguments, unwrap_custom_tool_arguments, validated_allowed_callers, validated_defer_loading, + without_malformed_tool_flags, ) from .reasoning_items import decode_thinking_blocks, encode_thinking_blocks, mint_reasoning_item_id from .tool_search import ( + CompletionRoute, anthropic_tool_search_chat_tool, build_tool_search_call, has_server_tool_search, + holds_tool_search_use, is_anthropic_tool_search_chat_tool, is_anthropic_tool_search_use, is_server_tool_search_tool, provider_builds_anthropic_request, + without_tool_search_uses, ) NamespaceNameMap: TypeAlias = Mapping[str, tuple[str, str]] @@ -242,14 +249,21 @@ def _input_item_role(input_item: Mapping[str, object]) -> str: return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated +SessionMessage: TypeAlias = ( + AllMessageValues + | GenericChatCompletionMessage + | ChatCompletionMessageToolCall + | ChatCompletionResponseMessage + | Message +) + + +def _without_stored_tool_search_uses(message: SessionMessage) -> SessionMessage | None: + return without_tool_search_uses(message) if isinstance(message, Message) else message + + class ChatCompletionSession(TypedDict, total=False): - messages: list[ - AllMessageValues - | GenericChatCompletionMessage - | ChatCompletionMessageToolCall - | ChatCompletionResponseMessage - | Message - ] + messages: list[SessionMessage] litellm_session_id: str | None instructions: ReadOnly[str | None] @@ -382,6 +396,12 @@ class LiteLLMCompletionResponsesConfig: ) return supported_params is not None and "web_search_options" not in supported_params + @staticmethod + def _without_malformed_tool_flags_when_dropping_params(tools: ResponseTools, drop_params: object) -> ResponseTools: + if litellm.drop_params is not True and normalize_drop_params(drop_params) is not True: + return tools + return [without_malformed_tool_flags(tool) for tool in tools or ()] + @staticmethod def _without_unsupported_tool_search( tools: ResponseTools, model: str, custom_llm_provider: str | None @@ -498,13 +518,18 @@ class LiteLLMCompletionResponsesConfig: custom_llm_provider: str | None = None, stream: bool | None = None, extra_headers: Mapping[str, object] | None = None, + drop_params: object = None, **kwargs, ) -> dict: """ Transform a Responses API request into a Chat Completion request """ request_tools: Final = LiteLLMCompletionResponsesConfig._without_unsupported_tool_search( - responses_api_request.get("tools") or [], model=model, custom_llm_provider=custom_llm_provider + LiteLLMCompletionResponsesConfig._without_malformed_tool_flags_when_dropping_params( + responses_api_request.get("tools") or [], drop_params + ), + model=model, + custom_llm_provider=custom_llm_provider, ) ( converted_tools, @@ -627,6 +652,14 @@ class LiteLLMCompletionResponsesConfig: return messages + @staticmethod + def _without_tool_search_uses_off_anthropic( + messages: Sequence[SessionMessage], route: CompletionRoute + ) -> tuple[SessionMessage, ...]: + if provider_builds_anthropic_request(route.model, route.custom_llm_provider): + return tuple(messages) + return tuple(kept for message in messages if (kept := _without_stored_tool_search_uses(message)) is not None) + @staticmethod async def async_responses_api_session_handler( previous_response_id: str, @@ -644,7 +677,14 @@ class LiteLLMCompletionResponsesConfig: ) ) _messages: Final = litellm_completion_request.get("messages") or [] - session_messages: Final = chat_completion_session.get("messages") or [] + stored_messages: Final = chat_completion_session.get("messages") or [] + session_messages: Final = ( + LiteLLMCompletionResponsesConfig._without_tool_search_uses_off_anthropic( + stored_messages, CompletionRoute.model_validate(litellm_completion_request) + ) + if any(holds_tool_search_use(message) for message in stored_messages) + else stored_messages + ) instructions_end: Final = 1 if instructions else 0 carried_instructions: Final = None if instructions else chat_completion_session.get("instructions") leading_system_messages: Final = ( @@ -658,7 +698,7 @@ class LiteLLMCompletionResponsesConfig: # Store original _messages before combining for safety check original_new_messages: Final = _messages.copy() if _messages else [] - combined_messages = leading_system_messages + session_messages + _messages[instructions_end:] + combined_messages = [*leading_system_messages, *session_messages, *_messages[instructions_end:]] # Fix: Ensure tool_results have corresponding tool_calls in previous assistant message # Pass tools parameter to help reconstruct tool_calls if not in cache @@ -680,7 +720,7 @@ class LiteLLMCompletionResponsesConfig: elif session_messages: # If we had session messages but they got filtered out, # restore them - combined_messages = session_messages + combined_messages = list(session_messages) else: # Both are empty - this likely means function_call_output had empty/invalid call_id # Provide a helpful error message @@ -2001,16 +2041,10 @@ class LiteLLMCompletionResponsesConfig: parameters=dict(normalized_parameters), strict=bool(namespace_tool.get("strict", False)), ) - allowed_callers: Final = validated_allowed_callers(namespace_tool.get("allowed_callers")) - deferred: Final = validated_defer_loading(namespace_tool.get("defer_loading")) - if allowed_callers is None and not deferred: - return ChatCompletionToolParam(type="function", function=function) - if allowed_callers is None: - return ChatCompletionToolParam(type="function", function=function, defer_loading=True) - if not deferred: - return ChatCompletionToolParam(type="function", function=function, allowed_callers=allowed_callers) - return ChatCompletionToolParam( - type="function", function=function, allowed_callers=allowed_callers, defer_loading=True + return function_chat_tool( + function, + validated_allowed_callers(namespace_tool.get("allowed_callers")), + validated_defer_loading(namespace_tool.get("defer_loading")), ) @staticmethod @@ -2230,7 +2264,7 @@ class LiteLLMCompletionResponsesConfig: def transform_chat_completion_tools_to_responses_tools( chat_completion_response: ModelResponse, responses_api_request: ResponsesAPIOptionalRequestParams | None = None, - ) -> list[ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem | ResponseToolSearchCall]: + ) -> list[ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem | OutputToolSearchCall]: """ Transform a Chat Completion tools into a Responses API tools. @@ -2238,7 +2272,7 @@ class LiteLLMCompletionResponsesConfig: with ``type="custom_tool_call"``. For regular function tools, returns ``ResponseFunctionToolCall`` with ``type="function_call"``. A provider-side tool search call made for a request's ``tool_search`` tool returns - ``ResponseToolSearchCall`` with ``type="tool_search_call"``. + ``OutputToolSearchCall`` with ``type="tool_search_call"``. """ all_chat_completion_tools: Final[list[ChatCompletionMessageToolCall]] = [] for choice in chat_completion_response.choices: @@ -2260,9 +2294,7 @@ class LiteLLMCompletionResponsesConfig: chat_completion_response ) responses_tools: Final[ - list[ - ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem | ResponseToolSearchCall - ] + list[ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem | OutputToolSearchCall] ] = [] for tool in all_chat_completion_tools: if tool.type == "function": @@ -2339,7 +2371,7 @@ class LiteLLMCompletionResponsesConfig: tool_arguments: str, web_search_calls: Mapping[str, ResponseFunctionWebSearch], tool_search_requested: bool, - ) -> ResponseFunctionWebSearch | ResponseToolSearchCall | None: + ) -> ResponseFunctionWebSearch | OutputToolSearchCall | None: web_search_call: Final = web_search_calls.get(tool_id) if web_search_call is not None: return web_search_call @@ -2614,7 +2646,7 @@ class LiteLLMCompletionResponsesConfig: | ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem - | ResponseToolSearchCall + | OutputToolSearchCall ]: responses_output: list[ GenericResponseOutputItem @@ -2624,7 +2656,7 @@ class LiteLLMCompletionResponsesConfig: | ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem - | ResponseToolSearchCall + | OutputToolSearchCall ] = [] responses_output.extend( diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 1acb448511e..33986d80df3 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -91,7 +91,7 @@ from litellm.types.responses.main import ( OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, - ResponseToolSearchCall, + OutputToolSearchCall, ) from .base import CachedTokensDetails @@ -1445,7 +1445,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): | ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem - | ResponseToolSearchCall + | OutputToolSearchCall ] ) parallel_tool_calls: bool | None = None diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 49b27c9494b..5588527c141 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -133,7 +133,7 @@ class CustomToolCallOutputItem(BaseLiteLLMOpenAIResponseObject): status: Literal["in_progress", "completed", "incomplete"] | None = None -class ResponseToolSearchCall(BaseLiteLLMOpenAIResponseObject): +class OutputToolSearchCall(BaseLiteLLMOpenAIResponseObject): model_config = ConfigDict(extra="allow", frozen=True, protected_namespaces=()) id: str diff --git a/tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py index 50ac31c6aa0..eb5ac2765fc 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py @@ -29,6 +29,7 @@ _CACHE_BUST: Final[Mapping[str, JsonValue]] = MappingProxyType({"cache": {"no-ca _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]]) _EMPTY_TURN: Final[Mapping[str, JsonValue]] = MappingProxyType({"role": "user", "content": ""}) +_EARLIER_PAIRS: Final = 200 _WEATHER_PARAMETERS: Final[dict[str, JsonValue]] = { "type": "object", "properties": {"city": {"type": "string"}}, @@ -283,6 +284,56 @@ def test_a_replayed_search_call_is_dropped_only_when_nothing_answers_it( _assert_wire(call, request, body) +def _earlier_pairs(marker: str) -> Iterator[JsonValue]: + for index in range(_EARLIER_PAIRS): + call_id = f"toolu_{marker}_{index}" + call: JsonValue = { + "id": call_id, + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + yield {"role": "assistant", "content": f"step {index}", "tool_calls": [call]} + yield _tool_message(call_id, "22C") + + +def _long_history(marker: str, answers: Sequence[Mapping[str, JsonValue]]) -> tuple[JsonValue, ...]: + first, *rest = _history(marker, answers) + return (first, *_earlier_pairs(marker), *rest) + + +def _last_assistant_tool_uses(messages: Sequence[Mapping[str, JsonValue]]) -> tuple[tuple[str, str], ...]: + last: Final = [turn for turn in messages if turn["role"] == "assistant"][-1] + blocks: Final = _ITEMS.validate_python(last["content"]) + return tuple(sorted((str(block["id"]), str(block["name"])) for block in blocks if block["type"] == "tool_use")) + + +@pytest.mark.parametrize("answer", ["unanswered", "answered"]) +def test_a_search_call_after_two_hundred_answered_tool_turns_is_paired_only_by_its_own_answers( + gateway: Gateway, answer: Answer +) -> None: + call: Final = _Call("anthropic", answer, False, uuid.uuid4().hex) + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url) + history: Final = _long_history(call.marker, _answers(call.marker, answer)) + response: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, call.marker, history, False)) + _assert_served(response, call.marker, False) + request, body = _only_request(wire) + assert request.target == "/v1/messages", request.target + _assert_tools_forwarded("anthropic", body) + messages: Final = _ITEMS.validate_python(body["messages"]) + earlier: Final = [f"toolu_{call.marker}_{index}" for index in range(_EARLIER_PAIRS)] + weather: Final = (f"toolu_{call.marker}", "get_weather") + search: Final = (f"srvtoolu_{call.marker}", _REGEX_TOOL_NAME) + match answer: + case "unanswered": + assert "srvtoolu_" not in json.dumps(messages), messages[-4:] + assert _last_assistant_tool_uses(messages) == (weather,), messages[-4:] + assert sorted(_tool_results(messages)) == sorted([*earlier, weather[0]]), messages[-4:] + case "answered": + assert _last_assistant_tool_uses(messages) == (search, weather), messages[-4:] + assert sorted(_tool_results(messages)) == sorted([*earlier, search[0], weather[0]]), messages[-4:] + + def test_a_function_role_answer_keeps_the_replayed_search_call(gateway: Gateway) -> None: call: Final = _Call("anthropic", "answered", False, uuid.uuid4().hex) answer: Final[dict[str, JsonValue]] = { diff --git a/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py index bc95861e467..af8765f4b7f 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py @@ -1,10 +1,11 @@ import asyncio +import itertools import json import uuid from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Final, Literal +from typing import Final, Literal, TypeAlias from urllib.parse import unquote import httpx @@ -23,6 +24,8 @@ _GEMINI: Final = "gemini-2.5-flash" _MISTRAL: Final = "mistral-large-latest" _REGEX_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" _REGEX_TOOL_NAME: Final = "tool_search_tool_regex" +_SEARCH_TOOL_TYPES: Final = frozenset({_REGEX_TOOL_TYPE, "tool_search_tool_bm25_20251119"}) +_AZURE_MESSAGES_PATH: Final = "/anthropic/v1/messages" _ALL_DEFERRED_ERROR: Final = "At least one tool must have defer_loading=false" _DEFERRED_NAMES: Final = ("get_weather", "finance__get_stock_price") _CACHE_BUST: Final[Mapping[str, JsonValue]] = MappingProxyType({"cache": {"no-cache": True}}) @@ -44,8 +47,9 @@ _EMAIL_PARAMETERS: Final[dict[str, JsonValue]] = { "required": ["to"], } -Route = Literal["gemini", "bedrock_converse", "mistral", "unlisted_claude", "anthropic"] +Route = Literal["gemini", "bedrock_converse", "mistral", "unlisted_claude", "anthropic", "azure_ai"] Search = Literal["absent", "dropped"] +NativeSearch: TypeAlias = Literal["regex", "bm25"] @dataclass(frozen=True, slots=True) @@ -70,6 +74,10 @@ def _tool_search(**fields: JsonValue) -> dict[str, JsonValue]: return {"type": "tool_search", **fields} +def _native_search_tool(kind: NativeSearch) -> Mapping[str, JsonValue]: + return MappingProxyType({"type": f"tool_search_tool_{kind}_20251119", "name": f"tool_search_tool_{kind}"}) + + def _function_tool(defer_loading: JsonValue = True) -> dict[str, JsonValue]: return { "type": "function", @@ -249,9 +257,9 @@ def _claude(request: Request) -> Reply: marker: Final = rv.newest_marker(request.body.decode()) or "health" body: Final = _JSON_OBJECT.validate_json(request.body) tools: Final = _request_tools(body) - regex_tools: Final = [tool for tool in tools if tool.get("type") == _REGEX_TOOL_TYPE] - function_tools: Final = [tool for tool in tools if tool.get("type") != _REGEX_TOOL_TYPE] - if not regex_tools and function_tools and all(tool.get("defer_loading") is True for tool in function_tools): + search_tools: Final = [tool for tool in tools if tool.get("type") in _SEARCH_TOOL_TYPES] + function_tools: Final = [tool for tool in tools if tool.get("type") not in _SEARCH_TOOL_TYPES] + if not search_tools and function_tools and all(tool.get("defer_loading") is True for tool in function_tools): return _anthropic_error(400, f"tools: {_ALL_DEFERRED_ERROR} marker-{marker}") if body.get("stream") is True: return Reply(content_type="text/event-stream", chunks=sse_chunks(_claude_events(marker))) @@ -266,7 +274,7 @@ def _vendor(route: Route) -> Callable[[Request], Reply]: return _converse case "mistral": return _mistral - case "unlisted_claude" | "anthropic": + case "unlisted_claude" | "anthropic" | "azure_ai": return _claude @@ -290,6 +298,10 @@ def _deployment(scenario: Scenario, route: Route, wire_url: str) -> str: ) case "anthropic": return scenario.model(model=f"anthropic/{_CLAUDE}", api_base=wire_url, api_key=cc.ANTHROPIC_API_KEY) + case "azure_ai": + return scenario.model( + model=f"azure_ai/{_CLAUDE}", api_base=f"{wire_url}{_AZURE_MESSAGES_PATH}", api_key="synthetic-azure-key" + ) def _responses_body( @@ -343,6 +355,9 @@ def _assert_target(route: Route, stream: bool, request: Request, body: Mapping[s case "unlisted_claude" | "anthropic": assert request.target == "/v1/messages", request.target assert body.get("stream", False) is stream, body + case "azure_ai": + assert request.target == _AZURE_MESSAGES_PATH, request.target + assert body.get("stream", False) is stream, body def _wire_tool_names(route: Route, body: Mapping[str, JsonValue]) -> list[str]: @@ -360,7 +375,7 @@ def _wire_tool_names(route: Route, body: Mapping[str, JsonValue]) -> list[str]: str(_JSON_OBJECT.validate_python(tool["function"])["name"]) for tool in _ITEMS.validate_python(body["tools"]) ] - case "unlisted_claude" | "anthropic": + case "unlisted_claude" | "anthropic" | "azure_ai": return [str(tool["name"]) for tool in _ITEMS.validate_python(body["tools"])] @@ -434,6 +449,48 @@ def test_the_supported_anthropic_route_keeps_each_tools_own_defer_loading_with_t assert "defer_loading" not in json.dumps(sent[3]), sent +def _native_cells() -> tuple[ParameterSet, ...]: + routes: Final[tuple[tuple[Route, NativeSearch], ...]] = ( + ("anthropic", "regex"), + ("anthropic", "bm25"), + ("azure_ai", "regex"), + ) + return tuple( + pytest.param(route, kind, stream, id=f"{route}-{kind}-{'stream' if stream else 'sync'}") + for (route, kind), stream in itertools.product(routes, (False, True)) + ) + + +@pytest.mark.parametrize(("route", "kind", "stream"), _native_cells()) +def test_a_native_anthropic_search_tool_reaches_the_wire_untouched_and_keeps_every_deferral( + gateway: Gateway, route: Route, kind: NativeSearch, stream: bool +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_claude) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, route, wire.url) + tools: Final = (_native_search_tool(kind), _function_tool(), _namespace_tool()) + response: Final = _raw(gateway, _responses_body(model, marker, tools, stream)) + _assert_served(response, marker, stream) + request, body = _only_request(wire) + _assert_target(route, stream, request, body) + sent: Final = _ITEMS.validate_python(body["tools"]) + assert sent[0] == dict(_native_search_tool(kind)), sent + assert [(tool["name"], tool.get("defer_loading")) for tool in sent[1:]] == [ + (name, True) for name in _DEFERRED_NAMES + ], sent + + +def test_bedrock_converse_skips_a_native_search_tool_and_serves_the_deferred_tools(gateway: Gateway) -> None: + call: Final = _Call("bedrock_converse", False, uuid.uuid4().hex) + with wire_server(_converse) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "bedrock_converse", wire.url) + tools: Final = (_native_search_tool("regex"), _function_tool(), _namespace_tool()) + response: Final = _raw(gateway, _responses_body(model, call.marker, tools, False)) + _assert_served(response, call.marker, False) + request, body = _only_request(wire) + _assert_undeferred_wire(call, request, body) + + @pytest.mark.parametrize("route", ["gemini", "mistral"]) def test_a_non_boolean_defer_loading_is_refused_with_400_on_a_route_without_tool_search( gateway: Gateway, route: Route diff --git a/tests/integration/providers/test_responses_bridge_tool_search_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_wire.py index 26fe899714b..054c2e136b9 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_wire.py @@ -35,6 +35,7 @@ _BEDROCK_REGEX_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( ) _REGEX_TOOL_TYPES: Final = frozenset({_REGEX_TOOL_TYPE, _REGEX_TOOL_NAME}) _TOOL_SEARCH_BETA_PREFIXES: Final = ("advanced-tool-use", "tool-search-tool") +_TOOL_SEARCH_CALL_KEYS: Final = frozenset({"id", "type", "call_id", "arguments", "execution", "status"}) _SEARCH_ARGUMENTS: Final = '{"pattern": "weather|stock"}' _TRUNCATED_ARGUMENTS: Final = '{"pattern": ' _ALL_DEFERRED_ERROR: Final = "At least one tool must have defer_loading=false" @@ -587,6 +588,24 @@ def test_bedrock_invoke_and_vertex_claude_keep_the_tool_search_tool( _assert_spend_rows(model, (str(answer.payload["id"]),)) +@pytest.mark.parametrize("provider", ["anthropic", "bedrock_invoke"]) +def test_a_non_streaming_tool_search_call_item_carries_only_its_own_keys(gateway: Gateway, provider: Provider) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, provider, wire.url, gateway.upstream_url) + tools: Final = (_tool_search(), _function_tool(), _namespace_tool()) + response: Final = _raw( + gateway, "/v1/responses", {**_responses_body(model, marker, tools, False), **_CACHE_BUST} + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + _assert_search_call_output(payload, marker) + (search,) = [item for item in _tool_items(payload) if item["type"] == "tool_search_call"] + assert frozenset(search) == _TOOL_SEARCH_CALL_KEYS, search + request, body = _only_request(wire) + _assert_claude_wire(provider, False, request, body, _DEFERRED_NAMES) + + def test_search_arguments_that_never_parse_stream_an_incomplete_tool_search_call(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(_Vendor(search_arguments=_TRUNCATED_ARGUMENTS).respond) as wire, gateway.scenario() as scenario: diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index 5563c2826bb..45d3f8996bf 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -17,7 +17,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.openai import ResponsesAPIStreamEvents -from litellm.types.utils import ModelResponse +from litellm.types.utils import Message, ModelResponse def test_response_api_handler_merges_metadata_and_service_tier_without_error(): @@ -304,6 +304,89 @@ async def test_previous_response_id_without_instructions_carries_the_latest_ones assert [block.text for block in anthropic.request.system] == [expected_system] +_SEARCH_CALL: Final = { + "id": "srvtoolu_search", + "type": "function", + "function": {"name": "tool_search_tool_regex", "arguments": '{"pattern": "weather"}'}, +} +_WEATHER_CALL: Final = { + "id": "toolu_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Tokyo"}'}, +} + + +def _search_turn(tool_calls: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + return { + "request_id": "chatcmpl-search-turn", + "call_type": "aresponses", + "session_id": "session-search", + "proxy_server_request": {"model": _MODEL, "input": "What is the weather in Tokyo?"}, + "response": { + "id": "chatcmpl-search-turn", + "object": "chat.completion", + "created": 0, + "model": _MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": {"role": "assistant", "content": None, "tool_calls": list(tool_calls)}, + } + ], + }, + } + + +async def _follow_up_messages( + tool_calls: Sequence[Mapping[str, object]], model: str, custom_llm_provider: str +) -> Sequence[object]: + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_search_turn(tool_calls)])): + request: Final = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( + previous_response_id="chatcmpl-search-turn", + litellm_completion_request={ + "model": model, + "custom_llm_provider": custom_llm_provider, + "messages": [{"role": "user", "content": "And in Osaka?"}], + }, + instructions=None, + ) + return request["messages"] + + +def _replayed_tool_call_ids(messages: Sequence[object]) -> list[str]: + assistant_turns: Final = [message for message in messages if isinstance(message, Message)] + return [tool_call.id for turn in assistant_turns for tool_call in turn.tool_calls or ()] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected_tool_call_ids"), + [ + ("mistral/mistral-small-latest", "mistral", ["toolu_weather"]), + ("gemini/gemini-2.5-flash", "gemini", ["toolu_weather"]), + (_MODEL, "anthropic", ["srvtoolu_search", "toolu_weather"]), + ], +) +async def test_previous_response_id_keeps_a_stored_claude_search_call_only_on_an_anthropic_route( + model: str, custom_llm_provider: str, expected_tool_call_ids: list[str] +) -> None: + """ + A follow-up that routes a Claude turn's history elsewhere must not replay Anthropic's server-side + search call as a client tool call, which Mistral answers 400 "Unexpected tool call id" + """ + messages: Final = await _follow_up_messages([_SEARCH_CALL, _WEATHER_CALL], model, custom_llm_provider) + + assert _replayed_tool_call_ids(messages) == expected_tool_call_ids + + +@pytest.mark.asyncio +async def test_previous_response_id_drops_a_claude_turn_that_held_only_a_search_call_off_anthropic() -> None: + messages: Final = await _follow_up_messages([_SEARCH_CALL], "mistral/mistral-small-latest", "mistral") + + assert [message["role"] for message in messages] == ["user", "user"] + + def _anthropic_sse(*events: Mapping[str, object]) -> bytes: return b"".join( f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 01c2638efcf..b81da5cba64 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -20,7 +20,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.base import HiddenParams -from litellm.types.responses.main import ResponseToolSearchCall, build_web_search_call +from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, Choices, @@ -1929,6 +1929,88 @@ class TestToolTransformation: assert [tool["function"]["name"] for tool in not_deferred] == ["finance__get_stock_price", "get_weather"] assert all("defer_loading" not in tool for tool in not_deferred) + @pytest.mark.parametrize("global_drop_params", [False, True]) + def test_malformed_tool_flags_answer_400_unless_drop_params_drops_them( + self, monkeypatch: pytest.MonkeyPatch, global_drop_params: bool + ) -> None: + monkeypatch.setattr(litellm, "drop_params", global_drop_params) + transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request + tools: Final = [ + {"type": "tool_search", "execution": "server"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}, "defer_loading": "false"}, + { + "type": "namespace", + "name": "finance", + "tools": [ + { + "type": "function", + "name": "get_stock_price", + "parameters": {"type": "object"}, + "defer_loading": 1, + "allowed_callers": "direct", + }, + { + "type": "function", + "name": "get_fx_rate", + "parameters": {"type": "object"}, + "defer_loading": True, + }, + ], + }, + {"type": "custom", "name": "apply_patch", "defer_loading": "yes"}, + ] + + if not global_drop_params: + with pytest.raises(litellm.BadRequestError): + transform( + model="anthropic/claude-sonnet-5-5", + input="What is the weather in Paris right now?", + responses_api_request={"tools": tools}, + custom_llm_provider="anthropic", + ) + result: Final = transform( + model="anthropic/claude-sonnet-5-5", + input="What is the weather in Paris right now?", + responses_api_request={"tools": tools}, + custom_llm_provider="anthropic", + drop_params=None if global_drop_params else True, + ) + + function_tools: Final = [tool for tool in result["tools"] if tool["type"] == "function"] + assert {tool["function"]["name"]: tool.get("defer_loading", False) for tool in function_tools} == { + "get_weather": False, + "finance__get_stock_price": False, + "finance__get_fx_rate": True, + "apply_patch": False, + } + assert all("allowed_callers" not in tool for tool in function_tools) + + def test_custom_tools_stay_deferred_when_tool_search_goes_out(self) -> None: + result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input="Patch the README", + responses_api_request={ + "tools": [ + {"type": "tool_search", "execution": "server"}, + {"type": "custom", "name": "apply_patch", "defer_loading": True}, + { + "type": "namespace", + "name": "editor", + "tools": [{"type": "custom", "name": "insert", "defer_loading": True}], + }, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + custom_llm_provider="anthropic", + ) + + function_tools: Final = [tool for tool in result["tools"] if tool["type"] == "function"] + assert {tool["function"]["name"]: tool.get("defer_loading", False) for tool in function_tools} == { + "apply_patch": True, + "insert": True, + "get_weather": False, + } + def test_anthropic_tool_search_call_becomes_tool_search_call_item(self) -> None: response: Final = _chat_response_with_tool_search_call('{"pattern": "weather|forecast"}') with_tool_search: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools( @@ -1944,7 +2026,7 @@ class TestToolTransformation: assert [item.type for item in with_tool_search] == ["tool_search_call", "function_call"] search_call: Final = with_tool_search[0] - assert isinstance(search_call, ResponseToolSearchCall) + assert isinstance(search_call, OutputToolSearchCall) assert search_call.id == "tsc_srvtoolu_01Search" assert search_call.call_id == "srvtoolu_01Search" assert search_call.arguments == {"pattern": "weather|forecast"} @@ -1969,7 +2051,7 @@ class TestToolTransformation: ) search_call: Final = items[0] - assert isinstance(search_call, ResponseToolSearchCall) + assert isinstance(search_call, OutputToolSearchCall) assert search_call.status == "incomplete" assert search_call.arguments == {} @@ -2130,9 +2212,7 @@ class TestToolTransformation: tools = [custom_tool] with pytest.raises(litellm.BadRequestError, match="allowed_callers must be a list of strings"): - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(tools=tools) def test_transform_web_search_tools_to_web_search_options(self): """Test that web_search tools are converted to web_search_options""" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 58126b12400..62a9d3943cf 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -39598,6 +39598,32 @@ export interface components { } & { [key: string]: unknown; }; + /** OutputToolSearchCall */ + OutputToolSearchCall: { + /** Arguments */ + arguments: unknown; + /** Call Id */ + call_id?: string | null; + /** + * Execution + * @enum {string} + */ + execution: "server" | "client"; + /** Id */ + id: string; + /** + * Status + * @enum {string} + */ + status: "in_progress" | "completed" | "incomplete"; + /** + * Type + * @constant + */ + type: "tool_search_call"; + } & { + [key: string]: unknown; + }; /** * PageLinks * @description Hypermedia for a paginated list. No `first`/`last`: without a total count the last page is unknown. @@ -43077,7 +43103,7 @@ export interface components { */ ResponseItemList: { /** Data */ - data: (components["schemas"]["ResponseInputMessageItem"] | components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseFunctionToolCallItem"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["openai__types__responses__response_tool_search_call__ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["McpCall"] | components["schemas"]["ResponseCustomToolCallItem"] | components["schemas"]["ResponseCustomToolCallOutputItem"])[]; + data: (components["schemas"]["ResponseInputMessageItem"] | components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseFunctionToolCallItem"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["McpCall"] | components["schemas"]["ResponseCustomToolCallItem"] | components["schemas"]["ResponseCustomToolCallOutputItem"])[]; /** First Id */ first_id: string; /** Has More */ @@ -43299,6 +43325,34 @@ export interface components { } & { [key: string]: unknown; }; + /** ResponseToolSearchCall */ + ResponseToolSearchCall: { + /** Arguments */ + arguments: unknown; + /** Call Id */ + call_id?: string | null; + /** Created By */ + created_by?: string | null; + /** + * Execution + * @enum {string} + */ + execution: "server" | "client"; + /** Id */ + id: string; + /** + * Status + * @enum {string} + */ + status: "in_progress" | "completed" | "incomplete"; + /** + * Type + * @constant + */ + type: "tool_search_call"; + } & { + [key: string]: unknown; + }; /** ResponseToolSearchOutputItem */ ResponseToolSearchOutputItem: { /** Call Id */ @@ -43533,9 +43587,9 @@ export interface components { /** Object */ object?: string | null; /** Output */ - output: (components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["openai__types__responses__response_tool_search_call__ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpCall"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["ResponseCustomToolCall"] | components["schemas"]["ResponseCustomToolCallOutputItem"] | { + output: (components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpCall"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["ResponseCustomToolCall"] | components["schemas"]["ResponseCustomToolCallOutputItem"] | { [key: string]: unknown; - })[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["litellm__types__responses__main__ResponseToolSearchCall"])[]; + })[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["OutputToolSearchCall"])[]; /** Parallel Tool Calls */ parallel_tool_calls?: boolean | null; /** Previous Response Id */ @@ -49368,32 +49422,6 @@ export interface components { /** Type */ type: "computer_use_preview" | string; }; - /** ResponseToolSearchCall */ - litellm__types__responses__main__ResponseToolSearchCall: { - /** Arguments */ - arguments: unknown; - /** Call Id */ - call_id?: string | null; - /** - * Execution - * @enum {string} - */ - execution: "server" | "client"; - /** Id */ - id: string; - /** - * Status - * @enum {string} - */ - status: "in_progress" | "completed" | "incomplete"; - /** - * Type - * @constant - */ - type: "tool_search_call"; - } & { - [key: string]: unknown; - }; /** ModelInfo */ litellm__types__router__ModelInfo: { /** Access Windows */ @@ -49476,34 +49504,6 @@ export interface components { */ type: "computer"; }; - /** ResponseToolSearchCall */ - openai__types__responses__response_tool_search_call__ResponseToolSearchCall: { - /** Arguments */ - arguments: unknown; - /** Call Id */ - call_id?: string | null; - /** Created By */ - created_by?: string | null; - /** - * Execution - * @enum {string} - */ - execution: "server" | "client"; - /** Id */ - id: string; - /** - * Status - * @enum {string} - */ - status: "in_progress" | "completed" | "incomplete"; - /** - * Type - * @constant - */ - type: "tool_search_call"; - } & { - [key: string]: unknown; - }; /** * UserLocation * @description The user's location.