From 418996391f5f785b4b491f249c7bb18b7b403ace Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:17:23 -0700 Subject: [PATCH] fix(responses): gate only the OpenAI tool_search shape and keep native Anthropic search tools intact A native tool_search_tool_regex_20251119 or tool_search_tool_bm25_20251119 tool the caller sends through /v1/responses passes through the bridge on every route as before, and the other tools keep defer_loading whenever any Anthropic search tool goes out. The route gate now drops the OpenAI tool_search item before translation instead of filtering the translated regex tool, so bm25 counts as a search tool. answered_tool_call_ids walks the history with islice instead of a slice copy. The tool_search_call output item drops its created_by field, which OpenAI rejects on a replayed input item and the streaming item never carried --- .../prompt_templates/factory.py | 4 +- litellm/proxy/_lazy_openapi_snapshot.json | 190 ++++--- .../tool_search.py | 4 +- .../transformation.py | 55 +- litellm/types/responses/main.py | 1 - ...esponses_bridge_tool_search_replay_wire.py | 401 ++++++++++++++ ...nses_bridge_tool_search_undeferred_wire.py | 508 ++++++++++++++++++ .../test_litellm_completion_responses.py | 36 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 88 +-- 9 files changed, 1156 insertions(+), 131 deletions(-) create mode 100644 tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py create mode 100644 tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index a9473e41e4b..e75442440e9 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1828,7 +1828,9 @@ def _tool_result_ids(answers: Iterable[AllMessageValues]) -> Iterator[str]: def answered_tool_call_ids(messages: Sequence[AllMessageValues], assistant_index: int) -> frozenset[str]: - following: Final = itertools.dropwhile(lambda message: message["role"] == "assistant", messages[assistant_index:]) + following: Final = itertools.dropwhile( + lambda message: message["role"] == "assistant", itertools.islice(messages, assistant_index, None) + ) answers: Final = itertools.takewhile(lambda message: message["role"] in ("tool", "function"), following) return frozenset(_tool_result_ids(answers)) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index bb2bdeb0fc6..9f81f4ceb32 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -25329,7 +25329,7 @@ "$ref": "#/components/schemas/ResponseFunctionToolCallOutputItem" }, { - "$ref": "#/components/schemas/ResponseToolSearchCall" + "$ref": "#/components/schemas/openai__types__responses__response_tool_search_call__ResponseToolSearchCall" }, { "$ref": "#/components/schemas/ResponseToolSearchOutputItem" @@ -25683,71 +25683,6 @@ "title": "ResponseTextConfigParam", "type": "object" }, - "ResponseToolSearchCall": { - "additionalProperties": true, - "properties": { - "arguments": { - "title": "Arguments" - }, - "call_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Call Id" - }, - "created_by": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Created By" - }, - "execution": { - "enum": [ - "server", - "client" - ], - "title": "Execution", - "type": "string" - }, - "id": { - "title": "Id", - "type": "string" - }, - "status": { - "enum": [ - "in_progress", - "completed", - "incomplete" - ], - "title": "Status", - "type": "string" - }, - "type": { - "const": "tool_search_call", - "title": "Type", - "type": "string" - } - }, - "required": [ - "id", - "arguments", - "execution", - "status", - "type" - ], - "title": "ResponseToolSearchCall", - "type": "object" - }, "ResponseToolSearchOutputItem": { "additionalProperties": true, "properties": { @@ -25982,7 +25917,7 @@ "$ref": "#/components/schemas/ResponseReasoningItem" }, { - "$ref": "#/components/schemas/ResponseToolSearchCall" + "$ref": "#/components/schemas/openai__types__responses__response_tool_search_call__ResponseToolSearchCall" }, { "$ref": "#/components/schemas/ResponseToolSearchOutputItem" @@ -26065,7 +26000,7 @@ "$ref": "#/components/schemas/CustomToolCallOutputItem" }, { - "$ref": "#/components/schemas/ResponseToolSearchCall" + "$ref": "#/components/schemas/litellm__types__responses__main__ResponseToolSearchCall" } ] }, @@ -27004,6 +26939,125 @@ "title": "WebSearchTool", "type": "object" }, + "litellm__types__responses__main__ResponseToolSearchCall": { + "additionalProperties": true, + "properties": { + "arguments": { + "title": "Arguments" + }, + "call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Call Id" + }, + "execution": { + "enum": [ + "server", + "client" + ], + "title": "Execution", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "status": { + "enum": [ + "in_progress", + "completed", + "incomplete" + ], + "title": "Status", + "type": "string" + }, + "type": { + "const": "tool_search_call", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "arguments", + "execution", + "status", + "type" + ], + "title": "ResponseToolSearchCall", + "type": "object" + }, + "openai__types__responses__response_tool_search_call__ResponseToolSearchCall": { + "additionalProperties": true, + "properties": { + "arguments": { + "title": "Arguments" + }, + "call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Call Id" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "execution": { + "enum": [ + "server", + "client" + ], + "title": "Execution", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "status": { + "enum": [ + "in_progress", + "completed", + "incomplete" + ], + "title": "Status", + "type": "string" + }, + "type": { + "const": "tool_search_call", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "arguments", + "execution", + "status", + "type" + ], + "title": "ResponseToolSearchCall", + "type": "object" + }, "openai__types__responses__web_search_preview_tool__UserLocation": { "additionalProperties": true, "description": "The user's location.", diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index 13d8191d6d5..bab28279176 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -4,7 +4,7 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, TypeAdapter, ValidationError from litellm.litellm_core_utils.prompt_templates.factory import is_anthropic_tool_search_use -from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex +from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AnthropicToolSearchToolRegex from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.responses.main import ResponseToolSearchCall from litellm.types.utils import LlmProviders @@ -55,7 +55,7 @@ def anthropic_tool_search_chat_tool() -> AnthropicToolSearchToolRegex: def is_anthropic_tool_search_chat_tool(tool: Mapping[str, object]) -> bool: - return tool.get("type") == ANTHROPIC_TOOL_SEARCH_TOOL_TYPE + return tool.get("type") in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES def provider_builds_anthropic_request(model: str, custom_llm_provider: str | None) -> bool: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 6d85a67acc4..0efdb3967a7 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -383,31 +383,31 @@ class LiteLLMCompletionResponsesConfig: return supported_params is not None and "web_search_options" not in supported_params @staticmethod - def _drop_tool_search_when_unsupported( - tools: Sequence[ChatToolParam], model: str, custom_llm_provider: str | None - ) -> Sequence[ChatToolParam]: + def _without_unsupported_tool_search( + tools: ResponseTools, model: str, custom_llm_provider: str | None + ) -> ResponseTools: from litellm.utils import supports_tool_search - has_search_tool: Final = any(is_anthropic_tool_search_chat_tool(tool) for tool in tools) - if ( - has_search_tool - and supports_tool_search(model=model, custom_llm_provider=custom_llm_provider) - and provider_builds_anthropic_request(model, custom_llm_provider) - ): + if not has_server_tool_search(tools): return tools - if has_search_tool: - verbose_logger.warning( - "Dropping Responses API tool of type 'tool_search' and the deferral of the other tools: model '%s' " - "(provider '%s') cannot run tool search on this route, so the target provider would reject or " - "ignore it.", - model, - custom_llm_provider, - ) - return [ - LiteLLMCompletionResponsesConfig._without_deferral(tool) - for tool in tools - if not is_anthropic_tool_search_chat_tool(tool) - ] + route_runs_tool_search: Final = supports_tool_search( + model=model, custom_llm_provider=custom_llm_provider + ) and provider_builds_anthropic_request(model, custom_llm_provider) + if route_runs_tool_search: + return tools + verbose_logger.warning( + "Dropping Responses API tool of type 'tool_search': model '%s' (provider '%s') cannot run tool search " + "on this route, so the target provider would reject or ignore it.", + model, + custom_llm_provider, + ) + return [tool for tool in tools or () if not is_server_tool_search_tool(tool)] + + @staticmethod + def _undeferred_unless_a_search_tool_goes_out(tools: Sequence[ChatToolParam]) -> Sequence[ChatToolParam]: + if any(is_anthropic_tool_search_chat_tool(tool) for tool in tools): + return tools + return [LiteLLMCompletionResponsesConfig._without_deferral(tool) for tool in tools] @staticmethod def _without_deferral(tool: ChatToolParam) -> ChatToolParam: @@ -503,15 +503,14 @@ class LiteLLMCompletionResponsesConfig: """ Transform a Responses API request into a Chat Completion request """ + request_tools: Final = LiteLLMCompletionResponsesConfig._without_unsupported_tool_search( + responses_api_request.get("tools") or [], model=model, custom_llm_provider=custom_llm_provider + ) ( converted_tools, web_search_options, - ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - responses_api_request.get("tools") or [] - ) - tools: Final = LiteLLMCompletionResponsesConfig._drop_tool_search_when_unsupported( - converted_tools, model=model, custom_llm_provider=custom_llm_provider - ) + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(request_tools) + tools: Final = LiteLLMCompletionResponsesConfig._undeferred_unless_a_search_tool_goes_out(converted_tools) if web_search_options is not None and LiteLLMCompletionResponsesConfig._should_drop_derived_web_search_options( model=model, custom_llm_provider=custom_llm_provider diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index d69e9ff3d6b..49b27c9494b 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -142,7 +142,6 @@ class ResponseToolSearchCall(BaseLiteLLMOpenAIResponseObject): execution: Literal["server", "client"] status: Literal["in_progress", "completed", "incomplete"] type: Literal["tool_search_call"] - created_by: str | None = None class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject): diff --git a/tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py new file mode 100644 index 00000000000..50ac31c6aa0 --- /dev/null +++ b/tests/integration/providers/test_responses_bridge_tool_search_replay_wire.py @@ -0,0 +1,401 @@ +import asyncio +import json +import uuid +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import unquote + +import httpx +import pytest +from integration._support import claude_code as cc +from integration._support import responses_vendor as rv +from integration._support.anthropic_thinking import BEDROCK_MODEL, EVENT_STREAM, Event, aws_chunks, sse_chunks +from integration._support.client import Gateway, Scenario +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_CLAUDE: Final = cc.OPUS +_REGEX_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" +_REGEX_TOOL_NAME: Final = "tool_search_tool_regex" +_REGEX_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType({"type": _REGEX_TOOL_TYPE, "name": _REGEX_TOOL_NAME}) +_BEDROCK_REGEX_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"type": _REGEX_TOOL_NAME, "name": _REGEX_TOOL_NAME} +) +_SEARCH_ARGUMENTS: Final = '{"pattern": "weather|stock"}' +_SEARCH_OUTPUT: Final = '{"tools": ["get_weather"]}' +_CACHE_BUST: Final[Mapping[str, JsonValue]] = MappingProxyType({"cache": {"no-cache": True}}) +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]]) +_EMPTY_TURN: Final[Mapping[str, JsonValue]] = MappingProxyType({"role": "user", "content": ""}) +_WEATHER_PARAMETERS: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], +} + +Provider = Literal["anthropic", "bedrock_invoke"] +Answer = Literal["unanswered", "answered"] + + +@dataclass(frozen=True, slots=True) +class _Call: + provider: Provider + answer: Answer + stream: bool + marker: str + + +@dataclass(frozen=True, slots=True) +class _Served: + call: _Call + status: int + text: str + + +def _prompt(marker: str) -> str: + return f"Weather in Paris and the ACME price marker-{marker}" + + +def _message(marker: str) -> dict[str, JsonValue]: + return { + "id": f"msg_{marker}", + "type": "message", + "role": "assistant", + "model": _CLAUDE, + "content": [{"type": "text", "text": f"tomorrow is sunny marker-{marker}"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 20, "output_tokens": 7}, + } + + +def _events(marker: str) -> tuple[Event, ...]: + opened: Final = {**_message(marker), "content": [], "stop_reason": None} + return ( + {"type": "message_start", "message": opened}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": f"tomorrow is sunny marker-{marker}"}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 7}, + }, + {"type": "message_stop"}, + ) + + +def _anthropic_error(status: int, message: str) -> Reply: + body: Final = {"type": "error", "error": {"type": "invalid_request_error", "message": message}} + return Reply(status=status, body=json.dumps(body).encode()) + + +def _block_ids(turn: Mapping[str, JsonValue], block_type: str, key: str) -> frozenset[str]: + content: Final = turn.get("content") + if not isinstance(content, list): + return frozenset() + blocks: Final = _ITEMS.validate_python(content) + return frozenset(str(block[key]) for block in blocks if block.get("type") == block_type) + + +def _mismatch(messages: Sequence[Mapping[str, JsonValue]], index: int) -> frozenset[str]: + following: Final = messages[index + 1] if index + 1 < len(messages) else _EMPTY_TURN + return _block_ids(messages[index], "tool_use", "id") ^ _block_ids(following, "tool_result", "tool_use_id") + + +def _unpaired(messages: Sequence[Mapping[str, JsonValue]]) -> frozenset[str]: + assistant_turns: Final = [index for index, turn in enumerate(messages) if turn["role"] == "assistant"] + return frozenset().union(*(_mismatch(messages, index) for index in assistant_turns)) + + +def _vendor(request: Request) -> Reply: + if request.method != "POST": + return Reply() + body: Final = _JSON_OBJECT.validate_json(request.body) + marker: Final = rv.newest_marker(request.body.decode()) or "health" + unpaired: Final = _unpaired(_ITEMS.validate_python(body.get("messages") or [])) + if unpaired: + return _anthropic_error(400, f"messages: unpaired tool ids {sorted(unpaired)} marker-{marker}") + if request.target.endswith("/invoke-with-response-stream"): + return Reply(content_type=EVENT_STREAM, chunks=aws_chunks(_events(marker))) + if body.get("stream") is True: + return Reply(content_type="text/event-stream", chunks=sse_chunks(_events(marker))) + return Reply(body=json.dumps(_message(marker)).encode()) + + +def _deployment(scenario: Scenario, provider: Provider, wire_url: str) -> str: + match provider: + case "anthropic": + return scenario.model(model=f"anthropic/{_CLAUDE}", api_base=wire_url, api_key=cc.ANTHROPIC_API_KEY) + case "bedrock_invoke": + return scenario.model( + model=f"bedrock/invoke/{BEDROCK_MODEL}", + api_base=wire_url, + aws_access_key_id="AKIASCRIPTEDPROVIDER", + aws_secret_access_key="scripted-secret", + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire_url, + ) + + +def _target(provider: Provider, stream: bool) -> str: + match provider: + case "anthropic": + return "/v1/messages" + case "bedrock_invoke": + return f"/model/{BEDROCK_MODEL}/invoke-with-response-stream" if stream else f"/model/{BEDROCK_MODEL}/invoke" + + +def _search_call(marker: str) -> dict[str, JsonValue]: + return { + "id": f"srvtoolu_{marker}", + "type": "function", + "function": {"name": _REGEX_TOOL_NAME, "arguments": _SEARCH_ARGUMENTS}, + } + + +def _weather_call(marker: str) -> dict[str, JsonValue]: + return { + "id": f"toolu_{marker}", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + + +def _tool_message(tool_call_id: JsonValue, content: str) -> dict[str, JsonValue]: + return {"role": "tool", "tool_call_id": tool_call_id, "content": content} + + +def _history(marker: str, answers: Sequence[Mapping[str, JsonValue]]) -> list[JsonValue]: + return [ + {"role": "user", "content": _prompt(marker)}, + { + "role": "assistant", + "content": f"searching marker-{marker}", + "tool_calls": [_search_call(marker), _weather_call(marker)], + }, + *(dict(answer) for answer in answers), + _tool_message(f"toolu_{marker}", "22C"), + {"role": "user", "content": f"And tomorrow? marker-{marker}"}, + ] + + +def _answers(marker: str, answer: Answer) -> tuple[dict[str, JsonValue], ...]: + return (_tool_message(f"srvtoolu_{marker}", _SEARCH_OUTPUT),) if answer == "answered" else () + + +def _chat_tools() -> list[JsonValue]: + return [ + dict(_REGEX_TOOL), + { + "type": "function", + "function": {"name": "get_weather", "description": "Weather for a city", "parameters": _WEATHER_PARAMETERS}, + "defer_loading": True, + }, + ] + + +def _chat_body(model: str, marker: str, messages: Sequence[JsonValue], stream: bool) -> dict[str, JsonValue]: + return {"model": model, "messages": list(messages), "tools": _chat_tools(), "stream": stream, **_CACHE_BUST} + + +def _raw(gateway: Gateway, path: str, body: Mapping[str, JsonValue]) -> httpx.Response: + with httpx.Client(base_url=str(gateway.client.base_url).rstrip("/"), trust_env=False, timeout=60) as client: + headers: Final = {"Authorization": f"Bearer {gateway.key}"} + with client.stream("POST", path, json=dict(body), headers=headers) as response: + response.read() + return response + + +def _assert_served(response: httpx.Response, marker: str, stream: bool) -> None: + assert response.status_code == 200, response.text + if stream: + assert "data: [DONE]" in response.text and f"marker-{marker}" in response.text, response.text + return + choice: Final = _ITEMS.validate_python(_JSON_OBJECT.validate_json(response.content)["choices"])[0] + assert f"marker-{marker}" in str(_JSON_OBJECT.validate_python(choice["message"])["content"]), response.text + + +def _only_request(wire: Wire) -> tuple[Request, dict[str, JsonValue]]: + received: Final = wire.drain() + assert len(received) == 1, [(request.method, request.target) for request in received] + return received[0], _JSON_OBJECT.validate_json(received[0].body) + + +def _assistant_tool_uses(messages: Sequence[Mapping[str, JsonValue]]) -> list[tuple[str, str]]: + (assistant,) = [turn for turn in messages if turn["role"] == "assistant"] + blocks: Final = _ITEMS.validate_python(assistant["content"]) + return sorted((str(block["id"]), str(block["name"])) for block in blocks if block["type"] == "tool_use") + + +def _tool_results(messages: Sequence[Mapping[str, JsonValue]]) -> Iterator[str]: + for turn in messages: + if turn["role"] != "user" or not isinstance(turn["content"], list): + continue + yield from ( + str(block["tool_use_id"]) + for block in _ITEMS.validate_python(turn["content"]) + if block["type"] == "tool_result" + ) + + +def _assert_tools_forwarded(provider: Provider, body: Mapping[str, JsonValue]) -> None: + tools: Final = _ITEMS.validate_python(body["tools"]) + assert tools[0] == dict(_BEDROCK_REGEX_TOOL if provider == "bedrock_invoke" else _REGEX_TOOL), tools + assert [(tool["name"], tool.get("defer_loading")) for tool in tools[1:]] == [("get_weather", True)], tools + + +def _assert_wire(call: _Call, request: Request, body: Mapping[str, JsonValue]) -> None: + assert unquote(request.target) == _target(call.provider, call.stream), request.target + _assert_tools_forwarded(call.provider, body) + messages: Final = _ITEMS.validate_python(body["messages"]) + weather: Final = (f"toolu_{call.marker}", "get_weather") + search: Final = (f"srvtoolu_{call.marker}", _REGEX_TOOL_NAME) + match call.answer: + case "unanswered": + assert "srvtoolu_" not in json.dumps(messages), messages + assert _assistant_tool_uses(messages) == [weather], messages + assert sorted(_tool_results(messages)) == [weather[0]], messages + case "answered": + assert _assistant_tool_uses(messages) == [search, weather], messages + assert sorted(_tool_results(messages)) == [search[0], weather[0]], messages + + +@pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"]) +@pytest.mark.parametrize("answer", ["unanswered", "answered"]) +@pytest.mark.parametrize("provider", ["anthropic", "bedrock_invoke"]) +def test_a_replayed_search_call_is_dropped_only_when_nothing_answers_it( + gateway: Gateway, provider: Provider, answer: Answer, stream: bool +) -> None: + call: Final = _Call(provider, answer, stream, uuid.uuid4().hex) + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, provider, wire.url) + history: Final = _history(call.marker, _answers(call.marker, answer)) + response: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, call.marker, history, stream)) + _assert_served(response, call.marker, stream) + request, body = _only_request(wire) + _assert_wire(call, request, body) + + +def test_a_function_role_answer_keeps_the_replayed_search_call(gateway: Gateway) -> None: + call: Final = _Call("anthropic", "answered", False, uuid.uuid4().hex) + answer: Final[dict[str, JsonValue]] = { + "role": "function", + "name": _REGEX_TOOL_NAME, + "tool_call_id": f"srvtoolu_{call.marker}", + "content": _SEARCH_OUTPUT, + } + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url) + history: Final = _history(call.marker, (answer,)) + response: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, call.marker, history, False)) + _assert_served(response, call.marker, False) + request, body = _only_request(wire) + _assert_wire(call, request, body) + + +def test_a_duplicated_answer_keeps_the_search_call_with_every_result(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + repeated: Final = _tool_message(f"srvtoolu_{marker}", _SEARCH_OUTPUT) + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url) + history: Final = _history(marker, (repeated, repeated)) + response: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, marker, history, False)) + _assert_served(response, marker, False) + request, body = _only_request(wire) + assert request.target == "/v1/messages", request.target + messages: Final = _ITEMS.validate_python(body["messages"]) + assert _assistant_tool_uses(messages) == [ + (f"srvtoolu_{marker}", _REGEX_TOOL_NAME), + (f"toolu_{marker}", "get_weather"), + ] + assert sorted(_tool_results(messages)) == [f"srvtoolu_{marker}", f"srvtoolu_{marker}", f"toolu_{marker}"], ( + messages + ) + + +@pytest.mark.parametrize("tool_call_id", ["", "x" * 5000], ids=["empty-string", "five-kilobyte-string"]) +def test_an_answer_whose_id_matches_nothing_leaves_the_search_call_dropped(gateway: Gateway, tool_call_id: str) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url) + history: Final = _history(marker, (_tool_message(tool_call_id, _SEARCH_OUTPUT),)) + response: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, marker, history, False)) + assert response.status_code == 400 and "unpaired tool ids" in response.text, response.text + _, body = _only_request(wire) + messages: Final = _ITEMS.validate_python(body["messages"]) + assert "srvtoolu_" not in json.dumps(messages), messages + assert _assistant_tool_uses(messages) == [(f"toolu_{marker}", "get_weather")], messages + + +@pytest.mark.parametrize("tool_call_id", [5, ["srvtoolu_"]], ids=["int", "list"]) +def test_a_non_string_answer_id_is_refused_with_400_before_the_wire_and_the_proxy_keeps_serving( + gateway: Gateway, tool_call_id: JsonValue +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url) + history: Final = _history(marker, (_tool_message(tool_call_id, _SEARCH_OUTPUT),)) + refused: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, marker, history, False)) + assert refused.status_code == 400, refused.text + assert wire.drain() == (), refused.text + follow_up: Final = uuid.uuid4().hex + history_served: Final = _history(follow_up, _answers(follow_up, "answered")) + served: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, follow_up, history_served, False)) + _assert_served(served, follow_up, False) + assert len(wire.drain()) == 1 + + +async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served: + body: Final = _chat_body(model, call.marker, _history(call.marker, _answers(call.marker, call.answer)), call.stream) + async with client.stream( + "POST", "/v1/chat/completions", json=body, headers={"Authorization": f"Bearer {key}"} + ) as response: + raw: Final = await response.aread() + return _Served(call, response.status_code, raw.decode()) + + +async def _burst( + base_url: str, key: str, models: Mapping[Provider, str], calls: Sequence[_Call] +) -> tuple[_Served, ...]: + async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client: + return tuple(await asyncio.gather(*(_send(client, key, models[call.provider], call) for call in calls))) + + +def _burst_calls(count: int) -> tuple[_Call, ...]: + providers: Final[tuple[Provider, ...]] = ("anthropic", "bedrock_invoke") + answers: Final[tuple[Answer, ...]] = ("unanswered", "answered") + return tuple( + _Call(providers[index % 2], answers[(index // 2) % 2], index % 4 >= 2, uuid.uuid4().hex) + for index in range(count) + ) + + +def _marked(received: Sequence[Request], marker: str) -> tuple[Request, dict[str, JsonValue]]: + (request,) = [request for request in received if marker in request.body.decode()] + return request, _JSON_OBJECT.validate_json(request.body) + + +def test_a_mixed_burst_of_replays_pairs_every_answered_call_and_drops_every_unanswered_one(gateway: Gateway) -> None: + calls: Final = _burst_calls(12) + with wire_server(_vendor) as anthropic_wire, wire_server(_vendor) as bedrock_wire, gateway.scenario() as scenario: + models: Final[dict[Provider, str]] = { + "anthropic": _deployment(scenario, "anthropic", anthropic_wire.url), + "bedrock_invoke": _deployment(scenario, "bedrock_invoke", bedrock_wire.url), + } + served: Final = asyncio.run(_burst(str(gateway.client.base_url).rstrip("/"), gateway.key, models, calls)) + assert [item.status for item in served] == [200] * len(calls), [ + (item.status, item.text[:200]) for item in served + ] + received: Final = {"anthropic": anthropic_wire.drain(), "bedrock_invoke": bedrock_wire.drain()} + assert {provider: len(posts) for provider, posts in received.items()} == {"anthropic": 6, "bedrock_invoke": 6} + for item in served: + assert f"marker-{item.call.marker}" in item.text, item.text + request, body = _marked(received[item.call.provider], item.call.marker) + _assert_wire(item.call, request, body) diff --git a/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py new file mode 100644 index 00000000000..bc95861e467 --- /dev/null +++ b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py @@ -0,0 +1,508 @@ +import asyncio +import json +import uuid +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import unquote + +import httpx +import pytest +from _pytest.mark.structures import ParameterSet +from integration._support import claude_code as cc +from integration._support import responses_vendor as rv +from integration._support.anthropic_thinking import BEDROCK_MODEL, Event, sse_chunks +from integration._support.client import Gateway, Scenario +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_CLAUDE: Final = cc.OPUS +_CLAUDE_WITHOUT_TOOL_SEARCH: Final = "claude-scripted-unlisted" +_GEMINI: Final = "gemini-2.5-flash" +_MISTRAL: Final = "mistral-large-latest" +_REGEX_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" +_REGEX_TOOL_NAME: Final = "tool_search_tool_regex" +_ALL_DEFERRED_ERROR: Final = "At least one tool must have defer_loading=false" +_DEFERRED_NAMES: Final = ("get_weather", "finance__get_stock_price") +_CACHE_BUST: Final[Mapping[str, JsonValue]] = MappingProxyType({"cache": {"no-cache": True}}) +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]]) +_WEATHER_PARAMETERS: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], +} +_TICKER_PARAMETERS: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"ticker": {"type": "string"}}, + "required": ["ticker"], +} +_EMAIL_PARAMETERS: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"to": {"type": "string"}}, + "required": ["to"], +} + +Route = Literal["gemini", "bedrock_converse", "mistral", "unlisted_claude", "anthropic"] +Search = Literal["absent", "dropped"] + + +@dataclass(frozen=True, slots=True) +class _Call: + route: Route + stream: bool + marker: str + + +@dataclass(frozen=True, slots=True) +class _Served: + call: _Call + status: int + text: str + + +def _prompt(marker: str) -> str: + return f"Weather in Paris and the ACME price marker-{marker}" + + +def _tool_search(**fields: JsonValue) -> dict[str, JsonValue]: + return {"type": "tool_search", **fields} + + +def _function_tool(defer_loading: JsonValue = True) -> dict[str, JsonValue]: + return { + "type": "function", + "name": "get_weather", + "description": "Weather for a city", + "parameters": _WEATHER_PARAMETERS, + "defer_loading": defer_loading, + } + + +def _loaded_tool() -> dict[str, JsonValue]: + return {"type": "function", "name": "send_email", "description": "Send an email", "parameters": _EMAIL_PARAMETERS} + + +def _namespace_tool() -> dict[str, JsonValue]: + return { + "type": "namespace", + "name": "finance", + "description": "Finance tools", + "tools": [ + { + "type": "function", + "name": "get_stock_price", + "description": "Price of a ticker", + "parameters": _TICKER_PARAMETERS, + "defer_loading": True, + } + ], + } + + +def _search_tools(route: Route, search: Search) -> tuple[dict[str, JsonValue], ...]: + if search == "absent": + return () + return (_tool_search(execution="client"),) if route == "anthropic" else (_tool_search(),) + + +def _gemini_frame(marker: str) -> dict[str, JsonValue]: + return { + "candidates": [ + {"content": {"role": "model", "parts": [{"text": f"loaded marker-{marker}"}]}, "finishReason": "STOP"} + ], + "usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 3, "totalTokenCount": 8}, + "modelVersion": _GEMINI, + } + + +def _chat_completion(marker: str) -> dict[str, JsonValue]: + return { + "id": f"cmpl-{marker}", + "object": "chat.completion", + "created": 1, + "model": _MISTRAL, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": f"loaded marker-{marker}"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + } + + +def _chat_chunks(marker: str) -> tuple[bytes, ...]: + head: Final[dict[str, JsonValue]] = { + "id": f"cmpl-{marker}", + "object": "chat.completion.chunk", + "created": 1, + "model": _MISTRAL, + } + return ( + rv.chat_sse( + { + **head, + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": f"loaded marker-{marker}"}, + "finish_reason": None, + } + ], + } + ), + rv.chat_sse( + { + **head, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + } + ), + b"data: [DONE]\n\n", + ) + + +def _claude_message(marker: str) -> dict[str, JsonValue]: + return { + "id": f"msg_{marker}", + "type": "message", + "role": "assistant", + "model": _CLAUDE, + "content": [{"type": "text", "text": f"loaded marker-{marker}"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 20, "output_tokens": 7}, + } + + +def _claude_events(marker: str) -> tuple[Event, ...]: + opened: Final = {**_claude_message(marker), "content": [], "stop_reason": None} + return ( + {"type": "message_start", "message": opened}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"loaded marker-{marker}"}}, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 7}, + }, + {"type": "message_stop"}, + ) + + +def _anthropic_error(status: int, message: str) -> Reply: + body: Final = {"type": "error", "error": {"type": "invalid_request_error", "message": message}} + return Reply(status=status, body=json.dumps(body).encode()) + + +def _request_tools(body: Mapping[str, JsonValue]) -> list[dict[str, JsonValue]]: + return _ITEMS.validate_python(body.get("tools") or []) + + +def _gemini(request: Request) -> Reply: + marker: Final = rv.newest_marker(request.body.decode()) or "health" + if f"/models/{_GEMINI}:streamGenerateContent" in request.target: + return Reply(content_type="text/event-stream", chunks=(rv.chat_sse(_gemini_frame(marker)),)) + if f"/models/{_GEMINI}:generateContent" in request.target: + return Reply(body=json.dumps(_gemini_frame(marker)).encode()) + return rv.error(404, f"unexpected target {request.target}", "not_found") + + +def _converse(request: Request) -> Reply: + marker: Final = rv.newest_marker(request.body.decode()) or "health" + if unquote(request.target) != f"/model/{BEDROCK_MODEL}/converse": + return rv.error(404, f"unexpected target {request.target}", "not_found") + return Reply( + body=json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": f"converse marker-{marker}"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + } + ).encode() + ) + + +def _mistral(request: Request) -> Reply: + marker: Final = rv.newest_marker(request.body.decode()) or "health" + if request.target != "/v1/chat/completions": + return rv.error(404, f"unexpected target {request.target}", "not_found") + body: Final = _JSON_OBJECT.validate_json(request.body) + deferred: Final = [index for index, tool in enumerate(_request_tools(body)) if "defer_loading" in tool] + if deferred: + return rv.error( + 422, f"tools.{deferred[0]}.defer_loading: Extra inputs are not permitted marker-{marker}", "extra_forbidden" + ) + if body.get("stream") is True: + return Reply(content_type="text/event-stream", chunks=_chat_chunks(marker)) + return Reply(body=json.dumps(_chat_completion(marker)).encode()) + + +def _claude(request: Request) -> Reply: + if request.method != "POST": + return Reply() + marker: Final = rv.newest_marker(request.body.decode()) or "health" + body: Final = _JSON_OBJECT.validate_json(request.body) + tools: Final = _request_tools(body) + regex_tools: Final = [tool for tool in tools if tool.get("type") == _REGEX_TOOL_TYPE] + function_tools: Final = [tool for tool in tools if tool.get("type") != _REGEX_TOOL_TYPE] + if not regex_tools and function_tools and all(tool.get("defer_loading") is True for tool in function_tools): + return _anthropic_error(400, f"tools: {_ALL_DEFERRED_ERROR} marker-{marker}") + if body.get("stream") is True: + return Reply(content_type="text/event-stream", chunks=sse_chunks(_claude_events(marker))) + return Reply(body=json.dumps(_claude_message(marker)).encode()) + + +def _vendor(route: Route) -> Callable[[Request], Reply]: + match route: + case "gemini": + return _gemini + case "bedrock_converse": + return _converse + case "mistral": + return _mistral + case "unlisted_claude" | "anthropic": + return _claude + + +def _deployment(scenario: Scenario, route: Route, wire_url: str) -> str: + match route: + case "gemini": + return scenario.model(model=f"gemini/{_GEMINI}", api_base=wire_url, api_key="synthetic-gemini-key") + case "bedrock_converse": + return scenario.model( + model=f"bedrock/converse/{BEDROCK_MODEL}", + aws_access_key_id="AKIASCRIPTEDPROVIDER", + aws_secret_access_key="scripted-secret", + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire_url, + ) + case "mistral": + return scenario.model(model=f"mistral/{_MISTRAL}", api_base=wire_url, api_key="synthetic-mistral-key") + case "unlisted_claude": + return scenario.model( + model=f"anthropic/{_CLAUDE_WITHOUT_TOOL_SEARCH}", api_base=wire_url, api_key=cc.ANTHROPIC_API_KEY + ) + case "anthropic": + return scenario.model(model=f"anthropic/{_CLAUDE}", api_base=wire_url, api_key=cc.ANTHROPIC_API_KEY) + + +def _responses_body( + model: str, marker: str, tools: Sequence[Mapping[str, JsonValue]], stream: bool +) -> dict[str, JsonValue]: + return { + "model": model, + "input": _prompt(marker), + "tools": [dict(tool) for tool in tools], + "stream": stream, + **_CACHE_BUST, + } + + +def _raw(gateway: Gateway, body: Mapping[str, JsonValue]) -> httpx.Response: + with httpx.Client(base_url=str(gateway.client.base_url).rstrip("/"), trust_env=False, timeout=60) as client: + headers: Final = {"Authorization": f"Bearer {gateway.key}"} + with client.stream("POST", "/v1/responses", json=dict(body), headers=headers) as response: + response.read() + return response + + +def _assert_served(response: httpx.Response, marker: str, stream: bool) -> None: + assert response.status_code == 200, response.text + assert f"marker-{marker}" in response.text, response.text + if stream: + assert "response.completed" in response.text, response.text + return + types: Final = [ + item["type"] for item in _ITEMS.validate_python(_JSON_OBJECT.validate_json(response.content)["output"]) + ] + assert "message" in types, response.text + + +def _only_request(wire: Wire) -> tuple[Request, dict[str, JsonValue]]: + received: Final = wire.drain() + assert len(received) == 1, [(request.method, request.target) for request in received] + return received[0], _JSON_OBJECT.validate_json(received[0].body) + + +def _assert_target(route: Route, stream: bool, request: Request, body: Mapping[str, JsonValue]) -> None: + match route: + case "gemini": + method: Final = "streamGenerateContent" if stream else "generateContent" + assert f"/models/{_GEMINI}:{method}" in request.target, request.target + case "bedrock_converse": + assert unquote(request.target) == f"/model/{BEDROCK_MODEL}/converse", request.target + case "mistral": + assert request.target == "/v1/chat/completions", request.target + assert body.get("stream", False) is stream, body + case "unlisted_claude" | "anthropic": + assert request.target == "/v1/messages", request.target + assert body.get("stream", False) is stream, body + + +def _wire_tool_names(route: Route, body: Mapping[str, JsonValue]) -> list[str]: + match route: + case "gemini": + declarations: Final = _ITEMS.validate_python( + _ITEMS.validate_python(body["tools"])[0]["function_declarations"] + ) + return [str(declaration["name"]) for declaration in declarations] + case "bedrock_converse": + specs: Final = _ITEMS.validate_python(_JSON_OBJECT.validate_python(body["toolConfig"])["tools"]) + return [str(_JSON_OBJECT.validate_python(spec["toolSpec"])["name"]) for spec in specs] + case "mistral": + return [ + str(_JSON_OBJECT.validate_python(tool["function"])["name"]) + for tool in _ITEMS.validate_python(body["tools"]) + ] + case "unlisted_claude" | "anthropic": + return [str(tool["name"]) for tool in _ITEMS.validate_python(body["tools"])] + + +def _assert_undeferred_wire(call: _Call, request: Request, body: Mapping[str, JsonValue]) -> None: + _assert_target(call.route, call.stream, request, body) + assert _wire_tool_names(call.route, body) == list(_DEFERRED_NAMES), body + assert "defer_loading" not in request.body.decode(), body + assert "tool_search" not in request.body.decode(), body + + +def _modes(route: Route) -> tuple[bool, ...]: + return (False,) if route == "bedrock_converse" else (False, True) + + +def _cells() -> tuple[ParameterSet, ...]: + routes: Final[tuple[Route, ...]] = ("gemini", "bedrock_converse", "mistral", "unlisted_claude", "anthropic") + searches: Final[tuple[Search, ...]] = ("absent", "dropped") + combos: Final = [ + (route, search, stream) for route in routes for search in searches for stream in _modes(route) + ] # comprehension-ok: a parametrize grid + return tuple( + pytest.param(route, search, stream, id=f"{route}-{search}-{'stream' if stream else 'sync'}") + for route, search, stream in combos + ) + + +@pytest.mark.parametrize(("route", "search", "stream"), _cells()) +def test_without_a_search_tool_on_the_wire_every_tool_arrives_without_defer_loading( + gateway: Gateway, route: Route, search: Search, stream: bool +) -> None: + call: Final = _Call(route, stream, uuid.uuid4().hex) + with wire_server(_vendor(route)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, route, wire.url) + tools: Final = (*_search_tools(route, search), _function_tool(), _namespace_tool()) + response: Final = _raw(gateway, _responses_body(model, call.marker, tools, stream)) + _assert_served(response, call.marker, stream) + request, body = _only_request(wire) + _assert_undeferred_wire(call, request, body) + + +def test_a_false_defer_loading_never_reaches_an_openai_compatible_wire_as_a_key(gateway: Gateway) -> None: + call: Final = _Call("mistral", False, uuid.uuid4().hex) + with wire_server(_mistral) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "mistral", wire.url) + tools: Final = (_function_tool(defer_loading=False), _namespace_tool()) + response: Final = _raw(gateway, _responses_body(model, call.marker, tools, False)) + _assert_served(response, call.marker, False) + request, body = _only_request(wire) + _assert_undeferred_wire(call, request, body) + + +@pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"]) +def test_the_supported_anthropic_route_keeps_each_tools_own_defer_loading_with_the_search_tool( + gateway: Gateway, stream: bool +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_claude) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url) + tools: Final = (_tool_search(), _function_tool(), _namespace_tool(), _loaded_tool()) + response: Final = _raw(gateway, _responses_body(model, marker, tools, stream)) + _assert_served(response, marker, stream) + request, body = _only_request(wire) + _assert_target("anthropic", stream, request, body) + sent: Final = _ITEMS.validate_python(body["tools"]) + assert sent[0] == {"type": _REGEX_TOOL_TYPE, "name": _REGEX_TOOL_NAME}, sent + assert [(tool["name"], tool.get("defer_loading")) for tool in sent[1:]] == [ + ("get_weather", True), + ("finance__get_stock_price", True), + ("send_email", None), + ], sent + assert "defer_loading" not in json.dumps(sent[3]), sent + + +@pytest.mark.parametrize("route", ["gemini", "mistral"]) +def test_a_non_boolean_defer_loading_is_refused_with_400_on_a_route_without_tool_search( + gateway: Gateway, route: Route +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_vendor(route)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, route, wire.url) + refused: Final = _raw( + gateway, _responses_body(model, marker, (_function_tool("true"), _namespace_tool()), False) + ) + assert refused.status_code == 400 and "defer_loading must be a boolean" in refused.text, refused.text + assert wire.drain() == (), refused.text + follow_up: Final = _Call(route, False, uuid.uuid4().hex) + served: Final = _raw( + gateway, _responses_body(model, follow_up.marker, (_function_tool(), _namespace_tool()), False) + ) + _assert_served(served, follow_up.marker, False) + request, body = _only_request(wire) + _assert_undeferred_wire(follow_up, request, body) + + +async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served: + body: Final = _responses_body(model, call.marker, (_function_tool(), _namespace_tool()), call.stream) + async with client.stream( + "POST", "/v1/responses", json=body, headers={"Authorization": f"Bearer {key}"} + ) as response: + raw: Final = await response.aread() + return _Served(call, response.status_code, raw.decode()) + + +async def _burst(base_url: str, key: str, models: Mapping[Route, str], calls: Sequence[_Call]) -> tuple[_Served, ...]: + async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client: + return tuple(await asyncio.gather(*(_send(client, key, models[call.route], call) for call in calls))) + + +def _burst_calls() -> tuple[_Call, ...]: + routes: Final[tuple[Route, ...]] = ("gemini", "bedrock_converse", "mistral", "unlisted_claude") + return tuple( + _Call(route, stream, uuid.uuid4().hex) for route in routes for stream in _modes(route) + ) # comprehension-ok: a two-axis burst grid + + +def _marked(received: Sequence[Request], marker: str) -> tuple[Request, dict[str, JsonValue]]: + (request,) = [request for request in received if marker in request.body.decode()] + return request, _JSON_OBJECT.validate_json(request.body) + + +def test_a_burst_across_four_routes_strips_every_deferral(gateway: Gateway) -> None: + calls: Final = _burst_calls() + with ( + wire_server(_gemini) as gemini_wire, + wire_server(_converse) as converse_wire, + wire_server(_mistral) as mistral_wire, + wire_server(_claude) as claude_wire, + gateway.scenario() as scenario, + ): + wires: Final[dict[Route, Wire]] = { + "gemini": gemini_wire, + "bedrock_converse": converse_wire, + "mistral": mistral_wire, + "unlisted_claude": claude_wire, + } + models: Final = {route: _deployment(scenario, route, wire.url) for route, wire in wires.items()} + served: Final = asyncio.run(_burst(str(gateway.client.base_url).rstrip("/"), gateway.key, models, calls)) + assert [item.status for item in served] == [200] * len(calls), [ + (item.status, item.text[:200]) for item in served + ] + received: Final = {route: wire.drain() for route, wire in wires.items()} + for item in served: + assert f"marker-{item.call.marker}" in item.text, item.text + request, body = _marked(received[item.call.route], item.call.marker) + _assert_undeferred_wire(item.call, request, body) diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 454edfab76f..08b8e6b2413 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1924,6 +1924,42 @@ class TestToolTransformation: assert [tool["function"]["name"] for tool in result["tools"]] == ["get_weather", "finance__get_stock_price"] assert all("defer_loading" not in tool for tool in result["tools"]) + def test_native_bm25_search_tool_keeps_the_other_tools_deferred(self) -> None: + tools: Final = [ + {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}, + *_deferred_tools_with_server_tool_search()[1:], + ] + + result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input="What is the weather in Paris right now?", + responses_api_request={"tools": tools}, + custom_llm_provider="anthropic", + ) + + chat_tools: Final = result["tools"] + assert chat_tools[0] == {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} + assert [tool["defer_loading"] for tool in chat_tools[1:]] == [True, True] + + def test_native_search_tool_passes_through_routes_without_tool_search(self) -> None: + model: Final = "azure_ai/claude-sonnet-5-5" + assert litellm.supports_tool_search(model, "azure_ai") is False + tools: Final = [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + *_deferred_tools_with_server_tool_search()[1:], + ] + + result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input="What is the weather in Paris right now?", + responses_api_request={"tools": tools}, + custom_llm_provider="azure_ai", + ) + + chat_tools: Final = result["tools"] + assert chat_tools[0] == {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"} + assert [tool["defer_loading"] for tool in chat_tools[1:]] == [True, True] + def test_server_tool_search_follows_the_bedrock_route(self) -> None: transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request converse: Final = transform( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bdf0c409716..9a5a7d625ac 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -42993,7 +42993,7 @@ export interface components { */ ResponseItemList: { /** Data */ - data: (components["schemas"]["ResponseInputMessageItem"] | components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseFunctionToolCallItem"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["McpCall"] | components["schemas"]["ResponseCustomToolCallItem"] | components["schemas"]["ResponseCustomToolCallOutputItem"])[]; + data: (components["schemas"]["ResponseInputMessageItem"] | components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseFunctionToolCallItem"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["openai__types__responses__response_tool_search_call__ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["McpCall"] | components["schemas"]["ResponseCustomToolCallItem"] | components["schemas"]["ResponseCustomToolCallOutputItem"])[]; /** First Id */ first_id: string; /** Has More */ @@ -43215,34 +43215,6 @@ export interface components { } & { [key: string]: unknown; }; - /** ResponseToolSearchCall */ - ResponseToolSearchCall: { - /** Arguments */ - arguments: unknown; - /** Call Id */ - call_id?: string | null; - /** Created By */ - created_by?: string | null; - /** - * Execution - * @enum {string} - */ - execution: "server" | "client"; - /** Id */ - id: string; - /** - * Status - * @enum {string} - */ - status: "in_progress" | "completed" | "incomplete"; - /** - * Type - * @constant - */ - type: "tool_search_call"; - } & { - [key: string]: unknown; - }; /** ResponseToolSearchOutputItem */ ResponseToolSearchOutputItem: { /** Call Id */ @@ -43477,9 +43449,9 @@ export interface components { /** Object */ object?: string | null; /** Output */ - output: (components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpCall"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["ResponseCustomToolCall"] | components["schemas"]["ResponseCustomToolCallOutputItem"] | { + output: (components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["openai__types__responses__response_tool_search_call__ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpCall"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["ResponseCustomToolCall"] | components["schemas"]["ResponseCustomToolCallOutputItem"] | { [key: string]: unknown; - })[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["ResponseToolSearchCall"])[]; + })[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["litellm__types__responses__main__ResponseToolSearchCall"])[]; /** Parallel Tool Calls */ parallel_tool_calls?: boolean | null; /** Previous Response Id */ @@ -49288,6 +49260,32 @@ export interface components { /** Type */ type: "computer_use_preview" | string; }; + /** ResponseToolSearchCall */ + litellm__types__responses__main__ResponseToolSearchCall: { + /** Arguments */ + arguments: unknown; + /** Call Id */ + call_id?: string | null; + /** + * Execution + * @enum {string} + */ + execution: "server" | "client"; + /** Id */ + id: string; + /** + * Status + * @enum {string} + */ + status: "in_progress" | "completed" | "incomplete"; + /** + * Type + * @constant + */ + type: "tool_search_call"; + } & { + [key: string]: unknown; + }; /** ModelInfo */ litellm__types__router__ModelInfo: { /** Access Windows */ @@ -49370,6 +49368,34 @@ export interface components { */ type: "computer"; }; + /** ResponseToolSearchCall */ + openai__types__responses__response_tool_search_call__ResponseToolSearchCall: { + /** Arguments */ + arguments: unknown; + /** Call Id */ + call_id?: string | null; + /** Created By */ + created_by?: string | null; + /** + * Execution + * @enum {string} + */ + execution: "server" | "client"; + /** Id */ + id: string; + /** + * Status + * @enum {string} + */ + status: "in_progress" | "completed" | "incomplete"; + /** + * Type + * @constant + */ + type: "tool_search_call"; + } & { + [key: string]: unknown; + }; /** * UserLocation * @description The user's location.