mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-17 04:34:25 +00:00
fix(responses): gate only the OpenAI tool_search shape and keep native Anthropic search tools intact
A native tool_search_tool_regex_20251119 or tool_search_tool_bm25_20251119 tool the caller sends through /v1/responses passes through the bridge on every route as before, and the other tools keep defer_loading whenever any Anthropic search tool goes out. The route gate now drops the OpenAI tool_search item before translation instead of filtering the translated regex tool, so bm25 counts as a search tool. answered_tool_call_ids walks the history with islice instead of a slice copy. The tool_search_call output item drops its created_by field, which OpenAI rejects on a replayed input item and the streaming item never carried
This commit is contained in:
mateo-berri
parent
c06dff683a
commit
418996391f
9 changed files with 1156 additions and 131 deletions
|
|
@ -1828,7 +1828,9 @@ def _tool_result_ids(answers: Iterable[AllMessageValues]) -> Iterator[str]:
|
|||
|
||||
|
||||
def answered_tool_call_ids(messages: Sequence[AllMessageValues], assistant_index: int) -> frozenset[str]:
|
||||
following: Final = itertools.dropwhile(lambda message: message["role"] == "assistant", messages[assistant_index:])
|
||||
following: Final = itertools.dropwhile(
|
||||
lambda message: message["role"] == "assistant", itertools.islice(messages, assistant_index, None)
|
||||
)
|
||||
answers: Final = itertools.takewhile(lambda message: message["role"] in ("tool", "function"), following)
|
||||
return frozenset(_tool_result_ids(answers))
|
||||
|
||||
|
|
|
|||
|
|
@ -25329,7 +25329,7 @@
|
|||
"$ref": "#/components/schemas/ResponseFunctionToolCallOutputItem"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseToolSearchCall"
|
||||
"$ref": "#/components/schemas/openai__types__responses__response_tool_search_call__ResponseToolSearchCall"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseToolSearchOutputItem"
|
||||
|
|
@ -25683,71 +25683,6 @@
|
|||
"title": "ResponseTextConfigParam",
|
||||
"type": "object"
|
||||
},
|
||||
"ResponseToolSearchCall": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"arguments": {
|
||||
"title": "Arguments"
|
||||
},
|
||||
"call_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Call Id"
|
||||
},
|
||||
"created_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Created By"
|
||||
},
|
||||
"execution": {
|
||||
"enum": [
|
||||
"server",
|
||||
"client"
|
||||
],
|
||||
"title": "Execution",
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"title": "Id",
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"enum": [
|
||||
"in_progress",
|
||||
"completed",
|
||||
"incomplete"
|
||||
],
|
||||
"title": "Status",
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"const": "tool_search_call",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"arguments",
|
||||
"execution",
|
||||
"status",
|
||||
"type"
|
||||
],
|
||||
"title": "ResponseToolSearchCall",
|
||||
"type": "object"
|
||||
},
|
||||
"ResponseToolSearchOutputItem": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
|
|
@ -25982,7 +25917,7 @@
|
|||
"$ref": "#/components/schemas/ResponseReasoningItem"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseToolSearchCall"
|
||||
"$ref": "#/components/schemas/openai__types__responses__response_tool_search_call__ResponseToolSearchCall"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseToolSearchOutputItem"
|
||||
|
|
@ -26065,7 +26000,7 @@
|
|||
"$ref": "#/components/schemas/CustomToolCallOutputItem"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseToolSearchCall"
|
||||
"$ref": "#/components/schemas/litellm__types__responses__main__ResponseToolSearchCall"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -27004,6 +26939,125 @@
|
|||
"title": "WebSearchTool",
|
||||
"type": "object"
|
||||
},
|
||||
"litellm__types__responses__main__ResponseToolSearchCall": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"arguments": {
|
||||
"title": "Arguments"
|
||||
},
|
||||
"call_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Call Id"
|
||||
},
|
||||
"execution": {
|
||||
"enum": [
|
||||
"server",
|
||||
"client"
|
||||
],
|
||||
"title": "Execution",
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"title": "Id",
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"enum": [
|
||||
"in_progress",
|
||||
"completed",
|
||||
"incomplete"
|
||||
],
|
||||
"title": "Status",
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"const": "tool_search_call",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"arguments",
|
||||
"execution",
|
||||
"status",
|
||||
"type"
|
||||
],
|
||||
"title": "ResponseToolSearchCall",
|
||||
"type": "object"
|
||||
},
|
||||
"openai__types__responses__response_tool_search_call__ResponseToolSearchCall": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"arguments": {
|
||||
"title": "Arguments"
|
||||
},
|
||||
"call_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Call Id"
|
||||
},
|
||||
"created_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Created By"
|
||||
},
|
||||
"execution": {
|
||||
"enum": [
|
||||
"server",
|
||||
"client"
|
||||
],
|
||||
"title": "Execution",
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"title": "Id",
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"enum": [
|
||||
"in_progress",
|
||||
"completed",
|
||||
"incomplete"
|
||||
],
|
||||
"title": "Status",
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"const": "tool_search_call",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"arguments",
|
||||
"execution",
|
||||
"status",
|
||||
"type"
|
||||
],
|
||||
"title": "ResponseToolSearchCall",
|
||||
"type": "object"
|
||||
},
|
||||
"openai__types__responses__web_search_preview_tool__UserLocation": {
|
||||
"additionalProperties": true,
|
||||
"description": "The user's location.",
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from typing import Final, Literal, TypeAlias
|
|||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import is_anthropic_tool_search_use
|
||||
from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AnthropicToolSearchToolRegex
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.responses.main import ResponseToolSearchCall
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
|
@ -55,7 +55,7 @@ def anthropic_tool_search_chat_tool() -> AnthropicToolSearchToolRegex:
|
|||
|
||||
|
||||
def is_anthropic_tool_search_chat_tool(tool: Mapping[str, object]) -> bool:
|
||||
return tool.get("type") == ANTHROPIC_TOOL_SEARCH_TOOL_TYPE
|
||||
return tool.get("type") in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
|
||||
|
||||
|
||||
def provider_builds_anthropic_request(model: str, custom_llm_provider: str | None) -> bool:
|
||||
|
|
|
|||
|
|
@ -383,31 +383,31 @@ class LiteLLMCompletionResponsesConfig:
|
|||
return supported_params is not None and "web_search_options" not in supported_params
|
||||
|
||||
@staticmethod
|
||||
def _drop_tool_search_when_unsupported(
|
||||
tools: Sequence[ChatToolParam], model: str, custom_llm_provider: str | None
|
||||
) -> Sequence[ChatToolParam]:
|
||||
def _without_unsupported_tool_search(
|
||||
tools: ResponseTools, model: str, custom_llm_provider: str | None
|
||||
) -> ResponseTools:
|
||||
from litellm.utils import supports_tool_search
|
||||
|
||||
has_search_tool: Final = any(is_anthropic_tool_search_chat_tool(tool) for tool in tools)
|
||||
if (
|
||||
has_search_tool
|
||||
and supports_tool_search(model=model, custom_llm_provider=custom_llm_provider)
|
||||
and provider_builds_anthropic_request(model, custom_llm_provider)
|
||||
):
|
||||
if not has_server_tool_search(tools):
|
||||
return tools
|
||||
if has_search_tool:
|
||||
verbose_logger.warning(
|
||||
"Dropping Responses API tool of type 'tool_search' and the deferral of the other tools: model '%s' "
|
||||
"(provider '%s') cannot run tool search on this route, so the target provider would reject or "
|
||||
"ignore it.",
|
||||
model,
|
||||
custom_llm_provider,
|
||||
)
|
||||
return [
|
||||
LiteLLMCompletionResponsesConfig._without_deferral(tool)
|
||||
for tool in tools
|
||||
if not is_anthropic_tool_search_chat_tool(tool)
|
||||
]
|
||||
route_runs_tool_search: Final = supports_tool_search(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
) and provider_builds_anthropic_request(model, custom_llm_provider)
|
||||
if route_runs_tool_search:
|
||||
return tools
|
||||
verbose_logger.warning(
|
||||
"Dropping Responses API tool of type 'tool_search': model '%s' (provider '%s') cannot run tool search "
|
||||
"on this route, so the target provider would reject or ignore it.",
|
||||
model,
|
||||
custom_llm_provider,
|
||||
)
|
||||
return [tool for tool in tools or () if not is_server_tool_search_tool(tool)]
|
||||
|
||||
@staticmethod
|
||||
def _undeferred_unless_a_search_tool_goes_out(tools: Sequence[ChatToolParam]) -> Sequence[ChatToolParam]:
|
||||
if any(is_anthropic_tool_search_chat_tool(tool) for tool in tools):
|
||||
return tools
|
||||
return [LiteLLMCompletionResponsesConfig._without_deferral(tool) for tool in tools]
|
||||
|
||||
@staticmethod
|
||||
def _without_deferral(tool: ChatToolParam) -> ChatToolParam:
|
||||
|
|
@ -503,15 +503,14 @@ class LiteLLMCompletionResponsesConfig:
|
|||
"""
|
||||
Transform a Responses API request into a Chat Completion request
|
||||
"""
|
||||
request_tools: Final = LiteLLMCompletionResponsesConfig._without_unsupported_tool_search(
|
||||
responses_api_request.get("tools") or [], model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
(
|
||||
converted_tools,
|
||||
web_search_options,
|
||||
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
|
||||
responses_api_request.get("tools") or []
|
||||
)
|
||||
tools: Final = LiteLLMCompletionResponsesConfig._drop_tool_search_when_unsupported(
|
||||
converted_tools, model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(request_tools)
|
||||
tools: Final = LiteLLMCompletionResponsesConfig._undeferred_unless_a_search_tool_goes_out(converted_tools)
|
||||
|
||||
if web_search_options is not None and LiteLLMCompletionResponsesConfig._should_drop_derived_web_search_options(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
|
|
|
|||
|
|
@ -142,7 +142,6 @@ class ResponseToolSearchCall(BaseLiteLLMOpenAIResponseObject):
|
|||
execution: Literal["server", "client"]
|
||||
status: Literal["in_progress", "completed", "incomplete"]
|
||||
type: Literal["tool_search_call"]
|
||||
created_by: str | None = None
|
||||
|
||||
|
||||
class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,401 @@
|
|||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import unquote
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support import claude_code as cc
|
||||
from integration._support import responses_vendor as rv
|
||||
from integration._support.anthropic_thinking import BEDROCK_MODEL, EVENT_STREAM, Event, aws_chunks, sse_chunks
|
||||
from integration._support.client import Gateway, Scenario
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_CLAUDE: Final = cc.OPUS
|
||||
_REGEX_TOOL_TYPE: Final = "tool_search_tool_regex_20251119"
|
||||
_REGEX_TOOL_NAME: Final = "tool_search_tool_regex"
|
||||
_REGEX_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType({"type": _REGEX_TOOL_TYPE, "name": _REGEX_TOOL_NAME})
|
||||
_BEDROCK_REGEX_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType(
|
||||
{"type": _REGEX_TOOL_NAME, "name": _REGEX_TOOL_NAME}
|
||||
)
|
||||
_SEARCH_ARGUMENTS: Final = '{"pattern": "weather|stock"}'
|
||||
_SEARCH_OUTPUT: Final = '{"tools": ["get_weather"]}'
|
||||
_CACHE_BUST: Final[Mapping[str, JsonValue]] = MappingProxyType({"cache": {"no-cache": True}})
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
_EMPTY_TURN: Final[Mapping[str, JsonValue]] = MappingProxyType({"role": "user", "content": ""})
|
||||
_WEATHER_PARAMETERS: Final[dict[str, JsonValue]] = {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
}
|
||||
|
||||
Provider = Literal["anthropic", "bedrock_invoke"]
|
||||
Answer = Literal["unanswered", "answered"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Call:
|
||||
provider: Provider
|
||||
answer: Answer
|
||||
stream: bool
|
||||
marker: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Served:
|
||||
call: _Call
|
||||
status: int
|
||||
text: str
|
||||
|
||||
|
||||
def _prompt(marker: str) -> str:
|
||||
return f"Weather in Paris and the ACME price marker-{marker}"
|
||||
|
||||
|
||||
def _message(marker: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"id": f"msg_{marker}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": _CLAUDE,
|
||||
"content": [{"type": "text", "text": f"tomorrow is sunny marker-{marker}"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 20, "output_tokens": 7},
|
||||
}
|
||||
|
||||
|
||||
def _events(marker: str) -> tuple[Event, ...]:
|
||||
opened: Final = {**_message(marker), "content": [], "stop_reason": None}
|
||||
return (
|
||||
{"type": "message_start", "message": opened},
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "text_delta", "text": f"tomorrow is sunny marker-{marker}"},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
|
||||
"usage": {"output_tokens": 7},
|
||||
},
|
||||
{"type": "message_stop"},
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_error(status: int, message: str) -> Reply:
|
||||
body: Final = {"type": "error", "error": {"type": "invalid_request_error", "message": message}}
|
||||
return Reply(status=status, body=json.dumps(body).encode())
|
||||
|
||||
|
||||
def _block_ids(turn: Mapping[str, JsonValue], block_type: str, key: str) -> frozenset[str]:
|
||||
content: Final = turn.get("content")
|
||||
if not isinstance(content, list):
|
||||
return frozenset()
|
||||
blocks: Final = _ITEMS.validate_python(content)
|
||||
return frozenset(str(block[key]) for block in blocks if block.get("type") == block_type)
|
||||
|
||||
|
||||
def _mismatch(messages: Sequence[Mapping[str, JsonValue]], index: int) -> frozenset[str]:
|
||||
following: Final = messages[index + 1] if index + 1 < len(messages) else _EMPTY_TURN
|
||||
return _block_ids(messages[index], "tool_use", "id") ^ _block_ids(following, "tool_result", "tool_use_id")
|
||||
|
||||
|
||||
def _unpaired(messages: Sequence[Mapping[str, JsonValue]]) -> frozenset[str]:
|
||||
assistant_turns: Final = [index for index, turn in enumerate(messages) if turn["role"] == "assistant"]
|
||||
return frozenset().union(*(_mismatch(messages, index) for index in assistant_turns))
|
||||
|
||||
|
||||
def _vendor(request: Request) -> Reply:
|
||||
if request.method != "POST":
|
||||
return Reply()
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
marker: Final = rv.newest_marker(request.body.decode()) or "health"
|
||||
unpaired: Final = _unpaired(_ITEMS.validate_python(body.get("messages") or []))
|
||||
if unpaired:
|
||||
return _anthropic_error(400, f"messages: unpaired tool ids {sorted(unpaired)} marker-{marker}")
|
||||
if request.target.endswith("/invoke-with-response-stream"):
|
||||
return Reply(content_type=EVENT_STREAM, chunks=aws_chunks(_events(marker)))
|
||||
if body.get("stream") is True:
|
||||
return Reply(content_type="text/event-stream", chunks=sse_chunks(_events(marker)))
|
||||
return Reply(body=json.dumps(_message(marker)).encode())
|
||||
|
||||
|
||||
def _deployment(scenario: Scenario, provider: Provider, wire_url: str) -> str:
|
||||
match provider:
|
||||
case "anthropic":
|
||||
return scenario.model(model=f"anthropic/{_CLAUDE}", api_base=wire_url, api_key=cc.ANTHROPIC_API_KEY)
|
||||
case "bedrock_invoke":
|
||||
return scenario.model(
|
||||
model=f"bedrock/invoke/{BEDROCK_MODEL}",
|
||||
api_base=wire_url,
|
||||
aws_access_key_id="AKIASCRIPTEDPROVIDER",
|
||||
aws_secret_access_key="scripted-secret",
|
||||
aws_region_name="us-east-1",
|
||||
aws_bedrock_runtime_endpoint=wire_url,
|
||||
)
|
||||
|
||||
|
||||
def _target(provider: Provider, stream: bool) -> str:
|
||||
match provider:
|
||||
case "anthropic":
|
||||
return "/v1/messages"
|
||||
case "bedrock_invoke":
|
||||
return f"/model/{BEDROCK_MODEL}/invoke-with-response-stream" if stream else f"/model/{BEDROCK_MODEL}/invoke"
|
||||
|
||||
|
||||
def _search_call(marker: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"id": f"srvtoolu_{marker}",
|
||||
"type": "function",
|
||||
"function": {"name": _REGEX_TOOL_NAME, "arguments": _SEARCH_ARGUMENTS},
|
||||
}
|
||||
|
||||
|
||||
def _weather_call(marker: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"id": f"toolu_{marker}",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
|
||||
}
|
||||
|
||||
|
||||
def _tool_message(tool_call_id: JsonValue, content: str) -> dict[str, JsonValue]:
|
||||
return {"role": "tool", "tool_call_id": tool_call_id, "content": content}
|
||||
|
||||
|
||||
def _history(marker: str, answers: Sequence[Mapping[str, JsonValue]]) -> list[JsonValue]:
|
||||
return [
|
||||
{"role": "user", "content": _prompt(marker)},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": f"searching marker-{marker}",
|
||||
"tool_calls": [_search_call(marker), _weather_call(marker)],
|
||||
},
|
||||
*(dict(answer) for answer in answers),
|
||||
_tool_message(f"toolu_{marker}", "22C"),
|
||||
{"role": "user", "content": f"And tomorrow? marker-{marker}"},
|
||||
]
|
||||
|
||||
|
||||
def _answers(marker: str, answer: Answer) -> tuple[dict[str, JsonValue], ...]:
|
||||
return (_tool_message(f"srvtoolu_{marker}", _SEARCH_OUTPUT),) if answer == "answered" else ()
|
||||
|
||||
|
||||
def _chat_tools() -> list[JsonValue]:
|
||||
return [
|
||||
dict(_REGEX_TOOL),
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "description": "Weather for a city", "parameters": _WEATHER_PARAMETERS},
|
||||
"defer_loading": True,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _chat_body(model: str, marker: str, messages: Sequence[JsonValue], stream: bool) -> dict[str, JsonValue]:
|
||||
return {"model": model, "messages": list(messages), "tools": _chat_tools(), "stream": stream, **_CACHE_BUST}
|
||||
|
||||
|
||||
def _raw(gateway: Gateway, path: str, body: Mapping[str, JsonValue]) -> httpx.Response:
|
||||
with httpx.Client(base_url=str(gateway.client.base_url).rstrip("/"), trust_env=False, timeout=60) as client:
|
||||
headers: Final = {"Authorization": f"Bearer {gateway.key}"}
|
||||
with client.stream("POST", path, json=dict(body), headers=headers) as response:
|
||||
response.read()
|
||||
return response
|
||||
|
||||
|
||||
def _assert_served(response: httpx.Response, marker: str, stream: bool) -> None:
|
||||
assert response.status_code == 200, response.text
|
||||
if stream:
|
||||
assert "data: [DONE]" in response.text and f"marker-{marker}" in response.text, response.text
|
||||
return
|
||||
choice: Final = _ITEMS.validate_python(_JSON_OBJECT.validate_json(response.content)["choices"])[0]
|
||||
assert f"marker-{marker}" in str(_JSON_OBJECT.validate_python(choice["message"])["content"]), response.text
|
||||
|
||||
|
||||
def _only_request(wire: Wire) -> tuple[Request, dict[str, JsonValue]]:
|
||||
received: Final = wire.drain()
|
||||
assert len(received) == 1, [(request.method, request.target) for request in received]
|
||||
return received[0], _JSON_OBJECT.validate_json(received[0].body)
|
||||
|
||||
|
||||
def _assistant_tool_uses(messages: Sequence[Mapping[str, JsonValue]]) -> list[tuple[str, str]]:
|
||||
(assistant,) = [turn for turn in messages if turn["role"] == "assistant"]
|
||||
blocks: Final = _ITEMS.validate_python(assistant["content"])
|
||||
return sorted((str(block["id"]), str(block["name"])) for block in blocks if block["type"] == "tool_use")
|
||||
|
||||
|
||||
def _tool_results(messages: Sequence[Mapping[str, JsonValue]]) -> Iterator[str]:
|
||||
for turn in messages:
|
||||
if turn["role"] != "user" or not isinstance(turn["content"], list):
|
||||
continue
|
||||
yield from (
|
||||
str(block["tool_use_id"])
|
||||
for block in _ITEMS.validate_python(turn["content"])
|
||||
if block["type"] == "tool_result"
|
||||
)
|
||||
|
||||
|
||||
def _assert_tools_forwarded(provider: Provider, body: Mapping[str, JsonValue]) -> None:
|
||||
tools: Final = _ITEMS.validate_python(body["tools"])
|
||||
assert tools[0] == dict(_BEDROCK_REGEX_TOOL if provider == "bedrock_invoke" else _REGEX_TOOL), tools
|
||||
assert [(tool["name"], tool.get("defer_loading")) for tool in tools[1:]] == [("get_weather", True)], tools
|
||||
|
||||
|
||||
def _assert_wire(call: _Call, request: Request, body: Mapping[str, JsonValue]) -> None:
|
||||
assert unquote(request.target) == _target(call.provider, call.stream), request.target
|
||||
_assert_tools_forwarded(call.provider, body)
|
||||
messages: Final = _ITEMS.validate_python(body["messages"])
|
||||
weather: Final = (f"toolu_{call.marker}", "get_weather")
|
||||
search: Final = (f"srvtoolu_{call.marker}", _REGEX_TOOL_NAME)
|
||||
match call.answer:
|
||||
case "unanswered":
|
||||
assert "srvtoolu_" not in json.dumps(messages), messages
|
||||
assert _assistant_tool_uses(messages) == [weather], messages
|
||||
assert sorted(_tool_results(messages)) == [weather[0]], messages
|
||||
case "answered":
|
||||
assert _assistant_tool_uses(messages) == [search, weather], messages
|
||||
assert sorted(_tool_results(messages)) == [search[0], weather[0]], messages
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"])
|
||||
@pytest.mark.parametrize("answer", ["unanswered", "answered"])
|
||||
@pytest.mark.parametrize("provider", ["anthropic", "bedrock_invoke"])
|
||||
def test_a_replayed_search_call_is_dropped_only_when_nothing_answers_it(
|
||||
gateway: Gateway, provider: Provider, answer: Answer, stream: bool
|
||||
) -> None:
|
||||
call: Final = _Call(provider, answer, stream, uuid.uuid4().hex)
|
||||
with wire_server(_vendor) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, provider, wire.url)
|
||||
history: Final = _history(call.marker, _answers(call.marker, answer))
|
||||
response: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, call.marker, history, stream))
|
||||
_assert_served(response, call.marker, stream)
|
||||
request, body = _only_request(wire)
|
||||
_assert_wire(call, request, body)
|
||||
|
||||
|
||||
def test_a_function_role_answer_keeps_the_replayed_search_call(gateway: Gateway) -> None:
|
||||
call: Final = _Call("anthropic", "answered", False, uuid.uuid4().hex)
|
||||
answer: Final[dict[str, JsonValue]] = {
|
||||
"role": "function",
|
||||
"name": _REGEX_TOOL_NAME,
|
||||
"tool_call_id": f"srvtoolu_{call.marker}",
|
||||
"content": _SEARCH_OUTPUT,
|
||||
}
|
||||
with wire_server(_vendor) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, "anthropic", wire.url)
|
||||
history: Final = _history(call.marker, (answer,))
|
||||
response: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, call.marker, history, False))
|
||||
_assert_served(response, call.marker, False)
|
||||
request, body = _only_request(wire)
|
||||
_assert_wire(call, request, body)
|
||||
|
||||
|
||||
def test_a_duplicated_answer_keeps_the_search_call_with_every_result(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
repeated: Final = _tool_message(f"srvtoolu_{marker}", _SEARCH_OUTPUT)
|
||||
with wire_server(_vendor) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, "anthropic", wire.url)
|
||||
history: Final = _history(marker, (repeated, repeated))
|
||||
response: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, marker, history, False))
|
||||
_assert_served(response, marker, False)
|
||||
request, body = _only_request(wire)
|
||||
assert request.target == "/v1/messages", request.target
|
||||
messages: Final = _ITEMS.validate_python(body["messages"])
|
||||
assert _assistant_tool_uses(messages) == [
|
||||
(f"srvtoolu_{marker}", _REGEX_TOOL_NAME),
|
||||
(f"toolu_{marker}", "get_weather"),
|
||||
]
|
||||
assert sorted(_tool_results(messages)) == [f"srvtoolu_{marker}", f"srvtoolu_{marker}", f"toolu_{marker}"], (
|
||||
messages
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool_call_id", ["", "x" * 5000], ids=["empty-string", "five-kilobyte-string"])
|
||||
def test_an_answer_whose_id_matches_nothing_leaves_the_search_call_dropped(gateway: Gateway, tool_call_id: str) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_vendor) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, "anthropic", wire.url)
|
||||
history: Final = _history(marker, (_tool_message(tool_call_id, _SEARCH_OUTPUT),))
|
||||
response: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, marker, history, False))
|
||||
assert response.status_code == 400 and "unpaired tool ids" in response.text, response.text
|
||||
_, body = _only_request(wire)
|
||||
messages: Final = _ITEMS.validate_python(body["messages"])
|
||||
assert "srvtoolu_" not in json.dumps(messages), messages
|
||||
assert _assistant_tool_uses(messages) == [(f"toolu_{marker}", "get_weather")], messages
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool_call_id", [5, ["srvtoolu_"]], ids=["int", "list"])
|
||||
def test_a_non_string_answer_id_is_refused_with_400_before_the_wire_and_the_proxy_keeps_serving(
|
||||
gateway: Gateway, tool_call_id: JsonValue
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_vendor) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, "anthropic", wire.url)
|
||||
history: Final = _history(marker, (_tool_message(tool_call_id, _SEARCH_OUTPUT),))
|
||||
refused: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, marker, history, False))
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert wire.drain() == (), refused.text
|
||||
follow_up: Final = uuid.uuid4().hex
|
||||
history_served: Final = _history(follow_up, _answers(follow_up, "answered"))
|
||||
served: Final = _raw(gateway, "/v1/chat/completions", _chat_body(model, follow_up, history_served, False))
|
||||
_assert_served(served, follow_up, False)
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
|
||||
async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served:
|
||||
body: Final = _chat_body(model, call.marker, _history(call.marker, _answers(call.marker, call.answer)), call.stream)
|
||||
async with client.stream(
|
||||
"POST", "/v1/chat/completions", json=body, headers={"Authorization": f"Bearer {key}"}
|
||||
) as response:
|
||||
raw: Final = await response.aread()
|
||||
return _Served(call, response.status_code, raw.decode())
|
||||
|
||||
|
||||
async def _burst(
|
||||
base_url: str, key: str, models: Mapping[Provider, str], calls: Sequence[_Call]
|
||||
) -> tuple[_Served, ...]:
|
||||
async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client:
|
||||
return tuple(await asyncio.gather(*(_send(client, key, models[call.provider], call) for call in calls)))
|
||||
|
||||
|
||||
def _burst_calls(count: int) -> tuple[_Call, ...]:
|
||||
providers: Final[tuple[Provider, ...]] = ("anthropic", "bedrock_invoke")
|
||||
answers: Final[tuple[Answer, ...]] = ("unanswered", "answered")
|
||||
return tuple(
|
||||
_Call(providers[index % 2], answers[(index // 2) % 2], index % 4 >= 2, uuid.uuid4().hex)
|
||||
for index in range(count)
|
||||
)
|
||||
|
||||
|
||||
def _marked(received: Sequence[Request], marker: str) -> tuple[Request, dict[str, JsonValue]]:
|
||||
(request,) = [request for request in received if marker in request.body.decode()]
|
||||
return request, _JSON_OBJECT.validate_json(request.body)
|
||||
|
||||
|
||||
def test_a_mixed_burst_of_replays_pairs_every_answered_call_and_drops_every_unanswered_one(gateway: Gateway) -> None:
|
||||
calls: Final = _burst_calls(12)
|
||||
with wire_server(_vendor) as anthropic_wire, wire_server(_vendor) as bedrock_wire, gateway.scenario() as scenario:
|
||||
models: Final[dict[Provider, str]] = {
|
||||
"anthropic": _deployment(scenario, "anthropic", anthropic_wire.url),
|
||||
"bedrock_invoke": _deployment(scenario, "bedrock_invoke", bedrock_wire.url),
|
||||
}
|
||||
served: Final = asyncio.run(_burst(str(gateway.client.base_url).rstrip("/"), gateway.key, models, calls))
|
||||
assert [item.status for item in served] == [200] * len(calls), [
|
||||
(item.status, item.text[:200]) for item in served
|
||||
]
|
||||
received: Final = {"anthropic": anthropic_wire.drain(), "bedrock_invoke": bedrock_wire.drain()}
|
||||
assert {provider: len(posts) for provider, posts in received.items()} == {"anthropic": 6, "bedrock_invoke": 6}
|
||||
for item in served:
|
||||
assert f"marker-{item.call.marker}" in item.text, item.text
|
||||
request, body = _marked(received[item.call.provider], item.call.marker)
|
||||
_assert_wire(item.call, request, body)
|
||||
|
|
@ -0,0 +1,508 @@
|
|||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import unquote
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from _pytest.mark.structures import ParameterSet
|
||||
from integration._support import claude_code as cc
|
||||
from integration._support import responses_vendor as rv
|
||||
from integration._support.anthropic_thinking import BEDROCK_MODEL, Event, sse_chunks
|
||||
from integration._support.client import Gateway, Scenario
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_CLAUDE: Final = cc.OPUS
|
||||
_CLAUDE_WITHOUT_TOOL_SEARCH: Final = "claude-scripted-unlisted"
|
||||
_GEMINI: Final = "gemini-2.5-flash"
|
||||
_MISTRAL: Final = "mistral-large-latest"
|
||||
_REGEX_TOOL_TYPE: Final = "tool_search_tool_regex_20251119"
|
||||
_REGEX_TOOL_NAME: Final = "tool_search_tool_regex"
|
||||
_ALL_DEFERRED_ERROR: Final = "At least one tool must have defer_loading=false"
|
||||
_DEFERRED_NAMES: Final = ("get_weather", "finance__get_stock_price")
|
||||
_CACHE_BUST: Final[Mapping[str, JsonValue]] = MappingProxyType({"cache": {"no-cache": True}})
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
_WEATHER_PARAMETERS: Final[dict[str, JsonValue]] = {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
}
|
||||
_TICKER_PARAMETERS: Final[dict[str, JsonValue]] = {
|
||||
"type": "object",
|
||||
"properties": {"ticker": {"type": "string"}},
|
||||
"required": ["ticker"],
|
||||
}
|
||||
_EMAIL_PARAMETERS: Final[dict[str, JsonValue]] = {
|
||||
"type": "object",
|
||||
"properties": {"to": {"type": "string"}},
|
||||
"required": ["to"],
|
||||
}
|
||||
|
||||
Route = Literal["gemini", "bedrock_converse", "mistral", "unlisted_claude", "anthropic"]
|
||||
Search = Literal["absent", "dropped"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Call:
|
||||
route: Route
|
||||
stream: bool
|
||||
marker: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Served:
|
||||
call: _Call
|
||||
status: int
|
||||
text: str
|
||||
|
||||
|
||||
def _prompt(marker: str) -> str:
|
||||
return f"Weather in Paris and the ACME price marker-{marker}"
|
||||
|
||||
|
||||
def _tool_search(**fields: JsonValue) -> dict[str, JsonValue]:
|
||||
return {"type": "tool_search", **fields}
|
||||
|
||||
|
||||
def _function_tool(defer_loading: JsonValue = True) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"description": "Weather for a city",
|
||||
"parameters": _WEATHER_PARAMETERS,
|
||||
"defer_loading": defer_loading,
|
||||
}
|
||||
|
||||
|
||||
def _loaded_tool() -> dict[str, JsonValue]:
|
||||
return {"type": "function", "name": "send_email", "description": "Send an email", "parameters": _EMAIL_PARAMETERS}
|
||||
|
||||
|
||||
def _namespace_tool() -> dict[str, JsonValue]:
|
||||
return {
|
||||
"type": "namespace",
|
||||
"name": "finance",
|
||||
"description": "Finance tools",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_stock_price",
|
||||
"description": "Price of a ticker",
|
||||
"parameters": _TICKER_PARAMETERS,
|
||||
"defer_loading": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _search_tools(route: Route, search: Search) -> tuple[dict[str, JsonValue], ...]:
|
||||
if search == "absent":
|
||||
return ()
|
||||
return (_tool_search(execution="client"),) if route == "anthropic" else (_tool_search(),)
|
||||
|
||||
|
||||
def _gemini_frame(marker: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"candidates": [
|
||||
{"content": {"role": "model", "parts": [{"text": f"loaded marker-{marker}"}]}, "finishReason": "STOP"}
|
||||
],
|
||||
"usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 3, "totalTokenCount": 8},
|
||||
"modelVersion": _GEMINI,
|
||||
}
|
||||
|
||||
|
||||
def _chat_completion(marker: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"id": f"cmpl-{marker}",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": _MISTRAL,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": f"loaded marker-{marker}"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
||||
}
|
||||
|
||||
|
||||
def _chat_chunks(marker: str) -> tuple[bytes, ...]:
|
||||
head: Final[dict[str, JsonValue]] = {
|
||||
"id": f"cmpl-{marker}",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": _MISTRAL,
|
||||
}
|
||||
return (
|
||||
rv.chat_sse(
|
||||
{
|
||||
**head,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"role": "assistant", "content": f"loaded marker-{marker}"},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
rv.chat_sse(
|
||||
{
|
||||
**head,
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
||||
}
|
||||
),
|
||||
b"data: [DONE]\n\n",
|
||||
)
|
||||
|
||||
|
||||
def _claude_message(marker: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"id": f"msg_{marker}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": _CLAUDE,
|
||||
"content": [{"type": "text", "text": f"loaded marker-{marker}"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 20, "output_tokens": 7},
|
||||
}
|
||||
|
||||
|
||||
def _claude_events(marker: str) -> tuple[Event, ...]:
|
||||
opened: Final = {**_claude_message(marker), "content": [], "stop_reason": None}
|
||||
return (
|
||||
{"type": "message_start", "message": opened},
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"loaded marker-{marker}"}},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
|
||||
"usage": {"output_tokens": 7},
|
||||
},
|
||||
{"type": "message_stop"},
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_error(status: int, message: str) -> Reply:
|
||||
body: Final = {"type": "error", "error": {"type": "invalid_request_error", "message": message}}
|
||||
return Reply(status=status, body=json.dumps(body).encode())
|
||||
|
||||
|
||||
def _request_tools(body: Mapping[str, JsonValue]) -> list[dict[str, JsonValue]]:
|
||||
return _ITEMS.validate_python(body.get("tools") or [])
|
||||
|
||||
|
||||
def _gemini(request: Request) -> Reply:
|
||||
marker: Final = rv.newest_marker(request.body.decode()) or "health"
|
||||
if f"/models/{_GEMINI}:streamGenerateContent" in request.target:
|
||||
return Reply(content_type="text/event-stream", chunks=(rv.chat_sse(_gemini_frame(marker)),))
|
||||
if f"/models/{_GEMINI}:generateContent" in request.target:
|
||||
return Reply(body=json.dumps(_gemini_frame(marker)).encode())
|
||||
return rv.error(404, f"unexpected target {request.target}", "not_found")
|
||||
|
||||
|
||||
def _converse(request: Request) -> Reply:
|
||||
marker: Final = rv.newest_marker(request.body.decode()) or "health"
|
||||
if unquote(request.target) != f"/model/{BEDROCK_MODEL}/converse":
|
||||
return rv.error(404, f"unexpected target {request.target}", "not_found")
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": f"converse marker-{marker}"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15},
|
||||
"metrics": {"latencyMs": 1},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def _mistral(request: Request) -> Reply:
|
||||
marker: Final = rv.newest_marker(request.body.decode()) or "health"
|
||||
if request.target != "/v1/chat/completions":
|
||||
return rv.error(404, f"unexpected target {request.target}", "not_found")
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
deferred: Final = [index for index, tool in enumerate(_request_tools(body)) if "defer_loading" in tool]
|
||||
if deferred:
|
||||
return rv.error(
|
||||
422, f"tools.{deferred[0]}.defer_loading: Extra inputs are not permitted marker-{marker}", "extra_forbidden"
|
||||
)
|
||||
if body.get("stream") is True:
|
||||
return Reply(content_type="text/event-stream", chunks=_chat_chunks(marker))
|
||||
return Reply(body=json.dumps(_chat_completion(marker)).encode())
|
||||
|
||||
|
||||
def _claude(request: Request) -> Reply:
|
||||
if request.method != "POST":
|
||||
return Reply()
|
||||
marker: Final = rv.newest_marker(request.body.decode()) or "health"
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
tools: Final = _request_tools(body)
|
||||
regex_tools: Final = [tool for tool in tools if tool.get("type") == _REGEX_TOOL_TYPE]
|
||||
function_tools: Final = [tool for tool in tools if tool.get("type") != _REGEX_TOOL_TYPE]
|
||||
if not regex_tools and function_tools and all(tool.get("defer_loading") is True for tool in function_tools):
|
||||
return _anthropic_error(400, f"tools: {_ALL_DEFERRED_ERROR} marker-{marker}")
|
||||
if body.get("stream") is True:
|
||||
return Reply(content_type="text/event-stream", chunks=sse_chunks(_claude_events(marker)))
|
||||
return Reply(body=json.dumps(_claude_message(marker)).encode())
|
||||
|
||||
|
||||
def _vendor(route: Route) -> Callable[[Request], Reply]:
|
||||
match route:
|
||||
case "gemini":
|
||||
return _gemini
|
||||
case "bedrock_converse":
|
||||
return _converse
|
||||
case "mistral":
|
||||
return _mistral
|
||||
case "unlisted_claude" | "anthropic":
|
||||
return _claude
|
||||
|
||||
|
||||
def _deployment(scenario: Scenario, route: Route, wire_url: str) -> str:
|
||||
match route:
|
||||
case "gemini":
|
||||
return scenario.model(model=f"gemini/{_GEMINI}", api_base=wire_url, api_key="synthetic-gemini-key")
|
||||
case "bedrock_converse":
|
||||
return scenario.model(
|
||||
model=f"bedrock/converse/{BEDROCK_MODEL}",
|
||||
aws_access_key_id="AKIASCRIPTEDPROVIDER",
|
||||
aws_secret_access_key="scripted-secret",
|
||||
aws_region_name="us-east-1",
|
||||
aws_bedrock_runtime_endpoint=wire_url,
|
||||
)
|
||||
case "mistral":
|
||||
return scenario.model(model=f"mistral/{_MISTRAL}", api_base=wire_url, api_key="synthetic-mistral-key")
|
||||
case "unlisted_claude":
|
||||
return scenario.model(
|
||||
model=f"anthropic/{_CLAUDE_WITHOUT_TOOL_SEARCH}", api_base=wire_url, api_key=cc.ANTHROPIC_API_KEY
|
||||
)
|
||||
case "anthropic":
|
||||
return scenario.model(model=f"anthropic/{_CLAUDE}", api_base=wire_url, api_key=cc.ANTHROPIC_API_KEY)
|
||||
|
||||
|
||||
def _responses_body(
|
||||
model: str, marker: str, tools: Sequence[Mapping[str, JsonValue]], stream: bool
|
||||
) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model": model,
|
||||
"input": _prompt(marker),
|
||||
"tools": [dict(tool) for tool in tools],
|
||||
"stream": stream,
|
||||
**_CACHE_BUST,
|
||||
}
|
||||
|
||||
|
||||
def _raw(gateway: Gateway, body: Mapping[str, JsonValue]) -> httpx.Response:
|
||||
with httpx.Client(base_url=str(gateway.client.base_url).rstrip("/"), trust_env=False, timeout=60) as client:
|
||||
headers: Final = {"Authorization": f"Bearer {gateway.key}"}
|
||||
with client.stream("POST", "/v1/responses", json=dict(body), headers=headers) as response:
|
||||
response.read()
|
||||
return response
|
||||
|
||||
|
||||
def _assert_served(response: httpx.Response, marker: str, stream: bool) -> None:
|
||||
assert response.status_code == 200, response.text
|
||||
assert f"marker-{marker}" in response.text, response.text
|
||||
if stream:
|
||||
assert "response.completed" in response.text, response.text
|
||||
return
|
||||
types: Final = [
|
||||
item["type"] for item in _ITEMS.validate_python(_JSON_OBJECT.validate_json(response.content)["output"])
|
||||
]
|
||||
assert "message" in types, response.text
|
||||
|
||||
|
||||
def _only_request(wire: Wire) -> tuple[Request, dict[str, JsonValue]]:
|
||||
received: Final = wire.drain()
|
||||
assert len(received) == 1, [(request.method, request.target) for request in received]
|
||||
return received[0], _JSON_OBJECT.validate_json(received[0].body)
|
||||
|
||||
|
||||
def _assert_target(route: Route, stream: bool, request: Request, body: Mapping[str, JsonValue]) -> None:
|
||||
match route:
|
||||
case "gemini":
|
||||
method: Final = "streamGenerateContent" if stream else "generateContent"
|
||||
assert f"/models/{_GEMINI}:{method}" in request.target, request.target
|
||||
case "bedrock_converse":
|
||||
assert unquote(request.target) == f"/model/{BEDROCK_MODEL}/converse", request.target
|
||||
case "mistral":
|
||||
assert request.target == "/v1/chat/completions", request.target
|
||||
assert body.get("stream", False) is stream, body
|
||||
case "unlisted_claude" | "anthropic":
|
||||
assert request.target == "/v1/messages", request.target
|
||||
assert body.get("stream", False) is stream, body
|
||||
|
||||
|
||||
def _wire_tool_names(route: Route, body: Mapping[str, JsonValue]) -> list[str]:
|
||||
match route:
|
||||
case "gemini":
|
||||
declarations: Final = _ITEMS.validate_python(
|
||||
_ITEMS.validate_python(body["tools"])[0]["function_declarations"]
|
||||
)
|
||||
return [str(declaration["name"]) for declaration in declarations]
|
||||
case "bedrock_converse":
|
||||
specs: Final = _ITEMS.validate_python(_JSON_OBJECT.validate_python(body["toolConfig"])["tools"])
|
||||
return [str(_JSON_OBJECT.validate_python(spec["toolSpec"])["name"]) for spec in specs]
|
||||
case "mistral":
|
||||
return [
|
||||
str(_JSON_OBJECT.validate_python(tool["function"])["name"])
|
||||
for tool in _ITEMS.validate_python(body["tools"])
|
||||
]
|
||||
case "unlisted_claude" | "anthropic":
|
||||
return [str(tool["name"]) for tool in _ITEMS.validate_python(body["tools"])]
|
||||
|
||||
|
||||
def _assert_undeferred_wire(call: _Call, request: Request, body: Mapping[str, JsonValue]) -> None:
|
||||
_assert_target(call.route, call.stream, request, body)
|
||||
assert _wire_tool_names(call.route, body) == list(_DEFERRED_NAMES), body
|
||||
assert "defer_loading" not in request.body.decode(), body
|
||||
assert "tool_search" not in request.body.decode(), body
|
||||
|
||||
|
||||
def _modes(route: Route) -> tuple[bool, ...]:
|
||||
return (False,) if route == "bedrock_converse" else (False, True)
|
||||
|
||||
|
||||
def _cells() -> tuple[ParameterSet, ...]:
|
||||
routes: Final[tuple[Route, ...]] = ("gemini", "bedrock_converse", "mistral", "unlisted_claude", "anthropic")
|
||||
searches: Final[tuple[Search, ...]] = ("absent", "dropped")
|
||||
combos: Final = [
|
||||
(route, search, stream) for route in routes for search in searches for stream in _modes(route)
|
||||
] # comprehension-ok: a parametrize grid
|
||||
return tuple(
|
||||
pytest.param(route, search, stream, id=f"{route}-{search}-{'stream' if stream else 'sync'}")
|
||||
for route, search, stream in combos
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("route", "search", "stream"), _cells())
|
||||
def test_without_a_search_tool_on_the_wire_every_tool_arrives_without_defer_loading(
|
||||
gateway: Gateway, route: Route, search: Search, stream: bool
|
||||
) -> None:
|
||||
call: Final = _Call(route, stream, uuid.uuid4().hex)
|
||||
with wire_server(_vendor(route)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, route, wire.url)
|
||||
tools: Final = (*_search_tools(route, search), _function_tool(), _namespace_tool())
|
||||
response: Final = _raw(gateway, _responses_body(model, call.marker, tools, stream))
|
||||
_assert_served(response, call.marker, stream)
|
||||
request, body = _only_request(wire)
|
||||
_assert_undeferred_wire(call, request, body)
|
||||
|
||||
|
||||
def test_a_false_defer_loading_never_reaches_an_openai_compatible_wire_as_a_key(gateway: Gateway) -> None:
|
||||
call: Final = _Call("mistral", False, uuid.uuid4().hex)
|
||||
with wire_server(_mistral) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, "mistral", wire.url)
|
||||
tools: Final = (_function_tool(defer_loading=False), _namespace_tool())
|
||||
response: Final = _raw(gateway, _responses_body(model, call.marker, tools, False))
|
||||
_assert_served(response, call.marker, False)
|
||||
request, body = _only_request(wire)
|
||||
_assert_undeferred_wire(call, request, body)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"])
|
||||
def test_the_supported_anthropic_route_keeps_each_tools_own_defer_loading_with_the_search_tool(
|
||||
gateway: Gateway, stream: bool
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_claude) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, "anthropic", wire.url)
|
||||
tools: Final = (_tool_search(), _function_tool(), _namespace_tool(), _loaded_tool())
|
||||
response: Final = _raw(gateway, _responses_body(model, marker, tools, stream))
|
||||
_assert_served(response, marker, stream)
|
||||
request, body = _only_request(wire)
|
||||
_assert_target("anthropic", stream, request, body)
|
||||
sent: Final = _ITEMS.validate_python(body["tools"])
|
||||
assert sent[0] == {"type": _REGEX_TOOL_TYPE, "name": _REGEX_TOOL_NAME}, sent
|
||||
assert [(tool["name"], tool.get("defer_loading")) for tool in sent[1:]] == [
|
||||
("get_weather", True),
|
||||
("finance__get_stock_price", True),
|
||||
("send_email", None),
|
||||
], sent
|
||||
assert "defer_loading" not in json.dumps(sent[3]), sent
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ["gemini", "mistral"])
|
||||
def test_a_non_boolean_defer_loading_is_refused_with_400_on_a_route_without_tool_search(
|
||||
gateway: Gateway, route: Route
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_vendor(route)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, route, wire.url)
|
||||
refused: Final = _raw(
|
||||
gateway, _responses_body(model, marker, (_function_tool("true"), _namespace_tool()), False)
|
||||
)
|
||||
assert refused.status_code == 400 and "defer_loading must be a boolean" in refused.text, refused.text
|
||||
assert wire.drain() == (), refused.text
|
||||
follow_up: Final = _Call(route, False, uuid.uuid4().hex)
|
||||
served: Final = _raw(
|
||||
gateway, _responses_body(model, follow_up.marker, (_function_tool(), _namespace_tool()), False)
|
||||
)
|
||||
_assert_served(served, follow_up.marker, False)
|
||||
request, body = _only_request(wire)
|
||||
_assert_undeferred_wire(follow_up, request, body)
|
||||
|
||||
|
||||
async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served:
|
||||
body: Final = _responses_body(model, call.marker, (_function_tool(), _namespace_tool()), call.stream)
|
||||
async with client.stream(
|
||||
"POST", "/v1/responses", json=body, headers={"Authorization": f"Bearer {key}"}
|
||||
) as response:
|
||||
raw: Final = await response.aread()
|
||||
return _Served(call, response.status_code, raw.decode())
|
||||
|
||||
|
||||
async def _burst(base_url: str, key: str, models: Mapping[Route, str], calls: Sequence[_Call]) -> tuple[_Served, ...]:
|
||||
async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client:
|
||||
return tuple(await asyncio.gather(*(_send(client, key, models[call.route], call) for call in calls)))
|
||||
|
||||
|
||||
def _burst_calls() -> tuple[_Call, ...]:
|
||||
routes: Final[tuple[Route, ...]] = ("gemini", "bedrock_converse", "mistral", "unlisted_claude")
|
||||
return tuple(
|
||||
_Call(route, stream, uuid.uuid4().hex) for route in routes for stream in _modes(route)
|
||||
) # comprehension-ok: a two-axis burst grid
|
||||
|
||||
|
||||
def _marked(received: Sequence[Request], marker: str) -> tuple[Request, dict[str, JsonValue]]:
|
||||
(request,) = [request for request in received if marker in request.body.decode()]
|
||||
return request, _JSON_OBJECT.validate_json(request.body)
|
||||
|
||||
|
||||
def test_a_burst_across_four_routes_strips_every_deferral(gateway: Gateway) -> None:
|
||||
calls: Final = _burst_calls()
|
||||
with (
|
||||
wire_server(_gemini) as gemini_wire,
|
||||
wire_server(_converse) as converse_wire,
|
||||
wire_server(_mistral) as mistral_wire,
|
||||
wire_server(_claude) as claude_wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
wires: Final[dict[Route, Wire]] = {
|
||||
"gemini": gemini_wire,
|
||||
"bedrock_converse": converse_wire,
|
||||
"mistral": mistral_wire,
|
||||
"unlisted_claude": claude_wire,
|
||||
}
|
||||
models: Final = {route: _deployment(scenario, route, wire.url) for route, wire in wires.items()}
|
||||
served: Final = asyncio.run(_burst(str(gateway.client.base_url).rstrip("/"), gateway.key, models, calls))
|
||||
assert [item.status for item in served] == [200] * len(calls), [
|
||||
(item.status, item.text[:200]) for item in served
|
||||
]
|
||||
received: Final = {route: wire.drain() for route, wire in wires.items()}
|
||||
for item in served:
|
||||
assert f"marker-{item.call.marker}" in item.text, item.text
|
||||
request, body = _marked(received[item.call.route], item.call.marker)
|
||||
_assert_undeferred_wire(item.call, request, body)
|
||||
|
|
@ -1924,6 +1924,42 @@ class TestToolTransformation:
|
|||
assert [tool["function"]["name"] for tool in result["tools"]] == ["get_weather", "finance__get_stock_price"]
|
||||
assert all("defer_loading" not in tool for tool in result["tools"])
|
||||
|
||||
def test_native_bm25_search_tool_keeps_the_other_tools_deferred(self) -> None:
|
||||
tools: Final = [
|
||||
{"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"},
|
||||
*_deferred_tools_with_server_tool_search()[1:],
|
||||
]
|
||||
|
||||
result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
model="anthropic/claude-sonnet-5-5",
|
||||
input="What is the weather in Paris right now?",
|
||||
responses_api_request={"tools": tools},
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
chat_tools: Final = result["tools"]
|
||||
assert chat_tools[0] == {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}
|
||||
assert [tool["defer_loading"] for tool in chat_tools[1:]] == [True, True]
|
||||
|
||||
def test_native_search_tool_passes_through_routes_without_tool_search(self) -> None:
|
||||
model: Final = "azure_ai/claude-sonnet-5-5"
|
||||
assert litellm.supports_tool_search(model, "azure_ai") is False
|
||||
tools: Final = [
|
||||
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
|
||||
*_deferred_tools_with_server_tool_search()[1:],
|
||||
]
|
||||
|
||||
result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
model=model,
|
||||
input="What is the weather in Paris right now?",
|
||||
responses_api_request={"tools": tools},
|
||||
custom_llm_provider="azure_ai",
|
||||
)
|
||||
|
||||
chat_tools: Final = result["tools"]
|
||||
assert chat_tools[0] == {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}
|
||||
assert [tool["defer_loading"] for tool in chat_tools[1:]] == [True, True]
|
||||
|
||||
def test_server_tool_search_follows_the_bedrock_route(self) -> None:
|
||||
transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request
|
||||
converse: Final = transform(
|
||||
|
|
|
|||
88
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
88
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -42993,7 +42993,7 @@ export interface components {
|
|||
*/
|
||||
ResponseItemList: {
|
||||
/** Data */
|
||||
data: (components["schemas"]["ResponseInputMessageItem"] | components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseFunctionToolCallItem"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["McpCall"] | components["schemas"]["ResponseCustomToolCallItem"] | components["schemas"]["ResponseCustomToolCallOutputItem"])[];
|
||||
data: (components["schemas"]["ResponseInputMessageItem"] | components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseFunctionToolCallItem"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["openai__types__responses__response_tool_search_call__ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["McpCall"] | components["schemas"]["ResponseCustomToolCallItem"] | components["schemas"]["ResponseCustomToolCallOutputItem"])[];
|
||||
/** First Id */
|
||||
first_id: string;
|
||||
/** Has More */
|
||||
|
|
@ -43215,34 +43215,6 @@ export interface components {
|
|||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/** ResponseToolSearchCall */
|
||||
ResponseToolSearchCall: {
|
||||
/** Arguments */
|
||||
arguments: unknown;
|
||||
/** Call Id */
|
||||
call_id?: string | null;
|
||||
/** Created By */
|
||||
created_by?: string | null;
|
||||
/**
|
||||
* Execution
|
||||
* @enum {string}
|
||||
*/
|
||||
execution: "server" | "client";
|
||||
/** Id */
|
||||
id: string;
|
||||
/**
|
||||
* Status
|
||||
* @enum {string}
|
||||
*/
|
||||
status: "in_progress" | "completed" | "incomplete";
|
||||
/**
|
||||
* Type
|
||||
* @constant
|
||||
*/
|
||||
type: "tool_search_call";
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/** ResponseToolSearchOutputItem */
|
||||
ResponseToolSearchOutputItem: {
|
||||
/** Call Id */
|
||||
|
|
@ -43477,9 +43449,9 @@ export interface components {
|
|||
/** Object */
|
||||
object?: string | null;
|
||||
/** Output */
|
||||
output: (components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpCall"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["ResponseCustomToolCall"] | components["schemas"]["ResponseCustomToolCallOutputItem"] | {
|
||||
output: (components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["openai__types__responses__response_tool_search_call__ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpCall"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["ResponseCustomToolCall"] | components["schemas"]["ResponseCustomToolCallOutputItem"] | {
|
||||
[key: string]: unknown;
|
||||
})[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["ResponseToolSearchCall"])[];
|
||||
})[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["litellm__types__responses__main__ResponseToolSearchCall"])[];
|
||||
/** Parallel Tool Calls */
|
||||
parallel_tool_calls?: boolean | null;
|
||||
/** Previous Response Id */
|
||||
|
|
@ -49288,6 +49260,32 @@ export interface components {
|
|||
/** Type */
|
||||
type: "computer_use_preview" | string;
|
||||
};
|
||||
/** ResponseToolSearchCall */
|
||||
litellm__types__responses__main__ResponseToolSearchCall: {
|
||||
/** Arguments */
|
||||
arguments: unknown;
|
||||
/** Call Id */
|
||||
call_id?: string | null;
|
||||
/**
|
||||
* Execution
|
||||
* @enum {string}
|
||||
*/
|
||||
execution: "server" | "client";
|
||||
/** Id */
|
||||
id: string;
|
||||
/**
|
||||
* Status
|
||||
* @enum {string}
|
||||
*/
|
||||
status: "in_progress" | "completed" | "incomplete";
|
||||
/**
|
||||
* Type
|
||||
* @constant
|
||||
*/
|
||||
type: "tool_search_call";
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/** ModelInfo */
|
||||
litellm__types__router__ModelInfo: {
|
||||
/** Access Windows */
|
||||
|
|
@ -49370,6 +49368,34 @@ export interface components {
|
|||
*/
|
||||
type: "computer";
|
||||
};
|
||||
/** ResponseToolSearchCall */
|
||||
openai__types__responses__response_tool_search_call__ResponseToolSearchCall: {
|
||||
/** Arguments */
|
||||
arguments: unknown;
|
||||
/** Call Id */
|
||||
call_id?: string | null;
|
||||
/** Created By */
|
||||
created_by?: string | null;
|
||||
/**
|
||||
* Execution
|
||||
* @enum {string}
|
||||
*/
|
||||
execution: "server" | "client";
|
||||
/** Id */
|
||||
id: string;
|
||||
/**
|
||||
* Status
|
||||
* @enum {string}
|
||||
*/
|
||||
status: "in_progress" | "completed" | "incomplete";
|
||||
/**
|
||||
* Type
|
||||
* @constant
|
||||
*/
|
||||
type: "tool_search_call";
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/**
|
||||
* UserLocation
|
||||
* @description The user's location.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue