mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge b859137b24 into d9c9d84298
This commit is contained in:
commit
9ce38e57a9
4 changed files with 218 additions and 6 deletions
|
|
@ -384,14 +384,17 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
(t for t in tools if is_anthropic_native_web_search_tool(t)),
|
||||
None,
|
||||
)
|
||||
emit_native_blocks: Final = native_tool is not None or (
|
||||
kwargs is not None and bool(kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY))
|
||||
)
|
||||
|
||||
outcome: Final = await self._short_circuit_search_outcome(query, kwargs=kwargs)
|
||||
search_result_text: Final = WebSearchTransformation.search_outcome_text(outcome)
|
||||
|
||||
content: Final[list[dict[str, object]]] = []
|
||||
if native_tool is not None:
|
||||
if emit_native_blocks:
|
||||
tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}"
|
||||
tool_name: Final = native_tool.get("name") or "web_search"
|
||||
tool_name: Final = (native_tool.get("name") if native_tool is not None else None) or "web_search"
|
||||
content.append(
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
|
|
@ -421,7 +424,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
verbose_logger.debug(
|
||||
"WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)",
|
||||
len(search_result_text),
|
||||
native_tool is not None,
|
||||
emit_native_blocks,
|
||||
)
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class FakeAnthropicMessagesStreamIterator:
|
|||
|
||||
def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]:
|
||||
"""Build SSE chunks for a single content block."""
|
||||
chunks: Final = []
|
||||
chunks: Final = list[bytes]()
|
||||
block_type: Final = block_dict.get("type")
|
||||
|
||||
if block_type == "text":
|
||||
|
|
@ -120,6 +120,29 @@ class FakeAnthropicMessagesStreamIterator:
|
|||
}
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
|
||||
|
||||
elif block_type == "server_tool_use":
|
||||
empty_input: Final[Mapping[str, object]] = {}
|
||||
server_tool_start: Final = {
|
||||
"type": "content_block_start",
|
||||
"index": index,
|
||||
"content_block": {
|
||||
"type": "server_tool_use",
|
||||
"id": block_dict.get("id"),
|
||||
"name": block_dict.get("name"),
|
||||
"input": empty_input,
|
||||
},
|
||||
}
|
||||
chunks.append(f"event: content_block_start\ndata: {json.dumps(server_tool_start)}\n\n".encode())
|
||||
server_tool_delta: Final = {
|
||||
"type": "content_block_delta",
|
||||
"index": index,
|
||||
"delta": {
|
||||
"type": "input_json_delta",
|
||||
"partial_json": json.dumps(block_dict.get("input", {})),
|
||||
},
|
||||
}
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(server_tool_delta)}\n\n".encode())
|
||||
|
||||
else:
|
||||
passthrough_start: Final = {
|
||||
"type": "content_block_start",
|
||||
|
|
|
|||
|
|
@ -72,6 +72,53 @@ class TestTryShortCircuitSearch:
|
|||
assert "Result" in text_block["text"]
|
||||
mock_search.assert_called_once_with("Search for Claude Code releases")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_emits_native_blocks_when_tools_already_converted(self):
|
||||
"""Pre-request/deployment hooks convert the native tool to the LiteLLM
|
||||
standard shape before the short-circuit runs and record that in
|
||||
WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY; the flag alone must be enough for
|
||||
native blocks to be emitted."""
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY,
|
||||
)
|
||||
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["hosted_vllm"])
|
||||
|
||||
with patch.object(
|
||||
logger, "_execute_search", new_callable=AsyncMock
|
||||
) as mock_search:
|
||||
mock_search.return_value = (
|
||||
"Title: Result\nURL: https://example.com\nSnippet: test",
|
||||
None,
|
||||
)
|
||||
|
||||
result = await logger.try_short_circuit_search(
|
||||
model="hosted_vllm/some-model",
|
||||
messages=[{"role": "user", "content": "Search for something"}],
|
||||
tools=[
|
||||
{
|
||||
"name": "litellm_web_search",
|
||||
"description": "Search the web for information.",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
},
|
||||
}
|
||||
],
|
||||
custom_llm_provider="hosted_vllm",
|
||||
kwargs={WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True},
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
block_types = [b["type"] for b in result["content"]]
|
||||
assert "server_tool_use" in block_types
|
||||
assert "web_search_tool_result" in block_types
|
||||
server_tool_use = next(
|
||||
b for b in result["content"] if b["type"] == "server_tool_use"
|
||||
)
|
||||
assert server_tool_use["name"] == "web_search"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_short_circuit_mixed_tools(self):
|
||||
"""Mix of web_search and other tools → NOT short-circuited"""
|
||||
|
|
@ -467,6 +514,68 @@ class TestShortCircuitEntryPoint:
|
|||
assert text_block["text"] == "results"
|
||||
|
||||
|
||||
class TestShortCircuitThroughMessagesEntryPoint:
|
||||
"""The pre-request hook renames the native tool before the short-circuit
|
||||
runs, so only a request through anthropic_messages shows whether native
|
||||
clients still get their server_tool_use and web_search_tool_result blocks."""
|
||||
|
||||
NATIVE_TOOL = {"type": "web_search_20250305", "name": "web_search"}
|
||||
|
||||
async def _send(self, *, tools: list[dict], stream: bool):
|
||||
import litellm
|
||||
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"])
|
||||
with patch.object(logger, "_execute_search", new_callable=AsyncMock) as mock_search:
|
||||
mock_search.return_value = ("Title: Result\nURL: https://example.com\nSnippet: test", None)
|
||||
with patch("litellm.callbacks", [logger]):
|
||||
response = await litellm.anthropic_messages(
|
||||
model="bedrock/converse/us.anthropic.claude-sonnet-5-5",
|
||||
messages=[{"role": "user", "content": "latest python release"}],
|
||||
tools=tools,
|
||||
max_tokens=64,
|
||||
stream=stream,
|
||||
)
|
||||
if not stream:
|
||||
return response
|
||||
return b"".join([chunk async for chunk in response])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_tool_gets_native_blocks(self):
|
||||
response = await self._send(tools=[self.NATIVE_TOOL], stream=False)
|
||||
|
||||
assert [block["type"] for block in response["content"]] == [
|
||||
"server_tool_use",
|
||||
"web_search_tool_result",
|
||||
"text",
|
||||
]
|
||||
server_tool_use, tool_result, _ = response["content"]
|
||||
assert server_tool_use["name"] == "web_search"
|
||||
assert server_tool_use["input"] == {"query": "latest python release"}
|
||||
assert tool_result["tool_use_id"] == server_tool_use["id"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_tool_streams_native_blocks(self):
|
||||
body = await self._send(tools=[self.NATIVE_TOOL], stream=True)
|
||||
|
||||
starts = [
|
||||
json.loads(line.removeprefix(b"data: "))["content_block"]
|
||||
for line in body.splitlines()
|
||||
if line.startswith(b"data: ") and b'"content_block_start"' in line
|
||||
]
|
||||
assert [block["type"] for block in starts] == ["server_tool_use", "web_search_tool_result", "text"]
|
||||
assert starts[1]["tool_use_id"] == starts[0]["id"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_litellm_standard_tool_stays_text_only(self):
|
||||
from litellm.integrations.websearch_interception.tools import (
|
||||
get_litellm_web_search_tool,
|
||||
)
|
||||
|
||||
response = await self._send(tools=[get_litellm_web_search_tool()], stream=False)
|
||||
|
||||
assert [block["type"] for block in response["content"]] == ["text"]
|
||||
|
||||
|
||||
class TestShortCircuitSafeguardResults:
|
||||
"""A short-circuited reply never reaches the backend, so the proxy answers Claude Code's safeguards itself."""
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import json
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Final
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Final, cast
|
||||
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
|
|
@ -91,3 +91,80 @@ def test_message_delta_omits_absent_optional_fields():
|
|||
"usage": {"input_tokens": 10, "output_tokens": 2},
|
||||
}
|
||||
assert _message_delta(response) == {"stop_reason": "end_turn", "stop_sequence": None}
|
||||
|
||||
|
||||
_WEB_SEARCH_RESULT_BLOCK: Final = {
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_01",
|
||||
"content": [
|
||||
{
|
||||
"type": "web_search_result",
|
||||
"url": "https://example.com/a",
|
||||
"title": "Example A",
|
||||
"encrypted_content": "abc123",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _content_events(content: Sequence[Mapping[str, object]]) -> tuple[tuple[str, dict[str, object]], ...]:
|
||||
response: Final = {
|
||||
"id": "msg_ws",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"content": list(content),
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
return tuple(
|
||||
event for event in _decoded_events(FakeAnthropicMessagesStreamIterator(response=response))
|
||||
if event[0].startswith("content_block_")
|
||||
)
|
||||
|
||||
|
||||
def test_server_tool_use_streams_empty_start_then_input_json_delta():
|
||||
"""A real Anthropic stream opens server_tool_use with an empty input and
|
||||
sends the query as an input_json_delta, which SDK accumulators read."""
|
||||
events: Final = _content_events(
|
||||
[{"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "example search"}}]
|
||||
)
|
||||
assert [event_type for event_type, _ in events] == [
|
||||
"content_block_start",
|
||||
"content_block_delta",
|
||||
"content_block_stop",
|
||||
]
|
||||
assert events[0][1]["content_block"] == {
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01",
|
||||
"name": "web_search",
|
||||
"input": {},
|
||||
}
|
||||
delta: Final = events[1][1]["delta"]
|
||||
assert isinstance(delta, dict)
|
||||
assert delta["type"] == "input_json_delta"
|
||||
assert json.loads(delta["partial_json"]) == {"query": "example search"}
|
||||
|
||||
|
||||
def test_web_search_tool_result_rides_whole_in_content_block_start():
|
||||
events: Final = _content_events([_WEB_SEARCH_RESULT_BLOCK])
|
||||
assert [event_type for event_type, _ in events] == ["content_block_start", "content_block_stop"]
|
||||
assert events[0][1]["content_block"] == _WEB_SEARCH_RESULT_BLOCK
|
||||
|
||||
|
||||
def test_native_search_blocks_keep_indices_aligned_with_text():
|
||||
events: Final = _content_events(
|
||||
[
|
||||
{"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "q"}},
|
||||
_WEB_SEARCH_RESULT_BLOCK,
|
||||
{"type": "text", "text": "Here is what I found."},
|
||||
]
|
||||
)
|
||||
starts: Final = tuple(payload for event_type, payload in events if event_type == "content_block_start")
|
||||
stops: Final = tuple(payload["index"] for event_type, payload in events if event_type == "content_block_stop")
|
||||
assert [(start["index"], cast(dict[str, object], start["content_block"])["type"]) for start in starts] == [
|
||||
(0, "server_tool_use"),
|
||||
(1, "web_search_tool_result"),
|
||||
(2, "text"),
|
||||
]
|
||||
assert stops == (0, 1, 2)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue