This commit is contained in:
Devon Krisman 2026-10-29 02:18:58 +00:00 • committed by GitHub
commit 9ce38e57a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 218 additions and 6 deletions

View file

@ -384,14 +384,17 @@ class WebSearchInterceptionLogger(CustomLogger):
(t for t in tools if is_anthropic_native_web_search_tool(t)),
None,
)
emit_native_blocks: Final = native_tool is not None or (
kwargs is not None and bool(kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY))
)
outcome: Final = await self._short_circuit_search_outcome(query, kwargs=kwargs)
search_result_text: Final = WebSearchTransformation.search_outcome_text(outcome)
content: Final[list[dict[str, object]]] = []
if native_tool is not None:
if emit_native_blocks:
tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}"
tool_name: Final = native_tool.get("name") or "web_search"
tool_name: Final = (native_tool.get("name") if native_tool is not None else None) or "web_search"
content.append(
{
"type": "server_tool_use",
@ -421,7 +424,7 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
"WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)",
len(search_result_text),
native_tool is not None,
emit_native_blocks,
)
return response

View file

@ -47,7 +47,7 @@ class FakeAnthropicMessagesStreamIterator:
def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]:
"""Build SSE chunks for a single content block."""
chunks: Final = []
chunks: Final = list[bytes]()
block_type: Final = block_dict.get("type")
if block_type == "text":
@ -120,6 +120,29 @@ class FakeAnthropicMessagesStreamIterator:
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
elif block_type == "server_tool_use":
empty_input: Final[Mapping[str, object]] = {}
server_tool_start: Final = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "server_tool_use",
"id": block_dict.get("id"),
"name": block_dict.get("name"),
"input": empty_input,
},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(server_tool_start)}\n\n".encode())
server_tool_delta: Final = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "input_json_delta",
"partial_json": json.dumps(block_dict.get("input", {})),
},
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(server_tool_delta)}\n\n".encode())
else:
passthrough_start: Final = {
"type": "content_block_start",

View file

@ -72,6 +72,53 @@ class TestTryShortCircuitSearch:
assert "Result" in text_block["text"]
mock_search.assert_called_once_with("Search for Claude Code releases")
@pytest.mark.asyncio
async def test_emits_native_blocks_when_tools_already_converted(self):
"""Pre-request/deployment hooks convert the native tool to the LiteLLM
standard shape before the short-circuit runs and record that in
WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY; the flag alone must be enough for
native blocks to be emitted."""
from litellm.integrations.websearch_interception.handler import (
WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY,
)
logger = WebSearchInterceptionLogger(enabled_providers=["hosted_vllm"])
with patch.object(
logger, "_execute_search", new_callable=AsyncMock
) as mock_search:
mock_search.return_value = (
"Title: Result\nURL: https://example.com\nSnippet: test",
None,
)
result = await logger.try_short_circuit_search(
model="hosted_vllm/some-model",
messages=[{"role": "user", "content": "Search for something"}],
tools=[
{
"name": "litellm_web_search",
"description": "Search the web for information.",
"input_schema": {
"type": "object",
"properties": {"query": {"type": "string"}},
"required": ["query"],
},
}
],
custom_llm_provider="hosted_vllm",
kwargs={WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True},
)
assert result is not None
block_types = [b["type"] for b in result["content"]]
assert "server_tool_use" in block_types
assert "web_search_tool_result" in block_types
server_tool_use = next(
b for b in result["content"] if b["type"] == "server_tool_use"
)
assert server_tool_use["name"] == "web_search"
@pytest.mark.asyncio
async def test_does_not_short_circuit_mixed_tools(self):
"""Mix of web_search and other tools → NOT short-circuited"""
@ -467,6 +514,68 @@ class TestShortCircuitEntryPoint:
assert text_block["text"] == "results"
class TestShortCircuitThroughMessagesEntryPoint:
"""The pre-request hook renames the native tool before the short-circuit
runs, so only a request through anthropic_messages shows whether native
clients still get their server_tool_use and web_search_tool_result blocks."""
NATIVE_TOOL = {"type": "web_search_20250305", "name": "web_search"}
async def _send(self, *, tools: list[dict], stream: bool):
import litellm
logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"])
with patch.object(logger, "_execute_search", new_callable=AsyncMock) as mock_search:
mock_search.return_value = ("Title: Result\nURL: https://example.com\nSnippet: test", None)
with patch("litellm.callbacks", [logger]):
response = await litellm.anthropic_messages(
model="bedrock/converse/us.anthropic.claude-sonnet-5-5",
messages=[{"role": "user", "content": "latest python release"}],
tools=tools,
max_tokens=64,
stream=stream,
)
if not stream:
return response
return b"".join([chunk async for chunk in response])
@pytest.mark.asyncio
async def test_native_tool_gets_native_blocks(self):
response = await self._send(tools=[self.NATIVE_TOOL], stream=False)
assert [block["type"] for block in response["content"]] == [
"server_tool_use",
"web_search_tool_result",
"text",
]
server_tool_use, tool_result, _ = response["content"]
assert server_tool_use["name"] == "web_search"
assert server_tool_use["input"] == {"query": "latest python release"}
assert tool_result["tool_use_id"] == server_tool_use["id"]
@pytest.mark.asyncio
async def test_native_tool_streams_native_blocks(self):
body = await self._send(tools=[self.NATIVE_TOOL], stream=True)
starts = [
json.loads(line.removeprefix(b"data: "))["content_block"]
for line in body.splitlines()
if line.startswith(b"data: ") and b'"content_block_start"' in line
]
assert [block["type"] for block in starts] == ["server_tool_use", "web_search_tool_result", "text"]
assert starts[1]["tool_use_id"] == starts[0]["id"]
@pytest.mark.asyncio
async def test_litellm_standard_tool_stays_text_only(self):
from litellm.integrations.websearch_interception.tools import (
get_litellm_web_search_tool,
)
response = await self._send(tools=[get_litellm_web_search_tool()], stream=False)
assert [block["type"] for block in response["content"]] == ["text"]
class TestShortCircuitSafeguardResults:
"""A short-circuited reply never reaches the backend, so the proxy answers Claude Code's safeguards itself."""

View file

@ -1,6 +1,6 @@
import json
from collections.abc import Iterable, Mapping
from typing import Final
from collections.abc import Iterable, Mapping, Sequence
from typing import Final, cast
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
@ -91,3 +91,80 @@ def test_message_delta_omits_absent_optional_fields():
"usage": {"input_tokens": 10, "output_tokens": 2},
}
assert _message_delta(response) == {"stop_reason": "end_turn", "stop_sequence": None}
_WEB_SEARCH_RESULT_BLOCK: Final = {
"type": "web_search_tool_result",
"tool_use_id": "srvtoolu_01",
"content": [
{
"type": "web_search_result",
"url": "https://example.com/a",
"title": "Example A",
"encrypted_content": "abc123",
}
],
}
def _content_events(content: Sequence[Mapping[str, object]]) -> tuple[tuple[str, dict[str, object]], ...]:
response: Final = {
"id": "msg_ws",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-6",
"content": list(content),
"stop_reason": "end_turn",
"usage": {"input_tokens": 10, "output_tokens": 5},
}
return tuple(
event for event in _decoded_events(FakeAnthropicMessagesStreamIterator(response=response))
if event[0].startswith("content_block_")
)
def test_server_tool_use_streams_empty_start_then_input_json_delta():
"""A real Anthropic stream opens server_tool_use with an empty input and
sends the query as an input_json_delta, which SDK accumulators read."""
events: Final = _content_events(
[{"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "example search"}}]
)
assert [event_type for event_type, _ in events] == [
"content_block_start",
"content_block_delta",
"content_block_stop",
]
assert events[0][1]["content_block"] == {
"type": "server_tool_use",
"id": "srvtoolu_01",
"name": "web_search",
"input": {},
}
delta: Final = events[1][1]["delta"]
assert isinstance(delta, dict)
assert delta["type"] == "input_json_delta"
assert json.loads(delta["partial_json"]) == {"query": "example search"}
def test_web_search_tool_result_rides_whole_in_content_block_start():
events: Final = _content_events([_WEB_SEARCH_RESULT_BLOCK])
assert [event_type for event_type, _ in events] == ["content_block_start", "content_block_stop"]
assert events[0][1]["content_block"] == _WEB_SEARCH_RESULT_BLOCK
def test_native_search_blocks_keep_indices_aligned_with_text():
events: Final = _content_events(
[
{"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "q"}},
_WEB_SEARCH_RESULT_BLOCK,
{"type": "text", "text": "Here is what I found."},
]
)
starts: Final = tuple(payload for event_type, payload in events if event_type == "content_block_start")
stops: Final = tuple(payload["index"] for event_type, payload in events if event_type == "content_block_stop")
assert [(start["index"], cast(dict[str, object], start["content_block"])["type"]) for start in starts] == [
(0, "server_tool_use"),
(1, "web_search_tool_result"),
(2, "text"),
]
assert stops == (0, 1, 2)