mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
fix(anthropic): keep pending server calls native and hide server calls from client tool paths
Replaying a server call that has no result yet now sends it back as server_tool_use, so Anthropic accepts the next turn. The tool permission guardrail checks server calls again and rewrite mode removes a denied one. The /v1/messages adapter keeps Claude's server blocks, and the Responses bridge never returns a server call as a client function_call.
This commit is contained in:
parent
8fedc17566
commit
d93d26ddec
11 changed files with 429 additions and 49 deletions
|
|
@ -2029,6 +2029,14 @@ def _assistant_tool_calls_with_server_tool_calls(
|
|||
)
|
||||
|
||||
|
||||
def _server_tool_call_ids(message: object) -> frozenset[str]:
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
|
||||
|
||||
return frozenset(
|
||||
tool_call["id"] for tool_call in anthropic_server_tool_call_params(message) if tool_call["id"] is not None
|
||||
)
|
||||
|
||||
|
||||
def _server_tool_answers_replaced_after(messages: Sequence[object], index: int) -> frozenset[str]:
|
||||
kept_server_result_ids: Final = _tool_call_ids_with_a_kept_server_result(messages[index])
|
||||
if not kept_server_result_ids:
|
||||
|
|
@ -2092,6 +2100,7 @@ def convert_to_anthropic_tool_invoke(
|
|||
web_search_results: Sequence[object] | None = None,
|
||||
tool_results: Sequence[object] | None = None,
|
||||
answered_tool_ids: Container[str] = frozenset(),
|
||||
server_tool_call_ids: Container[str] = frozenset(),
|
||||
) -> list[_AnthropicToolInvokeItem]:
|
||||
"""
|
||||
OpenAI tool invokes:
|
||||
|
|
@ -2169,6 +2178,10 @@ def convert_to_anthropic_tool_invoke(
|
|||
anthropic_tool_invoke.append(server_tool_result)
|
||||
elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
|
||||
continue
|
||||
elif tool_id in server_tool_call_ids and tool_id not in answered_tool_ids:
|
||||
anthropic_tool_invoke.append(
|
||||
{"type": "server_tool_use", "id": tool_id, "name": tool_name, "input": tool_input}
|
||||
)
|
||||
else:
|
||||
sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
|
||||
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
|
||||
|
|
@ -3161,6 +3174,7 @@ def anthropic_messages_pt(
|
|||
web_search_results=_web_search_results_tc,
|
||||
tool_results=_tool_results_tc,
|
||||
answered_tool_ids=client_answered_tool_ids,
|
||||
server_tool_call_ids=_server_tool_call_ids(assistant_content_block),
|
||||
)
|
||||
|
||||
# Group tool invoke results into (server_tool_use, result) pairs
|
||||
|
|
@ -3326,6 +3340,7 @@ def anthropic_messages_pt(
|
|||
web_search_results=_web_search_results,
|
||||
tool_results=_tool_results,
|
||||
answered_tool_ids=client_answered_tool_ids,
|
||||
server_tool_call_ids=_server_tool_call_ids(assistant_content_block),
|
||||
)
|
||||
|
||||
# Prevent "tool_use ids must be unique" errors by filtering duplicates
|
||||
|
|
|
|||
|
|
@ -1128,6 +1128,15 @@ class ModelResponseIterator:
|
|||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
server_tool_input: Final = self._server_tool_inputs.get(self._current_server_tool_id, {})
|
||||
if self._current_server_tool_id in self._web_search_calls:
|
||||
self._web_search_calls[self._current_server_tool_id] = build_web_search_call(
|
||||
self._current_server_tool_id,
|
||||
server_tool_input,
|
||||
{"content": []},
|
||||
status="in_progress",
|
||||
tool_name=self._current_server_tool_name,
|
||||
)
|
||||
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
|
||||
self._server_tool_calls = (
|
||||
*self._server_tool_calls,
|
||||
ChatCompletionToolCallChunk(
|
||||
|
|
|
|||
|
|
@ -45,6 +45,25 @@ def _thought_signature(provider_specific_fields: object) -> str | None:
|
|||
return signature if isinstance(signature, str) else None
|
||||
|
||||
|
||||
def _object_sequence(value: object) -> Sequence[object] | None:
|
||||
return value if isinstance(value, (list, tuple)) else None
|
||||
|
||||
|
||||
def _server_tool_blocks(message: object) -> tuple[Mapping[str, object], ...]:
|
||||
server_tool_calls: Final = anthropic_server_tool_call_params(message)
|
||||
if not server_tool_calls:
|
||||
return ()
|
||||
fields: Final = _as_string_mapping(_optional_attr(message, "provider_specific_fields")) or {}
|
||||
return tuple(
|
||||
convert_to_anthropic_tool_invoke(
|
||||
server_tool_calls,
|
||||
web_search_results=_object_sequence(fields.get("web_search_results")),
|
||||
tool_results=_object_sequence(fields.get("tool_results")),
|
||||
server_tool_call_ids=frozenset(tool_call["id"] for tool_call in server_tool_calls),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]:
|
||||
fields: Final = _as_string_mapping(provider_specific_fields)
|
||||
raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None
|
||||
|
|
@ -129,11 +148,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
THOUGHT_SIGNATURE_SEPARATOR,
|
||||
convert_to_anthropic_tool_invoke,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import (
|
||||
reasoning_effort_from_thinking,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
anthropic_server_tool_call_params,
|
||||
eager_input_streaming_flag,
|
||||
is_empty_unsigned_thinking_block,
|
||||
normalize_anthropic_tool_use_id,
|
||||
|
|
@ -1410,6 +1431,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
).model_dump()
|
||||
)
|
||||
|
||||
new_content.extend(dict(block) for block in _server_tool_blocks(choice.message))
|
||||
# Handle text content
|
||||
if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks):
|
||||
new_content.append(
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.integrations.custom_guardrail import (
|
|||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params, anthropic_server_tool_calls
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.callback_utils import (
|
||||
add_guardrail_to_applied_guardrails_header,
|
||||
|
|
@ -515,7 +516,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
|
||||
for choice_index, choice in enumerate(response.choices):
|
||||
if isinstance(choice, Choices):
|
||||
for tool in choice.message.tool_calls or []:
|
||||
for tool in (*(choice.message.tool_calls or ()), *anthropic_server_tool_calls(choice.message)):
|
||||
tool_calls.append(tool)
|
||||
legacy_tool_call = self._legacy_function_call_to_tool_call(
|
||||
getattr(choice.message, "function_call", None), choice_index
|
||||
|
|
@ -784,6 +785,23 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
|
||||
choice.message.tool_calls = filtered_tool_calls if filtered_tool_calls else None
|
||||
|
||||
if (server_tool_calls := anthropic_server_tool_call_params(choice.message)) and any(
|
||||
server_tool_call["id"] in error_results for server_tool_call in server_tool_calls
|
||||
):
|
||||
error_messages.extend(
|
||||
error_results[server_tool_call["id"]].content
|
||||
for server_tool_call in server_tool_calls
|
||||
if server_tool_call["id"] in error_results
|
||||
)
|
||||
choice.message.provider_specific_fields = {
|
||||
**(choice.message.provider_specific_fields or {}),
|
||||
"server_tool_calls": [
|
||||
server_tool_call
|
||||
for server_tool_call in server_tool_calls
|
||||
if server_tool_call["id"] not in error_results
|
||||
],
|
||||
}
|
||||
|
||||
legacy_tool_call = self._legacy_function_call_to_tool_call(
|
||||
getattr(choice.message, "function_call", None), choice_index
|
||||
)
|
||||
|
|
|
|||
|
|
@ -471,16 +471,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
|
||||
try:
|
||||
message: Final = litellm_complete_object.choices[0].message
|
||||
tool_calls = [
|
||||
*anthropic_server_tool_calls(message),
|
||||
*(getattr(message, "tool_calls", None) or ()),
|
||||
]
|
||||
server_tool_calls: Final = anthropic_server_tool_calls(message)
|
||||
tool_calls: Final = [*server_tool_calls, *(getattr(message, "tool_calls", None) or ())]
|
||||
except Exception:
|
||||
tool_calls = None
|
||||
return
|
||||
|
||||
if not tool_calls:
|
||||
return
|
||||
|
||||
server_tool_call_ids: Final = frozenset(tool_call.id for tool_call in server_tool_calls)
|
||||
|
||||
for tc in tool_calls:
|
||||
call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
|
||||
if not call_id_raw:
|
||||
|
|
@ -506,6 +506,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if self._is_tool_search_call(call_id, fn_name):
|
||||
self._queue_tool_search_call_done(call_id, output_index, fn_args)
|
||||
continue
|
||||
if call_id in server_tool_call_ids:
|
||||
continue
|
||||
|
||||
# Track if this is a new tool call that wasn't streamed
|
||||
is_new_tool_call = call_id not in self._tool_item_id_by_call_id
|
||||
|
|
@ -580,10 +582,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
}
|
||||
),
|
||||
)
|
||||
finished_events: Final = (
|
||||
()
|
||||
if item.status == "in_progress"
|
||||
else ((ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),)
|
||||
)
|
||||
for event_type, event_class in (
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
|
||||
*finished_events,
|
||||
):
|
||||
self._sequence_number += 1
|
||||
event = event_class(type=event_type, output_index=output_index, item_id=item.id)
|
||||
|
|
|
|||
|
|
@ -3370,7 +3370,22 @@ class LiteLLMCompletionResponsesConfig:
|
|||
for item in responses_output
|
||||
]
|
||||
|
||||
return responses_output
|
||||
server_tool_call_ids: Final = LiteLLMCompletionResponsesConfig._server_tool_call_ids(choices)
|
||||
return [
|
||||
item
|
||||
for item in responses_output
|
||||
if not (
|
||||
getattr(item, "type", None) == "function_call"
|
||||
and getattr(item, "call_id", None) in server_tool_call_ids
|
||||
)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _server_tool_call_ids(choices: Sequence[Choices]) -> frozenset[str]:
|
||||
server_tool_calls: Final = chain.from_iterable(
|
||||
anthropic_server_tool_calls(choice.message) for choice in choices
|
||||
)
|
||||
return frozenset(tool_call.id for tool_call in server_tool_calls)
|
||||
|
||||
@staticmethod
|
||||
def _extract_tool_result_output_items(
|
||||
|
|
|
|||
|
|
@ -3735,6 +3735,33 @@ def test_server_tool_result_round_trips_right_after_its_call(
|
|||
assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"]
|
||||
|
||||
|
||||
_FETCH_STILL_WAITING_ON_ITS_RESULT: Final = {
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01Fetch",
|
||||
"name": "web_fetch",
|
||||
"input": {"url": "https://example.com/"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("modify_params", [False, True])
|
||||
def test_server_call_still_waiting_on_its_result_is_replayed_as_a_server_call(
|
||||
modify_params: bool, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
content: Final = [_FETCH_STILL_WAITING_ON_ITS_RESULT, _WEATHER_CALL]
|
||||
message: Final = _parsed_assistant_message(content, "tool_use")
|
||||
monkeypatch.setattr(litellm, "modify_params", modify_params)
|
||||
|
||||
replayed: Final = _replayed(
|
||||
{"role": "user", "content": "Fetch example.com, then tell me the weather in Paris"},
|
||||
message.model_dump(),
|
||||
{"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"},
|
||||
)
|
||||
|
||||
assert replayed[1]["content"] == content
|
||||
assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"]
|
||||
assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"]
|
||||
|
||||
|
||||
_PARIS_TEXT: Final = "Paris is the capital of France."
|
||||
_PARIS_ANSWER: Final[Mapping[str, str]] = {"type": "text", "text": _PARIS_TEXT}
|
||||
|
||||
|
|
|
|||
|
|
@ -80,6 +80,59 @@ def test_native_compaction_response_roundtrip(text: str | None, count: int, expe
|
|||
assert result["stop_reason"] == expected_stop
|
||||
|
||||
|
||||
_SERVER_AND_CLIENT_TOOL_TURN: Final[tuple[dict[str, object], ...]] = (
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01Fetch",
|
||||
"name": "web_fetch",
|
||||
"input": {"url": "https://example.com/"},
|
||||
},
|
||||
{
|
||||
"type": "web_fetch_tool_result",
|
||||
"tool_use_id": "srvtoolu_01Fetch",
|
||||
"content": {"type": "web_fetch_result", "url": "https://example.com/"},
|
||||
},
|
||||
{"type": "server_tool_use", "id": "srvtoolu_02Search", "name": "web_search", "input": {"query": "Paris"}},
|
||||
{"type": "text", "text": "The page is titled Example."},
|
||||
{"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}},
|
||||
)
|
||||
|
||||
|
||||
def test_claude_server_tool_blocks_survive_the_round_trip_through_chat_completions() -> None:
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
anthropic_response: Final = {
|
||||
"id": "msg_01Fetch",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": list(_SERVER_AND_CLIENT_TOOL_TURN),
|
||||
"stop_reason": "tool_use",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 100, "output_tokens": 50},
|
||||
}
|
||||
raw_response: Final = MagicMock(spec=httpx.Response)
|
||||
raw_response.json.return_value = anthropic_response
|
||||
raw_response.status_code = 200
|
||||
raw_response.headers = {}
|
||||
chat_response: Final = AnthropicConfig().transform_parsed_response(
|
||||
completion_response=anthropic_response,
|
||||
raw_response=raw_response,
|
||||
model_response=ModelResponse(),
|
||||
json_mode=False,
|
||||
prefix_prompt=None,
|
||||
)
|
||||
|
||||
result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(chat_response)
|
||||
|
||||
assert [(block["type"], block.get("id") or block.get("tool_use_id")) for block in result["content"]] == [
|
||||
(block["type"], block.get("id") or block.get("tool_use_id")) for block in _SERVER_AND_CLIENT_TOOL_TURN
|
||||
]
|
||||
assert result["stop_reason"] == "tool_use"
|
||||
|
||||
|
||||
def test_translate_chat_refusal_to_anthropic_response():
|
||||
response = ModelResponse(
|
||||
id="chatcmpl-refusal",
|
||||
|
|
|
|||
|
|
@ -27,11 +27,21 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
|
|||
from litellm.types.utils import (
|
||||
ChatCompletionMessageToolCall,
|
||||
Choices,
|
||||
Delta,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
)
|
||||
|
||||
|
||||
_SERVER_WEB_SEARCH_CALL: Final = {
|
||||
"id": "srvtoolu_01Search",
|
||||
"type": "function",
|
||||
"function": {"name": "web_search", "arguments": '{"query": "weather in Paris"}'},
|
||||
}
|
||||
_ALLOWED_BASH_CALL: Final = {"id": "toolu_01Bash", "type": "function", "function": {"name": "Bash", "arguments": "{}"}}
|
||||
|
||||
|
||||
class TestToolPermissionGuardrail:
|
||||
"""Test class for Tool Permission Guardrail functionality"""
|
||||
|
||||
|
|
@ -952,6 +962,71 @@ class TestToolPermissionGuardrail:
|
|||
assert "Permission denied" in choice.message.content
|
||||
|
||||
|
||||
def _server_search_response(self) -> ModelResponse:
|
||||
return ModelResponse(
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
message={
|
||||
"content": "Searching.",
|
||||
"tool_calls": [_ALLOWED_BASH_CALL],
|
||||
"provider_specific_fields": {"server_tool_calls": [_SERVER_WEB_SEARCH_CALL]},
|
||||
},
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_denied_server_tool_call_is_blocked(self):
|
||||
with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
|
||||
with pytest.raises(GuardrailRaisedException):
|
||||
await self.guardrail.async_post_call_success_hook(
|
||||
data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response()
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_denied_server_tool_call_in_a_chat_stream_is_blocked(self):
|
||||
async def _stream():
|
||||
yield ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(
|
||||
content="Searching.",
|
||||
provider_specific_fields={"server_tool_calls": [{**_SERVER_WEB_SEARCH_CALL, "index": 0}]},
|
||||
),
|
||||
finish_reason="stop",
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
|
||||
with pytest.raises(GuardrailRaisedException):
|
||||
async for _ in self.guardrail.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(), response=_stream(), request_data={}
|
||||
):
|
||||
pass
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rewrite_mode_removes_the_denied_server_tool_call(self):
|
||||
rewriting: Final = ToolPermissionGuardrail(
|
||||
guardrail_name="test-tool-permission",
|
||||
rules=self.test_rules,
|
||||
default_action="deny",
|
||||
on_disallowed_action="rewrite",
|
||||
)
|
||||
|
||||
with patch.object(rewriting, "should_run_guardrail", return_value=True):
|
||||
result: Final = await rewriting.async_post_call_success_hook(
|
||||
data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response()
|
||||
)
|
||||
|
||||
message: Final = result.choices[0].message
|
||||
assert message.provider_specific_fields["server_tool_calls"] == []
|
||||
assert [tool_call.id for tool_call in message.tool_calls] == [_ALLOWED_BASH_CALL["id"]]
|
||||
assert "Permission denied" in message.content
|
||||
|
||||
|
||||
class TestToolPermissionGuardrailIntegration:
|
||||
"""Integration tests for Tool Permission Guardrail"""
|
||||
|
||||
|
|
|
|||
|
|
@ -5064,6 +5064,47 @@ def test_anthropic_server_web_call_without_a_result_is_an_in_progress_hosted_cal
|
|||
assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"]
|
||||
|
||||
|
||||
def test_anthropic_code_execution_is_never_a_client_function_call_on_responses() -> None:
|
||||
response: Final = _anthropic_turn_on_responses(
|
||||
(
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01Bash",
|
||||
"name": "bash_code_execution",
|
||||
"input": {"command": "ls"},
|
||||
},
|
||||
{
|
||||
"type": "bash_code_execution_tool_result",
|
||||
"tool_use_id": "srvtoolu_01Bash",
|
||||
"content": {
|
||||
"type": "bash_code_execution_result",
|
||||
"stdout": "notes.txt\n",
|
||||
"stderr": "",
|
||||
"return_code": 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_02Edit",
|
||||
"name": "text_editor_code_execution",
|
||||
"input": {"command": "view", "path": "notes.txt"},
|
||||
},
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_03Bash",
|
||||
"name": "bash_code_execution",
|
||||
"input": {"command": "pwd"},
|
||||
},
|
||||
_WEATHER_TOOL_USE,
|
||||
)
|
||||
)
|
||||
|
||||
assert [(item.type, getattr(item, "call_id", item.id)) for item in response.output if item.type != "message"] == [
|
||||
("code_interpreter_call", "srvtoolu_01Bash"),
|
||||
("function_call", "toolu_01Weather"),
|
||||
]
|
||||
|
||||
|
||||
class TestHostedWebSearchReplay:
|
||||
def test_emitted_hosted_search_output_round_trips_with_client_tool_result(self) -> None:
|
||||
search_result: Final = {
|
||||
|
|
|
|||
|
|
@ -355,53 +355,59 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to
|
|||
)
|
||||
|
||||
|
||||
def _anthropic_web_fetch_stream_events() -> tuple[dict[str, object], ...]:
|
||||
_STREAMED_WEB_FETCH_CALL: Final[tuple[dict[str, object], ...]] = (
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_01Fetch",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [],
|
||||
"usage": {"input_tokens": 10, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"},
|
||||
},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
)
|
||||
_STREAMED_WEB_FETCH_RESULT: Final[tuple[dict[str, object], ...]] = (
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 1,
|
||||
"content_block": {
|
||||
"type": "web_fetch_tool_result",
|
||||
"tool_use_id": "srvtoolu_01Fetch",
|
||||
"content": {"type": "web_fetch_result", "url": "https://example.com/"},
|
||||
},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 1},
|
||||
)
|
||||
|
||||
|
||||
def _streamed_text_then_weather_call(first_index: int) -> tuple[dict[str, object], ...]:
|
||||
return (
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_01Fetch",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [],
|
||||
"usage": {"input_tokens": 10, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
{"type": "content_block_start", "index": first_index, "content_block": {"type": "text", "text": ""}},
|
||||
{"type": "content_block_delta", "index": first_index, "delta": {"type": "text_delta", "text": "Fetched."}},
|
||||
{"type": "content_block_stop", "index": first_index},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"},
|
||||
},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 1,
|
||||
"content_block": {
|
||||
"type": "web_fetch_tool_result",
|
||||
"tool_use_id": "srvtoolu_01Fetch",
|
||||
"content": {"type": "web_fetch_result", "url": "https://example.com/"},
|
||||
},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 1},
|
||||
{"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}},
|
||||
{"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Fetched."}},
|
||||
{"type": "content_block_stop", "index": 2},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 3,
|
||||
"index": first_index + 1,
|
||||
"content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}},
|
||||
},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 3,
|
||||
"index": first_index + 1,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 3},
|
||||
{"type": "content_block_stop", "index": first_index + 1},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "tool_use", "stop_sequence": None},
|
||||
|
|
@ -410,13 +416,52 @@ def _anthropic_web_fetch_stream_events() -> tuple[dict[str, object], ...]:
|
|||
)
|
||||
|
||||
|
||||
def _anthropic_web_fetch_stream_events(fetch_finished: bool) -> tuple[dict[str, object], ...]:
|
||||
if fetch_finished:
|
||||
return (*_STREAMED_WEB_FETCH_CALL, *_STREAMED_WEB_FETCH_RESULT, *_streamed_text_then_weather_call(2))
|
||||
return (*_STREAMED_WEB_FETCH_CALL, *_streamed_text_then_weather_call(1))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
("fetch_finished", "expected_status", "expected_search_events"),
|
||||
[
|
||||
pytest.param(
|
||||
True,
|
||||
"completed",
|
||||
[
|
||||
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED,
|
||||
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
],
|
||||
id="fetch_finished_in_this_turn",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
"in_progress",
|
||||
[
|
||||
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING,
|
||||
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
],
|
||||
id="fetch_still_waiting_on_its_result",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(sync_mode: bool) -> None:
|
||||
async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(
|
||||
sync_mode: bool,
|
||||
fetch_finished: bool,
|
||||
expected_status: str,
|
||||
expected_search_events: list[ResponsesAPIStreamEvents],
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
|
||||
|
||||
parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False)
|
||||
chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events()]
|
||||
chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events(fetch_finished)]
|
||||
iterator: Final = LiteLLMCompletionStreamingIterator(
|
||||
model="claude-sonnet-5-5",
|
||||
litellm_custom_stream_wrapper=_FakeStreamWrapper(chunks),
|
||||
|
|
@ -439,7 +484,12 @@ async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(sync_mode
|
|||
(item.id, item.status, item.action.model_dump())
|
||||
for item in completed.response.output
|
||||
if item.type == "web_search_call"
|
||||
] == [("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": "https://example.com/"})]
|
||||
] == [("ws_srvtoolu_01Fetch", expected_status, {"type": "open_page", "url": "https://example.com/"})]
|
||||
assert [
|
||||
event.type
|
||||
for event in events
|
||||
if getattr(event, "item_id", getattr(getattr(event, "item", None), "id", None)) == "ws_srvtoolu_01Fetch"
|
||||
] == expected_search_events
|
||||
assert [item.name for item in completed.response.output if item.type == "function_call"] == ["get_weather"]
|
||||
assert not any(
|
||||
event.item.name == "web_fetch"
|
||||
|
|
@ -449,6 +499,54 @@ async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(sync_mode
|
|||
)
|
||||
|
||||
|
||||
_STREAMED_PENDING_CODE_EXECUTION: Final[tuple[dict[str, object], ...]] = (
|
||||
_STREAMED_WEB_FETCH_CALL[0],
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Bash", "name": "bash_code_execution"},
|
||||
},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '{"command": "ls"}'},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
*_streamed_text_then_weather_call(1),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_anthropic_code_execution_is_never_a_client_function_call(sync_mode: bool) -> None:
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
|
||||
|
||||
parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False)
|
||||
iterator: Final = LiteLLMCompletionStreamingIterator(
|
||||
model="claude-sonnet-5-5",
|
||||
litellm_custom_stream_wrapper=_FakeStreamWrapper(
|
||||
[parser.chunk_parser(event) for event in _STREAMED_PENDING_CODE_EXECUTION]
|
||||
),
|
||||
request_input="List the files, then check the weather in Paris.",
|
||||
responses_api_request={
|
||||
"tools": [
|
||||
{"type": "code_execution_20250825", "name": "code_execution"},
|
||||
{"type": "function", "name": "get_weather", "parameters": {"type": "object"}},
|
||||
]
|
||||
},
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
events: Final = await _collect_events(iterator, sync_mode)
|
||||
|
||||
item_events: Final = [event for event in events if getattr(event, "item", None) is not None]
|
||||
assert {event.item.call_id for event in item_events if event.item.type == "function_call"} == {"toolu_01Weather"}
|
||||
completed: Final = next(
|
||||
event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
|
||||
)
|
||||
assert [item.call_id for item in completed.response.output if item.type == "function_call"] == ["toolu_01Weather"]
|
||||
|
||||
|
||||
def test_tool_calls_present_only_in_final_response_are_emitted_before_completed():
|
||||
iterator = LiteLLMCompletionStreamingIterator(
|
||||
model="test-model",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue