fix(anthropic): apply streamed guardrail tool call rewrites to their own block when a server tool call follows

This commit is contained in:
mateo-berri 2026-11-05 23:49:31 -08:00
parent 418409e9d7
commit b1d1f99b1c
3 changed files with 80 additions and 18 deletions

View file

@ -518,6 +518,13 @@ def _rewritten_event(event: Mapping[str, object], rewrite_event: _SSEEventRewrit
return {**event, rewrite.section: {**section, rewrite.field: rewrite.value}}
def _in_stream_block_order(
tool_calls: Sequence[ChatCompletionMessageToolCall], blocks: Sequence[tuple[int, Mapping[str, object]]]
) -> tuple[ChatCompletionMessageToolCall, ...]:
positions: Final = MappingProxyType({block.get("id"): position for position, (_, block) in enumerate(blocks)})
return tuple(sorted(tool_calls, key=lambda tool_call: positions.get(tool_call.id, len(positions))))
def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...]:
"""The guardrail-visible shape of each tool call, whether the guardrail handed
back the ``ChatCompletionMessageToolCall`` objects it was given or plain dicts."""
@ -1572,15 +1579,18 @@ class AnthropicMessagesHandler(BaseTranslation):
if built_response is not None and hasattr(built_response, "choices") and built_response.choices:
model_response: Final = cast(ModelResponse, built_response)
first_choice: Final = cast(Choices, model_response.choices[0])
tool_calls_list: Final = (
*anthropic_server_tool_calls(first_choice.message),
*cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []),
tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far)
tool_calls_list: Final = _in_stream_block_order(
(
*anthropic_server_tool_calls(first_choice.message),
*cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []),
),
tool_call_blocks,
)
string_so_far = first_choice.message.content
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ())
guardrail_inputs: Final = GenericGuardrailAPIInputs()
leading_texts: Final = (string_so_far,) if isinstance(string_so_far, str) and string_so_far else ()
tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far)
streamed_query_strings: Final = _streamed_query_strings(tool_call_blocks, pre_guardrail_tool_calls)
streamed_result_strings: Final = self._streamed_web_search_result_strings(responses_so_far)
streamed_web_search_strings: Final = (*streamed_result_strings, *streamed_query_strings)

View file

@ -594,6 +594,57 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing:
assert '"id": "srvtoolu_1"' in raw and '"id": "toolu_1"' in raw
assert "persim" not in raw and '"query"' not in raw
@pytest.mark.asyncio
async def test_deliver_ended_stream_tool_use_rewrite_lands_on_its_own_block_when_server_tool_use_follows(self):
class MaskFruitLookup(CustomGuardrail):
async def apply_guardrail(
self,
inputs: GenericGuardrailAPIInputs,
request_data: dict[str, object],
input_type: Literal["request", "response"],
logging_obj: LiteLLMLoggingObj | None = None,
) -> GenericGuardrailAPIInputs:
for tool_call in inputs.get("tool_calls", []):
if tool_call.function.name == "lookup_fruit":
tool_call.function.arguments = '{"fruit": "[MASKED]"}'
return inputs
handler: Final = AnthropicMessagesHandler()
tool_use: Final = self._ended_tool_use_sse_chunks()
server_tool_use: Final = [
(
"content_block_start",
{
"type": "content_block_start",
"index": 1,
"content_block": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}},
},
),
(
"content_block_delta",
{
"type": "content_block_delta",
"index": 1,
"delta": {"type": "input_json_delta", "partial_json": '{"query": "fruit"}'},
},
),
("content_block_stop", {"type": "content_block_stop", "index": 1}),
]
chunks: Final = (
tool_use[:-2]
+ [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in server_tool_use]
+ tool_use[-2:]
)
await handler.process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=MaskFruitLookup(guardrail_name="test"),
litellm_logging_obj=MagicMock(),
deliver_ended_stream_rewrites=True,
)
assert self._partial_jsons(chunks) == ['{"fruit": "[MASKED]"}', "", "", '{"query": "fruit"}']
@pytest.mark.asyncio
async def test_deliver_ended_stream_tool_use_rewrite_on_an_unindexed_tool_block_fails_closed(self):
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite

View file

@ -1,6 +1,7 @@
import asyncio
import json
import ssl
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -2239,11 +2240,11 @@ async def test_post_call_success_hook_redacts_both_content_and_tool_arguments():
@pytest.mark.asyncio
async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_arguments():
guard = _make_guardrail()
request_data = {"messages": [{"role": "user", "content": "look up persimmon"}]}
guard: Final = _make_guardrail()
request_data: Final = {"messages": [{"role": "user", "content": "look up persimmon"}]}
def side_effect(url, *args, **kwargs):
last = kwargs["json"]["messages"][-1]["content"]
def side_effect(url: str, *args: object, **kwargs: dict[str, list[dict[str, str]]]) -> Response:
last: Final = kwargs["json"]["messages"][-1]["content"]
return _make_response(
{
"analysis_result": {"policy_drill_down": {"PII": {}}},
@ -2257,7 +2258,7 @@ async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_argu
}
)
llm_response = ModelResponse(
llm_response: Final = ModelResponse(
choices=[
{
"finish_reason": "tool_calls",
@ -2289,18 +2290,18 @@ async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_argu
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
side_effect=side_effect,
) as mock_post:
result = await guard.async_post_call_success_hook(
result: Final = await guard.async_post_call_success_hook(
data=request_data,
response=llm_response,
user_api_key_dict=UserAPIKeyAuth(),
)
inspected = [call.kwargs["json"]["messages"][-1] for call in mock_post.call_args_list]
inspected: Final = [call.kwargs["json"]["messages"][-1] for call in mock_post.call_args_list]
assert inspected == [
{"role": "assistant", "content": '{"query": "persimmon"}'},
{"role": "assistant", "content": '{"city": "Paris"}'},
]
message = result.choices[0].message
message: Final = result.choices[0].message
assert message.provider_specific_fields == {
"server_tool_calls": [
{
@ -2414,9 +2415,9 @@ async def test_post_call_success_hook_redacts_claude_hosted_web_fetch_url():
from openai.types.responses import ResponseFunctionWebSearch
from openai.types.responses.response_function_web_search import ActionOpenPage
guard = _make_guardrail()
request_data = {"messages": [{"role": "user", "content": "read the page"}]}
anonymize_response = _make_response(
guard: Final = _make_guardrail()
request_data: Final = {"messages": [{"role": "user", "content": "read the page"}]}
anonymize_response: Final = _make_response(
{
"analysis_result": {"policy_drill_down": {"PII": {}}},
"required_action": {"action_type": "anonymize_action", "policy_name": "PII"},
@ -2428,7 +2429,7 @@ async def test_post_call_success_hook_redacts_claude_hosted_web_fetch_url():
},
}
)
response = _make_responses_api_response(
response: Final = _make_responses_api_response(
[
ResponseFunctionWebSearch(
id="ws_srvtoolu_01",
@ -2443,12 +2444,12 @@ async def test_post_call_success_hook_redacts_claude_hosted_web_fetch_url():
new_callable=AsyncMock,
) as mock_post:
mock_post.return_value = anonymize_response
result = await guard.async_post_call_success_hook(
result: Final = await guard.async_post_call_success_hook(
data=request_data,
response=response,
user_api_key_dict=UserAPIKeyAuth(),
)
posted = mock_post.call_args.kwargs["json"]["messages"]
posted: Final = mock_post.call_args.kwargs["json"]["messages"]
assert posted[-1] == {"role": "assistant", "content": '{"url": "https://example.com/?ssn=123-45-6789"}'}
assert (result.output[0].id, result.output[0].action.url) == (
"ws_srvtoolu_01",