From b1d1f99b1cee2a24be649c43094d2552afee079b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:49:31 -0800 Subject: [PATCH] fix(anthropic): apply streamed guardrail tool call rewrites to their own block when a server tool call follows --- .../chat/guardrail_translation/handler.py | 18 +++++-- .../test_anthropic_guardrail_handler.py | 51 +++++++++++++++++++ .../guardrail_hooks/test_cato_networks.py | 29 ++++++----- 3 files changed, 80 insertions(+), 18 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index afee5c58958..8f1a08d2964 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -518,6 +518,13 @@ def _rewritten_event(event: Mapping[str, object], rewrite_event: _SSEEventRewrit return {**event, rewrite.section: {**section, rewrite.field: rewrite.value}} +def _in_stream_block_order( + tool_calls: Sequence[ChatCompletionMessageToolCall], blocks: Sequence[tuple[int, Mapping[str, object]]] +) -> tuple[ChatCompletionMessageToolCall, ...]: + positions: Final = MappingProxyType({block.get("id"): position for position, (_, block) in enumerate(blocks)}) + return tuple(sorted(tool_calls, key=lambda tool_call: positions.get(tool_call.id, len(positions)))) + + def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...]: """The guardrail-visible shape of each tool call, whether the guardrail handed back the ``ChatCompletionMessageToolCall`` objects it was given or plain dicts.""" @@ -1572,15 +1579,18 @@ class AnthropicMessagesHandler(BaseTranslation): if built_response is not None and hasattr(built_response, "choices") and built_response.choices: model_response: Final = cast(ModelResponse, built_response) first_choice: Final = cast(Choices, model_response.choices[0]) - tool_calls_list: Final = ( - *anthropic_server_tool_calls(first_choice.message), - *cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []), + tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far) + tool_calls_list: Final = _in_stream_block_order( + ( + *anthropic_server_tool_calls(first_choice.message), + *cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []), + ), + tool_call_blocks, ) string_so_far = first_choice.message.content pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ()) guardrail_inputs: Final = GenericGuardrailAPIInputs() leading_texts: Final = (string_so_far,) if isinstance(string_so_far, str) and string_so_far else () - tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far) streamed_query_strings: Final = _streamed_query_strings(tool_call_blocks, pre_guardrail_tool_calls) streamed_result_strings: Final = self._streamed_web_search_result_strings(responses_so_far) streamed_web_search_strings: Final = (*streamed_result_strings, *streamed_query_strings) diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index c2ee58b67ed..8affe2e136e 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -594,6 +594,57 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: assert '"id": "srvtoolu_1"' in raw and '"id": "toolu_1"' in raw assert "persim" not in raw and '"query"' not in raw + @pytest.mark.asyncio + async def test_deliver_ended_stream_tool_use_rewrite_lands_on_its_own_block_when_server_tool_use_follows(self): + class MaskFruitLookup(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + for tool_call in inputs.get("tool_calls", []): + if tool_call.function.name == "lookup_fruit": + tool_call.function.arguments = '{"fruit": "[MASKED]"}' + return inputs + + handler: Final = AnthropicMessagesHandler() + tool_use: Final = self._ended_tool_use_sse_chunks() + server_tool_use: Final = [ + ( + "content_block_start", + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "input_json_delta", "partial_json": '{"query": "fruit"}'}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 1}), + ] + chunks: Final = ( + tool_use[:-2] + + [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in server_tool_use] + + tool_use[-2:] + ) + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=MaskFruitLookup(guardrail_name="test"), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + + assert self._partial_jsons(chunks) == ['{"fruit": "[MASKED]"}', "", "", '{"query": "fruit"}'] + @pytest.mark.asyncio async def test_deliver_ended_stream_tool_use_rewrite_on_an_unindexed_tool_block_fails_closed(self): from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py index de4f0853df1..a8f68f038c0 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py @@ -1,6 +1,7 @@ import asyncio import json import ssl +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -2239,11 +2240,11 @@ async def test_post_call_success_hook_redacts_both_content_and_tool_arguments(): @pytest.mark.asyncio async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_arguments(): - guard = _make_guardrail() - request_data = {"messages": [{"role": "user", "content": "look up persimmon"}]} + guard: Final = _make_guardrail() + request_data: Final = {"messages": [{"role": "user", "content": "look up persimmon"}]} - def side_effect(url, *args, **kwargs): - last = kwargs["json"]["messages"][-1]["content"] + def side_effect(url: str, *args: object, **kwargs: dict[str, list[dict[str, str]]]) -> Response: + last: Final = kwargs["json"]["messages"][-1]["content"] return _make_response( { "analysis_result": {"policy_drill_down": {"PII": {}}}, @@ -2257,7 +2258,7 @@ async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_argu } ) - llm_response = ModelResponse( + llm_response: Final = ModelResponse( choices=[ { "finish_reason": "tool_calls", @@ -2289,18 +2290,18 @@ async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_argu "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", side_effect=side_effect, ) as mock_post: - result = await guard.async_post_call_success_hook( + result: Final = await guard.async_post_call_success_hook( data=request_data, response=llm_response, user_api_key_dict=UserAPIKeyAuth(), ) - inspected = [call.kwargs["json"]["messages"][-1] for call in mock_post.call_args_list] + inspected: Final = [call.kwargs["json"]["messages"][-1] for call in mock_post.call_args_list] assert inspected == [ {"role": "assistant", "content": '{"query": "persimmon"}'}, {"role": "assistant", "content": '{"city": "Paris"}'}, ] - message = result.choices[0].message + message: Final = result.choices[0].message assert message.provider_specific_fields == { "server_tool_calls": [ { @@ -2414,9 +2415,9 @@ async def test_post_call_success_hook_redacts_claude_hosted_web_fetch_url(): from openai.types.responses import ResponseFunctionWebSearch from openai.types.responses.response_function_web_search import ActionOpenPage - guard = _make_guardrail() - request_data = {"messages": [{"role": "user", "content": "read the page"}]} - anonymize_response = _make_response( + guard: Final = _make_guardrail() + request_data: Final = {"messages": [{"role": "user", "content": "read the page"}]} + anonymize_response: Final = _make_response( { "analysis_result": {"policy_drill_down": {"PII": {}}}, "required_action": {"action_type": "anonymize_action", "policy_name": "PII"}, @@ -2428,7 +2429,7 @@ async def test_post_call_success_hook_redacts_claude_hosted_web_fetch_url(): }, } ) - response = _make_responses_api_response( + response: Final = _make_responses_api_response( [ ResponseFunctionWebSearch( id="ws_srvtoolu_01", @@ -2443,12 +2444,12 @@ async def test_post_call_success_hook_redacts_claude_hosted_web_fetch_url(): new_callable=AsyncMock, ) as mock_post: mock_post.return_value = anonymize_response - result = await guard.async_post_call_success_hook( + result: Final = await guard.async_post_call_success_hook( data=request_data, response=response, user_api_key_dict=UserAPIKeyAuth(), ) - posted = mock_post.call_args.kwargs["json"]["messages"] + posted: Final = mock_post.call_args.kwargs["json"]["messages"] assert posted[-1] == {"role": "assistant", "content": '{"url": "https://example.com/?ssn=123-45-6789"}'} assert (result.output[0].id, result.output[0].action.url) == ( "ws_srvtoolu_01",