mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(anthropic): apply streamed guardrail tool call rewrites to their own block when a server tool call follows
This commit is contained in:
parent
418409e9d7
commit
b1d1f99b1c
3 changed files with 80 additions and 18 deletions
|
|
@ -518,6 +518,13 @@ def _rewritten_event(event: Mapping[str, object], rewrite_event: _SSEEventRewrit
|
|||
return {**event, rewrite.section: {**section, rewrite.field: rewrite.value}}
|
||||
|
||||
|
||||
def _in_stream_block_order(
|
||||
tool_calls: Sequence[ChatCompletionMessageToolCall], blocks: Sequence[tuple[int, Mapping[str, object]]]
|
||||
) -> tuple[ChatCompletionMessageToolCall, ...]:
|
||||
positions: Final = MappingProxyType({block.get("id"): position for position, (_, block) in enumerate(blocks)})
|
||||
return tuple(sorted(tool_calls, key=lambda tool_call: positions.get(tool_call.id, len(positions))))
|
||||
|
||||
|
||||
def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...]:
|
||||
"""The guardrail-visible shape of each tool call, whether the guardrail handed
|
||||
back the ``ChatCompletionMessageToolCall`` objects it was given or plain dicts."""
|
||||
|
|
@ -1572,15 +1579,18 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
if built_response is not None and hasattr(built_response, "choices") and built_response.choices:
|
||||
model_response: Final = cast(ModelResponse, built_response)
|
||||
first_choice: Final = cast(Choices, model_response.choices[0])
|
||||
tool_calls_list: Final = (
|
||||
*anthropic_server_tool_calls(first_choice.message),
|
||||
*cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []),
|
||||
tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far)
|
||||
tool_calls_list: Final = _in_stream_block_order(
|
||||
(
|
||||
*anthropic_server_tool_calls(first_choice.message),
|
||||
*cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []),
|
||||
),
|
||||
tool_call_blocks,
|
||||
)
|
||||
string_so_far = first_choice.message.content
|
||||
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ())
|
||||
guardrail_inputs: Final = GenericGuardrailAPIInputs()
|
||||
leading_texts: Final = (string_so_far,) if isinstance(string_so_far, str) and string_so_far else ()
|
||||
tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far)
|
||||
streamed_query_strings: Final = _streamed_query_strings(tool_call_blocks, pre_guardrail_tool_calls)
|
||||
streamed_result_strings: Final = self._streamed_web_search_result_strings(responses_so_far)
|
||||
streamed_web_search_strings: Final = (*streamed_result_strings, *streamed_query_strings)
|
||||
|
|
|
|||
|
|
@ -594,6 +594,57 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing:
|
|||
assert '"id": "srvtoolu_1"' in raw and '"id": "toolu_1"' in raw
|
||||
assert "persim" not in raw and '"query"' not in raw
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_tool_use_rewrite_lands_on_its_own_block_when_server_tool_use_follows(self):
|
||||
class MaskFruitLookup(CustomGuardrail):
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"],
|
||||
logging_obj: LiteLLMLoggingObj | None = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
for tool_call in inputs.get("tool_calls", []):
|
||||
if tool_call.function.name == "lookup_fruit":
|
||||
tool_call.function.arguments = '{"fruit": "[MASKED]"}'
|
||||
return inputs
|
||||
|
||||
handler: Final = AnthropicMessagesHandler()
|
||||
tool_use: Final = self._ended_tool_use_sse_chunks()
|
||||
server_tool_use: Final = [
|
||||
(
|
||||
"content_block_start",
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 1,
|
||||
"content_block": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}},
|
||||
},
|
||||
),
|
||||
(
|
||||
"content_block_delta",
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 1,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '{"query": "fruit"}'},
|
||||
},
|
||||
),
|
||||
("content_block_stop", {"type": "content_block_stop", "index": 1}),
|
||||
]
|
||||
chunks: Final = (
|
||||
tool_use[:-2]
|
||||
+ [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in server_tool_use]
|
||||
+ tool_use[-2:]
|
||||
)
|
||||
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=MaskFruitLookup(guardrail_name="test"),
|
||||
litellm_logging_obj=MagicMock(),
|
||||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
|
||||
assert self._partial_jsons(chunks) == ['{"fruit": "[MASKED]"}', "", "", '{"query": "fruit"}']
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_tool_use_rewrite_on_an_unindexed_tool_block_fails_closed(self):
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
import ssl
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -2239,11 +2240,11 @@ async def test_post_call_success_hook_redacts_both_content_and_tool_arguments():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_arguments():
|
||||
guard = _make_guardrail()
|
||||
request_data = {"messages": [{"role": "user", "content": "look up persimmon"}]}
|
||||
guard: Final = _make_guardrail()
|
||||
request_data: Final = {"messages": [{"role": "user", "content": "look up persimmon"}]}
|
||||
|
||||
def side_effect(url, *args, **kwargs):
|
||||
last = kwargs["json"]["messages"][-1]["content"]
|
||||
def side_effect(url: str, *args: object, **kwargs: dict[str, list[dict[str, str]]]) -> Response:
|
||||
last: Final = kwargs["json"]["messages"][-1]["content"]
|
||||
return _make_response(
|
||||
{
|
||||
"analysis_result": {"policy_drill_down": {"PII": {}}},
|
||||
|
|
@ -2257,7 +2258,7 @@ async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_argu
|
|||
}
|
||||
)
|
||||
|
||||
llm_response = ModelResponse(
|
||||
llm_response: Final = ModelResponse(
|
||||
choices=[
|
||||
{
|
||||
"finish_reason": "tool_calls",
|
||||
|
|
@ -2289,18 +2290,18 @@ async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_argu
|
|||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
||||
side_effect=side_effect,
|
||||
) as mock_post:
|
||||
result = await guard.async_post_call_success_hook(
|
||||
result: Final = await guard.async_post_call_success_hook(
|
||||
data=request_data,
|
||||
response=llm_response,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
)
|
||||
|
||||
inspected = [call.kwargs["json"]["messages"][-1] for call in mock_post.call_args_list]
|
||||
inspected: Final = [call.kwargs["json"]["messages"][-1] for call in mock_post.call_args_list]
|
||||
assert inspected == [
|
||||
{"role": "assistant", "content": '{"query": "persimmon"}'},
|
||||
{"role": "assistant", "content": '{"city": "Paris"}'},
|
||||
]
|
||||
message = result.choices[0].message
|
||||
message: Final = result.choices[0].message
|
||||
assert message.provider_specific_fields == {
|
||||
"server_tool_calls": [
|
||||
{
|
||||
|
|
@ -2414,9 +2415,9 @@ async def test_post_call_success_hook_redacts_claude_hosted_web_fetch_url():
|
|||
from openai.types.responses import ResponseFunctionWebSearch
|
||||
from openai.types.responses.response_function_web_search import ActionOpenPage
|
||||
|
||||
guard = _make_guardrail()
|
||||
request_data = {"messages": [{"role": "user", "content": "read the page"}]}
|
||||
anonymize_response = _make_response(
|
||||
guard: Final = _make_guardrail()
|
||||
request_data: Final = {"messages": [{"role": "user", "content": "read the page"}]}
|
||||
anonymize_response: Final = _make_response(
|
||||
{
|
||||
"analysis_result": {"policy_drill_down": {"PII": {}}},
|
||||
"required_action": {"action_type": "anonymize_action", "policy_name": "PII"},
|
||||
|
|
@ -2428,7 +2429,7 @@ async def test_post_call_success_hook_redacts_claude_hosted_web_fetch_url():
|
|||
},
|
||||
}
|
||||
)
|
||||
response = _make_responses_api_response(
|
||||
response: Final = _make_responses_api_response(
|
||||
[
|
||||
ResponseFunctionWebSearch(
|
||||
id="ws_srvtoolu_01",
|
||||
|
|
@ -2443,12 +2444,12 @@ async def test_post_call_success_hook_redacts_claude_hosted_web_fetch_url():
|
|||
new_callable=AsyncMock,
|
||||
) as mock_post:
|
||||
mock_post.return_value = anonymize_response
|
||||
result = await guard.async_post_call_success_hook(
|
||||
result: Final = await guard.async_post_call_success_hook(
|
||||
data=request_data,
|
||||
response=response,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
)
|
||||
posted = mock_post.call_args.kwargs["json"]["messages"]
|
||||
posted: Final = mock_post.call_args.kwargs["json"]["messages"]
|
||||
assert posted[-1] == {"role": "assistant", "content": '{"url": "https://example.com/?ssn=123-45-6789"}'}
|
||||
assert (result.output[0].id, result.output[0].action.url) == (
|
||||
"ws_srvtoolu_01",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue