mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
test: annotate the new server tool call test locals with Final and type their parameters
This commit is contained in:
parent
b1d1f99b1c
commit
47fda7eece
18 changed files with 134 additions and 110 deletions
|
|
@ -112,7 +112,7 @@ def test_output_tool_calls_use_the_datadog_tool_call_schema(logger: DataDogLLMOb
|
|||
|
||||
|
||||
def test_claude_server_tool_calls_are_reported_before_client_tool_calls(logger: DataDogLLMObsLogger) -> None:
|
||||
payload = build(
|
||||
payload: Final = build(
|
||||
logger,
|
||||
response_message={
|
||||
"role": "assistant",
|
||||
|
|
|
|||
|
|
@ -378,7 +378,9 @@ class TestExtractMessageContent:
|
|||
],
|
||||
ids=["dict", "message"],
|
||||
)
|
||||
def test_server_tool_call_only_message_serializes_the_server_call(self, message):
|
||||
def test_server_tool_call_only_message_serializes_the_server_call(
|
||||
self, message: dict[str, object] | litellm.Message
|
||||
) -> None:
|
||||
assert json.loads(self.logger._extract_message_content(message)) == [_SERVER_CALL]
|
||||
|
||||
def test_multimodal_list_serialized_as_json(self):
|
||||
|
|
|
|||
|
|
@ -800,24 +800,26 @@ def test_responses_tool_calls_fold_into_the_assistant_message_with_tool_calls_fi
|
|||
|
||||
|
||||
def test_responses_claude_web_calls_lead_the_tool_calls_without_ending_the_turn_on_tool_calls():
|
||||
claude_fetch = build_web_search_call("srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch")
|
||||
openai_search = {
|
||||
claude_fetch: Final = (
|
||||
build_web_search_call("srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch")
|
||||
)
|
||||
openai_search: Final = {
|
||||
"type": "web_search_call",
|
||||
"id": "ws_1",
|
||||
"status": "completed",
|
||||
"action": {"type": "search", "query": "news"},
|
||||
}
|
||||
client_call = {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": "{}"}
|
||||
claude_tool_call = {
|
||||
client_call: Final = {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": "{}"}
|
||||
claude_tool_call: Final = {
|
||||
"id": "srvtoolu_01",
|
||||
"type": "function",
|
||||
"function": {"name": "web_fetch", "arguments": '{"url": "https://example.com/"}'},
|
||||
}
|
||||
|
||||
hosted_only = LLMCallSpanData.from_standard_logging_payload(
|
||||
hosted_only: Final = LLMCallSpanData.from_standard_logging_payload(
|
||||
_responses_payload([_RESPONSES_TEXT_ITEM, claude_fetch.model_dump(), openai_search]), capture_content=True
|
||||
)
|
||||
with_client_call = LLMCallSpanData.from_standard_logging_payload(
|
||||
with_client_call: Final = LLMCallSpanData.from_standard_logging_payload(
|
||||
_responses_payload([_RESPONSES_TEXT_ITEM, claude_fetch.model_dump(), client_call]), capture_content=True
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ def _claude_mapping(messages, response_obj):
|
|||
|
||||
|
||||
@pytest.fixture()
|
||||
def anthropic_prompts(monkeypatch):
|
||||
def anthropic_prompts(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""
|
||||
Stub the anthropic module unconditionally: the SDK may be absent (it lives in the
|
||||
proxy-runtime extra), and the tests/unit/llms/anthropic test package can
|
||||
|
|
@ -65,7 +65,7 @@ def test_claude_mapping_serializes_custom_tool_calls():
|
|||
|
||||
@pytest.mark.usefixtures("anthropic_prompts")
|
||||
def test_claude_mapping_puts_server_tool_calls_first_as_tool_use_blocks():
|
||||
response_obj = {
|
||||
response_obj: Final = {
|
||||
"id": "chatcmpl-1",
|
||||
"choices": [
|
||||
{
|
||||
|
|
@ -94,7 +94,7 @@ def test_claude_mapping_puts_server_tool_calls_first_as_tool_use_blocks():
|
|||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 2},
|
||||
}
|
||||
mapped = _claude_mapping([{"role": "user", "content": "hi"}], response_obj)
|
||||
mapped: Final = _claude_mapping([{"role": "user", "content": "hi"}], response_obj)
|
||||
assert mapped["content"] == [
|
||||
{"type": "tool_use", "id": "srvtoolu_01", "name": "web_search", "input": '{"query": "persimmon"}'},
|
||||
{"type": "tool_use", "id": "call_f", "name": "read_file", "input": '{"path": "a.py"}'},
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
|
||||
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.lunary import parse_messages, parse_tool_calls
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -39,7 +41,7 @@ def test_parse_tool_calls_none_passthrough():
|
|||
|
||||
|
||||
def test_parse_messages_lists_server_tool_calls_before_client_calls():
|
||||
response = litellm.ModelResponse(
|
||||
response: Final = litellm.ModelResponse(
|
||||
choices=[
|
||||
litellm.Choices(
|
||||
index=0,
|
||||
|
|
|
|||
|
|
@ -1906,7 +1906,7 @@ class TestShadowPipeline:
|
|||
assert "Let me look that up. [tool call] Read({})" in judge_prompt
|
||||
|
||||
async def test_a_server_tool_call_reaches_the_judge_ahead_of_client_tool_calls(self):
|
||||
router = _shadow_reply_router(
|
||||
router: Final = _shadow_reply_router(
|
||||
{
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
|
|
@ -1930,7 +1930,7 @@ class TestShadowPipeline:
|
|||
)
|
||||
await self._judged_shadow_row(router)
|
||||
|
||||
judge_prompt = next(
|
||||
judge_prompt: Final = next(
|
||||
call.kwargs["messages"][-1]["content"]
|
||||
for call in router.acompletion.call_args_list
|
||||
if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN
|
||||
|
|
|
|||
|
|
@ -957,7 +957,7 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_guardrail_rewrite_of_a_replayed_server_tool_call_lands_on_the_request(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
data: Final = {
|
||||
"messages": [
|
||||
{"role": "user", "content": "look up jane and the weather"},
|
||||
|
|
@ -984,9 +984,9 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_output_guardrail_sees_a_response_that_only_carries_server_tool_calls(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
guardrail = TestOpenAIChatCompletionsHandlerStreamingOutput._recording_guardrail()
|
||||
response = ModelResponse(
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
guardrail: Final = TestOpenAIChatCompletionsHandlerStreamingOutput._recording_guardrail()
|
||||
response: Final = ModelResponse(
|
||||
model="claude-sonnet-4-5",
|
||||
choices=[
|
||||
Choices(
|
||||
|
|
@ -1011,9 +1011,9 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_output_guardrail_rewrite_of_a_server_tool_call_lands_in_server_tool_calls_only(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
weather_args = json.dumps({"city": "Paris", "query": "forecast"})
|
||||
response = ModelResponse(
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
weather_args: Final = json.dumps({"city": "Paris", "query": "forecast"})
|
||||
response: Final = ModelResponse(
|
||||
model="claude-sonnet-4-5",
|
||||
choices=[
|
||||
Choices(
|
||||
|
|
@ -1070,8 +1070,8 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_output_guardrail_rewrite_of_a_server_web_call_reaches_its_hosted_copy(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
response = ModelResponse(
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
response: Final = ModelResponse(
|
||||
model="claude-sonnet-4-5",
|
||||
choices=[
|
||||
Choices(
|
||||
|
|
@ -1677,7 +1677,9 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)],
|
||||
)
|
||||
|
||||
search = {**_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}), "index": 0}
|
||||
search: Final = (
|
||||
{**_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}), "index": 0}
|
||||
)
|
||||
return [
|
||||
chunk(Delta(content="Searching")),
|
||||
chunk(Delta(content=None, provider_specific_fields={"server_tool_calls": [search]})),
|
||||
|
|
@ -1704,7 +1706,7 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
)
|
||||
}
|
||||
|
||||
fetch = {**_server_tool_call("srvtoolu_01Fetch", "web_fetch", _FETCH_INPUT), "index": 0}
|
||||
fetch: Final = {**_server_tool_call("srvtoolu_01Fetch", "web_fetch", _FETCH_INPUT), "index": 0}
|
||||
return [
|
||||
chunk(
|
||||
Delta(
|
||||
|
|
@ -1734,8 +1736,8 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_rewrites_reach_every_hosted_copy_of_a_server_web_call(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
chunks = self._ended_server_fetch_stream_chunks()
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
chunks: Final = self._ended_server_fetch_stream_chunks()
|
||||
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
|
|
@ -1755,8 +1757,8 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ended_stream_guardrail_sees_streamed_server_tool_calls(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
guardrail = self._recording_guardrail()
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
guardrail: Final = self._recording_guardrail()
|
||||
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=self._ended_server_tool_call_stream_chunks(),
|
||||
|
|
@ -1771,8 +1773,8 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_rewrites_writes_server_tool_call_arguments_back_into_chunks(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
chunks = self._ended_server_tool_call_stream_chunks()
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
chunks: Final = self._ended_server_tool_call_stream_chunks()
|
||||
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
|
|
@ -1781,7 +1783,7 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
|
||||
delta = chunks[1].choices[0].delta
|
||||
delta: Final = chunks[1].choices[0].delta
|
||||
assert _tool_call_ids_names_and_arguments(delta.provider_specific_fields["server_tool_calls"]) == [
|
||||
("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}')
|
||||
]
|
||||
|
|
@ -2652,8 +2654,8 @@ class TestStreamingScanKey:
|
|||
def test_server_tool_calls_hold_the_window_and_enter_the_key_once_the_stream_has_ended(self):
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
server_chunk = ModelResponseStream(
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
server_chunk: Final = ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
|
|
@ -2666,8 +2668,8 @@ class TestStreamingScanKey:
|
|||
)
|
||||
]
|
||||
)
|
||||
open_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk])
|
||||
ended_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk, self._chunk(None, "stop")])
|
||||
open_key: Final = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk])
|
||||
ended_key: Final = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk, self._chunk(None, "stop")])
|
||||
assert open_key.tool_calls_in_flight is True
|
||||
assert len(ended_key.tool_calls) == 1 and "web_search" in ended_key.tool_calls[0]
|
||||
assert ended_key != open_key
|
||||
|
|
|
|||
|
|
@ -1852,8 +1852,8 @@ class TestOpenAIResponsesHandlerToolCallExtraction:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_output_response_scans_and_rewrites_claude_web_calls_but_not_native_search(self):
|
||||
guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
|
||||
response = ResponsesAPIResponse(
|
||||
guardrail: Final = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
|
||||
response: Final = ResponsesAPIResponse(
|
||||
id="resp_1",
|
||||
created_at=1,
|
||||
model="claude-sonnet-5-5",
|
||||
|
|
@ -1862,7 +1862,7 @@ class TestOpenAIResponsesHandlerToolCallExtraction:
|
|||
output=_claude_web_calls_and_a_native_search(),
|
||||
)
|
||||
|
||||
result = await OpenAIResponsesHandler().process_output_response(response, guardrail)
|
||||
result: Final = await OpenAIResponsesHandler().process_output_response(response, guardrail)
|
||||
|
||||
assert guardrail.seen_tool_calls == [
|
||||
("web_search", '{"query": "persimmon"}'),
|
||||
|
|
@ -2816,8 +2816,8 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
|
|||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
|
||||
items = _claude_web_calls_and_a_native_search()
|
||||
added = [
|
||||
items: Final = _claude_web_calls_and_a_native_search()
|
||||
added: Final = [
|
||||
OutputItemAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
output_index=index,
|
||||
|
|
@ -2825,7 +2825,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
|
|||
)
|
||||
for index, item in enumerate(items)
|
||||
]
|
||||
done = [
|
||||
done: Final = [
|
||||
OutputItemDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=index,
|
||||
|
|
@ -2834,7 +2834,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
|
|||
)
|
||||
for index, item in enumerate(items)
|
||||
]
|
||||
completed = ResponseCompletedEvent(
|
||||
completed: Final = ResponseCompletedEvent(
|
||||
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
|
||||
response=ResponsesAPIResponse(
|
||||
id="resp_1",
|
||||
|
|
@ -2845,7 +2845,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
|
|||
output=_claude_web_calls_and_a_native_search(),
|
||||
),
|
||||
)
|
||||
guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
|
||||
guardrail: Final = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
|
||||
|
||||
await OpenAIResponsesHandler().process_output_streaming_response(
|
||||
responses_so_far=[*added, *done, completed],
|
||||
|
|
@ -2864,12 +2864,12 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_rewrites_dict_shaped_claude_web_calls_but_not_native_search(self):
|
||||
items = [item.model_dump() for item in _claude_web_calls_and_a_native_search()]
|
||||
done = [
|
||||
items: Final = [item.model_dump() for item in _claude_web_calls_and_a_native_search()]
|
||||
done: Final = [
|
||||
{"type": "response.output_item.done", "output_index": index, "item": dict(item)}
|
||||
for index, item in enumerate(items)
|
||||
]
|
||||
completed = {
|
||||
completed: Final = {
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_1",
|
||||
|
|
@ -2878,7 +2878,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
|
|||
"output": [dict(item) for item in items],
|
||||
},
|
||||
}
|
||||
guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
|
||||
guardrail: Final = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
|
||||
|
||||
await OpenAIResponsesHandler().process_output_streaming_response(
|
||||
responses_so_far=[*done, completed],
|
||||
|
|
@ -2899,7 +2899,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
|
|||
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
|
||||
from litellm.types.llms.openai import OutputItemDoneEvent, ResponsesAPIStreamEvents
|
||||
|
||||
done = OutputItemDoneEvent(
|
||||
done: Final = OutputItemDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=0,
|
||||
sequence_number=1,
|
||||
|
|
@ -2907,7 +2907,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
|
|||
**build_web_search_call("srvtoolu_1", {"query": "persimmon"}, None, "completed").model_dump()
|
||||
),
|
||||
)
|
||||
guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
|
||||
guardrail: Final = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
|
||||
|
||||
await OpenAIResponsesHandler().process_output_streaming_response(
|
||||
responses_so_far=[done],
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ plus the LiteLLM_DailyToolSpend rollup in one transaction.
|
|||
|
||||
from collections.abc import Awaitable, Callable
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from typing import Any, Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
|
|
@ -231,7 +231,7 @@ class TestResponseToolCallNames:
|
|||
},
|
||||
]
|
||||
}
|
||||
completion_response = ModelResponse(**response) if as_object else response
|
||||
completion_response: Final = ModelResponse(**response) if as_object else response
|
||||
|
||||
assert response_tool_call_names(completion_response) == ("web_search", "web_fetch", "get_weather")
|
||||
|
||||
|
|
|
|||
|
|
@ -540,9 +540,9 @@ class TestCiscoAIDefenseResponsesAPIOutput:
|
|||
|
||||
@pytest.mark.parametrize("action", _CLAUDE_HOSTED_WEB_ACTIONS)
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_scans_claude_hosted_web_call(self, action):
|
||||
g = _make_guardrail(event_hook="post_call")
|
||||
post_mock = AsyncMock(return_value=_safe_response())
|
||||
async def test_post_call_scans_claude_hosted_web_call(self, action: Action) -> None:
|
||||
g: Final = _make_guardrail(event_hook="post_call")
|
||||
post_mock: Final = AsyncMock(return_value=_safe_response())
|
||||
|
||||
with _patch_inspection_post(g, post_mock):
|
||||
await g.async_post_call_success_hook(
|
||||
|
|
@ -551,8 +551,8 @@ class TestCiscoAIDefenseResponsesAPIOutput:
|
|||
response=_claude_hosted_web_call_response(action),
|
||||
)
|
||||
|
||||
sent = post_mock.call_args.kwargs["json"]
|
||||
joined = " ".join(m.get("content", "") for m in (sent.get("messages") or []))
|
||||
sent: Final = post_mock.call_args.kwargs["json"]
|
||||
joined: Final = " ".join(m.get("content", "") for m in (sent.get("messages") or []))
|
||||
assert "123-45-6789" in joined
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -625,18 +625,18 @@ class TestCiscoAIDefenseResponsesAPIOutputRedaction:
|
|||
|
||||
@pytest.mark.parametrize("action", _CLAUDE_HOSTED_WEB_ACTIONS)
|
||||
@pytest.mark.asyncio
|
||||
async def test_redact_clears_claude_hosted_web_call(self, action):
|
||||
g = _make_guardrail(event_hook="post_call", on_flagged_action="monitor")
|
||||
cisco_resp = _redact_response(sanitized_text="[REDACTED]")
|
||||
async def test_redact_clears_claude_hosted_web_call(self, action: Action) -> None:
|
||||
g: Final = _make_guardrail(event_hook="post_call", on_flagged_action="monitor")
|
||||
cisco_resp: Final = _redact_response(sanitized_text="[REDACTED]")
|
||||
|
||||
with _patch_inspection_post(g, AsyncMock(return_value=cisco_resp)):
|
||||
result = await g.async_post_call_success_hook(
|
||||
result: Final = await g.async_post_call_success_hook(
|
||||
data={"input": [{"role": "user", "content": "look it up"}]},
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
response=_claude_hosted_web_call_response(action),
|
||||
)
|
||||
|
||||
call = result.output[0]
|
||||
call: Final = result.output[0]
|
||||
assert (call.id, call.action.type) == ("ws_srvtoolu_01", action.type)
|
||||
assert "123-45-6789" not in call.action.model_dump_json()
|
||||
|
||||
|
|
@ -1730,8 +1730,8 @@ class TestCiscoAIDefenseToolCallBypass:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_scans_function_tool_call_that_also_carries_a_custom_key(self):
|
||||
g = _make_guardrail(event_hook="pre_call")
|
||||
post_mock = AsyncMock(return_value=_safe_response())
|
||||
g: Final = _make_guardrail(event_hook="pre_call")
|
||||
post_mock: Final = AsyncMock(return_value=_safe_response())
|
||||
data: Final = {
|
||||
"messages": [
|
||||
{
|
||||
|
|
@ -1757,7 +1757,7 @@ class TestCiscoAIDefenseToolCallBypass:
|
|||
call_type="completion",
|
||||
)
|
||||
|
||||
sent = post_mock.call_args.kwargs["json"]
|
||||
sent: Final = post_mock.call_args.kwargs["json"]
|
||||
assert "123-45-6789" in " ".join(m.get("content", "") for m in (sent.get("messages") or []))
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -1912,8 +1912,8 @@ class TestCiscoAIDefenseToolCallBypass:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_scan_sends_server_tool_call_arguments_to_cisco(self):
|
||||
g = _make_guardrail(event_hook="post_call")
|
||||
post_mock = AsyncMock(return_value=_safe_response())
|
||||
g: Final = _make_guardrail(event_hook="post_call")
|
||||
post_mock: Final = AsyncMock(return_value=_safe_response())
|
||||
|
||||
with _patch_inspection_post(g, post_mock):
|
||||
await g.async_post_call_success_hook(
|
||||
|
|
@ -1931,17 +1931,17 @@ class TestCiscoAIDefenseToolCallBypass:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_redact_clears_server_tool_call_arguments(self):
|
||||
g = _make_guardrail(event_hook="post_call", on_flagged_action="monitor")
|
||||
cisco_resp = _redact_response(sanitized_text="[REDACTED]")
|
||||
g: Final = _make_guardrail(event_hook="post_call", on_flagged_action="monitor")
|
||||
cisco_resp: Final = _redact_response(sanitized_text="[REDACTED]")
|
||||
|
||||
with _patch_inspection_post(g, AsyncMock(return_value=cisco_resp)):
|
||||
result = await g.async_post_call_success_hook(
|
||||
result: Final = await g.async_post_call_success_hook(
|
||||
data={"messages": [{"role": "user", "content": "look up persimmon"}]},
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
response=self._server_tool_call_response(),
|
||||
)
|
||||
|
||||
message = result.choices[0].message
|
||||
message: Final = result.choices[0].message
|
||||
assert message.provider_specific_fields == {
|
||||
"server_tool_calls": [
|
||||
{"id": "srvtoolu_01", "type": "function", "function": {"name": "web_search", "arguments": "{}"}}
|
||||
|
|
|
|||
|
|
@ -896,7 +896,7 @@ class TestRestoration:
|
|||
@pytest.mark.asyncio
|
||||
async def test_server_tool_call_arguments_are_restored(self):
|
||||
guardrail, _ = _shielded({"[EMAIL_1]": "a@b.com", "[URL_1]": "https://b.com/a"})
|
||||
response = ModelResponse(
|
||||
response: Final = ModelResponse(
|
||||
choices=[
|
||||
Choices(
|
||||
index=0,
|
||||
|
|
@ -923,7 +923,7 @@ class TestRestoration:
|
|||
]
|
||||
)
|
||||
|
||||
result = await guardrail.async_post_call_success_hook(
|
||||
result: Final = await guardrail.async_post_call_success_hook(
|
||||
data={"messages": []}, user_api_key_dict=None, response=response
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2186,8 +2186,8 @@ class TestCompletionResponseTextPartsToolCalls:
|
|||
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
guardrail = _make_guardrail()
|
||||
response = ResponsesAPIResponse(
|
||||
guardrail: Final = _make_guardrail()
|
||||
response: Final = ResponsesAPIResponse(
|
||||
id="resp-ws-1",
|
||||
created_at=0,
|
||||
output=[
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Any, Final
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch
|
||||
|
|
@ -373,7 +375,7 @@ async def test_pangea_ai_guard_response_transformed(pangea_guardrail):
|
|||
assert response.choices[0]["message"]["content"] == "Yes, here is an SSN: <US_SSN>"
|
||||
|
||||
|
||||
def _claude_fetch_call(url):
|
||||
def _claude_fetch_call(url: str) -> dict[str, object]:
|
||||
return {
|
||||
"id": "srvtoolu_01",
|
||||
"type": "function",
|
||||
|
|
@ -381,20 +383,22 @@ def _claude_fetch_call(url):
|
|||
}
|
||||
|
||||
|
||||
def _with_masked_tool_calls(choice):
|
||||
message = choice["message"]
|
||||
masked_calls = [_claude_fetch_call("<URL>") for _ in message["tool_calls"]]
|
||||
def _with_masked_tool_calls(choice: Mapping[str, Any]) -> dict[str, object]:
|
||||
message: Final = choice["message"]
|
||||
masked_calls: Final = [_claude_fetch_call("<URL>") for _ in message["tool_calls"]]
|
||||
return {**choice, "message": {**message, "tool_calls": masked_calls}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pangea_ai_guard_response_masks_claude_server_tool_call_inputs(pangea_guardrail):
|
||||
data = {"messages": [{"role": "user", "content": "Read https://example.com/"}]}
|
||||
fetch = build_web_search_call("srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch")
|
||||
sent_choices = []
|
||||
async def test_pangea_ai_guard_response_masks_claude_server_tool_call_inputs(pangea_guardrail: PangeaHandler) -> None:
|
||||
data: Final = {"messages": [{"role": "user", "content": "Read https://example.com/"}]}
|
||||
fetch: Final = build_web_search_call("srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch")
|
||||
sent_choices: Final = []
|
||||
|
||||
async def _mask_tool_call_urls(_handler, **kwargs):
|
||||
choices = kwargs["json"]["input"]["choices"]
|
||||
async def _mask_tool_call_urls(
|
||||
_handler: object, url: str, json: Mapping[str, Any], **_kwargs: object
|
||||
) -> httpx.Response:
|
||||
choices: Final = json["input"]["choices"]
|
||||
sent_choices.extend(choices)
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
|
|
@ -405,11 +409,11 @@ async def test_pangea_ai_guard_response_masks_claude_server_tool_call_inputs(pan
|
|||
"output": {"choices": [_with_masked_tool_calls(choice) for choice in choices]},
|
||||
}
|
||||
},
|
||||
request=httpx.Request(method="POST", url=kwargs["url"]),
|
||||
request=httpx.Request(method="POST", url=url),
|
||||
)
|
||||
|
||||
with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=_mask_tool_call_urls):
|
||||
response = await pangea_guardrail.async_post_call_success_hook(
|
||||
response: Final = await pangea_guardrail.async_post_call_success_hook(
|
||||
data=data,
|
||||
user_api_key_dict=None,
|
||||
response=ModelResponse(
|
||||
|
|
@ -429,8 +433,10 @@ async def test_pangea_ai_guard_response_masks_claude_server_tool_call_inputs(pan
|
|||
)
|
||||
|
||||
assert sent_choices[0]["message"]["tool_calls"] == [_claude_fetch_call("https://example.com/")]
|
||||
message = response.choices[0]["message"]
|
||||
message: Final = response.choices[0]["message"]
|
||||
assert message["tool_calls"] is None
|
||||
assert message["provider_specific_fields"]["server_tool_calls"] == [_claude_fetch_call("<URL>")]
|
||||
hosted_copy = ResponseFunctionWebSearch.model_validate(message["provider_specific_fields"]["web_search_calls"][0])
|
||||
hosted_copy: Final = (
|
||||
ResponseFunctionWebSearch.model_validate(message["provider_specific_fields"]["web_search_calls"][0])
|
||||
)
|
||||
assert hosted_copy.action.url == "<URL>"
|
||||
|
|
|
|||
|
|
@ -6548,7 +6548,9 @@ class TestPanwAirsServerToolCalls:
|
|||
return handler, transport
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_scan_sends_server_tool_call_query_to_airs(self, user_api_key_dict):
|
||||
async def test_post_call_scan_sends_server_tool_call_query_to_airs(
|
||||
self, user_api_key_dict: UserAPIKeyAuth
|
||||
) -> None:
|
||||
handler, transport = self._airs({"action": "allow", "category": "benign", "scan_id": "s1"})
|
||||
response: Final = self._response(
|
||||
ChatCompletionMessageToolCall(
|
||||
|
|
@ -6568,7 +6570,9 @@ class TestPanwAirsServerToolCalls:
|
|||
assert sent_contents == [[{"response": '{"query": "persimmon"} {"city": "Paris"}'}]]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_masking_rewrites_server_tool_call_arguments(self, user_api_key_dict):
|
||||
async def test_post_call_masking_rewrites_server_tool_call_arguments(
|
||||
self, user_api_key_dict: UserAPIKeyAuth
|
||||
) -> None:
|
||||
masked: Final = '{"query": "*********"}'
|
||||
handler, _ = self._airs(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1784,7 +1784,7 @@ async def test_pii_tokens_in_metadata_used_for_unmasking():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_output_unmasking_restores_server_tool_call_arguments(mock_user_api_key):
|
||||
async def test_output_unmasking_restores_server_tool_call_arguments(mock_user_api_key: UserAPIKeyAuth) -> None:
|
||||
guardrail: Final = OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
|
||||
response: Final = ModelResponse(
|
||||
choices=[
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import os
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -853,9 +855,9 @@ class TestRepelloAIPostCall:
|
|||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_tool_call_arguments_scanned(self, monkeypatch):
|
||||
guardrail = _guardrail(event_hook="post_call")
|
||||
response = ModelResponse(
|
||||
async def test_server_tool_call_arguments_scanned(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
guardrail: Final = _guardrail(event_hook="post_call")
|
||||
response: Final = ModelResponse(
|
||||
choices=[
|
||||
Choices(
|
||||
index=0,
|
||||
|
|
@ -889,9 +891,11 @@ class TestRepelloAIPostCall:
|
|||
)
|
||||
]
|
||||
)
|
||||
captured = {}
|
||||
captured: Final = {}
|
||||
|
||||
async def capture(url, headers, json, **_kwargs):
|
||||
async def capture(
|
||||
url: str, headers: Mapping[str, str], json: Mapping[str, object], **_kwargs: object
|
||||
) -> Response:
|
||||
captured["json"] = json
|
||||
return _verdict_response("passed", url)
|
||||
|
||||
|
|
|
|||
|
|
@ -1291,7 +1291,7 @@ class TestStreamingTransform:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claude_server_tool_call_chunks_keep_their_provider_fields(self):
|
||||
server_fields = {
|
||||
server_fields: Final = {
|
||||
"server_tool_calls": [
|
||||
{
|
||||
"id": "srvtoolu_01",
|
||||
|
|
@ -1305,17 +1305,19 @@ class TestStreamingTransform:
|
|||
).model_dump()
|
||||
},
|
||||
}
|
||||
server_chunk = ModelResponseStream(
|
||||
server_chunk: Final = ModelResponseStream(
|
||||
choices=[StreamingChoices(index=0, delta=Delta(content=None, provider_specific_fields=server_fields))],
|
||||
)
|
||||
|
||||
out = await _drive_stream(
|
||||
out: Final = await _drive_stream(
|
||||
UnifiedLLMGuardrails(),
|
||||
_StreamingTextGuardrail(),
|
||||
[server_chunk, _stream_chunk("done", finish_reason="stop")],
|
||||
)
|
||||
|
||||
delivered = [getattr(item.choices[0].delta, "provider_specific_fields", None) for item in out if item.choices]
|
||||
delivered: Final = (
|
||||
[getattr(item.choices[0].delta, "provider_specific_fields", None) for item in out if item.choices]
|
||||
)
|
||||
assert server_fields in delivered
|
||||
assert "".join(_delta_text(item) for item in out) == "DONE"
|
||||
|
||||
|
|
|
|||
|
|
@ -672,18 +672,18 @@ async def test_post_call_hook_with_tool_calls(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_hook_blocks_a_flagged_claude_server_tool_call(
|
||||
env_setup,
|
||||
sample_request_data,
|
||||
user_api_key_dict,
|
||||
pillar_clean_response,
|
||||
pillar_flagged_response,
|
||||
):
|
||||
server_call = {
|
||||
env_setup: None,
|
||||
sample_request_data: dict[str, object],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
pillar_clean_response: Response,
|
||||
pillar_flagged_response: Response,
|
||||
) -> None:
|
||||
server_call: Final = {
|
||||
"id": "srvtoolu_01",
|
||||
"type": "function",
|
||||
"function": {"name": "web_fetch", "arguments": '{"url": "https://example.com/"}'},
|
||||
}
|
||||
response = ModelResponse(
|
||||
response: Final = ModelResponse(
|
||||
choices=[
|
||||
Choices(
|
||||
message=Message(
|
||||
|
|
@ -709,7 +709,7 @@ async def test_post_call_hook_blocks_a_flagged_claude_server_tool_call(
|
|||
scanned_tool_calls: Final = messages[-1].get("tool_calls") or []
|
||||
return pillar_flagged_response if server_call in scanned_tool_calls else pillar_clean_response
|
||||
|
||||
guardrail = PillarGuardrail(
|
||||
guardrail: Final = PillarGuardrail(
|
||||
guardrail_name="pillar-test",
|
||||
api_key="test-pillar-key",
|
||||
api_base="https://api.pillar.security",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue