mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
test(guardrails): drive the failure-log masking tests through in-memory Presidio and Guardrails AI transports
This commit is contained in:
parent
d16efa05d6
commit
0232e11705
2 changed files with 80 additions and 52 deletions
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -232,6 +233,7 @@ async def test_guardrails_ai_process_input():
|
|||
@pytest.mark.asyncio
|
||||
async def test_guardrails_ai_logging_only_masks_the_failure_log(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy.guardrails.guardrail_hooks.guardrails_ai.guardrails_ai import GuardrailsAIResponse
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
|
|
@ -249,21 +251,32 @@ async def test_guardrails_ai_logging_only_masks_the_failure_log(monkeypatch: pyt
|
|||
api_base="http://test.example.com",
|
||||
event_hook=GuardrailEventHooks.logging_only,
|
||||
)
|
||||
validated: list[object] = []
|
||||
|
||||
def _validate(request: httpx.Request) -> httpx.Response:
|
||||
validated.append((request.url.path, json.loads(request.content)["llmOutput"]))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json=GuardrailsAIResponse(
|
||||
callId="call-1",
|
||||
rawLlmOutput="My name is John Smith",
|
||||
validatedOutput="My name is <PERSON>",
|
||||
validationPassed=True,
|
||||
),
|
||||
)
|
||||
|
||||
for name in ("input_callback", "success_callback", "failure_callback", "_async_success_callback"):
|
||||
monkeypatch.setattr(litellm, name, [])
|
||||
monkeypatch.setattr(litellm, "_async_failure_callback", [])
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail, _FailureLogRecorder()])
|
||||
monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler(transport=httpx.MockTransport(_validate)))
|
||||
|
||||
with patch.object(
|
||||
guardrail,
|
||||
"make_guardrails_ai_api_request",
|
||||
AsyncMock(return_value=GuardrailsAIResponse(validatedOutput="My name is <PERSON>")),
|
||||
):
|
||||
with pytest.raises(litellm.AuthenticationError):
|
||||
await litellm.acompletion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "My name is John Smith"}],
|
||||
mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
|
||||
)
|
||||
with pytest.raises(litellm.AuthenticationError):
|
||||
await litellm.acompletion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "My name is John Smith"}],
|
||||
mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
|
||||
)
|
||||
|
||||
assert validated == [("/guards/pii-guard/validate", "My name is John Smith")]
|
||||
assert logged == [[{"role": "user", "content": "My name is <PERSON>"}]]
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Tests PII detection and masking for different message formats
|
|||
|
||||
import asyncio
|
||||
import copy
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
|
@ -4796,45 +4797,6 @@ async def test_pii_masking_replays_a_byte_identical_prefix_across_turns(mock_use
|
|||
assert later[3]["content"] == "Now compare against <PERSON> too."
|
||||
|
||||
|
||||
class _FailureLogRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.logged: list[tuple[object, object]] = []
|
||||
|
||||
async def async_log_failure_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
payload: Final = cast(StandardLoggingPayload, kwargs["standard_logging_object"])
|
||||
self.logged.append((copy.deepcopy(kwargs["messages"]), copy.deepcopy(payload["messages"])))
|
||||
|
||||
|
||||
async def test_logging_only_presidio_masks_the_prompt_in_the_failure_log(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A failed call is logged with the prompt a logging_only Presidio guardrail masked, the same as a
|
||||
successful one. The analyzer and anonymizer are the in-process fake handed over through api_base."""
|
||||
recorder: Final = _FailureLogRecorder()
|
||||
for name in ("input_callback", "success_callback", "failure_callback", "_async_success_callback"):
|
||||
monkeypatch.setattr(litellm, name, [])
|
||||
monkeypatch.setattr(litellm, "_async_failure_callback", [])
|
||||
async with TestServer(_fake_presidio_app()) as server:
|
||||
guardrail: Final = OPTIONAL_PresidioPIIMasking(
|
||||
presidio_analyzer_api_base=str(server.make_url("/")),
|
||||
presidio_anonymizer_api_base=str(server.make_url("/")),
|
||||
pii_entities_config={PiiEntityType.PERSON: PiiAction.MASK},
|
||||
logging_only=True,
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail, recorder])
|
||||
with pytest.raises(litellm.AuthenticationError):
|
||||
await litellm.acompletion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "My name is John Smith"}],
|
||||
mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
|
||||
)
|
||||
await guardrail._close_http_session()
|
||||
|
||||
masked: Final = [{"role": "user", "content": "My name is <PERSON>"}]
|
||||
assert recorder.logged == [(masked, masked)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("surface", ["mcp_arguments", "mcp_result", "llm_output"])
|
||||
@pytest.mark.parametrize("action", [PiiAction.MASK, PiiAction.BLOCK])
|
||||
|
|
@ -5240,8 +5202,15 @@ _BLOCK_CARD_MASK_EMAIL: Final = {
|
|||
}
|
||||
|
||||
|
||||
class _AnonymizedSpan(TypedDict):
|
||||
entity_type: ReadOnly[str]
|
||||
start: ReadOnly[int]
|
||||
end: ReadOnly[int]
|
||||
|
||||
|
||||
class _PresidioAnonymizeRequest(TypedDict):
|
||||
text: ReadOnly[str]
|
||||
analyzer_results: ReadOnly[tuple[_AnonymizedSpan, ...]]
|
||||
|
||||
|
||||
class _PresidioAnonymizeReply(TypedDict):
|
||||
|
|
@ -5253,6 +5222,14 @@ _ANALYZE_REQUEST: Final = TypeAdapter(PresidioAnalyzeRequest)
|
|||
_ANONYMIZE_REQUEST: Final = TypeAdapter(_PresidioAnonymizeRequest)
|
||||
|
||||
|
||||
def _anonymized(text: str, spans: Sequence[_AnonymizedSpan]) -> str:
|
||||
return functools.reduce(
|
||||
lambda anonymized, span: f"{anonymized[: span['start']]}<{span['entity_type']}>{anonymized[span['end'] :]}",
|
||||
sorted(spans, key=lambda span: span["start"], reverse=True),
|
||||
text,
|
||||
)
|
||||
|
||||
|
||||
def _card_and_email_spans(text: str) -> tuple[PresidioAnalyzeResponseItem, ...]:
|
||||
return tuple(
|
||||
PresidioAnalyzeResponseItem(
|
||||
|
|
@ -5310,7 +5287,10 @@ class _InMemoryPresidioTransport(asyncio.Transport):
|
|||
self._analyzed.put_nowait(analyze_request)
|
||||
return _card_and_email_spans(analyze_request.get("text") or "")
|
||||
assert path == "/anonymize", path
|
||||
return _PresidioAnonymizeReply(text=_ANONYMIZE_REQUEST.validate_json(body)["text"], items=())
|
||||
anonymize_request: Final = _ANONYMIZE_REQUEST.validate_json(body)
|
||||
return _PresidioAnonymizeReply(
|
||||
text=_anonymized(anonymize_request["text"], anonymize_request["analyzer_results"]), items=()
|
||||
)
|
||||
|
||||
def writelines(self, list_of_data: Iterable[bytes | bytearray | memoryview]) -> None:
|
||||
self.write(b"".join(bytes(chunk) for chunk in list_of_data))
|
||||
|
|
@ -5361,12 +5341,13 @@ def _drain(analyzed: asyncio.Queue[PresidioAnalyzeRequest]) -> tuple[PresidioAna
|
|||
|
||||
|
||||
def _guardrail_with_in_memory_presidio(
|
||||
analyzed: asyncio.Queue[PresidioAnalyzeRequest], replies_wait_for: int = 0
|
||||
analyzed: asyncio.Queue[PresidioAnalyzeRequest], replies_wait_for: int = 0, logging_only: bool = False
|
||||
) -> OPTIONAL_PresidioPIIMasking:
|
||||
guardrail: Final = OPTIONAL_PresidioPIIMasking(
|
||||
pii_entities_config=_BLOCK_CARD_MASK_EMAIL,
|
||||
presidio_analyzer_api_base="http://presidio-analyzer.test/",
|
||||
presidio_anonymizer_api_base="http://presidio-anonymizer.test/",
|
||||
logging_only=logging_only,
|
||||
)
|
||||
guardrail._http_session = aiohttp.ClientSession(connector=_InMemoryPresidioConnector(analyzed, replies_wait_for))
|
||||
return guardrail
|
||||
|
|
@ -5392,6 +5373,40 @@ async def test_presidio_scans_every_message_of_a_request_at_once() -> None:
|
|||
assert sorted(request.get("text") or "" for request in _drain(analyzed)) == sorted(texts)
|
||||
|
||||
|
||||
class _FailureLogRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.logged: list[tuple[object, object]] = []
|
||||
|
||||
async def async_log_failure_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
payload: Final = cast(StandardLoggingPayload, kwargs["standard_logging_object"])
|
||||
self.logged.append((copy.deepcopy(kwargs["messages"]), copy.deepcopy(payload["messages"])))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logging_only_presidio_masks_the_prompt_in_the_failure_log(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
recorder: Final = _FailureLogRecorder()
|
||||
for name in ("input_callback", "success_callback", "failure_callback", "_async_success_callback"):
|
||||
monkeypatch.setattr(litellm, name, [])
|
||||
monkeypatch.setattr(litellm, "_async_failure_callback", [])
|
||||
guardrail: Final = _guardrail_with_in_memory_presidio(asyncio.Queue(), logging_only=True)
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail, recorder])
|
||||
try:
|
||||
with pytest.raises(litellm.AuthenticationError):
|
||||
await litellm.acompletion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": f"My email is {_EMAIL}"}],
|
||||
mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
|
||||
)
|
||||
finally:
|
||||
await guardrail._close_http_session()
|
||||
|
||||
masked: Final = [{"role": "user", "content": "My email is <EMAIL_ADDRESS>"}]
|
||||
assert recorder.logged == [(masked, masked)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_pii_raises_blocked_entity_for_card() -> None:
|
||||
text: Final = f"My credit card number is {_CARD_NUMBER} and my email is {_EMAIL}"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue