test(guardrails): drive the failure-log masking tests through in-memory Presidio and Guardrails AI transports

This commit is contained in:
mateo-berri 2026-10-31 13:32:57 -07:00
parent d16efa05d6
commit 0232e11705
2 changed files with 80 additions and 52 deletions

View file

@ -1,3 +1,4 @@
import json
from unittest.mock import AsyncMock, patch
import httpx
@ -232,6 +233,7 @@ async def test_guardrails_ai_process_input():
@pytest.mark.asyncio
async def test_guardrails_ai_logging_only_masks_the_failure_log(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy.guardrails.guardrail_hooks.guardrails_ai.guardrails_ai import GuardrailsAIResponse
from litellm.types.guardrails import GuardrailEventHooks
@ -249,21 +251,32 @@ async def test_guardrails_ai_logging_only_masks_the_failure_log(monkeypatch: pyt
api_base="http://test.example.com",
event_hook=GuardrailEventHooks.logging_only,
)
validated: list[object] = []
def _validate(request: httpx.Request) -> httpx.Response:
validated.append((request.url.path, json.loads(request.content)["llmOutput"]))
return httpx.Response(
200,
json=GuardrailsAIResponse(
callId="call-1",
rawLlmOutput="My name is John Smith",
validatedOutput="My name is <PERSON>",
validationPassed=True,
),
)
for name in ("input_callback", "success_callback", "failure_callback", "_async_success_callback"):
monkeypatch.setattr(litellm, name, [])
monkeypatch.setattr(litellm, "_async_failure_callback", [])
monkeypatch.setattr(litellm, "callbacks", [guardrail, _FailureLogRecorder()])
monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler(transport=httpx.MockTransport(_validate)))
with patch.object(
guardrail,
"make_guardrails_ai_api_request",
AsyncMock(return_value=GuardrailsAIResponse(validatedOutput="My name is <PERSON>")),
):
with pytest.raises(litellm.AuthenticationError):
await litellm.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "My name is John Smith"}],
mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
)
with pytest.raises(litellm.AuthenticationError):
await litellm.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "My name is John Smith"}],
mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
)
assert validated == [("/guards/pii-guard/validate", "My name is John Smith")]
assert logged == [[{"role": "user", "content": "My name is <PERSON>"}]]

View file

@ -5,6 +5,7 @@ Tests PII detection and masking for different message formats
import asyncio
import copy
import functools
import json
import os
import re
@ -4796,45 +4797,6 @@ async def test_pii_masking_replays_a_byte_identical_prefix_across_turns(mock_use
assert later[3]["content"] == "Now compare against <PERSON> too."
class _FailureLogRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.logged: list[tuple[object, object]] = []
async def async_log_failure_event(
self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
payload: Final = cast(StandardLoggingPayload, kwargs["standard_logging_object"])
self.logged.append((copy.deepcopy(kwargs["messages"]), copy.deepcopy(payload["messages"])))
async def test_logging_only_presidio_masks_the_prompt_in_the_failure_log(monkeypatch: pytest.MonkeyPatch) -> None:
"""A failed call is logged with the prompt a logging_only Presidio guardrail masked, the same as a
successful one. The analyzer and anonymizer are the in-process fake handed over through api_base."""
recorder: Final = _FailureLogRecorder()
for name in ("input_callback", "success_callback", "failure_callback", "_async_success_callback"):
monkeypatch.setattr(litellm, name, [])
monkeypatch.setattr(litellm, "_async_failure_callback", [])
async with TestServer(_fake_presidio_app()) as server:
guardrail: Final = OPTIONAL_PresidioPIIMasking(
presidio_analyzer_api_base=str(server.make_url("/")),
presidio_anonymizer_api_base=str(server.make_url("/")),
pii_entities_config={PiiEntityType.PERSON: PiiAction.MASK},
logging_only=True,
)
monkeypatch.setattr(litellm, "callbacks", [guardrail, recorder])
with pytest.raises(litellm.AuthenticationError):
await litellm.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "My name is John Smith"}],
mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
)
await guardrail._close_http_session()
masked: Final = [{"role": "user", "content": "My name is <PERSON>"}]
assert recorder.logged == [(masked, masked)]
@pytest.mark.asyncio
@pytest.mark.parametrize("surface", ["mcp_arguments", "mcp_result", "llm_output"])
@pytest.mark.parametrize("action", [PiiAction.MASK, PiiAction.BLOCK])
@ -5240,8 +5202,15 @@ _BLOCK_CARD_MASK_EMAIL: Final = {
}
class _AnonymizedSpan(TypedDict):
entity_type: ReadOnly[str]
start: ReadOnly[int]
end: ReadOnly[int]
class _PresidioAnonymizeRequest(TypedDict):
text: ReadOnly[str]
analyzer_results: ReadOnly[tuple[_AnonymizedSpan, ...]]
class _PresidioAnonymizeReply(TypedDict):
@ -5253,6 +5222,14 @@ _ANALYZE_REQUEST: Final = TypeAdapter(PresidioAnalyzeRequest)
_ANONYMIZE_REQUEST: Final = TypeAdapter(_PresidioAnonymizeRequest)
def _anonymized(text: str, spans: Sequence[_AnonymizedSpan]) -> str:
return functools.reduce(
lambda anonymized, span: f"{anonymized[: span['start']]}<{span['entity_type']}>{anonymized[span['end'] :]}",
sorted(spans, key=lambda span: span["start"], reverse=True),
text,
)
def _card_and_email_spans(text: str) -> tuple[PresidioAnalyzeResponseItem, ...]:
return tuple(
PresidioAnalyzeResponseItem(
@ -5310,7 +5287,10 @@ class _InMemoryPresidioTransport(asyncio.Transport):
self._analyzed.put_nowait(analyze_request)
return _card_and_email_spans(analyze_request.get("text") or "")
assert path == "/anonymize", path
return _PresidioAnonymizeReply(text=_ANONYMIZE_REQUEST.validate_json(body)["text"], items=())
anonymize_request: Final = _ANONYMIZE_REQUEST.validate_json(body)
return _PresidioAnonymizeReply(
text=_anonymized(anonymize_request["text"], anonymize_request["analyzer_results"]), items=()
)
def writelines(self, list_of_data: Iterable[bytes | bytearray | memoryview]) -> None:
self.write(b"".join(bytes(chunk) for chunk in list_of_data))
@ -5361,12 +5341,13 @@ def _drain(analyzed: asyncio.Queue[PresidioAnalyzeRequest]) -> tuple[PresidioAna
def _guardrail_with_in_memory_presidio(
analyzed: asyncio.Queue[PresidioAnalyzeRequest], replies_wait_for: int = 0
analyzed: asyncio.Queue[PresidioAnalyzeRequest], replies_wait_for: int = 0, logging_only: bool = False
) -> OPTIONAL_PresidioPIIMasking:
guardrail: Final = OPTIONAL_PresidioPIIMasking(
pii_entities_config=_BLOCK_CARD_MASK_EMAIL,
presidio_analyzer_api_base="http://presidio-analyzer.test/",
presidio_anonymizer_api_base="http://presidio-anonymizer.test/",
logging_only=logging_only,
)
guardrail._http_session = aiohttp.ClientSession(connector=_InMemoryPresidioConnector(analyzed, replies_wait_for))
return guardrail
@ -5392,6 +5373,40 @@ async def test_presidio_scans_every_message_of_a_request_at_once() -> None:
assert sorted(request.get("text") or "" for request in _drain(analyzed)) == sorted(texts)
class _FailureLogRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.logged: list[tuple[object, object]] = []
async def async_log_failure_event(
self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
payload: Final = cast(StandardLoggingPayload, kwargs["standard_logging_object"])
self.logged.append((copy.deepcopy(kwargs["messages"]), copy.deepcopy(payload["messages"])))
@pytest.mark.asyncio
async def test_logging_only_presidio_masks_the_prompt_in_the_failure_log(monkeypatch: pytest.MonkeyPatch) -> None:
recorder: Final = _FailureLogRecorder()
for name in ("input_callback", "success_callback", "failure_callback", "_async_success_callback"):
monkeypatch.setattr(litellm, name, [])
monkeypatch.setattr(litellm, "_async_failure_callback", [])
guardrail: Final = _guardrail_with_in_memory_presidio(asyncio.Queue(), logging_only=True)
monkeypatch.setattr(litellm, "callbacks", [guardrail, recorder])
try:
with pytest.raises(litellm.AuthenticationError):
await litellm.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": f"My email is {_EMAIL}"}],
mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
)
finally:
await guardrail._close_http_session()
masked: Final = [{"role": "user", "content": "My email is <EMAIL_ADDRESS>"}]
assert recorder.logged == [(masked, masked)]
@pytest.mark.asyncio
async def test_check_pii_raises_blocked_entity_for_card() -> None:
text: Final = f"My credit card number is {_CARD_NUMBER} and my email is {_EMAIL}"