diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/guardrails_ai/test_guardrails_ai.py b/tests/unit/proxy/guardrails/guardrail_hooks/guardrails_ai/test_guardrails_ai.py index 608acc01865..4b9ac1187cf 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/guardrails_ai/test_guardrails_ai.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/guardrails_ai/test_guardrails_ai.py @@ -1,3 +1,4 @@ +import json from unittest.mock import AsyncMock, patch import httpx @@ -232,6 +233,7 @@ async def test_guardrails_ai_process_input(): @pytest.mark.asyncio async def test_guardrails_ai_logging_only_masks_the_failure_log(monkeypatch: pytest.MonkeyPatch) -> None: from litellm.integrations.custom_logger import CustomLogger + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy.guardrails.guardrail_hooks.guardrails_ai.guardrails_ai import GuardrailsAIResponse from litellm.types.guardrails import GuardrailEventHooks @@ -249,21 +251,32 @@ async def test_guardrails_ai_logging_only_masks_the_failure_log(monkeypatch: pyt api_base="http://test.example.com", event_hook=GuardrailEventHooks.logging_only, ) + validated: list[object] = [] + + def _validate(request: httpx.Request) -> httpx.Response: + validated.append((request.url.path, json.loads(request.content)["llmOutput"])) + return httpx.Response( + 200, + json=GuardrailsAIResponse( + callId="call-1", + rawLlmOutput="My name is John Smith", + validatedOutput="My name is ", + validationPassed=True, + ), + ) + for name in ("input_callback", "success_callback", "failure_callback", "_async_success_callback"): monkeypatch.setattr(litellm, name, []) monkeypatch.setattr(litellm, "_async_failure_callback", []) monkeypatch.setattr(litellm, "callbacks", [guardrail, _FailureLogRecorder()]) + monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler(transport=httpx.MockTransport(_validate))) - with patch.object( - guardrail, - "make_guardrails_ai_api_request", - AsyncMock(return_value=GuardrailsAIResponse(validatedOutput="My name is ")), - ): - with pytest.raises(litellm.AuthenticationError): - await litellm.acompletion( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "My name is John Smith"}], - mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"), - ) + with pytest.raises(litellm.AuthenticationError): + await litellm.acompletion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "My name is John Smith"}], + mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"), + ) + assert validated == [("/guards/pii-guard/validate", "My name is John Smith")] assert logged == [[{"role": "user", "content": "My name is "}]] diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py index 4ad8fbf1c20..621e3d56de6 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -5,6 +5,7 @@ Tests PII detection and masking for different message formats import asyncio import copy +import functools import json import os import re @@ -4796,45 +4797,6 @@ async def test_pii_masking_replays_a_byte_identical_prefix_across_turns(mock_use assert later[3]["content"] == "Now compare against too." -class _FailureLogRecorder(CustomLogger): - def __init__(self) -> None: - super().__init__() - self.logged: list[tuple[object, object]] = [] - - async def async_log_failure_event( - self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object - ) -> None: - payload: Final = cast(StandardLoggingPayload, kwargs["standard_logging_object"]) - self.logged.append((copy.deepcopy(kwargs["messages"]), copy.deepcopy(payload["messages"]))) - - -async def test_logging_only_presidio_masks_the_prompt_in_the_failure_log(monkeypatch: pytest.MonkeyPatch) -> None: - """A failed call is logged with the prompt a logging_only Presidio guardrail masked, the same as a - successful one. The analyzer and anonymizer are the in-process fake handed over through api_base.""" - recorder: Final = _FailureLogRecorder() - for name in ("input_callback", "success_callback", "failure_callback", "_async_success_callback"): - monkeypatch.setattr(litellm, name, []) - monkeypatch.setattr(litellm, "_async_failure_callback", []) - async with TestServer(_fake_presidio_app()) as server: - guardrail: Final = OPTIONAL_PresidioPIIMasking( - presidio_analyzer_api_base=str(server.make_url("/")), - presidio_anonymizer_api_base=str(server.make_url("/")), - pii_entities_config={PiiEntityType.PERSON: PiiAction.MASK}, - logging_only=True, - ) - monkeypatch.setattr(litellm, "callbacks", [guardrail, recorder]) - with pytest.raises(litellm.AuthenticationError): - await litellm.acompletion( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "My name is John Smith"}], - mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"), - ) - await guardrail._close_http_session() - - masked: Final = [{"role": "user", "content": "My name is "}] - assert recorder.logged == [(masked, masked)] - - @pytest.mark.asyncio @pytest.mark.parametrize("surface", ["mcp_arguments", "mcp_result", "llm_output"]) @pytest.mark.parametrize("action", [PiiAction.MASK, PiiAction.BLOCK]) @@ -5240,8 +5202,15 @@ _BLOCK_CARD_MASK_EMAIL: Final = { } +class _AnonymizedSpan(TypedDict): + entity_type: ReadOnly[str] + start: ReadOnly[int] + end: ReadOnly[int] + + class _PresidioAnonymizeRequest(TypedDict): text: ReadOnly[str] + analyzer_results: ReadOnly[tuple[_AnonymizedSpan, ...]] class _PresidioAnonymizeReply(TypedDict): @@ -5253,6 +5222,14 @@ _ANALYZE_REQUEST: Final = TypeAdapter(PresidioAnalyzeRequest) _ANONYMIZE_REQUEST: Final = TypeAdapter(_PresidioAnonymizeRequest) +def _anonymized(text: str, spans: Sequence[_AnonymizedSpan]) -> str: + return functools.reduce( + lambda anonymized, span: f"{anonymized[: span['start']]}<{span['entity_type']}>{anonymized[span['end'] :]}", + sorted(spans, key=lambda span: span["start"], reverse=True), + text, + ) + + def _card_and_email_spans(text: str) -> tuple[PresidioAnalyzeResponseItem, ...]: return tuple( PresidioAnalyzeResponseItem( @@ -5310,7 +5287,10 @@ class _InMemoryPresidioTransport(asyncio.Transport): self._analyzed.put_nowait(analyze_request) return _card_and_email_spans(analyze_request.get("text") or "") assert path == "/anonymize", path - return _PresidioAnonymizeReply(text=_ANONYMIZE_REQUEST.validate_json(body)["text"], items=()) + anonymize_request: Final = _ANONYMIZE_REQUEST.validate_json(body) + return _PresidioAnonymizeReply( + text=_anonymized(anonymize_request["text"], anonymize_request["analyzer_results"]), items=() + ) def writelines(self, list_of_data: Iterable[bytes | bytearray | memoryview]) -> None: self.write(b"".join(bytes(chunk) for chunk in list_of_data)) @@ -5361,12 +5341,13 @@ def _drain(analyzed: asyncio.Queue[PresidioAnalyzeRequest]) -> tuple[PresidioAna def _guardrail_with_in_memory_presidio( - analyzed: asyncio.Queue[PresidioAnalyzeRequest], replies_wait_for: int = 0 + analyzed: asyncio.Queue[PresidioAnalyzeRequest], replies_wait_for: int = 0, logging_only: bool = False ) -> OPTIONAL_PresidioPIIMasking: guardrail: Final = OPTIONAL_PresidioPIIMasking( pii_entities_config=_BLOCK_CARD_MASK_EMAIL, presidio_analyzer_api_base="http://presidio-analyzer.test/", presidio_anonymizer_api_base="http://presidio-anonymizer.test/", + logging_only=logging_only, ) guardrail._http_session = aiohttp.ClientSession(connector=_InMemoryPresidioConnector(analyzed, replies_wait_for)) return guardrail @@ -5392,6 +5373,40 @@ async def test_presidio_scans_every_message_of_a_request_at_once() -> None: assert sorted(request.get("text") or "" for request in _drain(analyzed)) == sorted(texts) +class _FailureLogRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.logged: list[tuple[object, object]] = [] + + async def async_log_failure_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + payload: Final = cast(StandardLoggingPayload, kwargs["standard_logging_object"]) + self.logged.append((copy.deepcopy(kwargs["messages"]), copy.deepcopy(payload["messages"]))) + + +@pytest.mark.asyncio +async def test_logging_only_presidio_masks_the_prompt_in_the_failure_log(monkeypatch: pytest.MonkeyPatch) -> None: + recorder: Final = _FailureLogRecorder() + for name in ("input_callback", "success_callback", "failure_callback", "_async_success_callback"): + monkeypatch.setattr(litellm, name, []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + guardrail: Final = _guardrail_with_in_memory_presidio(asyncio.Queue(), logging_only=True) + monkeypatch.setattr(litellm, "callbacks", [guardrail, recorder]) + try: + with pytest.raises(litellm.AuthenticationError): + await litellm.acompletion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": f"My email is {_EMAIL}"}], + mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"), + ) + finally: + await guardrail._close_http_session() + + masked: Final = [{"role": "user", "content": "My email is "}] + assert recorder.logged == [(masked, masked)] + + @pytest.mark.asyncio async def test_check_pii_raises_blocked_entity_for_card() -> None: text: Final = f"My credit card number is {_CARD_NUMBER} and my email is {_EMAIL}"