scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

test(e2e): remove the Presidio guardrail suite (#35129)

Browse source 
Drops tests/e2e/guardrails/test_presidio_guardrail_e2e.py and the
PresidioParamsBody it was the only caller of.

Both cases were red on most stage runs between 07-25 and 07-29: pre_call
failed 6 of 11 runs, post_call 6 of 11, with post_call reporting the raw
address reaching the caller while apply_to_output was set.

The cause was propagation, not masking. GuardrailsClient.register() posts
/guardrails and returns immediately with no readiness wait, unlike
ProxyClient._await_model_servable or GuardrailsClient._await_team, and the
data plane only picks a new guardrail up on its next periodic DB sync. Calls
issued before that sync pass the raw value through. #34833 has since made
both cases poll to the deadline, and on the current build each masks on the
first attempt, so the suite is expected to be green now; it is being removed
because it spends real provider money on every retry and because a pod
replaced mid-poll still reproduces the old failure.

The three guardrail.presidio.* rows stay in coverage_registry/guardrail.yaml
and go uncovered on purpose, so Presidio reads as a tier-P0 gap in Grafana
rather than dropping out of the denominator.
This commit is contained in:
mubashir1osmani 2026-07-29 14:20:39 -07:00 • committed by GitHub
parent 440b1bcf65
commit 5182dfa66b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 0 additions and 152 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
tests/e2e/guardrails/guardrails_client.py
View file

@ -63,16 +63,6 @@ class OpenAIModerationParamsBody(GuardrailParamsBase):
model: str | None = None
class PresidioParamsBody(GuardrailParamsBase):
guardrail: Literal["presidio"] = "presidio"
presidio_analyzer_api_base: str | None = None
presidio_anonymizer_api_base: str | None = None
# apply_to_output masks PII the model itself emitted, which also makes the
# guardrail run post_call. logging_only masks what the proxy logs.
apply_to_output: bool | None = None
logging_only: bool | None = None
class BlockCodeExecutionParamsBody(GuardrailParamsBase):
guardrail: Literal["block_code_execution"] = "block_code_execution"
@ -81,7 +71,6 @@ GuardrailParamsBody = (
ContentFilterParamsBody
| BedrockGuardrailParamsBody
| OpenAIModerationParamsBody
| PresidioParamsBody
| BlockCodeExecutionParamsBody
)

141
tests/e2e/guardrails/test_presidio_guardrail_e2e.py
View file

@ -1,141 +0,0 @@
"""Live e2e: the built-in Presidio PII guardrail masks PII on the request and on
the model output.
Presidio replaces detected PII with `<ENTITY_TYPE>` placeholders (e.g.
`<EMAIL_ADDRESS>`) via a real analyzer + anonymizer. Two modes are checked
independently, each opted into per request (default_on=False) so it never touches
unrelated traffic:
- pre_call: the prompt is anonymized before it reaches the model, so a
repeat-verbatim request comes back with the placeholder, never the raw email
- post_call (apply_to_output): PII the model itself emits is masked on the way
out, so the caller never receives the raw value the model produced
A third mode, logging_only, is not covered here: the raw email stayed in the OTEL
span's `gen_ai.input.messages` on every attempt over a full poll deadline while
these two modes masked correctly, so that cell is tracked in LIT-4841 rather than
asserted against known-failing behavior.
Analyzer/anonymizer bases come from PRESIDIO_ANALYZER_API_BASE /
PRESIDIO_ANONYMIZER_API_BASE (compose provides the in-network hosts; point them at
locally published container ports for a host run). The chat backend is a gemini
deployment created for the test.
"""
from __future__ import annotations
import os
import time
from collections.abc import Callable
import pytest
from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker
from e2e_http import unwrap
from guardrails_client import GuardrailMode, GuardrailsClient, PresidioParamsBody
from lifecycle import ResourceManager
from models import ChatResponse
pytestmark = pytest.mark.e2e
RAW_EMAIL = "alice.example.person@example.com"
PLACEHOLDER = "<EMAIL_ADDRESS>"
ECHO_REQUEST = f"Repeat the following text back exactly, verbatim, with no changes: My email is {RAW_EMAIL}"
EMIT_REQUEST = f"Output exactly this one line and nothing else: Please contact {RAW_EMAIL} today"
def _content(response: ChatResponse) -> str:
if not response.choices:
return ""
message = response.choices[0].message
return (message.content if message else None) or ""
def _presidio_params(
mode: GuardrailMode, *, apply_to_output: bool = False, logging_only: bool = False
) -> PresidioParamsBody:
analyzer = os.environ["PRESIDIO_ANALYZER_API_BASE"]
anonymizer = os.environ["PRESIDIO_ANONYMIZER_API_BASE"]
return PresidioParamsBody(
mode=mode,
default_on=False,
presidio_analyzer_api_base=analyzer,
presidio_anonymizer_api_base=anonymizer,
apply_to_output=apply_to_output,
logging_only=logging_only,
)
def _poll_until_masked(call: Callable[[], str]) -> str:
"""Retry a call until the guardrail masks its PII, returning the last content.
Registering a guardrail is a control-plane write; the data-plane worker that
serves /chat/completions only picks it up on its next periodic DB sync (~30s
in proxy_server.py), so a call issued the instant after the create runs
against a worker that has no guardrail yet and passes the raw value through.
That is in-flight propagation, not a masking failure. Polling to the deadline
waits it out, so the assertions that follow judge the synced state; if the
mask never lands the last unmasked content is returned and they still fail.
"""
deadline = time.monotonic() + POLL_TIMEOUT
last = call()
while time.monotonic() < deadline:
if PLACEHOLDER in last and RAW_EMAIL not in last:
return last
time.sleep(POLL_INTERVAL)
last = call()
return last
class TestPresidioGuardrail:
@pytest.mark.covers(
"guardrail.presidio.pre_call.masks",
exercised_on=["chat_completions"],
)
def test_pre_call_masks_pii_before_the_model_sees_it(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
model = client.create_backend_model(resources, prefix="e2e-presidio-pre")
name = f"e2e-presidio-pre-{unique_marker()}"
guardrail_id = client.register(name, _presidio_params("pre_call"))
resources.defer(lambda: client.delete_guardrail(guardrail_id))
echoed = _poll_until_masked(
lambda: _content(
unwrap(client.chat(scoped_key, model, ECHO_REQUEST, guardrails=[name], max_tokens=128))
)
)
assert RAW_EMAIL not in echoed, (
"pre_call masking must strip the raw email before the model sees it, but the "
f"model echoed it back: {echoed[:300]!r}"
)
assert PLACEHOLDER in echoed, (
"the model should have echoed the masked placeholder the guardrail substituted, "
f"got: {echoed[:300]!r}"
)
@pytest.mark.covers(
"guardrail.presidio.post_call.masks",
exercised_on=["chat_completions"],
)
def test_post_call_masks_pii_in_model_output(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
model = client.create_backend_model(resources, prefix="e2e-presidio-post")
name = f"e2e-presidio-post-{unique_marker()}"
guardrail_id = client.register(name, _presidio_params("post_call", apply_to_output=True))
resources.defer(lambda: client.delete_guardrail(guardrail_id))
out = _poll_until_masked(
lambda: _content(
unwrap(client.chat(scoped_key, model, EMIT_REQUEST, guardrails=[name], max_tokens=128))
)
)
assert RAW_EMAIL not in out, (
"post_call masking must strip PII the model emitted, but the raw email reached the "
f"caller: {out[:300]!r}"
)
assert PLACEHOLDER in out, (
f"the masked placeholder should replace the model's PII output, got: {out[:300]!r}"
)