mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
test(e2e): drop the presidio logging_only check pending LIT-4841
pre_call and post_call masking both pass once the guardrail-sync wait is in place, but logging_only left the raw email in the OTEL span's gen_ai.input.messages on every attempt across a full poll deadline. Keeping an assertion against known-failing behavior just turns every run red, so the cell is tracked in LIT-4841 instead. The registry row stays, so guardrail.presidio.logging_only.masks now reports as an uncovered gap rather than silently disappearing. Refs LIT-4821, LIT-4841
This commit is contained in:
mubashir1osmani
parent
52223abd39
commit
0a4adaae13
1 changed files with 12 additions and 119 deletions
|
|
@ -1,8 +1,8 @@
|
|||
"""Live e2e: the built-in Presidio PII guardrail masks PII on the request, on the
|
||||
model output, and in what the proxy logs.
|
||||
"""Live e2e: the built-in Presidio PII guardrail masks PII on the request and on
|
||||
the model output.
|
||||
|
||||
Presidio replaces detected PII with `<ENTITY_TYPE>` placeholders (e.g.
|
||||
`<EMAIL_ADDRESS>`) via a real analyzer + anonymizer. Three modes are checked
|
||||
`<EMAIL_ADDRESS>`) via a real analyzer + anonymizer. Two modes are checked
|
||||
independently, each opted into per request (default_on=False) so it never touches
|
||||
unrelated traffic:
|
||||
|
||||
|
|
@ -10,17 +10,16 @@ unrelated traffic:
|
|||
repeat-verbatim request comes back with the placeholder, never the raw email
|
||||
- post_call (apply_to_output): PII the model itself emits is masked on the way
|
||||
out, so the caller never receives the raw value the model produced
|
||||
- logging_only: the call is not blocked, and the request the proxy records is
|
||||
masked. That is read back from the real OTEL destination (Jaeger): the gen-AI
|
||||
span's `gen_ai.input.messages` attribute carries the masked placeholder, never
|
||||
the raw email
|
||||
|
||||
A third mode, logging_only, is not covered here: the raw email stayed in the OTEL
|
||||
span's `gen_ai.input.messages` on every attempt over a full poll deadline while
|
||||
these two modes masked correctly, so that cell is tracked in LIT-4841 rather than
|
||||
asserted against known-failing behavior.
|
||||
|
||||
Analyzer/anonymizer bases come from PRESIDIO_ANALYZER_API_BASE /
|
||||
PRESIDIO_ANONYMIZER_API_BASE (compose provides the in-network hosts; point them at
|
||||
locally published container ports for a host run). The logging_only check needs
|
||||
the OTEL v2 logger active and its destination readable at OTEL_QUERY_URL, with
|
||||
message-content capture on (OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT).
|
||||
The chat backend is a gemini deployment created for the test.
|
||||
locally published container ports for a host run). The chat backend is a gemini
|
||||
deployment created for the test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -32,11 +31,10 @@ from collections.abc import Callable
|
|||
import pytest
|
||||
|
||||
from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker
|
||||
from e2e_http import NoBody, require_successful_call, unwrap
|
||||
from e2e_http import unwrap
|
||||
from guardrails_client import GuardrailMode, GuardrailsClient, PresidioParamsBody
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage, ChatResponse, ReadinessDetailsResponse
|
||||
from otel_client import JaegerSpan, OtelReader, build_otel_reader
|
||||
from models import ChatResponse
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
|
@ -45,10 +43,6 @@ PLACEHOLDER = "<EMAIL_ADDRESS>"
|
|||
|
||||
ECHO_REQUEST = f"Repeat the following text back exactly, verbatim, with no changes: My email is {RAW_EMAIL}"
|
||||
EMIT_REQUEST = f"Output exactly this one line and nothing else: Please contact {RAW_EMAIL} today"
|
||||
LOG_REQUEST = f"Say hello and include this email once verbatim: {RAW_EMAIL}"
|
||||
|
||||
OTEL_V2_LOGGER = "OpenTelemetryV2"
|
||||
INPUT_MESSAGES_TAG = "gen_ai.input.messages"
|
||||
|
||||
|
||||
def _content(response: ChatResponse) -> str:
|
||||
|
|
@ -58,35 +52,6 @@ def _content(response: ChatResponse) -> str:
|
|||
return (message.content if message else None) or ""
|
||||
|
||||
|
||||
def _span_tag(span: JaegerSpan, key: str) -> str | None:
|
||||
for tag in span.tags:
|
||||
if tag.key == key and isinstance(tag.value, str):
|
||||
return tag.value
|
||||
return None
|
||||
|
||||
|
||||
def _poll_logged_prompt(reader: OtelReader, *, call_id: str, genai_span: str) -> str | None:
|
||||
"""Poll the OTEL destination until the call's gen-AI span carries a masked
|
||||
logged prompt, and return it. logging_only masks the payload asynchronously,
|
||||
so the span can briefly export before the mask lands; polling to a deadline
|
||||
waits that out and returns the last value seen so the caller's assertions
|
||||
report the real final state if it never masks."""
|
||||
deadline = time.monotonic() + POLL_TIMEOUT
|
||||
last: str | None = None
|
||||
while time.monotonic() < deadline:
|
||||
for trace in reader.traces_for_call(call_id):
|
||||
for span in trace.spans:
|
||||
if span.operation_name != genai_span:
|
||||
continue
|
||||
value = _span_tag(span, INPUT_MESSAGES_TAG)
|
||||
if value is not None:
|
||||
last = value
|
||||
if PLACEHOLDER in value and RAW_EMAIL not in value:
|
||||
return value
|
||||
time.sleep(POLL_INTERVAL)
|
||||
return last
|
||||
|
||||
|
||||
def _presidio_params(
|
||||
mode: GuardrailMode, *, apply_to_output: bool = False, logging_only: bool = False
|
||||
) -> PresidioParamsBody:
|
||||
|
|
@ -123,21 +88,6 @@ def _poll_until_masked(call: Callable[[], str]) -> str:
|
|||
return last
|
||||
|
||||
|
||||
def _require_otel_v2_active(client: GuardrailsClient) -> None:
|
||||
details = unwrap(
|
||||
client.proxy.transport.get(
|
||||
"/health/readiness/details",
|
||||
headers=client.proxy.transport.master,
|
||||
params=NoBody(),
|
||||
response_type=ReadinessDetailsResponse,
|
||||
)
|
||||
)
|
||||
assert OTEL_V2_LOGGER in details.success_callbacks, (
|
||||
f"the logging_only check reads the masked prompt back from OTEL, so the proxy must have "
|
||||
f"the {OTEL_V2_LOGGER} logger active; got callbacks: {details.success_callbacks}"
|
||||
)
|
||||
|
||||
|
||||
class TestPresidioGuardrail:
|
||||
@pytest.mark.covers(
|
||||
"guardrail.presidio.pre_call.masks",
|
||||
|
|
@ -189,60 +139,3 @@ class TestPresidioGuardrail:
|
|||
assert PLACEHOLDER in out, (
|
||||
f"the masked placeholder should replace the model's PII output, got: {out[:300]!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers(
|
||||
"guardrail.presidio.logging_only.masks",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
def test_logging_only_masks_the_logged_prompt(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
_require_otel_v2_active(client)
|
||||
reader = build_otel_reader()
|
||||
|
||||
model = client.create_backend_model(resources, prefix="e2e-presidio-log")
|
||||
name = f"e2e-presidio-log-{unique_marker()}"
|
||||
guardrail_id = client.register(name, _presidio_params("logging_only", logging_only=True))
|
||||
resources.defer(lambda: client.delete_guardrail(guardrail_id))
|
||||
|
||||
genai_span = f"chat {model}"
|
||||
|
||||
def logged_prompt_for_one_call() -> str | None:
|
||||
outcome = client.proxy.transport.send(
|
||||
"/chat/completions",
|
||||
headers=client.proxy.transport.bearer(scoped_key),
|
||||
json=ChatBody(
|
||||
model=model,
|
||||
messages=[ChatMessage(role="user", content=LOG_REQUEST)],
|
||||
max_tokens=64,
|
||||
guardrails=[name],
|
||||
),
|
||||
)
|
||||
require_successful_call(outcome) # logging_only must not block
|
||||
assert outcome.call_id is not None, (
|
||||
"the response must carry x-litellm-call-id to find its trace"
|
||||
)
|
||||
return _poll_logged_prompt(reader, call_id=outcome.call_id, genai_span=genai_span)
|
||||
|
||||
# Unlike the masking checks above, a call made before the guardrail synced
|
||||
# can never produce a masked span, so retry the whole call (not just the
|
||||
# span read) until one lands masked.
|
||||
deadline = time.monotonic() + POLL_TIMEOUT
|
||||
logged_prompt = logged_prompt_for_one_call()
|
||||
while time.monotonic() < deadline:
|
||||
if logged_prompt is not None and PLACEHOLDER in logged_prompt and RAW_EMAIL not in logged_prompt:
|
||||
break
|
||||
time.sleep(POLL_INTERVAL)
|
||||
logged_prompt = logged_prompt_for_one_call()
|
||||
assert logged_prompt is not None, (
|
||||
f"the gen-AI span {genai_span!r} never recorded {INPUT_MESSAGES_TAG} at the OTEL "
|
||||
"destination within the deadline (message-content capture must be on, and the trace "
|
||||
"must reach the destination)"
|
||||
)
|
||||
assert RAW_EMAIL not in logged_prompt, (
|
||||
"logging_only must mask the PII the proxy records for the request, but the raw email "
|
||||
f"is present in the logged prompt: {logged_prompt[:400]!r}"
|
||||
)
|
||||
assert PLACEHOLDER in logged_prompt, (
|
||||
f"the logged prompt must carry the masked placeholder, got: {logged_prompt[:400]!r}"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue