From a88002c0348cc6b5bc5805dc27fd4d7e87d87db2 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 7 Sep 2026 11:43:32 -0700 Subject: [PATCH] test(guardrails): document audit prompt storage prerequisite --- tests/e2e/CONTRIBUTING.md | 9 +++++++++ tests/e2e/guardrails/test_presidio_masking_e2e.py | 13 ++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index b83300cc8d5..1183096b81e 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -52,6 +52,15 @@ The suites run against a live proxy, so bring one up first by running the litell Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE` +The Presidio spend-log audit test also requires prompt storage on the proxy, because its assertions inspect the detected entities and their scores. Add the following to the proxy config before starting it: + +```yaml +general_settings: + store_prompts_in_spend_logs: true +``` + +Alternatively, start the proxy with `STORE_PROMPTS_IN_SPEND_LOGS=true litellm --config .yml --port 4000`. Setting the variable only on the pytest process does not configure the proxy. With prompt storage disabled, the proxy correctly redacts `guardrail_response`, so that configuration cannot exercise this test's entity-detail assertions. Enable this only on a test stack using synthetic prompts + A couple of logging destinations are configured on the proxy rather than by the test. The Weave tests scope their callback to the key they create, but litellm builds the `weave_otel` logger from `WANDB_API_KEY` and `WANDB_PROJECT_ID` before it applies the per-key vars, so the proxy needs both in its own environment or the key-scoped callback never initializes and nothing ships ### The pull request check diff --git a/tests/e2e/guardrails/test_presidio_masking_e2e.py b/tests/e2e/guardrails/test_presidio_masking_e2e.py index 4edf6c4ceb6..49d698938ce 100644 --- a/tests/e2e/guardrails/test_presidio_masking_e2e.py +++ b/tests/e2e/guardrails/test_presidio_masking_e2e.py @@ -19,6 +19,12 @@ PRESIDIO_ANONYMIZER_API_BASE; missing env is a hard failure, never a skip. Each guardrail registers with an explicit presidio_filter_scope so only the configured hook's callback exists (the default "both" registers input masking AND a post_call output masker), and is deleted on teardown. + +The spend-log audit test requires general_settings.store_prompts_in_spend_logs: +true in the proxy config, or STORE_PROMPTS_IN_SPEND_LOGS=true in the proxy +process environment before startup. Setting it only on pytest has no effect. +Without this opt-in, redacting guardrail_response is expected proxy behavior; +this suite deliberately requires the detected-entity details to remain visible. """ from __future__ import annotations @@ -374,9 +380,10 @@ class TestPresidioSpendLogRecord: ) assert not isinstance(record.guardrail_response, str), ( - "guardrail_response must be the detected-entity list, but it is the string " - f"{record.guardrail_response!r}. A redaction marker here means the proxy is not storing " - "prompts in spend logs, which also strips the per-entity scores from the guardrail panel" + "This audit test requires general_settings.store_prompts_in_spend_logs: true in the proxy config " + "or STORE_PROMPTS_IN_SPEND_LOGS=true in the proxy process environment before startup " + "(not just the pytest environment). Default prompt redaction is valid proxy behavior, " + f"but prevents entity-detail assertions; got guardrail_response={record.guardrail_response!r}" ) entities = _ENTITY_LIST_ADAPTER.validate_python(record.guardrail_response) assert entities, (