mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
f0fadb7f99
* test: add logging e2e coverage (s3_v2, gcs_bucket, team langfuse callback, datadog failure) Five new live e2e scenarios raising Logging & Guardrails registry coverage: s3_v2 success and failure objects read back from the real S3 bucket, gcs_bucket success record read back through the GCS JSON API (with nextPageToken pagination and per-request bearer minting), team-scoped Langfuse callback delivery with non-team isolation, and DataDog failure event delivery queried by indexed model_group. datadog_reader gains query-based variants of the marker search; the langfuse cell is a new registry row. Bucket readers settle past a full flush interval so a late duplicate cannot hide from the exactly-one assertions * test: cover clock-skew day prefix in gcs read-back and retry team callback propagation * test: key the s3 failure read-back on the provider error, not payload absence * chore: rerun ci * chore: rerun ci after config sync * chore: rerun ci with pr lane env * chore: rerun ci * chore: rerun ci * chore: rerun ci * chore: rerun ci * chore: rerun ci * test: add guardrail e2e coverage (presidio masking, bedrock post and during call, moderation on messages) (#38553) * test: add guardrail e2e coverage (presidio masking, bedrock post/during, moderation on messages) * test: require the phone placeholder positively in the presidio masking predicate * test: count only the 400 verdict body as a bedrock post_call block * test(e2e): exempt the guardrail config echo from the post_call leak assertion * test(e2e): pin the fail-closed contract for an unknown guardrail name (skipped, product gap) * test(e2e): tolerate the readiness 503 from a transient db blip in the callback-config probes
41 lines
1.7 KiB
Python
41 lines
1.7 KiB
Python
"""Live e2e: the per-request `guardrails` selector must fail closed.
|
|
|
|
A request that names a guardrail is a caller asking for protection. When the
|
|
proxy does not serve that name (a typo, a deleted guardrail, or a worker that
|
|
never loaded it), answering 200 silently drops the protection the caller asked
|
|
for; the contract this test pins is a 4xx naming the unknown guardrail.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from e2e_config import unique_marker
|
|
from e2e_http import UnknownApiError, ValidationError
|
|
from guardrails_client import GuardrailsClient
|
|
|
|
pytestmark = pytest.mark.e2e
|
|
|
|
MODEL = "gemini-2.5-flash"
|
|
|
|
|
|
@pytest.mark.skip(
|
|
reason=(
|
|
"stage red: product gap, a request naming a guardrail the proxy does not "
|
|
"serve is silently served unguarded (200) instead of failing closed"
|
|
)
|
|
)
|
|
@pytest.mark.covers(
|
|
"guardrail.dispatch.pre_call.rejects_unknown_name",
|
|
exercised_on=["chat_completions"],
|
|
)
|
|
def test_request_naming_an_unknown_guardrail_fails_closed(client: GuardrailsClient, scoped_key: str) -> None:
|
|
result = client.chat(scoped_key, MODEL, "say hi", guardrails=[f"e2e-no-such-guardrail-{unique_marker()}"])
|
|
|
|
match result:
|
|
case UnknownApiError(status_code=status, body=body):
|
|
assert status == 400, f"expected a 400 for an unknown guardrail name, got {status}: {body[:400]}"
|
|
assert "guardrail" in body.lower(), f"the rejection should name the guardrail; got: {body[:400]}"
|
|
case ValidationError(message=message):
|
|
assert "guardrail" in message.lower(), f"the rejection should name the guardrail; got: {message[:400]}"
|
|
case _:
|
|
pytest.fail(f"a request naming an unknown guardrail must fail closed with a 4xx; got {result}")
|