mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
test(e2e): pin the fail-closed contract for an unknown guardrail name (skipped, product gap)
This commit is contained in:
parent
4f7db40587
commit
3e036dcdca
2 changed files with 42 additions and 0 deletions
|
|
@ -32,3 +32,4 @@
|
|||
- {id: guardrail.mcp_security.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [mcp_operations], source: "guardrail_hooks/mcp_security", rationale: "MCP protocol security"}
|
||||
- {id: guardrail.llm_as_a_judge.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/llm_as_a_judge", rationale: "LLM-based judgment guardrail"}
|
||||
- {id: guardrail.litellm_content_filter.pre_mcp_call.blocks, module: guardrail, tier: P1, hook_point: pre_mcp_call, assertions: [blocks], exercised_on: [mcp_operations], source: "guardrail_hooks/litellm_content_filter/content_filter.py:_scan_mcp_tool_call_arguments", rationale: "A general content-filter guardrail configured mode=pre_mcp_call blocks a banned keyword in an MCP tool call's arguments before it reaches the upstream MCP server; a clean argument passes"}
|
||||
- {id: guardrail.dispatch.pre_call.rejects_unknown_name, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "proxy guardrail dispatch (per-request `guardrails` selector)", rationale: "A request naming a guardrail this proxy does not serve must fail closed with a 4xx; today it is silently served unguarded, so a typo'd name drops the protection the caller asked for"}
|
||||
|
|
|
|||
41
tests/e2e/guardrails/test_guardrail_dispatch_e2e.py
Normal file
41
tests/e2e/guardrails/test_guardrail_dispatch_e2e.py
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
"""Live e2e: the per-request `guardrails` selector must fail closed.
|
||||
|
||||
A request that names a guardrail is a caller asking for protection. When the
|
||||
proxy does not serve that name (a typo, a deleted guardrail, or a worker that
|
||||
never loaded it), answering 200 silently drops the protection the caller asked
|
||||
for; the contract this test pins is a 4xx naming the unknown guardrail.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import UnknownApiError, ValidationError
|
||||
from guardrails_client import GuardrailsClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
MODEL = "gemini-2.5-flash"
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason=(
|
||||
"stage red: product gap, a request naming a guardrail the proxy does not "
|
||||
"serve is silently served unguarded (200) instead of failing closed"
|
||||
)
|
||||
)
|
||||
@pytest.mark.covers(
|
||||
"guardrail.dispatch.pre_call.rejects_unknown_name",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
def test_request_naming_an_unknown_guardrail_fails_closed(client: GuardrailsClient, scoped_key: str) -> None:
|
||||
result = client.chat(scoped_key, MODEL, "say hi", guardrails=[f"e2e-no-such-guardrail-{unique_marker()}"])
|
||||
|
||||
match result:
|
||||
case UnknownApiError(status_code=status, body=body):
|
||||
assert status == 400, f"expected a 400 for an unknown guardrail name, got {status}: {body[:400]}"
|
||||
assert "guardrail" in body.lower(), f"the rejection should name the guardrail; got: {body[:400]}"
|
||||
case ValidationError(message=message):
|
||||
assert "guardrail" in message.lower(), f"the rejection should name the guardrail; got: {message[:400]}"
|
||||
case _:
|
||||
pytest.fail(f"a request naming an unknown guardrail must fail closed with a 4xx; got {result}")
|
||||
Loading…
Add table
Reference in a new issue