mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-28 06:23:07 +00:00
test(bedrock): reject version-less guardrails in the scripted Bedrock and cover the 400
The mantle and fallback fixtures named a guardrail with no guardrailVersion, which real Bedrock refuses, yet the scripted Bedrock answered them as guarded successes. The fake now answers a version-less guardrail with a 400 ValidationException, the fixtures carry versions, and a new case checks a version-less deployment guardrail reaches Bedrock and the caller gets its rejection on all four actions
This commit is contained in:
parent
2346258f26
commit
92fdfee72e
1 changed files with 48 additions and 5 deletions
|
|
@ -11,6 +11,7 @@ import boto3
|
|||
import pytest
|
||||
import yaml
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from integration._support.budget import case_timeout_seconds
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
|
|
@ -24,6 +25,7 @@ _MODEL_ID: Final = "anthropic.claude-sonnet-5-v1:0"
|
|||
_EVENT_STREAM: Final = "application/vnd.amazon.eventstream"
|
||||
_BLOCKED_WORD: Final = "zzzforbidden"
|
||||
_BLOCKED_TEXT: Final = "Blocked by guardrail"
|
||||
_VERSIONLESS_GUARDRAIL_MESSAGE: Final = "guardrailIdentifier and guardrailVersion must be provided together."
|
||||
_ACTIONS: Final = ("converse", "converse-stream", "invoke", "invoke-with-response-stream")
|
||||
_DEPLOYMENT_GUARDRAIL: Final[Mapping[str, str]] = {
|
||||
"guardrailIdentifier": "deployment-guardrail",
|
||||
|
|
@ -105,8 +107,16 @@ def _assessment(applied: Applied) -> Mapping[str, JsonValue]:
|
|||
|
||||
def _bedrock_with_guardrails(request: Request) -> Reply:
|
||||
"""Bedrock's runtime as the caller sees it: the guardrail the request names blocks a prompt holding the blocked
|
||||
word, answering with its message and, when the request enabled tracing, a trace naming that guardrail."""
|
||||
word, answering with its message and, when the request enabled tracing, a trace naming that guardrail. A guardrail
|
||||
named without a version is refused with a 400 ValidationException, as real Bedrock in us-east-1 refused it on
|
||||
every one of these actions on 2026-10-19."""
|
||||
applied: Final = _applied(request)
|
||||
if applied is not None and applied.version is None:
|
||||
return Reply(
|
||||
status=400,
|
||||
body=json.dumps({"message": _VERSIONLESS_GUARDRAIL_MESSAGE}).encode(),
|
||||
headers={"x-amzn-errortype": "ValidationException"},
|
||||
)
|
||||
blocked: Final = applied is not None and _BLOCKED_WORD.encode() in request.body
|
||||
traced: Final = blocked and applied is not None and (applied.trace or "").lower() == "enabled"
|
||||
text: Final = _BLOCKED_TEXT if blocked else "ok"
|
||||
|
|
@ -257,6 +267,7 @@ class Rig:
|
|||
failing_wire: Wire
|
||||
fallback_wire: Wire
|
||||
mantle_model: str
|
||||
versionless_model: str
|
||||
key: str
|
||||
|
||||
def runtime( # noqa: ANN201 # boto3 clients are untyped
|
||||
|
|
@ -288,6 +299,7 @@ def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> I
|
|||
failing_model: Final = f"bedrock-failing-{suffix}"
|
||||
fallback_model: Final = f"bedrock-fallback-{suffix}"
|
||||
mantle_model: Final = f"bedrock-mantle-{suffix}"
|
||||
versionless_model: Final = f"bedrock-versionless-{suffix}"
|
||||
directory: Final = tmp_path_factory.mktemp("bedrock-guardrail-config")
|
||||
base_config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
with (
|
||||
|
|
@ -321,13 +333,21 @@ def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> I
|
|||
{
|
||||
"model_name": failing_model,
|
||||
"litellm_params": _deployment(
|
||||
failing_wire, {"guardrailIdentifier": "failing-guardrail", "trace": "enabled"}
|
||||
failing_wire,
|
||||
{"guardrailIdentifier": "failing-guardrail", "guardrailVersion": "5", "trace": "enabled"},
|
||||
),
|
||||
},
|
||||
{
|
||||
"model_name": fallback_model,
|
||||
"litellm_params": _deployment(
|
||||
fallback_wire, {"guardrailIdentifier": "fallback-guardrail", "trace": "enabled"}
|
||||
fallback_wire,
|
||||
{"guardrailIdentifier": "fallback-guardrail", "guardrailVersion": "6", "trace": "enabled"},
|
||||
),
|
||||
},
|
||||
{
|
||||
"model_name": versionless_model,
|
||||
"litellm_params": _deployment(
|
||||
wire, {"guardrailIdentifier": "versionless-guardrail", "trace": "enabled"}
|
||||
),
|
||||
},
|
||||
{
|
||||
|
|
@ -338,7 +358,11 @@ def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> I
|
|||
"api_key": _MANTLE_KEY,
|
||||
"aws_region_name": "us-east-1",
|
||||
"use_in_pass_through": True,
|
||||
"guardrailConfig": {"guardrailIdentifier": "mantle-guardrail", "trace": "enabled"},
|
||||
"guardrailConfig": {
|
||||
"guardrailIdentifier": "mantle-guardrail",
|
||||
"guardrailVersion": "4",
|
||||
"trace": "enabled",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
|
|
@ -364,6 +388,7 @@ def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> I
|
|||
failing_wire=failing_wire,
|
||||
fallback_wire=fallback_wire,
|
||||
mantle_model=mantle_model,
|
||||
versionless_model=versionless_model,
|
||||
key=scenario.key(key_alias=f"guardrail-key-{suffix}"),
|
||||
)
|
||||
|
||||
|
|
@ -426,10 +451,28 @@ def test_mantle_deployment_passthrough_sends_its_guardrail_with_bearer_auth(
|
|||
(request,) = rig.received(rig.wire, marker)
|
||||
assert request.headers["authorization"] == f"Bearer {_MANTLE_KEY}", request.headers
|
||||
assert _applied(request) == Applied(
|
||||
"mantle-guardrail", None, "ENABLED" if action.startswith("invoke") else "enabled", None
|
||||
"mantle-guardrail", "4", "ENABLED" if action.startswith("invoke") else "enabled", None
|
||||
), (request.headers, request.body)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", _ACTIONS)
|
||||
def test_deployment_guardrail_without_a_version_returns_the_bedrock_rejection(
|
||||
rig: Rig, monkeypatch: pytest.MonkeyPatch, action: str
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
runtime: Final = rig.runtime(monkeypatch)
|
||||
|
||||
with pytest.raises(ClientError) as raised:
|
||||
_send(runtime, action, rig.versionless_model, f"{marker} say hello", caller_guardrail=False)
|
||||
|
||||
error: Final = raised.value.response.get("Error", {})
|
||||
assert (error.get("Code"), error.get("Message")) == ("ValidationException", _VERSIONLESS_GUARDRAIL_MESSAGE), error
|
||||
received: Final = rig.received(rig.wire, marker)
|
||||
assert {_applied(request) for request in received} == {
|
||||
Applied("versionless-guardrail", None, "ENABLED" if action.startswith("invoke") else "enabled", None)
|
||||
}, [(request.headers, request.body) for request in received]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", _ACTIONS)
|
||||
def test_deployment_without_a_guardrail_leaves_the_caller_guardrail_as_sent(
|
||||
rig: Rig, monkeypatch: pytest.MonkeyPatch, action: str
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue