diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml index f66a73e7daf..9a2d16b16e1 100644 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ b/tests/e2e/coverage_registry/guardrail.yaml @@ -30,5 +30,5 @@ - {id: guardrail.niche_providers.pre_call.allows, module: guardrail, tier: P2, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: grammar, rationale: "SMOKE niche allow-path passthrough"} - {id: guardrail.tool_policy.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/tool_policy/tool_policy_guardrail.py", rationale: "Tool-use policy enforcement"} - {id: guardrail.mcp_security.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [mcp_operations], source: "guardrail_hooks/mcp_security", rationale: "MCP protocol security"} -- {id: guardrail.llm_as_a_judge.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/llm_as_a_judge", rationale: "LLM-based judgment guardrail"} +- {id: guardrail.llm_as_a_judge.post_call.blocks, module: guardrail, tier: P2, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/llm_as_a_judge", rationale: "LLM-based judgment guardrail"} - {id: guardrail.litellm_content_filter.pre_mcp_call.blocks, module: guardrail, tier: P1, hook_point: pre_mcp_call, assertions: [blocks], exercised_on: [mcp_operations], source: "guardrail_hooks/litellm_content_filter/content_filter.py:_scan_mcp_tool_call_arguments", rationale: "A general content-filter guardrail configured mode=pre_mcp_call blocks a banned keyword in an MCP tool call's arguments before it reaches the upstream MCP server; a clean argument passes"} diff --git a/tests/e2e/guardrails/native_guardrails.py b/tests/e2e/guardrails/native_guardrails.py new file mode 100644 index 00000000000..ed3c46db762 --- /dev/null +++ b/tests/e2e/guardrails/native_guardrails.py @@ -0,0 +1,255 @@ +"""Typed bodies and helpers for the guardrails litellm implements itself. + +`tool_permission` and `tool_policy` decide, before the upstream model runs, +whether the tools a request carries may be used at all: the first from a regex +allow-list configured on the guardrail, the second from the per-key blocked-tool +overrides admins set through POST /v1/tool/policy. `llm_as_a_judge` runs after +the model instead, scoring the response against weighted criteria with a second +LLM. All three register through the same POST /guardrails route the rest of the +suite uses and only their params differ, so the params compose onto the shared +`GuardrailParamsBase` and the create body is written once here. +""" + +from __future__ import annotations + +import time +from typing import Literal, Sequence + +from pydantic import BaseModel + +from e2e_config import CHEAP_OPENAI_MODEL, POLL_INTERVAL, POLL_TIMEOUT +from e2e_http import NoBody, Result, Success, unwrap +from guardrails_client import ( + GuardrailCreateResponse, + GuardrailParamsBase, + GuardrailsClient, +) +from models import ChatBody, ChatMessage, ChatResponse, ChatTool, ChatToolFunction + +ToolInputPolicy = Literal["blocked", "trusted", "untrusted"] + + +class ToolPermissionRuleBody(BaseModel): + """One allow/deny rule: `tool_name` is a regex matched against the tool's + function name.""" + + id: str + tool_name: str + decision: Literal["allow", "deny"] + + +class ToolPermissionParamsBody(GuardrailParamsBase): + guardrail: Literal["tool_permission"] = "tool_permission" + rules: list[ToolPermissionRuleBody] + default_action: Literal["allow", "deny"] + on_disallowed_action: Literal["block", "rewrite"] + + +class ToolPolicyParamsBody(GuardrailParamsBase): + """tool_policy reads its decisions from the tool registry and the caller's + blocked-tool overrides, so the guardrail itself carries no rules.""" + + guardrail: Literal["tool_policy"] = "tool_policy" + + +class JudgeCriterionBody(BaseModel): + """One scoring criterion. Weights across a guardrail's criteria must sum to + 100 or the proxy rejects the registration.""" + + name: str + weight: int + description: str + + +class LLMAsAJudgeParamsBody(GuardrailParamsBase): + guardrail: Literal["llm_as_a_judge"] = "llm_as_a_judge" + judge_model: str + criteria: list[JudgeCriterionBody] + overall_threshold: float + on_failure: Literal["block", "log"] + + +NativeGuardrailParamsBody = ( + ToolPermissionParamsBody | ToolPolicyParamsBody | LLMAsAJudgeParamsBody +) + + +class NativeGuardrailSpecBody(BaseModel): + guardrail_name: str + litellm_params: NativeGuardrailParamsBody + + +class NativeGuardrailCreateBody(BaseModel): + guardrail: NativeGuardrailSpecBody + + +class ToolPolicyOverrideBody(BaseModel): + """POST /v1/tool/policy scoped to one virtual key: sets that key's policy for + a single tool without touching the global registry entry.""" + + tool_name: str + input_policy: ToolInputPolicy + key_hash: str + + +class ToolPolicyUpdateResponse(BaseModel): + tool_name: str + updated: bool + + +class ToolOverrideParams(BaseModel): + key_hash: str + + +class ToolOverrideDeleteResponse(BaseModel): + deleted: bool + tool_name: str + + +class GuardrailUsageLogEntry(BaseModel): + """One row of GET /guardrails/usage/logs. `action` is the guardrail's own + verdict on that request: `passed` when it ran and approved, `blocked` when it + intervened, and `flagged` when it errored and let the request through anyway. + `id` is the completion id for a request the guardrail approved.""" + + id: str + action: Literal["passed", "blocked", "flagged"] + + +class GuardrailUsageLogs(BaseModel): + logs: list[GuardrailUsageLogEntry] + total: int + + +class GuardrailUsageLogsParams(BaseModel): + guardrail_id: str + + +def register_guardrail( + client: GuardrailsClient, name: str, params: NativeGuardrailParamsBody +) -> str: + """Register a native guardrail and return its id. Registered with + `default_on=False` and opted into per request, so it never intercepts + unrelated traffic on the shared proxy.""" + return unwrap( + client.proxy.transport.post( + "/guardrails", + headers=client.proxy.transport.master, + json=NativeGuardrailCreateBody( + guardrail=NativeGuardrailSpecBody(guardrail_name=name, litellm_params=params) + ), + response_type=GuardrailCreateResponse, + ) + ).guardrail_id + + +def block_tool_for_key(client: GuardrailsClient, *, tool_name: str, key: str) -> None: + """Mark one tool blocked for one virtual key. The proxy resolves the raw key + to its object permission (creating one when the key has none) and resyncs the + in-memory tool-policy registry before answering.""" + _ = unwrap( + client.proxy.transport.post( + "/v1/tool/policy", + headers=client.proxy.transport.master, + json=ToolPolicyOverrideBody( + tool_name=tool_name, input_policy="blocked", key_hash=key + ), + response_type=ToolPolicyUpdateResponse, + ) + ) + + +def unblock_tool_for_key(client: GuardrailsClient, *, tool_name: str, key: str) -> None: + _ = client.proxy.transport.delete( + f"/v1/tool/{tool_name}/overrides", + headers=client.proxy.transport.master, + json=NoBody(), + params=ToolOverrideParams(key_hash=key), + response_type=ToolOverrideDeleteResponse, + ) + + +def function_tool(name: str) -> ChatTool: + return ChatTool( + function=ChatToolFunction( + name=name, + description="Look up the current weather for a city.", + parameters={ + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + ) + ) + + +def chat( + client: GuardrailsClient, + key: str, + *, + prompt: str, + guardrail_name: str, + tools: Sequence[ChatTool] = (), + force_tool_call: bool = False, +) -> Result[ChatResponse]: + """Drive a real chat completion, opting into one guardrail for this request + only. `tools` carries the function definitions a tool-gating guardrail reads; + `force_tool_call` sets tool_choice=required so an allowed tool is provably + reachable rather than left to the model's mood. No max_tokens is sent: gpt-5.5 + spends tokens on reasoning before it writes anything, and a truncated empty + response would leave a post-call guardrail with nothing to judge.""" + return client.proxy.chat( + key, + ChatBody( + model=CHEAP_OPENAI_MODEL, + messages=[ChatMessage(role="user", content=prompt)], + tools=list(tools) or None, + tool_choice="required" if force_tool_call else None, + guardrails=[guardrail_name], + ), + ) + + +def called_tool_names(response: ChatResponse) -> list[str]: + if not response.choices: + return [] + message = response.choices[0].message + calls = (message.tool_calls if message else None) or [] + return [call.function.name for call in calls if call.function.name] + + +def poll_guardrail_usage_logs( + client: GuardrailsClient, guardrail_id: str, *, min_rows: int +) -> list[GuardrailUsageLogEntry]: + """Poll the guardrail's own run log until it has recorded `min_rows` requests. + + This is the only caller-visible record of what a guardrail decided. It matters + for a guardrail that adjudicates with a second LLM, because that call fails + open on any internal error: the request then returns a normal 200 that looks + exactly like an approval. `action` separates the two, so a test can assert the + guardrail actually ran rather than that the response merely came back. Rows are + fetched by guardrail_id, and each test registers its own guardrail, so the log + holds that test's requests and nothing else.""" + deadline = time.monotonic() + POLL_TIMEOUT + last: Result[GuardrailUsageLogs] | None = None + while time.monotonic() < deadline: + last = client.proxy.transport.get( + "/guardrails/usage/logs", + headers=client.proxy.transport.master, + params=GuardrailUsageLogsParams(guardrail_id=guardrail_id), + response_type=GuardrailUsageLogs, + ) + if isinstance(last, Success) and len(last.data.logs) >= min_rows: + return last.data.logs + time.sleep(POLL_INTERVAL) + raise AssertionError( + f"guardrail {guardrail_id!r} never recorded {min_rows} run(s) in " + f"/guardrails/usage/logs; last response was {last}" + ) + + +def response_text(response: ChatResponse) -> str: + if not response.choices: + return "" + message = response.choices[0].message + return (message.content if message else None) or "" diff --git a/tests/e2e/guardrails/test_llm_as_a_judge_guardrail_e2e.py b/tests/e2e/guardrails/test_llm_as_a_judge_guardrail_e2e.py new file mode 100644 index 00000000000..e10b21f70fc --- /dev/null +++ b/tests/e2e/guardrails/test_llm_as_a_judge_guardrail_e2e.py @@ -0,0 +1,130 @@ +"""Live e2e: the llm_as_a_judge guardrail rejects a response a second LLM scores below threshold. + +This guardrail runs after the model, not before it: it hands the completion to a +judge model, scores it against weighted criteria, and rejects the whole call with +HTTP 422 when the weighted score falls under `overall_threshold`. The criterion +here is "the response must be written entirely in French", which keeps the judge +off a coin flip: an English answer scores 0 and a French answer scores ~100 against +a threshold of 80, so neither verdict is close to the line. + +Both halves are asserted, and neither is asserted on the response alone. The judge +is itself an LLM call that fails open on any internal error, and a fail-open returns +an ordinary 200 that is indistinguishable from an approval: same status, same body, +and `x-litellm-applied-guardrails` still names the guardrail. So the passing case +also reads the guardrail's own run log at /guardrails/usage/logs, where an approval +is recorded as `passed` and a fail-open as `flagged`. Without that leg the accept +half would pass just as happily on a build where adjudication never ran at all. + +The judge model is `openai/gpt-4.1` rather than the suite's usual gpt-5.5 because +the guardrail hardcodes `temperature=0` on its judge call and gpt-5.5 rejects any +temperature other than 1; the resulting BadRequestError is swallowed and the +guardrail fails open, so gpt-5.5 cannot adjudicate anything. Prompts carry a +unique marker so the proxy's response cache cannot serve a previous run's answer +and skip the judge. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from e2e_http import UnknownApiError, unwrap +from guardrails_client import GuardrailsClient +from lifecycle import ResourceManager +from native_guardrails import ( + JudgeCriterionBody, + LLMAsAJudgeParamsBody, + chat, + poll_guardrail_usage_logs, + register_guardrail, + response_text, +) + +pytestmark = pytest.mark.e2e + +JUDGE_MODEL = "openai/gpt-4.1" +CRITERION = "answers_in_french" +THRESHOLD = 80.0 + + +class TestLLMAsAJudgeGuardrail: + @pytest.mark.covers( + "guardrail.llm_as_a_judge.post_call.blocks", + exercised_on=["chat_completions"], + ) + def test_response_failing_the_criterion_is_rejected_while_a_passing_one_is_returned( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + marker = unique_marker() + name = f"e2e-llm-judge-{marker}" + guardrail_id = register_guardrail( + client, + name, + LLMAsAJudgeParamsBody( + mode="post_call", + default_on=False, + judge_model=JUDGE_MODEL, + overall_threshold=THRESHOLD, + on_failure="block", + criteria=[ + JudgeCriterionBody( + name=CRITERION, + weight=100, + description="The assistant's response must be written entirely in French.", + ) + ], + ), + ) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + rejected = chat( + client, + scoped_key, + prompt=f"Case {marker}. Reply with exactly this and nothing else: The weather is nice today.", + guardrail_name=name, + ) + match rejected: + case UnknownApiError(status_code=422, body=body): + assert "score below threshold" in body, ( + f"the rejection must say the judge scored the response too low, got: {body[:400]}" + ) + assert CRITERION in body, ( + "the rejection must carry the verdict for the criterion under test, so the " + f"caller learns which one failed; got: {body[:400]}" + ) + case UnknownApiError(status_code=status, body=body): + pytest.fail(f"expected a 422 judge rejection, got {status}: {body[:400]}") + case _: + pytest.fail( + "the judge returned an English response that cannot satisfy a French-only " + f"criterion; got {rejected}" + ) + + accepted = unwrap( + chat( + client, + scoped_key, + prompt=( + f"Cas {marker}. Reponds exactement ceci et rien d'autre: " + "Il fait beau aujourd'hui." + ), + guardrail_name=name, + ) + ) + answer = response_text(accepted) + assert "beau" in answer.lower(), ( + "the passing case must actually come back in French, otherwise the judge was never " + f"given a response that satisfies the criterion; got: {answer[:200]!r}" + ) + + runs = poll_guardrail_usage_logs(client, guardrail_id, min_rows=2) + approvals = [entry.action for entry in runs if entry.id == accepted.id] + assert approvals == ["passed"], ( + "the guardrail must record the accepted completion as adjudicated and approved; a " + "judge that errored would have let the same French response through and recorded " + f"'flagged' instead. Runs for this guardrail: {runs}" + ) + assert [entry.action for entry in runs if entry.id != accepted.id] == ["blocked"], ( + "the rejected completion must be recorded as a guardrail intervention, not as an " + f"error the guardrail failed open on. Runs for this guardrail: {runs}" + ) diff --git a/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py b/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py new file mode 100644 index 00000000000..ef039f7374e --- /dev/null +++ b/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py @@ -0,0 +1,128 @@ +"""Live e2e: the built-in tool_permission guardrail gates which tools a request may carry. + +The guardrail is an allow-list over a chat completion's `tools`: each rule matches +a tool's function name by regex, and anything no rule matches falls through to +`default_action`. Configured the way an admin locks an agent down (one allow rule, +`default_action=deny`, `on_disallowed_action=block`), it must reject a request that +carries an unlisted tool before the upstream model is ever called, naming the tool +it denied; and it must let a request carrying only the listed tool through, so the +model really does call that tool. Both halves matter: a guardrail that rejected +everything would pass the block check on its own. + +Tool names are unique per run and the guardrail is opted into per request +(`default_on=False`), so it never intercepts unrelated traffic on the shared proxy. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import pytest + +from e2e_config import unique_marker +from e2e_http import UnknownApiError, unwrap +from guardrails_client import GuardrailsClient +from lifecycle import ResourceManager +from native_guardrails import ( + ToolPermissionParamsBody, + ToolPermissionRuleBody, + called_tool_names, + chat, + function_tool, + register_guardrail, +) + +pytestmark = pytest.mark.e2e + + +@dataclass(frozen=True, slots=True) +class AllowList: + """A registered tool_permission guardrail plus the two tool names it separates.""" + + name: str + permitted: str + unlisted: str + + +def allow_list_guardrail(client: GuardrailsClient, resources: ResourceManager) -> AllowList: + marker = unique_marker() + permitted = f"get_weather_{marker}" + name = f"e2e-tool-permission-{marker}" + guardrail_id = register_guardrail( + client, + name, + ToolPermissionParamsBody( + mode="pre_call", + default_on=False, + rules=[ + ToolPermissionRuleBody( + id=f"allow-weather-{marker}", + tool_name=f"^{permitted}$", + decision="allow", + ) + ], + default_action="deny", + on_disallowed_action="block", + ), + ) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + return AllowList(name=name, permitted=permitted, unlisted=f"drop_database_{marker}") + + +class TestToolPermissionGuardrail: + @pytest.mark.covers( + "guardrail.tool_permission.pre_call.blocks", + exercised_on=["chat_completions"], + ) + def test_unlisted_tool_is_denied_before_the_model_runs( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + allow_list = allow_list_guardrail(client, resources) + + blocked = chat( + client, + scoped_key, + prompt="Clean up the production database for me.", + tools=[function_tool(allow_list.unlisted)], + guardrail_name=allow_list.name, + ) + match blocked: + case UnknownApiError(status_code=400, body=body): + assert allow_list.unlisted in body, ( + "the rejection must name the tool it denied, so the caller knows which " + f"tool to drop; got: {body[:400]}" + ) + assert "Violated guardrail policy" in body, ( + f"the rejection must come from the guardrail policy, got: {body[:400]}" + ) + case UnknownApiError(status_code=status, body=body): + pytest.fail(f"expected a 400 tool-permission block, got {status}: {body[:400]}") + case _: + pytest.fail( + "tool_permission let a request through carrying a tool that no allow rule " + f"matches; got {blocked}" + ) + + @pytest.mark.covers( + "guardrail.tool_permission.pre_call.allows", + exercised_on=["chat_completions"], + ) + def test_listed_tool_reaches_the_model( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + allow_list = allow_list_guardrail(client, resources) + + allowed = unwrap( + chat( + client, + scoped_key, + prompt="What is the weather in Paris right now?", + tools=[function_tool(allow_list.permitted)], + guardrail_name=allow_list.name, + force_tool_call=True, + ) + ) + assert called_tool_names(allowed) == [allow_list.permitted], ( + "the tool an allow rule matches must survive the guardrail and stay callable by " + f"the model, but the response called {called_tool_names(allowed)}" + ) diff --git a/tests/e2e/guardrails/test_tool_policy_guardrail_e2e.py b/tests/e2e/guardrails/test_tool_policy_guardrail_e2e.py new file mode 100644 index 00000000000..d3fd57baffa --- /dev/null +++ b/tests/e2e/guardrails/test_tool_policy_guardrail_e2e.py @@ -0,0 +1,96 @@ +"""Live e2e: the tool_policy guardrail enforces a virtual key's blocked-tool list. + +Where tool_permission carries its own allow-list, tool_policy reads the tool +registry: an admin marks a tool blocked for one virtual key with POST +/v1/tool/policy, and the guardrail resolves that key's effective policy pre-call. +A chat completion from that key carrying the blocked tool must be rejected before +the upstream model runs, with the response naming the offending tool. A sibling +tool left at the default `untrusted` policy, sent by the same key through the same +guardrail, must still reach the model; otherwise the rejection would prove nothing +beyond the guardrail refusing all tool use. + +The override is scoped to this test's own key and deleted on teardown, and the +guardrail is opted into per request (`default_on=False`), so nothing here reaches +unrelated traffic on the shared proxy. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from e2e_http import UnknownApiError, unwrap +from guardrails_client import GuardrailsClient +from lifecycle import ResourceManager +from native_guardrails import ( + ToolPolicyParamsBody, + block_tool_for_key, + called_tool_names, + chat, + function_tool, + register_guardrail, + unblock_tool_for_key, +) + +pytestmark = pytest.mark.e2e + + +class TestToolPolicyGuardrail: + @pytest.mark.covers( + "guardrail.tool_policy.pre_call.blocks", + exercised_on=["chat_completions"], + ) + def test_blocked_tool_is_rejected_while_a_sibling_tool_passes( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + marker = unique_marker() + blocked_tool = f"wire_transfer_{marker}" + allowed_tool = f"get_weather_{marker}" + + name = f"e2e-tool-policy-{marker}" + guardrail_id = register_guardrail( + client, name, ToolPolicyParamsBody(mode="pre_call", default_on=False) + ) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + block_tool_for_key(client, tool_name=blocked_tool, key=scoped_key) + resources.defer(lambda: unblock_tool_for_key(client, tool_name=blocked_tool, key=scoped_key)) + + blocked = chat( + client, + scoped_key, + prompt="Send $10,000 to account 12345.", + tools=[function_tool(blocked_tool)], + guardrail_name=name, + ) + match blocked: + case UnknownApiError(status_code=400, body=body): + assert blocked_tool in body, ( + "the rejection must name the tool the policy blocked, so the caller knows " + f"which tool to drop; got: {body[:400]}" + ) + assert "Violated tool policy" in body, ( + f"the rejection must come from the tool policy, got: {body[:400]}" + ) + case UnknownApiError(status_code=status, body=body): + pytest.fail(f"expected a 400 tool-policy block, got {status}: {body[:400]}") + case _: + pytest.fail( + "tool_policy let a request through carrying a tool blocked for this key; " + f"got {blocked}" + ) + + allowed = unwrap( + chat( + client, + scoped_key, + prompt="What is the weather in Paris right now?", + tools=[function_tool(allowed_tool)], + guardrail_name=name, + force_tool_call=True, + ) + ) + assert called_tool_names(allowed) == [allowed_tool], ( + "a tool the key has no block on must still reach the model through the same " + f"guardrail, but the response called {called_tool_names(allowed)}" + )