Merge 2fd7929dc3 into 982d3a5476

tests/e2e/coverage_registry/guardrail.yaml

@@ -30,5 +30,5 @@
 - {id: guardrail.niche_providers.pre_call.allows, module: guardrail, tier: P2, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: grammar, rationale: "SMOKE niche allow-path passthrough"}
 - {id: guardrail.tool_policy.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/tool_policy/tool_policy_guardrail.py", rationale: "Tool-use policy enforcement"}
 - {id: guardrail.mcp_security.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [mcp_operations], source: "guardrail_hooks/mcp_security", rationale: "MCP protocol security"}
-- {id: guardrail.llm_as_a_judge.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/llm_as_a_judge", rationale: "LLM-based judgment guardrail"}
+- {id: guardrail.llm_as_a_judge.post_call.blocks, module: guardrail, tier: P2, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/llm_as_a_judge", rationale: "LLM-based judgment guardrail"}
 - {id: guardrail.litellm_content_filter.pre_mcp_call.blocks, module: guardrail, tier: P1, hook_point: pre_mcp_call, assertions: [blocks], exercised_on: [mcp_operations], source: "guardrail_hooks/litellm_content_filter/content_filter.py:_scan_mcp_tool_call_arguments", rationale: "A general content-filter guardrail configured mode=pre_mcp_call blocks a banned keyword in an MCP tool call's arguments before it reaches the upstream MCP server; a clean argument passes"}

tests/e2e/guardrails/native_guardrails.py

@@ -0,0 +1,255 @@
+"""Typed bodies and helpers for the guardrails litellm implements itself.
+
+`tool_permission` and `tool_policy` decide, before the upstream model runs,
+whether the tools a request carries may be used at all: the first from a regex
+allow-list configured on the guardrail, the second from the per-key blocked-tool
+overrides admins set through POST /v1/tool/policy. `llm_as_a_judge` runs after
+the model instead, scoring the response against weighted criteria with a second
+LLM. All three register through the same POST /guardrails route the rest of the
+suite uses and only their params differ, so the params compose onto the shared
+`GuardrailParamsBase` and the create body is written once here.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Literal, Sequence
+
+from pydantic import BaseModel
+
+from e2e_config import CHEAP_OPENAI_MODEL, POLL_INTERVAL, POLL_TIMEOUT
+from e2e_http import NoBody, Result, Success, unwrap
+from guardrails_client import (
+    GuardrailCreateResponse,
+    GuardrailParamsBase,
+    GuardrailsClient,
+)
+from models import ChatBody, ChatMessage, ChatResponse, ChatTool, ChatToolFunction
+
+ToolInputPolicy = Literal["blocked", "trusted", "untrusted"]
+
+
+class ToolPermissionRuleBody(BaseModel):
+    """One allow/deny rule: `tool_name` is a regex matched against the tool's
+    function name."""
+
+    id: str
+    tool_name: str
+    decision: Literal["allow", "deny"]
+
+
+class ToolPermissionParamsBody(GuardrailParamsBase):
+    guardrail: Literal["tool_permission"] = "tool_permission"
+    rules: list[ToolPermissionRuleBody]
+    default_action: Literal["allow", "deny"]
+    on_disallowed_action: Literal["block", "rewrite"]
+
+
+class ToolPolicyParamsBody(GuardrailParamsBase):
+    """tool_policy reads its decisions from the tool registry and the caller's
+    blocked-tool overrides, so the guardrail itself carries no rules."""
+
+    guardrail: Literal["tool_policy"] = "tool_policy"
+
+
+class JudgeCriterionBody(BaseModel):
+    """One scoring criterion. Weights across a guardrail's criteria must sum to
+    100 or the proxy rejects the registration."""
+
+    name: str
+    weight: int
+    description: str
+
+
+class LLMAsAJudgeParamsBody(GuardrailParamsBase):
+    guardrail: Literal["llm_as_a_judge"] = "llm_as_a_judge"
+    judge_model: str
+    criteria: list[JudgeCriterionBody]
+    overall_threshold: float
+    on_failure: Literal["block", "log"]
+
+
+NativeGuardrailParamsBody = (
+    ToolPermissionParamsBody | ToolPolicyParamsBody | LLMAsAJudgeParamsBody
+)
+
+
+class NativeGuardrailSpecBody(BaseModel):
+    guardrail_name: str
+    litellm_params: NativeGuardrailParamsBody
+
+
+class NativeGuardrailCreateBody(BaseModel):
+    guardrail: NativeGuardrailSpecBody
+
+
+class ToolPolicyOverrideBody(BaseModel):
+    """POST /v1/tool/policy scoped to one virtual key: sets that key's policy for
+    a single tool without touching the global registry entry."""
+
+    tool_name: str
+    input_policy: ToolInputPolicy
+    key_hash: str
+
+
+class ToolPolicyUpdateResponse(BaseModel):
+    tool_name: str
+    updated: bool
+
+
+class ToolOverrideParams(BaseModel):
+    key_hash: str
+
+
+class ToolOverrideDeleteResponse(BaseModel):
+    deleted: bool
+    tool_name: str
+
+
+class GuardrailUsageLogEntry(BaseModel):
+    """One row of GET /guardrails/usage/logs. `action` is the guardrail's own
+    verdict on that request: `passed` when it ran and approved, `blocked` when it
+    intervened, and `flagged` when it errored and let the request through anyway.
+    `id` is the completion id for a request the guardrail approved."""
+
+    id: str
+    action: Literal["passed", "blocked", "flagged"]
+
+
+class GuardrailUsageLogs(BaseModel):
+    logs: list[GuardrailUsageLogEntry]
+    total: int
+
+
+class GuardrailUsageLogsParams(BaseModel):
+    guardrail_id: str
+
+
+def register_guardrail(
+    client: GuardrailsClient, name: str, params: NativeGuardrailParamsBody
+) -> str:
+    """Register a native guardrail and return its id. Registered with
+    `default_on=False` and opted into per request, so it never intercepts
+    unrelated traffic on the shared proxy."""
+    return unwrap(
+        client.proxy.transport.post(
+            "/guardrails",
+            headers=client.proxy.transport.master,
+            json=NativeGuardrailCreateBody(
+                guardrail=NativeGuardrailSpecBody(guardrail_name=name, litellm_params=params)
+            ),
+            response_type=GuardrailCreateResponse,
+        )
+    ).guardrail_id
+
+
+def block_tool_for_key(client: GuardrailsClient, *, tool_name: str, key: str) -> None:
+    """Mark one tool blocked for one virtual key. The proxy resolves the raw key
+    to its object permission (creating one when the key has none) and resyncs the
+    in-memory tool-policy registry before answering."""
+    _ = unwrap(
+        client.proxy.transport.post(
+            "/v1/tool/policy",
+            headers=client.proxy.transport.master,
+            json=ToolPolicyOverrideBody(
+                tool_name=tool_name, input_policy="blocked", key_hash=key
+            ),
+            response_type=ToolPolicyUpdateResponse,
+        )
+    )
+
+
+def unblock_tool_for_key(client: GuardrailsClient, *, tool_name: str, key: str) -> None:
+    _ = client.proxy.transport.delete(
+        f"/v1/tool/{tool_name}/overrides",
+        headers=client.proxy.transport.master,
+        json=NoBody(),
+        params=ToolOverrideParams(key_hash=key),
+        response_type=ToolOverrideDeleteResponse,
+    )
+
+
+def function_tool(name: str) -> ChatTool:
+    return ChatTool(
+        function=ChatToolFunction(
+            name=name,
+            description="Look up the current weather for a city.",
+            parameters={
+                "type": "object",
+                "properties": {"city": {"type": "string"}},
+                "required": ["city"],
+            },
+        )
+    )
+
+
+def chat(
+    client: GuardrailsClient,
+    key: str,
+    *,
+    prompt: str,
+    guardrail_name: str,
+    tools: Sequence[ChatTool] = (),
+    force_tool_call: bool = False,
+) -> Result[ChatResponse]:
+    """Drive a real chat completion, opting into one guardrail for this request
+    only. `tools` carries the function definitions a tool-gating guardrail reads;
+    `force_tool_call` sets tool_choice=required so an allowed tool is provably
+    reachable rather than left to the model's mood. No max_tokens is sent: gpt-5.5
+    spends tokens on reasoning before it writes anything, and a truncated empty
+    response would leave a post-call guardrail with nothing to judge."""
+    return client.proxy.chat(
+        key,
+        ChatBody(
+            model=CHEAP_OPENAI_MODEL,
+            messages=[ChatMessage(role="user", content=prompt)],
+            tools=list(tools) or None,
+            tool_choice="required" if force_tool_call else None,
+            guardrails=[guardrail_name],
+        ),
+    )
+
+
+def called_tool_names(response: ChatResponse) -> list[str]:
+    if not response.choices:
+        return []
+    message = response.choices[0].message
+    calls = (message.tool_calls if message else None) or []
+    return [call.function.name for call in calls if call.function.name]
+
+
+def poll_guardrail_usage_logs(
+    client: GuardrailsClient, guardrail_id: str, *, min_rows: int
+) -> list[GuardrailUsageLogEntry]:
+    """Poll the guardrail's own run log until it has recorded `min_rows` requests.
+
+    This is the only caller-visible record of what a guardrail decided. It matters
+    for a guardrail that adjudicates with a second LLM, because that call fails
+    open on any internal error: the request then returns a normal 200 that looks
+    exactly like an approval. `action` separates the two, so a test can assert the
+    guardrail actually ran rather than that the response merely came back. Rows are
+    fetched by guardrail_id, and each test registers its own guardrail, so the log
+    holds that test's requests and nothing else."""
+    deadline = time.monotonic() + POLL_TIMEOUT
+    last: Result[GuardrailUsageLogs] | None = None
+    while time.monotonic() < deadline:
+        last = client.proxy.transport.get(
+            "/guardrails/usage/logs",
+            headers=client.proxy.transport.master,
+            params=GuardrailUsageLogsParams(guardrail_id=guardrail_id),
+            response_type=GuardrailUsageLogs,
+        )
+        if isinstance(last, Success) and len(last.data.logs) >= min_rows:
+            return last.data.logs
+        time.sleep(POLL_INTERVAL)
+    raise AssertionError(
+        f"guardrail {guardrail_id!r} never recorded {min_rows} run(s) in "
+        f"/guardrails/usage/logs; last response was {last}"
+    )
+
+
+def response_text(response: ChatResponse) -> str:
+    if not response.choices:
+        return ""
+    message = response.choices[0].message
+    return (message.content if message else None) or ""

tests/e2e/guardrails/test_llm_as_a_judge_guardrail_e2e.py

@@ -0,0 +1,130 @@
+"""Live e2e: the llm_as_a_judge guardrail rejects a response a second LLM scores below threshold.
+
+This guardrail runs after the model, not before it: it hands the completion to a
+judge model, scores it against weighted criteria, and rejects the whole call with
+HTTP 422 when the weighted score falls under `overall_threshold`. The criterion
+here is "the response must be written entirely in French", which keeps the judge
+off a coin flip: an English answer scores 0 and a French answer scores ~100 against
+a threshold of 80, so neither verdict is close to the line.
+
+Both halves are asserted, and neither is asserted on the response alone. The judge
+is itself an LLM call that fails open on any internal error, and a fail-open returns
+an ordinary 200 that is indistinguishable from an approval: same status, same body,
+and `x-litellm-applied-guardrails` still names the guardrail. So the passing case
+also reads the guardrail's own run log at /guardrails/usage/logs, where an approval
+is recorded as `passed` and a fail-open as `flagged`. Without that leg the accept
+half would pass just as happily on a build where adjudication never ran at all.
+
+The judge model is `openai/gpt-4.1` rather than the suite's usual gpt-5.5 because
+the guardrail hardcodes `temperature=0` on its judge call and gpt-5.5 rejects any
+temperature other than 1; the resulting BadRequestError is swallowed and the
+guardrail fails open, so gpt-5.5 cannot adjudicate anything. Prompts carry a
+unique marker so the proxy's response cache cannot serve a previous run's answer
+and skip the judge.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from e2e_config import unique_marker
+from e2e_http import UnknownApiError, unwrap
+from guardrails_client import GuardrailsClient
+from lifecycle import ResourceManager
+from native_guardrails import (
+    JudgeCriterionBody,
+    LLMAsAJudgeParamsBody,
+    chat,
+    poll_guardrail_usage_logs,
+    register_guardrail,
+    response_text,
+)
+
+pytestmark = pytest.mark.e2e
+
+JUDGE_MODEL = "openai/gpt-4.1"
+CRITERION = "answers_in_french"
+THRESHOLD = 80.0
+
+
+class TestLLMAsAJudgeGuardrail:
+    @pytest.mark.covers(
+        "guardrail.llm_as_a_judge.post_call.blocks",
+        exercised_on=["chat_completions"],
+    )
+    def test_response_failing_the_criterion_is_rejected_while_a_passing_one_is_returned(
+        self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        marker = unique_marker()
+        name = f"e2e-llm-judge-{marker}"
+        guardrail_id = register_guardrail(
+            client,
+            name,
+            LLMAsAJudgeParamsBody(
+                mode="post_call",
+                default_on=False,
+                judge_model=JUDGE_MODEL,
+                overall_threshold=THRESHOLD,
+                on_failure="block",
+                criteria=[
+                    JudgeCriterionBody(
+                        name=CRITERION,
+                        weight=100,
+                        description="The assistant's response must be written entirely in French.",
+                    )
+                ],
+            ),
+        )
+        resources.defer(lambda: client.delete_guardrail(guardrail_id))
+
+        rejected = chat(
+            client,
+            scoped_key,
+            prompt=f"Case {marker}. Reply with exactly this and nothing else: The weather is nice today.",
+            guardrail_name=name,
+        )
+        match rejected:
+            case UnknownApiError(status_code=422, body=body):
+                assert "score below threshold" in body, (
+                    f"the rejection must say the judge scored the response too low, got: {body[:400]}"
+                )
+                assert CRITERION in body, (
+                    "the rejection must carry the verdict for the criterion under test, so the "
+                    f"caller learns which one failed; got: {body[:400]}"
+                )
+            case UnknownApiError(status_code=status, body=body):
+                pytest.fail(f"expected a 422 judge rejection, got {status}: {body[:400]}")
+            case _:
+                pytest.fail(
+                    "the judge returned an English response that cannot satisfy a French-only "
+                    f"criterion; got {rejected}"
+                )
+
+        accepted = unwrap(
+            chat(
+                client,
+                scoped_key,
+                prompt=(
+                    f"Cas {marker}. Reponds exactement ceci et rien d'autre: "
+                    "Il fait beau aujourd'hui."
+                ),
+                guardrail_name=name,
+            )
+        )
+        answer = response_text(accepted)
+        assert "beau" in answer.lower(), (
+            "the passing case must actually come back in French, otherwise the judge was never "
+            f"given a response that satisfies the criterion; got: {answer[:200]!r}"
+        )
+
+        runs = poll_guardrail_usage_logs(client, guardrail_id, min_rows=2)
+        approvals = [entry.action for entry in runs if entry.id == accepted.id]
+        assert approvals == ["passed"], (
+            "the guardrail must record the accepted completion as adjudicated and approved; a "
+            "judge that errored would have let the same French response through and recorded "
+            f"'flagged' instead. Runs for this guardrail: {runs}"
+        )
+        assert [entry.action for entry in runs if entry.id != accepted.id] == ["blocked"], (
+            "the rejected completion must be recorded as a guardrail intervention, not as an "
+            f"error the guardrail failed open on. Runs for this guardrail: {runs}"
+        )

tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py

@@ -0,0 +1,128 @@
+"""Live e2e: the built-in tool_permission guardrail gates which tools a request may carry.
+
+The guardrail is an allow-list over a chat completion's `tools`: each rule matches
+a tool's function name by regex, and anything no rule matches falls through to
+`default_action`. Configured the way an admin locks an agent down (one allow rule,
+`default_action=deny`, `on_disallowed_action=block`), it must reject a request that
+carries an unlisted tool before the upstream model is ever called, naming the tool
+it denied; and it must let a request carrying only the listed tool through, so the
+model really does call that tool. Both halves matter: a guardrail that rejected
+everything would pass the block check on its own.
+
+Tool names are unique per run and the guardrail is opted into per request
+(`default_on=False`), so it never intercepts unrelated traffic on the shared proxy.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pytest
+
+from e2e_config import unique_marker
+from e2e_http import UnknownApiError, unwrap
+from guardrails_client import GuardrailsClient
+from lifecycle import ResourceManager
+from native_guardrails import (
+    ToolPermissionParamsBody,
+    ToolPermissionRuleBody,
+    called_tool_names,
+    chat,
+    function_tool,
+    register_guardrail,
+)
+
+pytestmark = pytest.mark.e2e
+
+
+@dataclass(frozen=True, slots=True)
+class AllowList:
+    """A registered tool_permission guardrail plus the two tool names it separates."""
+
+    name: str
+    permitted: str
+    unlisted: str
+
+
+def allow_list_guardrail(client: GuardrailsClient, resources: ResourceManager) -> AllowList:
+    marker = unique_marker()
+    permitted = f"get_weather_{marker}"
+    name = f"e2e-tool-permission-{marker}"
+    guardrail_id = register_guardrail(
+        client,
+        name,
+        ToolPermissionParamsBody(
+            mode="pre_call",
+            default_on=False,
+            rules=[
+                ToolPermissionRuleBody(
+                    id=f"allow-weather-{marker}",
+                    tool_name=f"^{permitted}$",
+                    decision="allow",
+                )
+            ],
+            default_action="deny",
+            on_disallowed_action="block",
+        ),
+    )
+    resources.defer(lambda: client.delete_guardrail(guardrail_id))
+    return AllowList(name=name, permitted=permitted, unlisted=f"drop_database_{marker}")
+
+
+class TestToolPermissionGuardrail:
+    @pytest.mark.covers(
+        "guardrail.tool_permission.pre_call.blocks",
+        exercised_on=["chat_completions"],
+    )
+    def test_unlisted_tool_is_denied_before_the_model_runs(
+        self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        allow_list = allow_list_guardrail(client, resources)
+
+        blocked = chat(
+            client,
+            scoped_key,
+            prompt="Clean up the production database for me.",
+            tools=[function_tool(allow_list.unlisted)],
+            guardrail_name=allow_list.name,
+        )
+        match blocked:
+            case UnknownApiError(status_code=400, body=body):
+                assert allow_list.unlisted in body, (
+                    "the rejection must name the tool it denied, so the caller knows which "
+                    f"tool to drop; got: {body[:400]}"
+                )
+                assert "Violated guardrail policy" in body, (
+                    f"the rejection must come from the guardrail policy, got: {body[:400]}"
+                )
+            case UnknownApiError(status_code=status, body=body):
+                pytest.fail(f"expected a 400 tool-permission block, got {status}: {body[:400]}")
+            case _:
+                pytest.fail(
+                    "tool_permission let a request through carrying a tool that no allow rule "
+                    f"matches; got {blocked}"
+                )
+
+    @pytest.mark.covers(
+        "guardrail.tool_permission.pre_call.allows",
+        exercised_on=["chat_completions"],
+    )
+    def test_listed_tool_reaches_the_model(
+        self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        allow_list = allow_list_guardrail(client, resources)
+
+        allowed = unwrap(
+            chat(
+                client,
+                scoped_key,
+                prompt="What is the weather in Paris right now?",
+                tools=[function_tool(allow_list.permitted)],
+                guardrail_name=allow_list.name,
+                force_tool_call=True,
+            )
+        )
+        assert called_tool_names(allowed) == [allow_list.permitted], (
+            "the tool an allow rule matches must survive the guardrail and stay callable by "
+            f"the model, but the response called {called_tool_names(allowed)}"
+        )

tests/e2e/guardrails/test_tool_policy_guardrail_e2e.py

@@ -0,0 +1,96 @@
+"""Live e2e: the tool_policy guardrail enforces a virtual key's blocked-tool list.
+
+Where tool_permission carries its own allow-list, tool_policy reads the tool
+registry: an admin marks a tool blocked for one virtual key with POST
+/v1/tool/policy, and the guardrail resolves that key's effective policy pre-call.
+A chat completion from that key carrying the blocked tool must be rejected before
+the upstream model runs, with the response naming the offending tool. A sibling
+tool left at the default `untrusted` policy, sent by the same key through the same
+guardrail, must still reach the model; otherwise the rejection would prove nothing
+beyond the guardrail refusing all tool use.
+
+The override is scoped to this test's own key and deleted on teardown, and the
+guardrail is opted into per request (`default_on=False`), so nothing here reaches
+unrelated traffic on the shared proxy.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from e2e_config import unique_marker
+from e2e_http import UnknownApiError, unwrap
+from guardrails_client import GuardrailsClient
+from lifecycle import ResourceManager
+from native_guardrails import (
+    ToolPolicyParamsBody,
+    block_tool_for_key,
+    called_tool_names,
+    chat,
+    function_tool,
+    register_guardrail,
+    unblock_tool_for_key,
+)
+
+pytestmark = pytest.mark.e2e
+
+
+class TestToolPolicyGuardrail:
+    @pytest.mark.covers(
+        "guardrail.tool_policy.pre_call.blocks",
+        exercised_on=["chat_completions"],
+    )
+    def test_blocked_tool_is_rejected_while_a_sibling_tool_passes(
+        self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        marker = unique_marker()
+        blocked_tool = f"wire_transfer_{marker}"
+        allowed_tool = f"get_weather_{marker}"
+
+        name = f"e2e-tool-policy-{marker}"
+        guardrail_id = register_guardrail(
+            client, name, ToolPolicyParamsBody(mode="pre_call", default_on=False)
+        )
+        resources.defer(lambda: client.delete_guardrail(guardrail_id))
+
+        block_tool_for_key(client, tool_name=blocked_tool, key=scoped_key)
+        resources.defer(lambda: unblock_tool_for_key(client, tool_name=blocked_tool, key=scoped_key))
+
+        blocked = chat(
+            client,
+            scoped_key,
+            prompt="Send $10,000 to account 12345.",
+            tools=[function_tool(blocked_tool)],
+            guardrail_name=name,
+        )
+        match blocked:
+            case UnknownApiError(status_code=400, body=body):
+                assert blocked_tool in body, (
+                    "the rejection must name the tool the policy blocked, so the caller knows "
+                    f"which tool to drop; got: {body[:400]}"
+                )
+                assert "Violated tool policy" in body, (
+                    f"the rejection must come from the tool policy, got: {body[:400]}"
+                )
+            case UnknownApiError(status_code=status, body=body):
+                pytest.fail(f"expected a 400 tool-policy block, got {status}: {body[:400]}")
+            case _:
+                pytest.fail(
+                    "tool_policy let a request through carrying a tool blocked for this key; "
+                    f"got {blocked}"
+                )
+
+        allowed = unwrap(
+            chat(
+                client,
+                scoped_key,
+                prompt="What is the weather in Paris right now?",
+                tools=[function_tool(allowed_tool)],
+                guardrail_name=name,
+                force_tool_call=True,
+            )
+        )
+        assert called_tool_names(allowed) == [allowed_tool], (
+            "a tool the key has no block on must still reach the model through the same "
+            f"guardrail, but the response called {called_tool_names(allowed)}"
+        )