mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-01 02:02:20 +00:00
feat(guardrails): agent 365 fails open by default and hides the production overrides from the UI form
Agent 365 sits in the runtime path of every MCP tool call, so an Entra or Agent 365 outage now lets the call through unscanned (logged at error level, recorded as Unscanned with guardrail_failed_to_respond) instead of blocking it. unreachable_fallback: fail_closed stays as the opt-in strict mode. Policy blocks, throttling, 4xx rejections and a rejected caller token still block The shared unreachable_fallback field becomes nullable so each guardrail owns its default; every sibling still resolves None to fail_closed and typesafe keeps failing open api_base, resource_app_id and agent_id have production defaults and leave the dashboard form (ui_hidden); they stay available in config.yaml and env. The authority_host override and its env keys are gone, the OBO exchange always uses login.microsoftonline.com. The integration suite keeps only the cells that need no Entra double, the evaluation paths live in unit tests with an injected handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng
parent
1dc8053ca2
commit
8cf5664ac1
17 changed files with 144 additions and 495 deletions
|
|
@ -11579,18 +11579,6 @@
|
|||
"description": "Authorization bearer token for IBM Guardrails API. Reads from IBM_GUARDRAILS_AUTH_TOKEN env var if None.",
|
||||
"title": "Auth Token"
|
||||
},
|
||||
"authority_host": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Microsoft Entra authority host that issues the On-Behalf-Of token, for sovereign clouds. Defaults to https://login.microsoftonline.com. Falls back to the AGENT365_AUTHORITY_HOST, then AZURE_AUTHORITY_HOST environment variables.",
|
||||
"title": "Authority Host"
|
||||
},
|
||||
"aws_access_key_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1838,6 +1838,11 @@ def _build_field_dict(
|
|||
return field_dict
|
||||
|
||||
|
||||
def _is_ui_hidden(field: "FieldInfo") -> bool:
|
||||
field_json_schema_extra: Final = field.json_schema_extra
|
||||
return isinstance(field_json_schema_extra, Mapping) and bool(field_json_schema_extra.get("ui_hidden"))
|
||||
|
||||
|
||||
def _extract_fields_recursive(
|
||||
model: type[BaseModel],
|
||||
depth: int = 0,
|
||||
|
|
@ -1858,6 +1863,9 @@ def _extract_fields_recursive(
|
|||
if _should_skip_optional_params(field_name=field_name, field_annotation=field_annotation):
|
||||
continue
|
||||
|
||||
if _is_ui_hidden(field):
|
||||
continue
|
||||
|
||||
# Handle Optional types and get the actual type
|
||||
if field_annotation is None:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from typing import TYPE_CHECKING, Final
|
|||
|
||||
from litellm.types.guardrails import SupportedGuardrailIntegrations
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.agent_365 import (
|
||||
AGENT_365_DEFAULT_AUTHORITY_HOST,
|
||||
AGENT_365_PROD_API_BASE,
|
||||
AGENT_365_PROD_RESOURCE_APP_ID,
|
||||
)
|
||||
|
|
@ -24,12 +23,6 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
|
|||
)
|
||||
api_base: Final = litellm_params.api_base or get_secret_str("AGENT365_API_BASE")
|
||||
resource_app_id: Final = litellm_params.resource_app_id or get_secret_str("AGENT365_RESOURCE_APP_ID")
|
||||
authority_host: Final = (
|
||||
litellm_params.authority_host
|
||||
or get_secret_str("AGENT365_AUTHORITY_HOST")
|
||||
or get_secret_str("AZURE_AUTHORITY_HOST")
|
||||
or AGENT_365_DEFAULT_AUTHORITY_HOST
|
||||
)
|
||||
|
||||
if not tenant_id:
|
||||
raise ValueError("Microsoft Agent 365: tenant_id is required")
|
||||
|
|
@ -52,9 +45,8 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
|
|||
api_base=api_base or AGENT_365_PROD_API_BASE,
|
||||
resource_app_id=resource_app_id or AGENT_365_PROD_RESOURCE_APP_ID,
|
||||
agent_id=litellm_params.agent_id,
|
||||
authority_host=authority_host,
|
||||
request_timeout=litellm_params.timeout if litellm_params.timeout is not None else 10.0,
|
||||
unreachable_fallback=litellm_params.unreachable_fallback,
|
||||
unreachable_fallback=litellm_params.unreachable_fallback or "fail_open",
|
||||
event_hook=litellm_params.mode,
|
||||
default_on=litellm_params.default_on,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -36,10 +36,10 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
)
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.agent_365 import (
|
||||
AGENT_365_DEFAULT_AUTHORITY_HOST,
|
||||
AGENT_365_PROD_API_BASE,
|
||||
AGENT_365_PROD_RESOURCE_APP_ID,
|
||||
AGENT_365_SCOPE_NAME,
|
||||
AGENT_365_TOKEN_URL_TEMPLATE,
|
||||
Agent365GuardrailConfigModel,
|
||||
)
|
||||
|
||||
|
|
@ -49,7 +49,6 @@ if TYPE_CHECKING:
|
|||
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
|
||||
from litellm.types.utils import GuardrailStatus
|
||||
|
||||
TOKEN_PATH_TEMPLATE: Final = "/{tenant_id}/oauth2/v2.0/token"
|
||||
EVALUATE_PATH: Final = "/agents/tool-evaluation/evaluate"
|
||||
MCP_SESSION_ID_HEADER: Final = "mcp-session-id"
|
||||
DEFENDER_STATUS_EVALUATED: Final = "Evaluated"
|
||||
|
|
@ -158,9 +157,8 @@ class Agent365Guardrail(CustomGuardrail):
|
|||
resource_app_id: str = AGENT_365_PROD_RESOURCE_APP_ID,
|
||||
agent_id: str | None = None,
|
||||
request_timeout: float = 10.0,
|
||||
unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed",
|
||||
unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_open",
|
||||
async_handler: AsyncHTTPHandler | None = None,
|
||||
authority_host: str = AGENT_365_DEFAULT_AUTHORITY_HOST,
|
||||
**kwargs, # noqa: ANN003 # kwargs-ok: forwarded verbatim to CustomGuardrail (event_hook, default_on)
|
||||
) -> None:
|
||||
super().__init__(
|
||||
|
|
@ -176,8 +174,6 @@ class Agent365Guardrail(CustomGuardrail):
|
|||
self.api_base = api_base.rstrip("/")
|
||||
self.resource_app_id = resource_app_id
|
||||
self.agent_id = agent_id
|
||||
authority: Final = authority_host.strip().rstrip("/")
|
||||
self.authority_host = authority if "://" in authority else f"https://{authority}"
|
||||
self.request_timeout = request_timeout
|
||||
self.unreachable_fallback: Literal["fail_closed", "fail_open"] = (
|
||||
"fail_open" if unreachable_fallback == "fail_open" else "fail_closed"
|
||||
|
|
@ -452,7 +448,7 @@ class Agent365Guardrail(CustomGuardrail):
|
|||
return cached[0]
|
||||
|
||||
response: Final = await self._post_allowing_error_status(
|
||||
url=f"{self.authority_host}{TOKEN_PATH_TEMPLATE.format(tenant_id=self.tenant_id)}",
|
||||
url=AGENT_365_TOKEN_URL_TEMPLATE.format(tenant_id=self.tenant_id),
|
||||
data={ # mutable-ok: OAuth form body; AsyncHTTPHandler.post requires dict
|
||||
"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
||||
"client_id": self.client_id,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
|
|||
akto_api_key=getattr(litellm_params, "akto_api_key", None),
|
||||
akto_account_id=getattr(litellm_params, "akto_account_id", None),
|
||||
akto_vxlan_id=getattr(litellm_params, "akto_vxlan_id", None),
|
||||
unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"),
|
||||
unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed",
|
||||
guardrail_timeout=getattr(litellm_params, "guardrail_timeout", None),
|
||||
guardrail_name=guardrail.get("guardrail_name", ""),
|
||||
event_hook=litellm_params.mode,
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
|
|||
_alice_guardrail_callback: Final = AliceGuardrail(
|
||||
api_key=litellm_params.api_key,
|
||||
api_base=litellm_params.api_base,
|
||||
unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"),
|
||||
unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed",
|
||||
guardrail_name=guardrail.get("guardrail_name", ""),
|
||||
event_hook=litellm_params.mode,
|
||||
default_on=litellm_params.default_on,
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ def initialize_guardrail(
|
|||
agent_token=litellm_params.api_key,
|
||||
workspace_id=extras.get("workspace_id"),
|
||||
tool_name=extras.get("tool_name", "llm_call"),
|
||||
unreachable_fallback=litellm_params.unreachable_fallback,
|
||||
unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed",
|
||||
timeout=DEFAULT_TIMEOUT_SECONDS if litellm_params.timeout is None else litellm_params.timeout,
|
||||
guardrail_name=guardrail.get("guardrail_name", ""),
|
||||
event_hook=litellm_params.mode,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
|
|||
api_base=litellm_params.api_base,
|
||||
api_key=litellm_params.api_key,
|
||||
firewall_id=getattr(litellm_params, "deepkeep_firewall_id", None),
|
||||
unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"),
|
||||
unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed",
|
||||
extra_headers=getattr(litellm_params, "extra_headers", None),
|
||||
guardrail_name=guardrail.get("guardrail_name", ""),
|
||||
event_hook=litellm_params.mode,
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
|
|||
api_key=litellm_params.api_key,
|
||||
headers=getattr(litellm_params, "headers", None),
|
||||
additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", {}),
|
||||
unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"),
|
||||
unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed",
|
||||
fail_on_error=getattr(litellm_params, "fail_on_error", True),
|
||||
extra_headers=getattr(litellm_params, "extra_headers", None),
|
||||
guardrail_name=guardrail.get("guardrail_name", ""),
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
|
|||
api_key=litellm_params.api_key,
|
||||
api_base=litellm_params.api_base,
|
||||
asset_id=litellm_params.asset_id,
|
||||
unreachable_fallback=litellm_params.unreachable_fallback,
|
||||
unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed",
|
||||
event_hook=_event_hook_from_mode(litellm_params.mode),
|
||||
default_on=litellm_params.default_on or False,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -55,9 +55,7 @@ def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) ->
|
|||
guardrail_name=guardrail["guardrail_name"],
|
||||
event_hook=_coerce_event_hook(litellm_params.mode),
|
||||
default_on=litellm_params.default_on or False,
|
||||
unreachable_fallback=(
|
||||
litellm_params.unreachable_fallback if "unreachable_fallback" in litellm_params.model_fields_set else None
|
||||
),
|
||||
unreachable_fallback=litellm_params.unreachable_fallback,
|
||||
)
|
||||
litellm.logging_callback_manager.add_litellm_callback( # pyright: ignore[reportUnknownMemberType] # callback manager is untyped
|
||||
_callback
|
||||
|
|
|
|||
|
|
@ -1055,12 +1055,13 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up
|
|||
description="Additional provider-specific parameters for generic guardrail APIs",
|
||||
)
|
||||
|
||||
unreachable_fallback: Literal["fail_closed", "fail_open"] = Field(
|
||||
default="fail_closed",
|
||||
unreachable_fallback: Literal["fail_closed", "fail_open"] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Behavior when a guardrail endpoint is unreachable due to network errors. "
|
||||
"Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. "
|
||||
"'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed."
|
||||
"'fail_closed' raises an error. 'fail_open' logs a critical error and allows the request to proceed. "
|
||||
"Unset applies the guardrail's own default: 'agent_365' and 'typesafe' fail open, the others fail closed."
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -1204,6 +1205,13 @@ class LitellmParams( # pyright: ignore[reportIncompatibleVariableOverride] # o
|
|||
mode: str | list[str] | Mode = Field(
|
||||
description="When to apply the guardrail (pre_call, post_call, during_call, logging_only)"
|
||||
)
|
||||
unreachable_fallback: Literal["fail_closed", "fail_open"] | None = Field( # pyright: ignore[reportIncompatibleVariableOverride] # mixins pin a default; unset defers to the guardrail's own
|
||||
default=None,
|
||||
description=(
|
||||
"Behavior when the guardrail endpoint is unreachable. 'fail_closed' blocks, 'fail_open' allows and logs. "
|
||||
"Unset applies the guardrail's own default: 'agent_365' and 'typesafe' fail open, the others fail closed."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("timeout", mode="before", check_fields=False)
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from .base import GuardrailConfigModel
|
|||
AGENT_365_PROD_API_BASE: Final = "https://agent365.svc.cloud.microsoft"
|
||||
AGENT_365_PROD_RESOURCE_APP_ID: Final = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
|
||||
AGENT_365_SCOPE_NAME: Final = "ThreatProtection.Evaluate.All"
|
||||
AGENT_365_DEFAULT_AUTHORITY_HOST: Final = "https://login.microsoftonline.com"
|
||||
AGENT_365_TOKEN_URL_TEMPLATE: Final = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
|
||||
|
||||
|
||||
class Agent365GuardrailConfigModel(GuardrailConfigModel):
|
||||
|
|
@ -42,6 +42,7 @@ class Agent365GuardrailConfigModel(GuardrailConfigModel):
|
|||
f"Defaults to the production endpoint {AGENT_365_PROD_API_BASE}. "
|
||||
"Falls back to the AGENT365_API_BASE environment variable."
|
||||
),
|
||||
json_schema_extra={"ui_hidden": True},
|
||||
)
|
||||
|
||||
resource_app_id: str | None = Field(
|
||||
|
|
@ -52,6 +53,7 @@ class Agent365GuardrailConfigModel(GuardrailConfigModel):
|
|||
"the Test and PreProd environments use a different id. "
|
||||
"Falls back to the AGENT365_RESOURCE_APP_ID environment variable."
|
||||
),
|
||||
json_schema_extra={"ui_hidden": True},
|
||||
)
|
||||
|
||||
agent_id: str | None = Field(
|
||||
|
|
@ -60,23 +62,15 @@ class Agent365GuardrailConfigModel(GuardrailConfigModel):
|
|||
"Agent identity reported to Agent 365 with every tool evaluation. "
|
||||
"When unset, the caller's key alias is used."
|
||||
),
|
||||
)
|
||||
|
||||
authority_host: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Microsoft Entra authority host that issues the On-Behalf-Of token, for sovereign clouds. "
|
||||
f"Defaults to {AGENT_365_DEFAULT_AUTHORITY_HOST}. "
|
||||
"Falls back to the AGENT365_AUTHORITY_HOST, then AZURE_AUTHORITY_HOST environment variables."
|
||||
),
|
||||
json_schema_extra={"ui_hidden": True},
|
||||
)
|
||||
|
||||
unreachable_fallback: Literal["fail_closed", "fail_open"] = Field(
|
||||
default="fail_closed",
|
||||
default="fail_open",
|
||||
description=(
|
||||
"Behavior when Agent 365 or Entra is unreachable, times out, returns 5xx, skips the evaluation, or "
|
||||
"rejects the gateway's own client credentials. 'fail_closed' (default) blocks the tool call with HTTP 503. "
|
||||
"'fail_open' allows it, logs an error and records it as Unscanned in the logs and OpenTelemetry. "
|
||||
"rejects the gateway's own client credentials. 'fail_open' (default) allows the tool call, logs an error "
|
||||
"and records it as Unscanned in the logs and OpenTelemetry. 'fail_closed' blocks it with HTTP 503. "
|
||||
"Policy blocks, 4xx rejections, throttling and a rejected caller token always block."
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,20 +1,16 @@
|
|||
"""Agent 365 guardrail when its own dependencies fail: Entra and Agent 365 are owned local doubles."""
|
||||
"""Agent 365 guardrail paths that end before the OBO exchange: no Entra, so no real tenant is ever contacted."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from dataclasses import dataclass, replace
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import parse_qs
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
|
|
@ -23,7 +19,6 @@ from integration._support.mcp import (
|
|||
EntryPoint,
|
||||
McpCaller,
|
||||
McpPeer,
|
||||
Outcome,
|
||||
echo_tool,
|
||||
register_mcp,
|
||||
scripted_peer,
|
||||
|
|
@ -33,68 +28,16 @@ from integration._support.process import owned_proxy_process
|
|||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
TENANT: Final = "00000000-0000-4000-8000-0000000a3650"
|
||||
EVALUATE_PATH: Final = "/agents/tool-evaluation/evaluate"
|
||||
CALLER_TOKEN: Final = "eyJhbGciOiJub25lIn0.eyJzdWIiOiJpbnRlZ3JhdGlvbiJ9.synthetic-signature"
|
||||
GUARDRAIL_TIMEOUT_SECONDS: Final = 1.0
|
||||
SLOW_REPLY_SECONDS: Final = 3.0
|
||||
GUARDRAIL_STATUSES: Final = (
|
||||
"SELECT metadata->'mcp_tool_call_metadata'->>'name' AS tool, gi->>'guardrail_status' AS status "
|
||||
"FROM \"LiteLLM_SpendLogs\", jsonb_array_elements(metadata->'guardrail_information') gi "
|
||||
"WHERE api_key = %s AND gi->>'guardrail_provider' = 'agent_365'"
|
||||
)
|
||||
REJECTED: Final = "Agent 365 guardrail rejected the tool call"
|
||||
GUARDRAIL_ROWS: Final = (
|
||||
"SELECT metadata->'guardrail_information' AS gi FROM \"LiteLLM_SpendLogs\" "
|
||||
'WHERE api_key = %s AND call_type = %s ORDER BY "startTime"'
|
||||
)
|
||||
TOOLS: Final = ("outage", "skipped", "denied", "add", "throttled", "rejected", "nonjson", "nobool", "slow")
|
||||
FALLBACKS: Final = (None, "fail_open", "fail_closed")
|
||||
|
||||
|
||||
def _caller_token(entra_case: str) -> str:
|
||||
"""A compact JWS whose signature segment tells the Entra double how to answer the OBO exchange."""
|
||||
return f"eyJhbGciOiJub25lIn0.eyJzdWIiOiJpbnRlZ3JhdGlvbiJ9.{entra_case}"
|
||||
|
||||
|
||||
def _entra(request: Request) -> Reply:
|
||||
assert request.target == f"/{TENANT}/oauth2/v2.0/token", request.target
|
||||
case: Final = parse_qs(request.body.decode())["assertion"][0].rsplit(".", 1)[-1]
|
||||
match case:
|
||||
case "entra-outage":
|
||||
return Reply(status=503, body=json.dumps({"error": "synthetic Entra outage"}).encode())
|
||||
case "entra-nonjson":
|
||||
return Reply(body=b"<html>synthetic gateway timeout</html>", content_type="text/html")
|
||||
case "entra-slow":
|
||||
time.sleep(SLOW_REPLY_SECONDS)
|
||||
case "entra-misconfigured":
|
||||
misconfigured: Final = {"error": "invalid_client", "error_codes": [7000215]}
|
||||
return Reply(status=401, body=json.dumps(misconfigured).encode())
|
||||
case "entra-rejected":
|
||||
rejected: Final = {"error": "invalid_grant", "error_codes": [50013]}
|
||||
return Reply(status=400, body=json.dumps(rejected).encode())
|
||||
return Reply(body=json.dumps({"access_token": "obo-" + uuid.uuid4().hex, "expires_in": 3599}).encode())
|
||||
|
||||
|
||||
def _agent_365(request: Request) -> Reply:
|
||||
assert request.target == EVALUATE_PATH, request.target
|
||||
tool: Final = str(json.loads(request.body)["tool"]["name"]).rsplit("-", 1)[-1]
|
||||
match tool:
|
||||
case "outage":
|
||||
return Reply(status=503, body=json.dumps({"error": "synthetic Agent 365 outage"}).encode())
|
||||
case "skipped":
|
||||
return Reply(body=json.dumps({"allowed": True, "defender": {"status": "Skipped"}}).encode())
|
||||
case "denied":
|
||||
verdict: Final = {"status": "Evaluated", "verdict": "Block", "message": "synthetic block"}
|
||||
return Reply(body=json.dumps({"allowed": False, "defender": verdict, "correlationId": "denied-1"}).encode())
|
||||
case "throttled":
|
||||
return Reply(status=429, body=json.dumps({"error": "synthetic throttle"}).encode())
|
||||
case "rejected":
|
||||
return Reply(status=400, body=json.dumps({"error": "synthetic malformed evaluation request"}).encode())
|
||||
case "nonjson":
|
||||
return Reply(body=b"<html>synthetic upstream error page</html>", content_type="text/html")
|
||||
case "nobool":
|
||||
return Reply(body=json.dumps({"allowed": "yes", "defender": {"status": "Evaluated"}}).encode())
|
||||
case "slow":
|
||||
time.sleep(SLOW_REPLY_SECONDS)
|
||||
return Reply(body=json.dumps({"allowed": True, "defender": {"status": "Evaluated"}}).encode())
|
||||
def _agent_365_never_reached(request: Request) -> Reply:
|
||||
return Reply(status=500, body=json.dumps({"error": f"unexpected evaluation request {request.target}"}).encode())
|
||||
|
||||
|
||||
def _generic_guardrail_outage(request: Request) -> Reply:
|
||||
|
|
@ -102,14 +45,7 @@ def _generic_guardrail_outage(request: Request) -> Reply:
|
|||
return Reply(status=503, body=json.dumps({"error": "synthetic sibling guardrail outage"}).encode())
|
||||
|
||||
|
||||
def _config(
|
||||
tmp_path: Path,
|
||||
name: str,
|
||||
entra_url: str | None,
|
||||
agent_365_url: str,
|
||||
fallback: str | None,
|
||||
sibling_url: str | None,
|
||||
) -> Path:
|
||||
def _config(tmp_path: Path, name: str, agent_365_url: str, fallback: str | None, sibling_url: str | None) -> Path:
|
||||
config: dict = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["guardrails"] = [
|
||||
{
|
||||
|
|
@ -122,8 +58,6 @@ def _config(
|
|||
"client_id": "synthetic-client-id",
|
||||
"client_secret": "synthetic-client-secret",
|
||||
"api_base": agent_365_url,
|
||||
"timeout": GUARDRAIL_TIMEOUT_SECONDS,
|
||||
**({"authority_host": entra_url} if entra_url else {}),
|
||||
**({"unreachable_fallback": fallback} if fallback else {}),
|
||||
},
|
||||
},
|
||||
|
|
@ -152,111 +86,47 @@ def _config(
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class Rig:
|
||||
candidate: Gateway
|
||||
caller: McpCaller
|
||||
key: str
|
||||
alias: str
|
||||
guardrail_name: str
|
||||
peer: McpPeer
|
||||
entra: Wire
|
||||
agent_365: Wire
|
||||
server_id: str
|
||||
|
||||
def caller_for(self, entry: EntryPoint, token: str = CALLER_TOKEN) -> McpCaller:
|
||||
return McpCaller(self.candidate, self.key, entry, self.alias, headers={"Authorization": f"Bearer {token}"})
|
||||
|
||||
def catalog_by_worker(self, samples: int = 8) -> frozenset[tuple[int, bool]]:
|
||||
"""A fresh connection stays on the worker that accepted it, so its self-reported pid pairs with its catalog."""
|
||||
|
||||
def probe(connection: httpx.Client) -> tuple[int, bool] | None:
|
||||
candidate: Final = replace(self.candidate, client=connection)
|
||||
caller: Final = McpCaller(
|
||||
candidate, self.key, "mcp", self.alias, headers={"Authorization": f"Bearer {CALLER_TOKEN}"}
|
||||
)
|
||||
try:
|
||||
summary: Final = connection.get(
|
||||
"/debug/memory/summary", headers={"Authorization": f"Bearer {candidate.key}"}
|
||||
)
|
||||
return int(summary.json()["worker_pid"]), f"{self.alias}-add" in caller.list_tools().tools
|
||||
except httpx.TransportError:
|
||||
return None
|
||||
|
||||
with ExitStack() as connections, ThreadPoolExecutor(max_workers=samples) as pool:
|
||||
fresh: Final = tuple(
|
||||
connections.enter_context(httpx.Client(base_url=str(self.candidate.client.base_url)))
|
||||
for _ in range(samples)
|
||||
)
|
||||
return frozenset(seen for seen in pool.map(probe, fresh) if seen is not None)
|
||||
|
||||
def every_worker_serves_the_catalog(self, workers: int, without: int | None = None) -> frozenset[int]:
|
||||
seen: Final = eventually(
|
||||
self.catalog_by_worker,
|
||||
lambda pairs: (
|
||||
len({pid for pid, _ in pairs}) == workers
|
||||
and without not in {pid for pid, _ in pairs}
|
||||
and all(served for _, served in pairs)
|
||||
),
|
||||
seconds=40,
|
||||
)
|
||||
return frozenset(pid for pid, _ in seen)
|
||||
|
||||
def call_without_bearer(self, tool: str) -> Outcome:
|
||||
return McpCaller(self.candidate, self.key, "mcp", self.alias).call(f"{self.alias}-{tool}", {"a": 0})
|
||||
def caller(self, entry: EntryPoint = "mcp", bearer: str | None = None) -> McpCaller:
|
||||
headers: Final = {"Authorization": f"Bearer {bearer}"} if bearer else {}
|
||||
return McpCaller(self.candidate, self.key, entry, self.alias, headers=headers)
|
||||
|
||||
def upstream_tool_names(self) -> tuple[str, ...]:
|
||||
return tuple(str(call["body"]["params"]["name"]) for call in tool_calls(self.peer.drain()))
|
||||
|
||||
def guardrail_statuses(self, call_type: str, at_least: int) -> list[str]:
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(GUARDRAIL_ROWS, (sha256(self.key.encode()).hexdigest(), call_type)),
|
||||
lambda seen: len(seen) >= at_least,
|
||||
seconds=70,
|
||||
)
|
||||
return [row["gi"][0]["guardrail_status"] if row["gi"] else "none" for row in rows]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _rig(
|
||||
gateway: Gateway,
|
||||
tmp_path: Path,
|
||||
fallback: str | None,
|
||||
*,
|
||||
workers: int = 1,
|
||||
environment: dict[str, str] | None = None,
|
||||
authority_env: str | None = None,
|
||||
sibling: bool = False,
|
||||
) -> Iterator[Rig]:
|
||||
"""``authority_env`` names the environment variable that carries the Entra double instead of the config."""
|
||||
def _rig(gateway: Gateway, tmp_path: Path, fallback: str | None, *, sibling: bool = False) -> Iterator[Rig]:
|
||||
alias: Final = "a365" + uuid.uuid4().hex[:8]
|
||||
with (
|
||||
wire_server(_entra) as entra,
|
||||
wire_server(_agent_365) as agent_365,
|
||||
wire_server(_agent_365_never_reached) as agent_365,
|
||||
wire_server(_generic_guardrail_outage) as sibling_outage,
|
||||
scripted_peer(*(echo_tool(tool) for tool in TOOLS)) as peer,
|
||||
scripted_peer(echo_tool("add")) as peer,
|
||||
owned_proxy_process(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{
|
||||
"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2",
|
||||
**(environment or {}),
|
||||
**({authority_env: entra.url} if authority_env else {}),
|
||||
},
|
||||
config=_config(
|
||||
tmp_path,
|
||||
alias,
|
||||
None if authority_env else entra.url,
|
||||
agent_365.url,
|
||||
fallback,
|
||||
sibling_outage.url if sibling else None,
|
||||
),
|
||||
workers=workers,
|
||||
{"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2"},
|
||||
config=_config(tmp_path, alias, agent_365.url, fallback, sibling_outage.url if sibling else None),
|
||||
) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
identity: Final = register_mcp(scenario, peer, alias)
|
||||
key: Final = scenario.key(object_permission={"mcp_servers": [identity]})
|
||||
caller: Final = McpCaller(owned.gateway, key, "mcp", alias, headers={"Authorization": f"Bearer {CALLER_TOKEN}"})
|
||||
rig: Final = Rig(owned.gateway, caller, key, alias, alias, peer, entra, agent_365, identity)
|
||||
if workers > 1:
|
||||
rig.every_worker_serves_the_catalog(workers)
|
||||
peer.drain()
|
||||
yield rig
|
||||
|
||||
|
||||
def _guardrail_statuses(key: str) -> dict[str, str]:
|
||||
rows: Final = read_rows(GUARDRAIL_STATUSES, (sha256(key.encode()).hexdigest(),))
|
||||
return {str(row["tool"]): str(row["status"]) for row in rows}
|
||||
yield Rig(owned.gateway, key, alias, peer, agent_365, identity)
|
||||
|
||||
|
||||
def _chat(rig: Rig, model: str, marker: str) -> httpx.Response:
|
||||
|
|
@ -265,217 +135,39 @@ def _chat(rig: Rig, model: str, marker: str) -> httpx.Response:
|
|||
)
|
||||
|
||||
|
||||
def test_default_blocks_with_503_and_never_reaches_upstream_when_agent_365_cannot_evaluate(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
@pytest.mark.parametrize("fallback", FALLBACKS)
|
||||
def test_a_missing_or_malformed_caller_bearer_blocks_on_every_entry_point_whatever_the_fallback(
|
||||
gateway: Gateway, tmp_path: Path, fallback: str | None
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path, fallback=None) as rig:
|
||||
unavailable: Final = "could not authorize the tool call"
|
||||
outage: Final = rig.caller.call(f"{rig.alias}-outage", {"a": 1})
|
||||
assert outage.error is not None and unavailable in outage.raw, (
|
||||
f"Agent 365 down must block by default: {outage.raw}"
|
||||
)
|
||||
skipped: Final = rig.caller.call(f"{rig.alias}-skipped", {"a": 2})
|
||||
assert skipped.error is not None and unavailable in skipped.raw, (
|
||||
f"Defender skip must block by default: {skipped.raw}"
|
||||
)
|
||||
denied: Final = rig.caller.call(f"{rig.alias}-denied", {"a": 3})
|
||||
assert denied.error is not None and "Blocked by Microsoft Defender" in denied.raw, denied.raw
|
||||
assert rig.caller.call(f"{rig.alias}-add", {"a": 4}).text == '{"a": 4}'
|
||||
assert rig.upstream_tool_names() == ("add",)
|
||||
statuses: Final = eventually(lambda: _guardrail_statuses(rig.key), lambda seen: len(seen) >= 4, seconds=70)
|
||||
assert statuses == {
|
||||
"outage": "guardrail_failed_to_respond",
|
||||
"skipped": "guardrail_failed_to_respond",
|
||||
"denied": "guardrail_intervened",
|
||||
"add": "success",
|
||||
}, statuses
|
||||
|
||||
|
||||
def test_explicit_fail_closed_matches_the_default(gateway: Gateway, tmp_path: Path) -> None:
|
||||
with _rig(gateway, tmp_path, fallback="fail_closed") as rig:
|
||||
outage: Final = rig.caller.call(f"{rig.alias}-outage", {"a": 1})
|
||||
assert outage.error is not None and "could not authorize the tool call" in outage.raw, outage.raw
|
||||
assert rig.caller.call(f"{rig.alias}-add", {"a": 2}).text == '{"a": 2}'
|
||||
assert rig.upstream_tool_names() == ("add",)
|
||||
statuses: Final = eventually(lambda: _guardrail_statuses(rig.key), lambda seen: len(seen) >= 2, seconds=70)
|
||||
assert statuses == {"outage": "guardrail_failed_to_respond", "add": "success"}, statuses
|
||||
|
||||
|
||||
def test_opted_in_fail_open_lets_the_call_through_unscanned_and_still_blocks_policy_denials(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path, fallback="fail_open") as rig:
|
||||
outage: Final = rig.caller.call(f"{rig.alias}-outage", {"a": 1})
|
||||
assert outage.text == '{"a": 1}', f"Agent 365 down must fail open once opted in: {outage.raw}"
|
||||
skipped: Final = rig.caller.call(f"{rig.alias}-skipped", {"a": 2})
|
||||
assert skipped.text == '{"a": 2}', f"Defender skipping the call must fail open once opted in: {skipped.raw}"
|
||||
denied: Final = rig.caller.call(f"{rig.alias}-denied", {"a": 3})
|
||||
assert denied.error is not None and "Blocked by Microsoft Defender" in denied.raw, denied.raw
|
||||
assert rig.upstream_tool_names() == ("outage", "skipped")
|
||||
statuses: Final = eventually(lambda: _guardrail_statuses(rig.key), lambda seen: len(seen) >= 3, seconds=70)
|
||||
assert statuses == {
|
||||
"outage": "guardrail_failed_to_respond",
|
||||
"skipped": "guardrail_failed_to_respond",
|
||||
"denied": "guardrail_intervened",
|
||||
}, statuses
|
||||
|
||||
|
||||
def test_opted_in_fail_open_covers_malformed_or_stalled_agent_365_replies(gateway: Gateway, tmp_path: Path) -> None:
|
||||
with _rig(gateway, tmp_path, fallback="fail_open") as rig:
|
||||
for index, tool in enumerate(("nonjson", "nobool", "slow")):
|
||||
outcome: Final = rig.caller.call(f"{rig.alias}-{tool}", {"a": index})
|
||||
assert outcome.text == json.dumps({"a": index}), f"{tool}: {outcome.raw}"
|
||||
assert rig.upstream_tool_names() == ("nonjson", "nobool", "slow")
|
||||
statuses: Final = eventually(lambda: _guardrail_statuses(rig.key), lambda seen: len(seen) >= 3, seconds=70)
|
||||
assert statuses == dict.fromkeys(("nonjson", "nobool", "slow"), "guardrail_failed_to_respond"), statuses
|
||||
|
||||
|
||||
def test_opted_in_fail_open_covers_entra_down_stalled_malformed_or_refusing_the_gateway_credentials(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path, fallback="fail_open") as rig:
|
||||
cases: Final = ("entra-outage", "entra-slow", "entra-nonjson", "entra-misconfigured")
|
||||
for index, case in enumerate(cases):
|
||||
outcome: Final = rig.caller_for("mcp", _caller_token(case)).call(f"{rig.alias}-add", {"a": index})
|
||||
assert outcome.text == json.dumps({"a": index}), f"{case}: {outcome.raw}"
|
||||
assert rig.upstream_tool_names() == ("add",) * len(cases)
|
||||
assert tuple(
|
||||
sorted(parse_qs(request.body.decode())["assertion"][0].rsplit(".", 1)[-1] for request in rig.entra.drain())
|
||||
) == tuple(sorted(cases))
|
||||
assert rig.agent_365.drain() == (), "no OBO token means no evaluation request"
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(GUARDRAIL_ROWS, (sha256(rig.key.encode()).hexdigest(), "call_mcp_tool")),
|
||||
lambda seen: len(seen) >= len(cases),
|
||||
seconds=70,
|
||||
)
|
||||
assert [row["gi"][0]["guardrail_status"] for row in rows] == ["guardrail_failed_to_respond"] * len(cases)
|
||||
|
||||
|
||||
def test_throttling_and_ordinary_4xx_from_agent_365_keep_blocking_even_when_opted_in_to_fail_open(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path, fallback="fail_open") as rig:
|
||||
throttled: Final = rig.caller.call(f"{rig.alias}-throttled", {"a": 1})
|
||||
assert throttled.error == "Error: Agent 365 guardrail could not authorize the tool call", throttled.raw
|
||||
rejected: Final = rig.caller.call(f"{rig.alias}-rejected", {"a": 2})
|
||||
assert rejected.error == "Error: Agent 365 rejected the tool evaluation request", rejected.raw
|
||||
assert rig.upstream_tool_names() == ()
|
||||
statuses: Final = eventually(lambda: _guardrail_statuses(rig.key), lambda seen: len(seen) >= 2, seconds=70)
|
||||
assert statuses == {"throttled": "guardrail_failed_to_respond", "rejected": "guardrail_intervened"}, statuses
|
||||
|
||||
|
||||
def test_caller_authentication_failures_keep_blocking_even_when_opted_in_to_fail_open(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path, fallback="fail_open") as rig:
|
||||
rejected: Final = "Error: Agent 365 guardrail rejected the tool call"
|
||||
missing: Final = rig.call_without_bearer("add")
|
||||
assert missing.error == rejected, missing.raw
|
||||
malformed: Final = rig.caller_for("mcp", "not-a-jws").call(f"{rig.alias}-add", {"a": 1})
|
||||
assert malformed.error == rejected, malformed.raw
|
||||
refused: Final = rig.caller_for("mcp", _caller_token("entra-rejected")).call(f"{rig.alias}-add", {"a": 2})
|
||||
assert refused.error == rejected, refused.raw
|
||||
assert len(rig.entra.drain()) == 1, "only the well-formed bearer reaches the OBO exchange"
|
||||
assert rig.upstream_tool_names() == ()
|
||||
assert rig.agent_365.drain() == ()
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(GUARDRAIL_ROWS, (sha256(rig.key.encode()).hexdigest(), "call_mcp_tool")),
|
||||
lambda seen: len(seen) >= 3,
|
||||
seconds=70,
|
||||
)
|
||||
assert [row["gi"][0]["guardrail_status"] for row in rows] == ["guardrail_intervened"] * 3
|
||||
|
||||
|
||||
def test_every_mcp_entry_point_honors_the_fail_open_opt_in_and_blocks_denials(gateway: Gateway, tmp_path: Path) -> None:
|
||||
with _rig(gateway, tmp_path, fallback="fail_open") as rig:
|
||||
with _rig(gateway, tmp_path, fallback) as rig:
|
||||
assert f"{rig.alias}-add" in rig.caller().list_tools().tools, "the catalog needs only the virtual key"
|
||||
for entry in ENTRY_POINTS:
|
||||
caller: Final = rig.caller_for(entry)
|
||||
passed: Final = caller.call(f"{rig.alias}-outage", {"entry": entry}, server_id=rig.server_id)
|
||||
assert passed.text == json.dumps({"entry": entry}), f"{entry}: {passed.raw}"
|
||||
denied: Final = caller.call(f"{rig.alias}-denied", {"entry": entry}, server_id=rig.server_id)
|
||||
assert denied.error is not None and "Blocked by Microsoft Defender" in denied.raw, f"{entry}: {denied.raw}"
|
||||
assert rig.upstream_tool_names() == ("outage",) * len(ENTRY_POINTS)
|
||||
|
||||
|
||||
def test_agent_365_fail_open_opt_in_does_not_leak_to_a_sibling_guardrail_on_chat_completions(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path, fallback="fail_open", sibling=True) as rig, rig.candidate.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
chat: Final = _chat(rig, model, "sibling-" + uuid.uuid4().hex)
|
||||
assert chat.status_code == 500 and "Generic Guardrail API failed" in chat.text, chat.text
|
||||
assert rig.entra.drain() == () and rig.agent_365.drain() == ()
|
||||
assert rig.caller.call(f"{rig.alias}-outage", {"a": 1}).text == '{"a": 1}'
|
||||
assert rig.upstream_tool_names() == ("outage",)
|
||||
missing: Final = rig.caller(entry).call(f"{rig.alias}-add", {"entry": entry}, server_id=rig.server_id)
|
||||
assert missing.error is not None and REJECTED in missing.raw, f"{entry} without a bearer: {missing.raw}"
|
||||
malformed: Final = rig.caller(entry, "not-a-jws").call(
|
||||
f"{rig.alias}-add", {"entry": entry}, server_id=rig.server_id
|
||||
)
|
||||
assert malformed.error is not None and REJECTED in malformed.raw, f"{entry} opaque bearer: {malformed.raw}"
|
||||
assert rig.upstream_tool_names() == ()
|
||||
assert rig.agent_365.drain() == (), "a rejected caller never produces an evaluation request"
|
||||
expected: Final = 2 * len(ENTRY_POINTS)
|
||||
assert rig.guardrail_statuses("call_mcp_tool", expected) == ["guardrail_intervened"] * expected
|
||||
|
||||
|
||||
def test_chat_completions_are_unaffected_by_the_mcp_guardrail(gateway: Gateway, tmp_path: Path) -> None:
|
||||
with _rig(gateway, tmp_path, fallback=None) as rig, rig.candidate.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
marker: Final = "unaffected-" + uuid.uuid4().hex
|
||||
chat: Final = _chat(rig, model, marker)
|
||||
chat: Final = _chat(rig, model, "unaffected-" + uuid.uuid4().hex)
|
||||
assert chat.status_code == 200, chat.text
|
||||
assert rig.entra.drain() == () and rig.agent_365.drain() == ()
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(GUARDRAIL_ROWS, (sha256(rig.key.encode()).hexdigest(), "acompletion")),
|
||||
lambda seen: len(seen) >= 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert rows[0]["gi"] is None, rows
|
||||
assert rig.agent_365.drain() == ()
|
||||
assert rig.guardrail_statuses("acompletion", 1) == ["none"]
|
||||
|
||||
|
||||
def test_agent365_authority_host_env_wins_over_azure_authority_host_when_config_has_none(
|
||||
def test_a_sibling_guardrail_keeps_its_own_fail_closed_default_next_to_agent_365(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path / "azure", fallback=None, authority_env="AZURE_AUTHORITY_HOST") as azure_only:
|
||||
assert azure_only.caller.call(f"{azure_only.alias}-add", {"a": 1}).text == '{"a": 1}'
|
||||
assert len(azure_only.entra.drain()) == 1
|
||||
with (
|
||||
wire_server(_entra) as decoy,
|
||||
_rig(
|
||||
gateway,
|
||||
tmp_path / "agent365",
|
||||
fallback=None,
|
||||
authority_env="AGENT365_AUTHORITY_HOST",
|
||||
environment={"AZURE_AUTHORITY_HOST": decoy.url},
|
||||
) as both,
|
||||
):
|
||||
assert both.caller.call(f"{both.alias}-add", {"a": 2}).text == '{"a": 2}'
|
||||
assert len(both.entra.drain()) == 1 and decoy.drain() == ()
|
||||
|
||||
|
||||
def test_thirty_call_burst_against_a_flapping_agent_365_reaches_upstream_exactly_once_each_on_two_workers(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path, fallback="fail_open", workers=2) as rig:
|
||||
markers: Final = tuple(("outage" if index % 2 else "add", uuid.uuid4().hex) for index in range(30))
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
outcomes: Final = tuple(
|
||||
pool.map(lambda pair: rig.caller.call(f"{rig.alias}-{pair[0]}", {"marker": pair[1]}), markers)
|
||||
)
|
||||
for (tool, marker), outcome in zip(markers, outcomes, strict=True):
|
||||
assert outcome.text == json.dumps({"marker": marker}), f"{tool}: {outcome.raw}"
|
||||
seen: Final = sorted(
|
||||
str(call["body"]["params"]["arguments"]["marker"]) for call in tool_calls(rig.peer.drain())
|
||||
)
|
||||
assert seen == sorted(marker for _, marker in markers)
|
||||
assert rig.caller.call(f"{rig.alias}-denied", {"a": 1}).error is not None
|
||||
assert rig.upstream_tool_names() == ()
|
||||
|
||||
|
||||
def test_opted_in_fail_open_survives_a_worker_kill_and_keeps_blocking_denials_on_two_workers(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path, fallback="fail_open", workers=2) as rig:
|
||||
before: Final = rig.every_worker_serves_the_catalog(2)
|
||||
victim: Final = min(before)
|
||||
os.kill(victim, signal.SIGKILL)
|
||||
after: Final = rig.every_worker_serves_the_catalog(2, without=victim)
|
||||
assert after - before, f"a replacement worker took over: before {before}, after {after}"
|
||||
rig.peer.drain()
|
||||
for index in range(10):
|
||||
passed: Final = rig.caller.call(f"{rig.alias}-outage", {"a": index})
|
||||
assert passed.text == json.dumps({"a": index}), passed.raw
|
||||
denied: Final = rig.caller.call(f"{rig.alias}-denied", {"a": index})
|
||||
assert denied.error is not None and "Blocked by Microsoft Defender" in denied.raw, denied.raw
|
||||
assert rig.upstream_tool_names() == ("outage",) * 10
|
||||
with _rig(gateway, tmp_path, fallback=None, sibling=True) as rig, rig.candidate.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
chat: Final = _chat(rig, model, "sibling-" + uuid.uuid4().hex)
|
||||
assert chat.status_code == 500 and "Generic Guardrail API failed" in chat.text, chat.text
|
||||
assert rig.agent_365.drain() == ()
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ from litellm.types.guardrails import (
|
|||
SupportedGuardrailIntegrations,
|
||||
)
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.agent_365 import (
|
||||
AGENT_365_DEFAULT_AUTHORITY_HOST,
|
||||
AGENT_365_PROD_API_BASE,
|
||||
AGENT_365_PROD_RESOURCE_APP_ID,
|
||||
Agent365GuardrailConfigModel,
|
||||
|
|
@ -125,7 +124,6 @@ def _make_guardrail(
|
|||
unreachable_fallback: str = "fail_closed",
|
||||
agent_id: str | None = None,
|
||||
api_base: str = AGENT_365_PROD_API_BASE,
|
||||
authority_host: str = AGENT_365_DEFAULT_AUTHORITY_HOST,
|
||||
) -> Agent365Guardrail:
|
||||
return Agent365Guardrail(
|
||||
guardrail_name="agent-365-guard",
|
||||
|
|
@ -134,7 +132,6 @@ def _make_guardrail(
|
|||
client_secret="secret-123",
|
||||
api_base=api_base,
|
||||
agent_id=agent_id,
|
||||
authority_host=authority_host,
|
||||
unreachable_fallback=unreachable_fallback,
|
||||
async_handler=handler,
|
||||
event_hook="pre_mcp_call",
|
||||
|
|
@ -234,62 +231,34 @@ class TestInitializeGuardrail:
|
|||
assert guardrail.client_secret == "env-secret"
|
||||
assert guardrail.api_base == "https://env.example.test"
|
||||
assert guardrail.resource_app_id == AGENT_365_PROD_RESOURCE_APP_ID
|
||||
assert guardrail.unreachable_fallback == "fail_closed"
|
||||
assert guardrail.unreachable_fallback == "fail_open"
|
||||
|
||||
def test_fail_open_is_opt_in_through_litellm_params(self):
|
||||
def test_fail_closed_is_opt_in_through_litellm_params(self):
|
||||
params: Final = LitellmParams(
|
||||
guardrail="agent_365",
|
||||
mode="pre_mcp_call",
|
||||
tenant_id="t",
|
||||
client_id="c",
|
||||
client_secret="s",
|
||||
unreachable_fallback="fail_open",
|
||||
unreachable_fallback="fail_closed",
|
||||
)
|
||||
guardrail: Final = initialize_guardrail(params, {"guardrail_name": "a365-open"})
|
||||
guardrail: Final = initialize_guardrail(params, {"guardrail_name": "a365-closed"})
|
||||
assert guardrail.unreachable_fallback == "fail_closed"
|
||||
|
||||
def test_unreachable_fallback_round_trips_through_a_stored_litellm_params_dump(self):
|
||||
stored: Final = LitellmParams(
|
||||
guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s"
|
||||
).model_dump()
|
||||
guardrail: Final = initialize_guardrail(LitellmParams(**stored), {"guardrail_name": "a365-stored"})
|
||||
assert guardrail.unreachable_fallback == "fail_open"
|
||||
|
||||
def test_authority_host_defaults_to_public_entra(self, monkeypatch):
|
||||
monkeypatch.delenv("AGENT365_AUTHORITY_HOST", raising=False)
|
||||
monkeypatch.delenv("AZURE_AUTHORITY_HOST", raising=False)
|
||||
params: Final = LitellmParams(
|
||||
guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s"
|
||||
)
|
||||
guardrail: Final = initialize_guardrail(params, {"guardrail_name": "a365-authority"})
|
||||
assert guardrail.authority_host == "https://login.microsoftonline.com"
|
||||
def test_ui_form_hides_the_production_default_overrides(self):
|
||||
from litellm.proxy.guardrails.guardrail_endpoints import _get_fields_from_model
|
||||
|
||||
def test_authority_host_precedence_param_then_agent365_env_then_azure_env(self, monkeypatch):
|
||||
monkeypatch.setenv("AZURE_AUTHORITY_HOST", "https://login.microsoftonline.us/")
|
||||
azure_only: Final = initialize_guardrail(
|
||||
LitellmParams(guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s"),
|
||||
{"guardrail_name": "a365-azure"},
|
||||
)
|
||||
assert azure_only.authority_host == "https://login.microsoftonline.us"
|
||||
monkeypatch.setenv("AGENT365_AUTHORITY_HOST", "https://login.partner.microsoftonline.cn")
|
||||
agent_env: Final = initialize_guardrail(
|
||||
LitellmParams(guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s"),
|
||||
{"guardrail_name": "a365-agentenv"},
|
||||
)
|
||||
assert agent_env.authority_host == "https://login.partner.microsoftonline.cn"
|
||||
explicit: Final = initialize_guardrail(
|
||||
LitellmParams(
|
||||
guardrail="agent_365",
|
||||
mode="pre_mcp_call",
|
||||
tenant_id="t",
|
||||
client_id="c",
|
||||
client_secret="s",
|
||||
authority_host="http://127.0.0.1:9",
|
||||
),
|
||||
{"guardrail_name": "a365-explicit"},
|
||||
)
|
||||
assert explicit.authority_host == "http://127.0.0.1:9"
|
||||
|
||||
def test_scheme_less_authority_host_env_gets_https(self, monkeypatch):
|
||||
monkeypatch.setenv("AZURE_AUTHORITY_HOST", " login.microsoftonline.us/ ")
|
||||
guardrail: Final = initialize_guardrail(
|
||||
LitellmParams(guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s"),
|
||||
{"guardrail_name": "a365-bare-host"},
|
||||
)
|
||||
assert guardrail.authority_host == "https://login.microsoftonline.us"
|
||||
fields: Final = _get_fields_from_model(Agent365GuardrailConfigModel)
|
||||
assert {"tenant_id", "client_id", "client_secret", "unreachable_fallback"} <= set(fields)
|
||||
assert not {"api_base", "resource_app_id", "agent_id"} & set(fields)
|
||||
assert fields["unreachable_fallback"]["default_value"] == "fail_open"
|
||||
|
||||
def test_explicit_params_win(self, monkeypatch):
|
||||
monkeypatch.setenv("AGENT365_TENANT_ID", "env-tenant")
|
||||
|
|
@ -300,14 +269,14 @@ class TestInitializeGuardrail:
|
|||
client_id="client-xyz",
|
||||
client_secret="param-secret",
|
||||
agent_id="agent-007",
|
||||
unreachable_fallback="fail_open",
|
||||
unreachable_fallback="fail_closed",
|
||||
timeout=5,
|
||||
)
|
||||
guardrail: Final = initialize_guardrail(params, {"guardrail_name": "a365-params"})
|
||||
assert guardrail.tenant_id == "param-tenant"
|
||||
assert guardrail.client_secret == "param-secret"
|
||||
assert guardrail.agent_id == "agent-007"
|
||||
assert guardrail.unreachable_fallback == "fail_open"
|
||||
assert guardrail.unreachable_fallback == "fail_closed"
|
||||
assert guardrail.request_timeout == 5.0
|
||||
|
||||
def test_wrong_mode_rejected(self):
|
||||
|
|
@ -357,21 +326,6 @@ class TestAllowFlow:
|
|||
assert token_call.data["client_secret"] == "secret-123"
|
||||
assert token_call.data["scope"] == f"{AGENT_365_PROD_RESOURCE_APP_ID}/ThreatProtection.Evaluate.All"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_obo_exchange_goes_to_the_configured_authority_host(self):
|
||||
handler: Final = FakeHandler([_token_response(), _allow_response()])
|
||||
guardrail: Final = _make_guardrail(handler, authority_host="https://login.microsoftonline.us/")
|
||||
await _run(guardrail, _mcp_data())
|
||||
assert handler.calls[0].url == "https://login.microsoftonline.us/tenant-abc/oauth2/v2.0/token"
|
||||
assert handler.calls[1].url == EVALUATE_URL
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_obo_exchange_adds_https_to_a_bare_authority_host(self):
|
||||
handler: Final = FakeHandler([_token_response(), _allow_response()])
|
||||
guardrail: Final = _make_guardrail(handler, authority_host="login.microsoftonline.us")
|
||||
await _run(guardrail, _mcp_data())
|
||||
assert handler.calls[0].url == "https://login.microsoftonline.us/tenant-abc/oauth2/v2.0/token"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evaluate_payload(self):
|
||||
handler: Final = FakeHandler([_token_response(), _allow_response()])
|
||||
|
|
@ -565,20 +519,12 @@ AVAILABILITY_FAILURES: Final = (
|
|||
)
|
||||
|
||||
|
||||
class TestOptInFailOpen:
|
||||
class TestFailOpenDefault:
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("responses", AVAILABILITY_FAILURES)
|
||||
async def test_constructor_default_blocks_each_availability_failure_with_503(self, responses):
|
||||
async def test_constructor_default_lets_each_availability_failure_through_as_failed_to_respond(self, responses):
|
||||
guardrail: Final = _default_fallback_guardrail(FakeHandler(responses))
|
||||
assert guardrail.unreachable_fallback == "fail_closed"
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await _run(guardrail, _mcp_data())
|
||||
assert exc_info.value.status_code == 503
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("responses", AVAILABILITY_FAILURES)
|
||||
async def test_opted_in_fail_open_lets_each_availability_failure_through_as_failed_to_respond(self, responses):
|
||||
guardrail: Final = _make_guardrail(FakeHandler(responses), unreachable_fallback="fail_open")
|
||||
assert guardrail.unreachable_fallback == "fail_open"
|
||||
data: Final = _mcp_data()
|
||||
assert await _run(guardrail, data) is data
|
||||
info: Final = _guardrail_info(data)
|
||||
|
|
@ -586,7 +532,16 @@ class TestOptInFailOpen:
|
|||
assert info["guardrail_response"]["verdict"] == "Unscanned"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_opted_in_fail_open_logs_the_unscanned_call_at_error_level(self, caplog):
|
||||
@pytest.mark.parametrize("responses", AVAILABILITY_FAILURES)
|
||||
async def test_opted_in_fail_closed_blocks_each_availability_failure_with_503(self, responses):
|
||||
guardrail: Final = _make_guardrail(FakeHandler(responses), unreachable_fallback="fail_closed")
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await _run(guardrail, _mcp_data())
|
||||
assert exc_info.value.status_code == 503
|
||||
assert "fail_closed" in exc_info.value.detail["message"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_fail_open_logs_the_unscanned_call_at_error_level(self, caplog):
|
||||
handler: Final = FakeHandler([_token_response(), httpx.ReadTimeout("timed out")])
|
||||
guardrail: Final = _make_guardrail(handler, unreachable_fallback="fail_open")
|
||||
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
|
||||
|
|
@ -595,9 +550,9 @@ class TestOptInFailOpen:
|
|||
assert [r.levelno for r in fail_open_logs] == [logging.ERROR], caplog.text
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_opted_in_fail_open_still_blocks_a_policy_block(self):
|
||||
async def test_default_fail_open_still_blocks_a_policy_block(self):
|
||||
handler: Final = FakeHandler([_token_response(), _block_response()])
|
||||
guardrail: Final = _make_guardrail(handler, unreachable_fallback="fail_open")
|
||||
guardrail: Final = _default_fallback_guardrail(handler)
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await _run(guardrail, _mcp_data())
|
||||
assert exc_info.value.status_code == 400
|
||||
|
|
|
|||
|
|
@ -167,6 +167,29 @@ def test_initialize_guardrail_sets_run_in_parallel(config_value, expected):
|
|||
assert custom_guardrail.run_in_parallel is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"guardrail, provider_params, expected",
|
||||
[
|
||||
("agent_365", {"tenant_id": "t", "client_id": "c", "client_secret": "s", "mode": "pre_mcp_call"}, "fail_open"),
|
||||
("typesafe", {"api_key": "k"}, "fail_open"),
|
||||
("generic_guardrail_api", {"api_base": "http://127.0.0.1:1/guard"}, "fail_closed"),
|
||||
("akto", {"akto_base_url": "http://127.0.0.1:1", "akto_api_key": "k", "akto_account_id": "1"}, "fail_closed"),
|
||||
("alice", {"api_key": "k", "api_base": "http://127.0.0.1:1"}, "fail_closed"),
|
||||
("deepkeep", {"api_base": "http://127.0.0.1:1", "api_key": "k", "deepkeep_firewall_id": "f"}, "fail_closed"),
|
||||
("repelloai", {"api_key": "k", "api_base": "http://127.0.0.1:1", "asset_id": "a"}, "fail_closed"),
|
||||
],
|
||||
)
|
||||
def test_unset_unreachable_fallback_applies_each_guardrails_own_default(guardrail, provider_params, expected):
|
||||
litellm_params = {"guardrail": guardrail, "mode": "pre_call", **provider_params}
|
||||
guardrail_handler = InMemoryGuardrailHandler()
|
||||
result = guardrail_handler.initialize_guardrail(
|
||||
guardrail={"guardrail_name": f"default-fallback-{guardrail}", "litellm_params": litellm_params},
|
||||
)
|
||||
|
||||
custom_guardrail = guardrail_handler.guardrail_id_to_custom_guardrail[result["guardrail_id"]]
|
||||
assert custom_guardrail.unreachable_fallback == expected, f"{guardrail} with unreachable_fallback unset"
|
||||
|
||||
|
||||
def test_initialize_presidio_forwards_analyze_chunk_size_bytes():
|
||||
"""Regression (LIT-4785): `presidio_analyze_chunk_size_bytes` set in
|
||||
config.yaml must reach the guardrail instance. The field lives on
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -34091,11 +34091,6 @@ export interface components {
|
|||
* @description Authorization bearer token for IBM Guardrails API. Reads from IBM_GUARDRAILS_AUTH_TOKEN env var if None.
|
||||
*/
|
||||
auth_token?: string | null;
|
||||
/**
|
||||
* Authority Host
|
||||
* @description Microsoft Entra authority host that issues the On-Behalf-Of token, for sovereign clouds. Defaults to https://login.microsoftonline.com. Falls back to the AGENT365_AUTHORITY_HOST, then AZURE_AUTHORITY_HOST environment variables.
|
||||
*/
|
||||
authority_host?: string | null;
|
||||
/**
|
||||
* Aws Access Key Id
|
||||
* @description AWS access key ID for authentication
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue