diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5225f33ae92..3d44315341b 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -11579,18 +11579,6 @@ "description": "Authorization bearer token for IBM Guardrails API. Reads from IBM_GUARDRAILS_AUTH_TOKEN env var if None.", "title": "Auth Token" }, - "authority_host": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "Microsoft Entra authority host that issues the On-Behalf-Of token, for sovereign clouds. Defaults to https://login.microsoftonline.com. Falls back to the AGENT365_AUTHORITY_HOST, then AZURE_AUTHORITY_HOST environment variables.", - "title": "Authority Host" - }, "aws_access_key_id": { "anyOf": [ { diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 6874d7aa73e..ca23af93b71 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -1838,6 +1838,11 @@ def _build_field_dict( return field_dict +def _is_ui_hidden(field: "FieldInfo") -> bool: + field_json_schema_extra: Final = field.json_schema_extra + return isinstance(field_json_schema_extra, Mapping) and bool(field_json_schema_extra.get("ui_hidden")) + + def _extract_fields_recursive( model: type[BaseModel], depth: int = 0, @@ -1858,6 +1863,9 @@ def _extract_fields_recursive( if _should_skip_optional_params(field_name=field_name, field_annotation=field_annotation): continue + if _is_ui_hidden(field): + continue + # Handle Optional types and get the actual type if field_annotation is None: continue diff --git a/litellm/proxy/guardrails/guardrail_hooks/agent_365/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/agent_365/__init__.py index 68fdb14e6c7..ef4b46e3dfd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/agent_365/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/agent_365/__init__.py @@ -2,7 +2,6 @@ from typing import TYPE_CHECKING, Final from litellm.types.guardrails import SupportedGuardrailIntegrations from litellm.types.proxy.guardrails.guardrail_hooks.agent_365 import ( - AGENT_365_DEFAULT_AUTHORITY_HOST, AGENT_365_PROD_API_BASE, AGENT_365_PROD_RESOURCE_APP_ID, ) @@ -24,12 +23,6 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" ) api_base: Final = litellm_params.api_base or get_secret_str("AGENT365_API_BASE") resource_app_id: Final = litellm_params.resource_app_id or get_secret_str("AGENT365_RESOURCE_APP_ID") - authority_host: Final = ( - litellm_params.authority_host - or get_secret_str("AGENT365_AUTHORITY_HOST") - or get_secret_str("AZURE_AUTHORITY_HOST") - or AGENT_365_DEFAULT_AUTHORITY_HOST - ) if not tenant_id: raise ValueError("Microsoft Agent 365: tenant_id is required") @@ -52,9 +45,8 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" api_base=api_base or AGENT_365_PROD_API_BASE, resource_app_id=resource_app_id or AGENT_365_PROD_RESOURCE_APP_ID, agent_id=litellm_params.agent_id, - authority_host=authority_host, request_timeout=litellm_params.timeout if litellm_params.timeout is not None else 10.0, - unreachable_fallback=litellm_params.unreachable_fallback, + unreachable_fallback=litellm_params.unreachable_fallback or "fail_open", event_hook=litellm_params.mode, default_on=litellm_params.default_on, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py b/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py index fc44d2c7532..fd1b8e821da 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py +++ b/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py @@ -36,10 +36,10 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.guardrails.guardrail_hooks.agent_365 import ( - AGENT_365_DEFAULT_AUTHORITY_HOST, AGENT_365_PROD_API_BASE, AGENT_365_PROD_RESOURCE_APP_ID, AGENT_365_SCOPE_NAME, + AGENT_365_TOKEN_URL_TEMPLATE, Agent365GuardrailConfigModel, ) @@ -49,7 +49,6 @@ if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel from litellm.types.utils import GuardrailStatus -TOKEN_PATH_TEMPLATE: Final = "/{tenant_id}/oauth2/v2.0/token" EVALUATE_PATH: Final = "/agents/tool-evaluation/evaluate" MCP_SESSION_ID_HEADER: Final = "mcp-session-id" DEFENDER_STATUS_EVALUATED: Final = "Evaluated" @@ -158,9 +157,8 @@ class Agent365Guardrail(CustomGuardrail): resource_app_id: str = AGENT_365_PROD_RESOURCE_APP_ID, agent_id: str | None = None, request_timeout: float = 10.0, - unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed", + unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_open", async_handler: AsyncHTTPHandler | None = None, - authority_host: str = AGENT_365_DEFAULT_AUTHORITY_HOST, **kwargs, # noqa: ANN003 # kwargs-ok: forwarded verbatim to CustomGuardrail (event_hook, default_on) ) -> None: super().__init__( @@ -176,8 +174,6 @@ class Agent365Guardrail(CustomGuardrail): self.api_base = api_base.rstrip("/") self.resource_app_id = resource_app_id self.agent_id = agent_id - authority: Final = authority_host.strip().rstrip("/") - self.authority_host = authority if "://" in authority else f"https://{authority}" self.request_timeout = request_timeout self.unreachable_fallback: Literal["fail_closed", "fail_open"] = ( "fail_open" if unreachable_fallback == "fail_open" else "fail_closed" @@ -452,7 +448,7 @@ class Agent365Guardrail(CustomGuardrail): return cached[0] response: Final = await self._post_allowing_error_status( - url=f"{self.authority_host}{TOKEN_PATH_TEMPLATE.format(tenant_id=self.tenant_id)}", + url=AGENT_365_TOKEN_URL_TEMPLATE.format(tenant_id=self.tenant_id), data={ # mutable-ok: OAuth form body; AsyncHTTPHandler.post requires dict "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", "client_id": self.client_id, diff --git a/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py index 1888b333748..8c3f22d3767 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py @@ -16,7 +16,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" akto_api_key=getattr(litellm_params, "akto_api_key", None), akto_account_id=getattr(litellm_params, "akto_account_id", None), akto_vxlan_id=getattr(litellm_params, "akto_vxlan_id", None), - unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", guardrail_timeout=getattr(litellm_params, "guardrail_timeout", None), guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, diff --git a/litellm/proxy/guardrails/guardrail_hooks/alice/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/alice/__init__.py index 75ea16f7a88..7117dbb5b23 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/alice/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/alice/__init__.py @@ -14,7 +14,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" _alice_guardrail_callback: Final = AliceGuardrail( api_key=litellm_params.api_key, api_base=litellm_params.api_base, - unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, default_on=litellm_params.default_on, diff --git a/litellm/proxy/guardrails/guardrail_hooks/conduct/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/conduct/__init__.py index 9eac143be88..07abc51672d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/conduct/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/conduct/__init__.py @@ -29,7 +29,7 @@ def initialize_guardrail( agent_token=litellm_params.api_key, workspace_id=extras.get("workspace_id"), tool_name=extras.get("tool_name", "llm_call"), - unreachable_fallback=litellm_params.unreachable_fallback, + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", timeout=DEFAULT_TIMEOUT_SECONDS if litellm_params.timeout is None else litellm_params.timeout, guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, diff --git a/litellm/proxy/guardrails/guardrail_hooks/deepkeep/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/deepkeep/__init__.py index 3b73883d290..6eae31c86e2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/deepkeep/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/deepkeep/__init__.py @@ -15,7 +15,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" api_base=litellm_params.api_base, api_key=litellm_params.api_key, firewall_id=getattr(litellm_params, "deepkeep_firewall_id", None), - unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", extra_headers=getattr(litellm_params, "extra_headers", None), guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py index e3511d46544..7b02cf5b04f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py @@ -30,7 +30,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" api_key=litellm_params.api_key, headers=getattr(litellm_params, "headers", None), additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", {}), - unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", fail_on_error=getattr(litellm_params, "fail_on_error", True), extra_headers=getattr(litellm_params, "extra_headers", None), guardrail_name=guardrail.get("guardrail_name", ""), diff --git a/litellm/proxy/guardrails/guardrail_hooks/repelloai/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/repelloai/__init__.py index 37788b35ec7..a20a3d8e1a9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/repelloai/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/repelloai/__init__.py @@ -30,7 +30,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" api_key=litellm_params.api_key, api_base=litellm_params.api_base, asset_id=litellm_params.asset_id, - unreachable_fallback=litellm_params.unreachable_fallback, + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", event_hook=_event_hook_from_mode(litellm_params.mode), default_on=litellm_params.default_on or False, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/typesafe/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/typesafe/__init__.py index dcea75d3a98..b521c2e03a3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/typesafe/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/typesafe/__init__.py @@ -55,9 +55,7 @@ def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) -> guardrail_name=guardrail["guardrail_name"], event_hook=_coerce_event_hook(litellm_params.mode), default_on=litellm_params.default_on or False, - unreachable_fallback=( - litellm_params.unreachable_fallback if "unreachable_fallback" in litellm_params.model_fields_set else None - ), + unreachable_fallback=litellm_params.unreachable_fallback, ) litellm.logging_callback_manager.add_litellm_callback( # pyright: ignore[reportUnknownMemberType] # callback manager is untyped _callback diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 579a3f6322f..4a79f41b030 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -1055,12 +1055,13 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up description="Additional provider-specific parameters for generic guardrail APIs", ) - unreachable_fallback: Literal["fail_closed", "fail_open"] = Field( - default="fail_closed", + unreachable_fallback: Literal["fail_closed", "fail_open"] | None = Field( + default=None, description=( "Behavior when a guardrail endpoint is unreachable due to network errors. " "Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. " - "'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed." + "'fail_closed' raises an error. 'fail_open' logs a critical error and allows the request to proceed. " + "Unset applies the guardrail's own default: 'agent_365' and 'typesafe' fail open, the others fail closed." ), ) @@ -1204,6 +1205,13 @@ class LitellmParams( # pyright: ignore[reportIncompatibleVariableOverride] # o mode: str | list[str] | Mode = Field( description="When to apply the guardrail (pre_call, post_call, during_call, logging_only)" ) + unreachable_fallback: Literal["fail_closed", "fail_open"] | None = Field( # pyright: ignore[reportIncompatibleVariableOverride] # mixins pin a default; unset defers to the guardrail's own + default=None, + description=( + "Behavior when the guardrail endpoint is unreachable. 'fail_closed' blocks, 'fail_open' allows and logs. " + "Unset applies the guardrail's own default: 'agent_365' and 'typesafe' fail open, the others fail closed." + ), + ) @field_validator("timeout", mode="before", check_fields=False) @classmethod diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/agent_365.py b/litellm/types/proxy/guardrails/guardrail_hooks/agent_365.py index d4d9452c380..a1ca3cf268d 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/agent_365.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/agent_365.py @@ -7,7 +7,7 @@ from .base import GuardrailConfigModel AGENT_365_PROD_API_BASE: Final = "https://agent365.svc.cloud.microsoft" AGENT_365_PROD_RESOURCE_APP_ID: Final = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1" AGENT_365_SCOPE_NAME: Final = "ThreatProtection.Evaluate.All" -AGENT_365_DEFAULT_AUTHORITY_HOST: Final = "https://login.microsoftonline.com" +AGENT_365_TOKEN_URL_TEMPLATE: Final = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" class Agent365GuardrailConfigModel(GuardrailConfigModel): @@ -42,6 +42,7 @@ class Agent365GuardrailConfigModel(GuardrailConfigModel): f"Defaults to the production endpoint {AGENT_365_PROD_API_BASE}. " "Falls back to the AGENT365_API_BASE environment variable." ), + json_schema_extra={"ui_hidden": True}, ) resource_app_id: str | None = Field( @@ -52,6 +53,7 @@ class Agent365GuardrailConfigModel(GuardrailConfigModel): "the Test and PreProd environments use a different id. " "Falls back to the AGENT365_RESOURCE_APP_ID environment variable." ), + json_schema_extra={"ui_hidden": True}, ) agent_id: str | None = Field( @@ -60,23 +62,15 @@ class Agent365GuardrailConfigModel(GuardrailConfigModel): "Agent identity reported to Agent 365 with every tool evaluation. " "When unset, the caller's key alias is used." ), - ) - - authority_host: str | None = Field( - default=None, - description=( - "Microsoft Entra authority host that issues the On-Behalf-Of token, for sovereign clouds. " - f"Defaults to {AGENT_365_DEFAULT_AUTHORITY_HOST}. " - "Falls back to the AGENT365_AUTHORITY_HOST, then AZURE_AUTHORITY_HOST environment variables." - ), + json_schema_extra={"ui_hidden": True}, ) unreachable_fallback: Literal["fail_closed", "fail_open"] = Field( - default="fail_closed", + default="fail_open", description=( "Behavior when Agent 365 or Entra is unreachable, times out, returns 5xx, skips the evaluation, or " - "rejects the gateway's own client credentials. 'fail_closed' (default) blocks the tool call with HTTP 503. " - "'fail_open' allows it, logs an error and records it as Unscanned in the logs and OpenTelemetry. " + "rejects the gateway's own client credentials. 'fail_open' (default) allows the tool call, logs an error " + "and records it as Unscanned in the logs and OpenTelemetry. 'fail_closed' blocks it with HTTP 503. " "Policy blocks, 4xx rejections, throttling and a rejected caller token always block." ), ) diff --git a/tests/integration/mcp/test_mcp_agent_365_guardrail.py b/tests/integration/mcp/test_mcp_agent_365_guardrail.py index 995fdc27e45..d2309cf4d5d 100644 --- a/tests/integration/mcp/test_mcp_agent_365_guardrail.py +++ b/tests/integration/mcp/test_mcp_agent_365_guardrail.py @@ -1,20 +1,16 @@ -"""Agent 365 guardrail when its own dependencies fail: Entra and Agent 365 are owned local doubles.""" +"""Agent 365 guardrail paths that end before the OBO exchange: no Entra, so no real tenant is ever contacted.""" import json -import os -import signal -import time import uuid from collections.abc import Iterator -from concurrent.futures import ThreadPoolExecutor -from contextlib import ExitStack, contextmanager -from dataclasses import dataclass, replace +from contextlib import contextmanager +from dataclasses import dataclass from hashlib import sha256 from pathlib import Path from typing import Final -from urllib.parse import parse_qs import httpx +import pytest import yaml from integration._support.client import Gateway, eventually from integration._support.database import read_rows @@ -23,7 +19,6 @@ from integration._support.mcp import ( EntryPoint, McpCaller, McpPeer, - Outcome, echo_tool, register_mcp, scripted_peer, @@ -33,68 +28,16 @@ from integration._support.process import owned_proxy_process from integration._support.wire import Reply, Request, Wire, wire_server TENANT: Final = "00000000-0000-4000-8000-0000000a3650" -EVALUATE_PATH: Final = "/agents/tool-evaluation/evaluate" -CALLER_TOKEN: Final = "eyJhbGciOiJub25lIn0.eyJzdWIiOiJpbnRlZ3JhdGlvbiJ9.synthetic-signature" -GUARDRAIL_TIMEOUT_SECONDS: Final = 1.0 -SLOW_REPLY_SECONDS: Final = 3.0 -GUARDRAIL_STATUSES: Final = ( - "SELECT metadata->'mcp_tool_call_metadata'->>'name' AS tool, gi->>'guardrail_status' AS status " - "FROM \"LiteLLM_SpendLogs\", jsonb_array_elements(metadata->'guardrail_information') gi " - "WHERE api_key = %s AND gi->>'guardrail_provider' = 'agent_365'" -) +REJECTED: Final = "Agent 365 guardrail rejected the tool call" GUARDRAIL_ROWS: Final = ( "SELECT metadata->'guardrail_information' AS gi FROM \"LiteLLM_SpendLogs\" " 'WHERE api_key = %s AND call_type = %s ORDER BY "startTime"' ) -TOOLS: Final = ("outage", "skipped", "denied", "add", "throttled", "rejected", "nonjson", "nobool", "slow") +FALLBACKS: Final = (None, "fail_open", "fail_closed") -def _caller_token(entra_case: str) -> str: - """A compact JWS whose signature segment tells the Entra double how to answer the OBO exchange.""" - return f"eyJhbGciOiJub25lIn0.eyJzdWIiOiJpbnRlZ3JhdGlvbiJ9.{entra_case}" - - -def _entra(request: Request) -> Reply: - assert request.target == f"/{TENANT}/oauth2/v2.0/token", request.target - case: Final = parse_qs(request.body.decode())["assertion"][0].rsplit(".", 1)[-1] - match case: - case "entra-outage": - return Reply(status=503, body=json.dumps({"error": "synthetic Entra outage"}).encode()) - case "entra-nonjson": - return Reply(body=b"synthetic gateway timeout", content_type="text/html") - case "entra-slow": - time.sleep(SLOW_REPLY_SECONDS) - case "entra-misconfigured": - misconfigured: Final = {"error": "invalid_client", "error_codes": [7000215]} - return Reply(status=401, body=json.dumps(misconfigured).encode()) - case "entra-rejected": - rejected: Final = {"error": "invalid_grant", "error_codes": [50013]} - return Reply(status=400, body=json.dumps(rejected).encode()) - return Reply(body=json.dumps({"access_token": "obo-" + uuid.uuid4().hex, "expires_in": 3599}).encode()) - - -def _agent_365(request: Request) -> Reply: - assert request.target == EVALUATE_PATH, request.target - tool: Final = str(json.loads(request.body)["tool"]["name"]).rsplit("-", 1)[-1] - match tool: - case "outage": - return Reply(status=503, body=json.dumps({"error": "synthetic Agent 365 outage"}).encode()) - case "skipped": - return Reply(body=json.dumps({"allowed": True, "defender": {"status": "Skipped"}}).encode()) - case "denied": - verdict: Final = {"status": "Evaluated", "verdict": "Block", "message": "synthetic block"} - return Reply(body=json.dumps({"allowed": False, "defender": verdict, "correlationId": "denied-1"}).encode()) - case "throttled": - return Reply(status=429, body=json.dumps({"error": "synthetic throttle"}).encode()) - case "rejected": - return Reply(status=400, body=json.dumps({"error": "synthetic malformed evaluation request"}).encode()) - case "nonjson": - return Reply(body=b"synthetic upstream error page", content_type="text/html") - case "nobool": - return Reply(body=json.dumps({"allowed": "yes", "defender": {"status": "Evaluated"}}).encode()) - case "slow": - time.sleep(SLOW_REPLY_SECONDS) - return Reply(body=json.dumps({"allowed": True, "defender": {"status": "Evaluated"}}).encode()) +def _agent_365_never_reached(request: Request) -> Reply: + return Reply(status=500, body=json.dumps({"error": f"unexpected evaluation request {request.target}"}).encode()) def _generic_guardrail_outage(request: Request) -> Reply: @@ -102,14 +45,7 @@ def _generic_guardrail_outage(request: Request) -> Reply: return Reply(status=503, body=json.dumps({"error": "synthetic sibling guardrail outage"}).encode()) -def _config( - tmp_path: Path, - name: str, - entra_url: str | None, - agent_365_url: str, - fallback: str | None, - sibling_url: str | None, -) -> Path: +def _config(tmp_path: Path, name: str, agent_365_url: str, fallback: str | None, sibling_url: str | None) -> Path: config: dict = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) config["guardrails"] = [ { @@ -122,8 +58,6 @@ def _config( "client_id": "synthetic-client-id", "client_secret": "synthetic-client-secret", "api_base": agent_365_url, - "timeout": GUARDRAIL_TIMEOUT_SECONDS, - **({"authority_host": entra_url} if entra_url else {}), **({"unreachable_fallback": fallback} if fallback else {}), }, }, @@ -152,111 +86,47 @@ def _config( @dataclass(frozen=True, slots=True) class Rig: candidate: Gateway - caller: McpCaller key: str alias: str - guardrail_name: str peer: McpPeer - entra: Wire agent_365: Wire server_id: str - def caller_for(self, entry: EntryPoint, token: str = CALLER_TOKEN) -> McpCaller: - return McpCaller(self.candidate, self.key, entry, self.alias, headers={"Authorization": f"Bearer {token}"}) - - def catalog_by_worker(self, samples: int = 8) -> frozenset[tuple[int, bool]]: - """A fresh connection stays on the worker that accepted it, so its self-reported pid pairs with its catalog.""" - - def probe(connection: httpx.Client) -> tuple[int, bool] | None: - candidate: Final = replace(self.candidate, client=connection) - caller: Final = McpCaller( - candidate, self.key, "mcp", self.alias, headers={"Authorization": f"Bearer {CALLER_TOKEN}"} - ) - try: - summary: Final = connection.get( - "/debug/memory/summary", headers={"Authorization": f"Bearer {candidate.key}"} - ) - return int(summary.json()["worker_pid"]), f"{self.alias}-add" in caller.list_tools().tools - except httpx.TransportError: - return None - - with ExitStack() as connections, ThreadPoolExecutor(max_workers=samples) as pool: - fresh: Final = tuple( - connections.enter_context(httpx.Client(base_url=str(self.candidate.client.base_url))) - for _ in range(samples) - ) - return frozenset(seen for seen in pool.map(probe, fresh) if seen is not None) - - def every_worker_serves_the_catalog(self, workers: int, without: int | None = None) -> frozenset[int]: - seen: Final = eventually( - self.catalog_by_worker, - lambda pairs: ( - len({pid for pid, _ in pairs}) == workers - and without not in {pid for pid, _ in pairs} - and all(served for _, served in pairs) - ), - seconds=40, - ) - return frozenset(pid for pid, _ in seen) - - def call_without_bearer(self, tool: str) -> Outcome: - return McpCaller(self.candidate, self.key, "mcp", self.alias).call(f"{self.alias}-{tool}", {"a": 0}) + def caller(self, entry: EntryPoint = "mcp", bearer: str | None = None) -> McpCaller: + headers: Final = {"Authorization": f"Bearer {bearer}"} if bearer else {} + return McpCaller(self.candidate, self.key, entry, self.alias, headers=headers) def upstream_tool_names(self) -> tuple[str, ...]: return tuple(str(call["body"]["params"]["name"]) for call in tool_calls(self.peer.drain())) + def guardrail_statuses(self, call_type: str, at_least: int) -> list[str]: + rows: Final = eventually( + lambda: read_rows(GUARDRAIL_ROWS, (sha256(self.key.encode()).hexdigest(), call_type)), + lambda seen: len(seen) >= at_least, + seconds=70, + ) + return [row["gi"][0]["guardrail_status"] if row["gi"] else "none" for row in rows] + @contextmanager -def _rig( - gateway: Gateway, - tmp_path: Path, - fallback: str | None, - *, - workers: int = 1, - environment: dict[str, str] | None = None, - authority_env: str | None = None, - sibling: bool = False, -) -> Iterator[Rig]: - """``authority_env`` names the environment variable that carries the Entra double instead of the config.""" +def _rig(gateway: Gateway, tmp_path: Path, fallback: str | None, *, sibling: bool = False) -> Iterator[Rig]: alias: Final = "a365" + uuid.uuid4().hex[:8] with ( - wire_server(_entra) as entra, - wire_server(_agent_365) as agent_365, + wire_server(_agent_365_never_reached) as agent_365, wire_server(_generic_guardrail_outage) as sibling_outage, - scripted_peer(*(echo_tool(tool) for tool in TOOLS)) as peer, + scripted_peer(echo_tool("add")) as peer, owned_proxy_process( gateway, tmp_path, - { - "PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2", - **(environment or {}), - **({authority_env: entra.url} if authority_env else {}), - }, - config=_config( - tmp_path, - alias, - None if authority_env else entra.url, - agent_365.url, - fallback, - sibling_outage.url if sibling else None, - ), - workers=workers, + {"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2"}, + config=_config(tmp_path, alias, agent_365.url, fallback, sibling_outage.url if sibling else None), ) as owned, owned.gateway.scenario() as scenario, ): identity: Final = register_mcp(scenario, peer, alias) key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) - caller: Final = McpCaller(owned.gateway, key, "mcp", alias, headers={"Authorization": f"Bearer {CALLER_TOKEN}"}) - rig: Final = Rig(owned.gateway, caller, key, alias, alias, peer, entra, agent_365, identity) - if workers > 1: - rig.every_worker_serves_the_catalog(workers) peer.drain() - yield rig - - -def _guardrail_statuses(key: str) -> dict[str, str]: - rows: Final = read_rows(GUARDRAIL_STATUSES, (sha256(key.encode()).hexdigest(),)) - return {str(row["tool"]): str(row["status"]) for row in rows} + yield Rig(owned.gateway, key, alias, peer, agent_365, identity) def _chat(rig: Rig, model: str, marker: str) -> httpx.Response: @@ -265,217 +135,39 @@ def _chat(rig: Rig, model: str, marker: str) -> httpx.Response: ) -def test_default_blocks_with_503_and_never_reaches_upstream_when_agent_365_cannot_evaluate( - gateway: Gateway, tmp_path: Path +@pytest.mark.parametrize("fallback", FALLBACKS) +def test_a_missing_or_malformed_caller_bearer_blocks_on_every_entry_point_whatever_the_fallback( + gateway: Gateway, tmp_path: Path, fallback: str | None ) -> None: - with _rig(gateway, tmp_path, fallback=None) as rig: - unavailable: Final = "could not authorize the tool call" - outage: Final = rig.caller.call(f"{rig.alias}-outage", {"a": 1}) - assert outage.error is not None and unavailable in outage.raw, ( - f"Agent 365 down must block by default: {outage.raw}" - ) - skipped: Final = rig.caller.call(f"{rig.alias}-skipped", {"a": 2}) - assert skipped.error is not None and unavailable in skipped.raw, ( - f"Defender skip must block by default: {skipped.raw}" - ) - denied: Final = rig.caller.call(f"{rig.alias}-denied", {"a": 3}) - assert denied.error is not None and "Blocked by Microsoft Defender" in denied.raw, denied.raw - assert rig.caller.call(f"{rig.alias}-add", {"a": 4}).text == '{"a": 4}' - assert rig.upstream_tool_names() == ("add",) - statuses: Final = eventually(lambda: _guardrail_statuses(rig.key), lambda seen: len(seen) >= 4, seconds=70) - assert statuses == { - "outage": "guardrail_failed_to_respond", - "skipped": "guardrail_failed_to_respond", - "denied": "guardrail_intervened", - "add": "success", - }, statuses - - -def test_explicit_fail_closed_matches_the_default(gateway: Gateway, tmp_path: Path) -> None: - with _rig(gateway, tmp_path, fallback="fail_closed") as rig: - outage: Final = rig.caller.call(f"{rig.alias}-outage", {"a": 1}) - assert outage.error is not None and "could not authorize the tool call" in outage.raw, outage.raw - assert rig.caller.call(f"{rig.alias}-add", {"a": 2}).text == '{"a": 2}' - assert rig.upstream_tool_names() == ("add",) - statuses: Final = eventually(lambda: _guardrail_statuses(rig.key), lambda seen: len(seen) >= 2, seconds=70) - assert statuses == {"outage": "guardrail_failed_to_respond", "add": "success"}, statuses - - -def test_opted_in_fail_open_lets_the_call_through_unscanned_and_still_blocks_policy_denials( - gateway: Gateway, tmp_path: Path -) -> None: - with _rig(gateway, tmp_path, fallback="fail_open") as rig: - outage: Final = rig.caller.call(f"{rig.alias}-outage", {"a": 1}) - assert outage.text == '{"a": 1}', f"Agent 365 down must fail open once opted in: {outage.raw}" - skipped: Final = rig.caller.call(f"{rig.alias}-skipped", {"a": 2}) - assert skipped.text == '{"a": 2}', f"Defender skipping the call must fail open once opted in: {skipped.raw}" - denied: Final = rig.caller.call(f"{rig.alias}-denied", {"a": 3}) - assert denied.error is not None and "Blocked by Microsoft Defender" in denied.raw, denied.raw - assert rig.upstream_tool_names() == ("outage", "skipped") - statuses: Final = eventually(lambda: _guardrail_statuses(rig.key), lambda seen: len(seen) >= 3, seconds=70) - assert statuses == { - "outage": "guardrail_failed_to_respond", - "skipped": "guardrail_failed_to_respond", - "denied": "guardrail_intervened", - }, statuses - - -def test_opted_in_fail_open_covers_malformed_or_stalled_agent_365_replies(gateway: Gateway, tmp_path: Path) -> None: - with _rig(gateway, tmp_path, fallback="fail_open") as rig: - for index, tool in enumerate(("nonjson", "nobool", "slow")): - outcome: Final = rig.caller.call(f"{rig.alias}-{tool}", {"a": index}) - assert outcome.text == json.dumps({"a": index}), f"{tool}: {outcome.raw}" - assert rig.upstream_tool_names() == ("nonjson", "nobool", "slow") - statuses: Final = eventually(lambda: _guardrail_statuses(rig.key), lambda seen: len(seen) >= 3, seconds=70) - assert statuses == dict.fromkeys(("nonjson", "nobool", "slow"), "guardrail_failed_to_respond"), statuses - - -def test_opted_in_fail_open_covers_entra_down_stalled_malformed_or_refusing_the_gateway_credentials( - gateway: Gateway, tmp_path: Path -) -> None: - with _rig(gateway, tmp_path, fallback="fail_open") as rig: - cases: Final = ("entra-outage", "entra-slow", "entra-nonjson", "entra-misconfigured") - for index, case in enumerate(cases): - outcome: Final = rig.caller_for("mcp", _caller_token(case)).call(f"{rig.alias}-add", {"a": index}) - assert outcome.text == json.dumps({"a": index}), f"{case}: {outcome.raw}" - assert rig.upstream_tool_names() == ("add",) * len(cases) - assert tuple( - sorted(parse_qs(request.body.decode())["assertion"][0].rsplit(".", 1)[-1] for request in rig.entra.drain()) - ) == tuple(sorted(cases)) - assert rig.agent_365.drain() == (), "no OBO token means no evaluation request" - rows: Final = eventually( - lambda: read_rows(GUARDRAIL_ROWS, (sha256(rig.key.encode()).hexdigest(), "call_mcp_tool")), - lambda seen: len(seen) >= len(cases), - seconds=70, - ) - assert [row["gi"][0]["guardrail_status"] for row in rows] == ["guardrail_failed_to_respond"] * len(cases) - - -def test_throttling_and_ordinary_4xx_from_agent_365_keep_blocking_even_when_opted_in_to_fail_open( - gateway: Gateway, tmp_path: Path -) -> None: - with _rig(gateway, tmp_path, fallback="fail_open") as rig: - throttled: Final = rig.caller.call(f"{rig.alias}-throttled", {"a": 1}) - assert throttled.error == "Error: Agent 365 guardrail could not authorize the tool call", throttled.raw - rejected: Final = rig.caller.call(f"{rig.alias}-rejected", {"a": 2}) - assert rejected.error == "Error: Agent 365 rejected the tool evaluation request", rejected.raw - assert rig.upstream_tool_names() == () - statuses: Final = eventually(lambda: _guardrail_statuses(rig.key), lambda seen: len(seen) >= 2, seconds=70) - assert statuses == {"throttled": "guardrail_failed_to_respond", "rejected": "guardrail_intervened"}, statuses - - -def test_caller_authentication_failures_keep_blocking_even_when_opted_in_to_fail_open( - gateway: Gateway, tmp_path: Path -) -> None: - with _rig(gateway, tmp_path, fallback="fail_open") as rig: - rejected: Final = "Error: Agent 365 guardrail rejected the tool call" - missing: Final = rig.call_without_bearer("add") - assert missing.error == rejected, missing.raw - malformed: Final = rig.caller_for("mcp", "not-a-jws").call(f"{rig.alias}-add", {"a": 1}) - assert malformed.error == rejected, malformed.raw - refused: Final = rig.caller_for("mcp", _caller_token("entra-rejected")).call(f"{rig.alias}-add", {"a": 2}) - assert refused.error == rejected, refused.raw - assert len(rig.entra.drain()) == 1, "only the well-formed bearer reaches the OBO exchange" - assert rig.upstream_tool_names() == () - assert rig.agent_365.drain() == () - rows: Final = eventually( - lambda: read_rows(GUARDRAIL_ROWS, (sha256(rig.key.encode()).hexdigest(), "call_mcp_tool")), - lambda seen: len(seen) >= 3, - seconds=70, - ) - assert [row["gi"][0]["guardrail_status"] for row in rows] == ["guardrail_intervened"] * 3 - - -def test_every_mcp_entry_point_honors_the_fail_open_opt_in_and_blocks_denials(gateway: Gateway, tmp_path: Path) -> None: - with _rig(gateway, tmp_path, fallback="fail_open") as rig: + with _rig(gateway, tmp_path, fallback) as rig: + assert f"{rig.alias}-add" in rig.caller().list_tools().tools, "the catalog needs only the virtual key" for entry in ENTRY_POINTS: - caller: Final = rig.caller_for(entry) - passed: Final = caller.call(f"{rig.alias}-outage", {"entry": entry}, server_id=rig.server_id) - assert passed.text == json.dumps({"entry": entry}), f"{entry}: {passed.raw}" - denied: Final = caller.call(f"{rig.alias}-denied", {"entry": entry}, server_id=rig.server_id) - assert denied.error is not None and "Blocked by Microsoft Defender" in denied.raw, f"{entry}: {denied.raw}" - assert rig.upstream_tool_names() == ("outage",) * len(ENTRY_POINTS) - - -def test_agent_365_fail_open_opt_in_does_not_leak_to_a_sibling_guardrail_on_chat_completions( - gateway: Gateway, tmp_path: Path -) -> None: - with _rig(gateway, tmp_path, fallback="fail_open", sibling=True) as rig, rig.candidate.scenario() as scenario: - model: Final = scenario.model() - chat: Final = _chat(rig, model, "sibling-" + uuid.uuid4().hex) - assert chat.status_code == 500 and "Generic Guardrail API failed" in chat.text, chat.text - assert rig.entra.drain() == () and rig.agent_365.drain() == () - assert rig.caller.call(f"{rig.alias}-outage", {"a": 1}).text == '{"a": 1}' - assert rig.upstream_tool_names() == ("outage",) + missing: Final = rig.caller(entry).call(f"{rig.alias}-add", {"entry": entry}, server_id=rig.server_id) + assert missing.error is not None and REJECTED in missing.raw, f"{entry} without a bearer: {missing.raw}" + malformed: Final = rig.caller(entry, "not-a-jws").call( + f"{rig.alias}-add", {"entry": entry}, server_id=rig.server_id + ) + assert malformed.error is not None and REJECTED in malformed.raw, f"{entry} opaque bearer: {malformed.raw}" + assert rig.upstream_tool_names() == () + assert rig.agent_365.drain() == (), "a rejected caller never produces an evaluation request" + expected: Final = 2 * len(ENTRY_POINTS) + assert rig.guardrail_statuses("call_mcp_tool", expected) == ["guardrail_intervened"] * expected def test_chat_completions_are_unaffected_by_the_mcp_guardrail(gateway: Gateway, tmp_path: Path) -> None: with _rig(gateway, tmp_path, fallback=None) as rig, rig.candidate.scenario() as scenario: model: Final = scenario.model() - marker: Final = "unaffected-" + uuid.uuid4().hex - chat: Final = _chat(rig, model, marker) + chat: Final = _chat(rig, model, "unaffected-" + uuid.uuid4().hex) assert chat.status_code == 200, chat.text - assert rig.entra.drain() == () and rig.agent_365.drain() == () - rows: Final = eventually( - lambda: read_rows(GUARDRAIL_ROWS, (sha256(rig.key.encode()).hexdigest(), "acompletion")), - lambda seen: len(seen) >= 1, - seconds=70, - ) - assert rows[0]["gi"] is None, rows + assert rig.agent_365.drain() == () + assert rig.guardrail_statuses("acompletion", 1) == ["none"] -def test_agent365_authority_host_env_wins_over_azure_authority_host_when_config_has_none( +def test_a_sibling_guardrail_keeps_its_own_fail_closed_default_next_to_agent_365( gateway: Gateway, tmp_path: Path ) -> None: - with _rig(gateway, tmp_path / "azure", fallback=None, authority_env="AZURE_AUTHORITY_HOST") as azure_only: - assert azure_only.caller.call(f"{azure_only.alias}-add", {"a": 1}).text == '{"a": 1}' - assert len(azure_only.entra.drain()) == 1 - with ( - wire_server(_entra) as decoy, - _rig( - gateway, - tmp_path / "agent365", - fallback=None, - authority_env="AGENT365_AUTHORITY_HOST", - environment={"AZURE_AUTHORITY_HOST": decoy.url}, - ) as both, - ): - assert both.caller.call(f"{both.alias}-add", {"a": 2}).text == '{"a": 2}' - assert len(both.entra.drain()) == 1 and decoy.drain() == () - - -def test_thirty_call_burst_against_a_flapping_agent_365_reaches_upstream_exactly_once_each_on_two_workers( - gateway: Gateway, tmp_path: Path -) -> None: - with _rig(gateway, tmp_path, fallback="fail_open", workers=2) as rig: - markers: Final = tuple(("outage" if index % 2 else "add", uuid.uuid4().hex) for index in range(30)) - with ThreadPoolExecutor(max_workers=10) as pool: - outcomes: Final = tuple( - pool.map(lambda pair: rig.caller.call(f"{rig.alias}-{pair[0]}", {"marker": pair[1]}), markers) - ) - for (tool, marker), outcome in zip(markers, outcomes, strict=True): - assert outcome.text == json.dumps({"marker": marker}), f"{tool}: {outcome.raw}" - seen: Final = sorted( - str(call["body"]["params"]["arguments"]["marker"]) for call in tool_calls(rig.peer.drain()) - ) - assert seen == sorted(marker for _, marker in markers) - assert rig.caller.call(f"{rig.alias}-denied", {"a": 1}).error is not None - assert rig.upstream_tool_names() == () - - -def test_opted_in_fail_open_survives_a_worker_kill_and_keeps_blocking_denials_on_two_workers( - gateway: Gateway, tmp_path: Path -) -> None: - with _rig(gateway, tmp_path, fallback="fail_open", workers=2) as rig: - before: Final = rig.every_worker_serves_the_catalog(2) - victim: Final = min(before) - os.kill(victim, signal.SIGKILL) - after: Final = rig.every_worker_serves_the_catalog(2, without=victim) - assert after - before, f"a replacement worker took over: before {before}, after {after}" - rig.peer.drain() - for index in range(10): - passed: Final = rig.caller.call(f"{rig.alias}-outage", {"a": index}) - assert passed.text == json.dumps({"a": index}), passed.raw - denied: Final = rig.caller.call(f"{rig.alias}-denied", {"a": index}) - assert denied.error is not None and "Blocked by Microsoft Defender" in denied.raw, denied.raw - assert rig.upstream_tool_names() == ("outage",) * 10 + with _rig(gateway, tmp_path, fallback=None, sibling=True) as rig, rig.candidate.scenario() as scenario: + model: Final = scenario.model() + chat: Final = _chat(rig, model, "sibling-" + uuid.uuid4().hex) + assert chat.status_code == 500 and "Generic Guardrail API failed" in chat.text, chat.text + assert rig.agent_365.drain() == () diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_agent_365.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_agent_365.py index 97d69e92aa3..e18b9aea979 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_agent_365.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_agent_365.py @@ -28,7 +28,6 @@ from litellm.types.guardrails import ( SupportedGuardrailIntegrations, ) from litellm.types.proxy.guardrails.guardrail_hooks.agent_365 import ( - AGENT_365_DEFAULT_AUTHORITY_HOST, AGENT_365_PROD_API_BASE, AGENT_365_PROD_RESOURCE_APP_ID, Agent365GuardrailConfigModel, @@ -125,7 +124,6 @@ def _make_guardrail( unreachable_fallback: str = "fail_closed", agent_id: str | None = None, api_base: str = AGENT_365_PROD_API_BASE, - authority_host: str = AGENT_365_DEFAULT_AUTHORITY_HOST, ) -> Agent365Guardrail: return Agent365Guardrail( guardrail_name="agent-365-guard", @@ -134,7 +132,6 @@ def _make_guardrail( client_secret="secret-123", api_base=api_base, agent_id=agent_id, - authority_host=authority_host, unreachable_fallback=unreachable_fallback, async_handler=handler, event_hook="pre_mcp_call", @@ -234,62 +231,34 @@ class TestInitializeGuardrail: assert guardrail.client_secret == "env-secret" assert guardrail.api_base == "https://env.example.test" assert guardrail.resource_app_id == AGENT_365_PROD_RESOURCE_APP_ID - assert guardrail.unreachable_fallback == "fail_closed" + assert guardrail.unreachable_fallback == "fail_open" - def test_fail_open_is_opt_in_through_litellm_params(self): + def test_fail_closed_is_opt_in_through_litellm_params(self): params: Final = LitellmParams( guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s", - unreachable_fallback="fail_open", + unreachable_fallback="fail_closed", ) - guardrail: Final = initialize_guardrail(params, {"guardrail_name": "a365-open"}) + guardrail: Final = initialize_guardrail(params, {"guardrail_name": "a365-closed"}) + assert guardrail.unreachable_fallback == "fail_closed" + + def test_unreachable_fallback_round_trips_through_a_stored_litellm_params_dump(self): + stored: Final = LitellmParams( + guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s" + ).model_dump() + guardrail: Final = initialize_guardrail(LitellmParams(**stored), {"guardrail_name": "a365-stored"}) assert guardrail.unreachable_fallback == "fail_open" - def test_authority_host_defaults_to_public_entra(self, monkeypatch): - monkeypatch.delenv("AGENT365_AUTHORITY_HOST", raising=False) - monkeypatch.delenv("AZURE_AUTHORITY_HOST", raising=False) - params: Final = LitellmParams( - guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s" - ) - guardrail: Final = initialize_guardrail(params, {"guardrail_name": "a365-authority"}) - assert guardrail.authority_host == "https://login.microsoftonline.com" + def test_ui_form_hides_the_production_default_overrides(self): + from litellm.proxy.guardrails.guardrail_endpoints import _get_fields_from_model - def test_authority_host_precedence_param_then_agent365_env_then_azure_env(self, monkeypatch): - monkeypatch.setenv("AZURE_AUTHORITY_HOST", "https://login.microsoftonline.us/") - azure_only: Final = initialize_guardrail( - LitellmParams(guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s"), - {"guardrail_name": "a365-azure"}, - ) - assert azure_only.authority_host == "https://login.microsoftonline.us" - monkeypatch.setenv("AGENT365_AUTHORITY_HOST", "https://login.partner.microsoftonline.cn") - agent_env: Final = initialize_guardrail( - LitellmParams(guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s"), - {"guardrail_name": "a365-agentenv"}, - ) - assert agent_env.authority_host == "https://login.partner.microsoftonline.cn" - explicit: Final = initialize_guardrail( - LitellmParams( - guardrail="agent_365", - mode="pre_mcp_call", - tenant_id="t", - client_id="c", - client_secret="s", - authority_host="http://127.0.0.1:9", - ), - {"guardrail_name": "a365-explicit"}, - ) - assert explicit.authority_host == "http://127.0.0.1:9" - - def test_scheme_less_authority_host_env_gets_https(self, monkeypatch): - monkeypatch.setenv("AZURE_AUTHORITY_HOST", " login.microsoftonline.us/ ") - guardrail: Final = initialize_guardrail( - LitellmParams(guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s"), - {"guardrail_name": "a365-bare-host"}, - ) - assert guardrail.authority_host == "https://login.microsoftonline.us" + fields: Final = _get_fields_from_model(Agent365GuardrailConfigModel) + assert {"tenant_id", "client_id", "client_secret", "unreachable_fallback"} <= set(fields) + assert not {"api_base", "resource_app_id", "agent_id"} & set(fields) + assert fields["unreachable_fallback"]["default_value"] == "fail_open" def test_explicit_params_win(self, monkeypatch): monkeypatch.setenv("AGENT365_TENANT_ID", "env-tenant") @@ -300,14 +269,14 @@ class TestInitializeGuardrail: client_id="client-xyz", client_secret="param-secret", agent_id="agent-007", - unreachable_fallback="fail_open", + unreachable_fallback="fail_closed", timeout=5, ) guardrail: Final = initialize_guardrail(params, {"guardrail_name": "a365-params"}) assert guardrail.tenant_id == "param-tenant" assert guardrail.client_secret == "param-secret" assert guardrail.agent_id == "agent-007" - assert guardrail.unreachable_fallback == "fail_open" + assert guardrail.unreachable_fallback == "fail_closed" assert guardrail.request_timeout == 5.0 def test_wrong_mode_rejected(self): @@ -357,21 +326,6 @@ class TestAllowFlow: assert token_call.data["client_secret"] == "secret-123" assert token_call.data["scope"] == f"{AGENT_365_PROD_RESOURCE_APP_ID}/ThreatProtection.Evaluate.All" - @pytest.mark.asyncio - async def test_obo_exchange_goes_to_the_configured_authority_host(self): - handler: Final = FakeHandler([_token_response(), _allow_response()]) - guardrail: Final = _make_guardrail(handler, authority_host="https://login.microsoftonline.us/") - await _run(guardrail, _mcp_data()) - assert handler.calls[0].url == "https://login.microsoftonline.us/tenant-abc/oauth2/v2.0/token" - assert handler.calls[1].url == EVALUATE_URL - - @pytest.mark.asyncio - async def test_obo_exchange_adds_https_to_a_bare_authority_host(self): - handler: Final = FakeHandler([_token_response(), _allow_response()]) - guardrail: Final = _make_guardrail(handler, authority_host="login.microsoftonline.us") - await _run(guardrail, _mcp_data()) - assert handler.calls[0].url == "https://login.microsoftonline.us/tenant-abc/oauth2/v2.0/token" - @pytest.mark.asyncio async def test_evaluate_payload(self): handler: Final = FakeHandler([_token_response(), _allow_response()]) @@ -565,20 +519,12 @@ AVAILABILITY_FAILURES: Final = ( ) -class TestOptInFailOpen: +class TestFailOpenDefault: @pytest.mark.asyncio @pytest.mark.parametrize("responses", AVAILABILITY_FAILURES) - async def test_constructor_default_blocks_each_availability_failure_with_503(self, responses): + async def test_constructor_default_lets_each_availability_failure_through_as_failed_to_respond(self, responses): guardrail: Final = _default_fallback_guardrail(FakeHandler(responses)) - assert guardrail.unreachable_fallback == "fail_closed" - with pytest.raises(HTTPException) as exc_info: - await _run(guardrail, _mcp_data()) - assert exc_info.value.status_code == 503 - - @pytest.mark.asyncio - @pytest.mark.parametrize("responses", AVAILABILITY_FAILURES) - async def test_opted_in_fail_open_lets_each_availability_failure_through_as_failed_to_respond(self, responses): - guardrail: Final = _make_guardrail(FakeHandler(responses), unreachable_fallback="fail_open") + assert guardrail.unreachable_fallback == "fail_open" data: Final = _mcp_data() assert await _run(guardrail, data) is data info: Final = _guardrail_info(data) @@ -586,7 +532,16 @@ class TestOptInFailOpen: assert info["guardrail_response"]["verdict"] == "Unscanned" @pytest.mark.asyncio - async def test_opted_in_fail_open_logs_the_unscanned_call_at_error_level(self, caplog): + @pytest.mark.parametrize("responses", AVAILABILITY_FAILURES) + async def test_opted_in_fail_closed_blocks_each_availability_failure_with_503(self, responses): + guardrail: Final = _make_guardrail(FakeHandler(responses), unreachable_fallback="fail_closed") + with pytest.raises(HTTPException) as exc_info: + await _run(guardrail, _mcp_data()) + assert exc_info.value.status_code == 503 + assert "fail_closed" in exc_info.value.detail["message"] + + @pytest.mark.asyncio + async def test_default_fail_open_logs_the_unscanned_call_at_error_level(self, caplog): handler: Final = FakeHandler([_token_response(), httpx.ReadTimeout("timed out")]) guardrail: Final = _make_guardrail(handler, unreachable_fallback="fail_open") with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): @@ -595,9 +550,9 @@ class TestOptInFailOpen: assert [r.levelno for r in fail_open_logs] == [logging.ERROR], caplog.text @pytest.mark.asyncio - async def test_opted_in_fail_open_still_blocks_a_policy_block(self): + async def test_default_fail_open_still_blocks_a_policy_block(self): handler: Final = FakeHandler([_token_response(), _block_response()]) - guardrail: Final = _make_guardrail(handler, unreachable_fallback="fail_open") + guardrail: Final = _default_fallback_guardrail(handler) with pytest.raises(HTTPException) as exc_info: await _run(guardrail, _mcp_data()) assert exc_info.value.status_code == 400 diff --git a/tests/test_litellm/proxy/guardrails/test_init_guardrails.py b/tests/test_litellm/proxy/guardrails/test_init_guardrails.py index fc2fb949143..b7cbb3b8038 100644 --- a/tests/test_litellm/proxy/guardrails/test_init_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_init_guardrails.py @@ -167,6 +167,29 @@ def test_initialize_guardrail_sets_run_in_parallel(config_value, expected): assert custom_guardrail.run_in_parallel is expected +@pytest.mark.parametrize( + "guardrail, provider_params, expected", + [ + ("agent_365", {"tenant_id": "t", "client_id": "c", "client_secret": "s", "mode": "pre_mcp_call"}, "fail_open"), + ("typesafe", {"api_key": "k"}, "fail_open"), + ("generic_guardrail_api", {"api_base": "http://127.0.0.1:1/guard"}, "fail_closed"), + ("akto", {"akto_base_url": "http://127.0.0.1:1", "akto_api_key": "k", "akto_account_id": "1"}, "fail_closed"), + ("alice", {"api_key": "k", "api_base": "http://127.0.0.1:1"}, "fail_closed"), + ("deepkeep", {"api_base": "http://127.0.0.1:1", "api_key": "k", "deepkeep_firewall_id": "f"}, "fail_closed"), + ("repelloai", {"api_key": "k", "api_base": "http://127.0.0.1:1", "asset_id": "a"}, "fail_closed"), + ], +) +def test_unset_unreachable_fallback_applies_each_guardrails_own_default(guardrail, provider_params, expected): + litellm_params = {"guardrail": guardrail, "mode": "pre_call", **provider_params} + guardrail_handler = InMemoryGuardrailHandler() + result = guardrail_handler.initialize_guardrail( + guardrail={"guardrail_name": f"default-fallback-{guardrail}", "litellm_params": litellm_params}, + ) + + custom_guardrail = guardrail_handler.guardrail_id_to_custom_guardrail[result["guardrail_id"]] + assert custom_guardrail.unreachable_fallback == expected, f"{guardrail} with unreachable_fallback unset" + + def test_initialize_presidio_forwards_analyze_chunk_size_bytes(): """Regression (LIT-4785): `presidio_analyze_chunk_size_bytes` set in config.yaml must reach the guardrail instance. The field lives on diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 500495b3797..6126733095b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -34091,11 +34091,6 @@ export interface components { * @description Authorization bearer token for IBM Guardrails API. Reads from IBM_GUARDRAILS_AUTH_TOKEN env var if None. */ auth_token?: string | null; - /** - * Authority Host - * @description Microsoft Entra authority host that issues the On-Behalf-Of token, for sovereign clouds. Defaults to https://login.microsoftonline.com. Falls back to the AGENT365_AUTHORITY_HOST, then AZURE_AUTHORITY_HOST environment variables. - */ - authority_host?: string | null; /** * Aws Access Key Id * @description AWS access key ID for authentication