diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index c7bf291a887..978cc549dea 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -3252,6 +3252,16 @@ class PrometheusLogger(CustomLogger): except Exception as e: verbose_logger.debug("Error recording guardrail metrics: %s", e) + def record_guardrail_fail_open(self, guardrail_name: str, hook_type: str) -> None: + try: + self.litellm_guardrail_errors_total.labels( + guardrail_name=guardrail_name, + error_type="fail_open", + hook_type=hook_type, + ).inc() + except Exception as e: + verbose_logger.debug("Error recording guardrail fail-open metric: %s", e) + ######################################## # Managed Batch Metric Recording Methods ######################################## diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 3d44315341b..3842a5cb504 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -10405,14 +10405,20 @@ "title": "Timeout" }, "unreachable_fallback": { - "default": "fail_closed", - "description": "Behavior when a guardrail endpoint is unreachable due to network errors. Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed.", - "enum": [ - "fail_closed", - "fail_open" + "anyOf": [ + { + "enum": [ + "fail_closed", + "fail_open" + ], + "type": "string" + }, + { + "type": "null" + } ], - "title": "Unreachable Fallback", - "type": "string" + "description": "Behavior when a guardrail endpoint is unreachable due to network errors. Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. 'fail_closed' raises an error. 'fail_open' logs a critical error and allows the request to proceed. Unset applies the guardrail's own default: 'agent_365' and 'typesafe' fail open, the others fail closed.", + "title": "Unreachable Fallback" }, "violation_message_template": { "anyOf": [ @@ -13261,14 +13267,20 @@ "title": "Tracker Api Key" }, "unreachable_fallback": { - "default": "fail_closed", - "description": "Behavior when the headroom compression service is unreachable or errors. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and forwards the request uncompressed instead of blocking it.", - "enum": [ - "fail_closed", - "fail_open" + "anyOf": [ + { + "enum": [ + "fail_closed", + "fail_open" + ], + "type": "string" + }, + { + "type": "null" + } ], - "title": "Unreachable Fallback", - "type": "string" + "description": "Behavior when the guardrail endpoint is unreachable. 'fail_closed' blocks, 'fail_open' allows and logs. Unset applies the guardrail's own default: 'agent_365' and 'typesafe' fail open, the others fail closed.", + "title": "Unreachable Fallback" }, "use_v2": { "anyOf": [ diff --git a/litellm/proxy/guardrails/guardrail_hooks/agent_365/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/agent_365/__init__.py index 9aacdec0602..ef4b46e3dfd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/agent_365/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/agent_365/__init__.py @@ -46,7 +46,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" resource_app_id=resource_app_id or AGENT_365_PROD_RESOURCE_APP_ID, agent_id=litellm_params.agent_id, request_timeout=litellm_params.timeout if litellm_params.timeout is not None else 10.0, - unreachable_fallback=litellm_params.unreachable_fallback, + unreachable_fallback=litellm_params.unreachable_fallback or "fail_open", event_hook=litellm_params.mode, default_on=litellm_params.default_on, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py b/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py index 975d321104d..3d8007fc480 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py +++ b/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py @@ -14,7 +14,7 @@ import threading import time import uuid from collections import OrderedDict -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, ClassVar, Final, Literal, NoReturn import httpx @@ -22,12 +22,14 @@ from fastapi import HTTPException from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict +import litellm from litellm._logging import verbose_proxy_logger from litellm.exceptions import Timeout as LitellmTimeout from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.integrations.prometheus import PrometheusLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -81,6 +83,10 @@ def _parse_aadsts_codes(raw: object) -> tuple[int, ...]: return () +def registered_prometheus_logger() -> PrometheusLogger | None: + return next((cb for cb in litellm.callbacks if isinstance(cb, PrometheusLogger)), None) + + def entra_assertion(value: object) -> str | None: """``value`` when it is a compact JWS, the only bearer shape the OBO exchange accepts as its assertion. A LiteLLM virtual key, session bearer, or opaque upstream token in ``Authorization`` yields ``None``.""" @@ -157,8 +163,9 @@ class Agent365Guardrail(CustomGuardrail): resource_app_id: str = AGENT_365_PROD_RESOURCE_APP_ID, agent_id: str | None = None, request_timeout: float = 10.0, - unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed", + unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_open", async_handler: AsyncHTTPHandler | None = None, + prometheus_logger_lookup: Callable[[], PrometheusLogger | None] = registered_prometheus_logger, **kwargs, # noqa: ANN003 # kwargs-ok: forwarded verbatim to CustomGuardrail (event_hook, default_on) ) -> None: super().__init__( @@ -176,11 +183,12 @@ class Agent365Guardrail(CustomGuardrail): self.agent_id = agent_id self.request_timeout = request_timeout self.unreachable_fallback: Literal["fail_closed", "fail_open"] = ( - "fail_open" if unreachable_fallback == "fail_open" else "fail_closed" + "fail_closed" if unreachable_fallback == "fail_closed" else "fail_open" ) self.async_handler = async_handler or get_async_httpx_client( llm_provider=httpxSpecialProvider.GuardrailCallback ) + self._prometheus_logger_lookup = prometheus_logger_lookup self._obo_token_cache: OrderedDict[str, tuple[str, float]] = OrderedDict() # mutable-ok: lock-guarded LRU self._obo_cache_lock = threading.Lock() verbose_proxy_logger.info("Initialized Microsoft Agent 365 guardrail: %s", guardrail_name) @@ -575,12 +583,13 @@ class Agent365Guardrail(CustomGuardrail): latency_ms: float | None = None, ) -> dict: # mutable-ok: returns the request data dict per hook contract if self.unreachable_fallback == "fail_open": - verbose_proxy_logger.warning( + verbose_proxy_logger.error( "Agent 365 guardrail (%s): %s; unreachable_fallback='fail_open', allowing tool call '%s' unscanned", self.guardrail_name, reason, tool_name, ) + self._count_fail_open() self._record_verdict( data=data, verdict="Unscanned", @@ -608,6 +617,13 @@ class Agent365Guardrail(CustomGuardrail): } raise HTTPException(status_code=503, detail=unavailable_detail) + def _count_fail_open(self) -> None: + prometheus: Final = self._prometheus_logger_lookup() + if prometheus is not None: + prometheus.record_guardrail_fail_open( + guardrail_name=self.guardrail_name or type(self).__name__, hook_type="pre_call" + ) + def _record_verdict( self, data: dict[str, object], # mutable-ok: standard guardrail logging appends into the request metadata in place diff --git a/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py index 1888b333748..8c3f22d3767 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py @@ -16,7 +16,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" akto_api_key=getattr(litellm_params, "akto_api_key", None), akto_account_id=getattr(litellm_params, "akto_account_id", None), akto_vxlan_id=getattr(litellm_params, "akto_vxlan_id", None), - unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", guardrail_timeout=getattr(litellm_params, "guardrail_timeout", None), guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, diff --git a/litellm/proxy/guardrails/guardrail_hooks/alice/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/alice/__init__.py index 75ea16f7a88..7117dbb5b23 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/alice/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/alice/__init__.py @@ -14,7 +14,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" _alice_guardrail_callback: Final = AliceGuardrail( api_key=litellm_params.api_key, api_base=litellm_params.api_base, - unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, default_on=litellm_params.default_on, diff --git a/litellm/proxy/guardrails/guardrail_hooks/conduct/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/conduct/__init__.py index 9eac143be88..07abc51672d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/conduct/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/conduct/__init__.py @@ -29,7 +29,7 @@ def initialize_guardrail( agent_token=litellm_params.api_key, workspace_id=extras.get("workspace_id"), tool_name=extras.get("tool_name", "llm_call"), - unreachable_fallback=litellm_params.unreachable_fallback, + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", timeout=DEFAULT_TIMEOUT_SECONDS if litellm_params.timeout is None else litellm_params.timeout, guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, diff --git a/litellm/proxy/guardrails/guardrail_hooks/deepkeep/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/deepkeep/__init__.py index 3b73883d290..6eae31c86e2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/deepkeep/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/deepkeep/__init__.py @@ -15,7 +15,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" api_base=litellm_params.api_base, api_key=litellm_params.api_key, firewall_id=getattr(litellm_params, "deepkeep_firewall_id", None), - unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", extra_headers=getattr(litellm_params, "extra_headers", None), guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py index e3511d46544..7b02cf5b04f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py @@ -30,7 +30,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" api_key=litellm_params.api_key, headers=getattr(litellm_params, "headers", None), additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", {}), - unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", fail_on_error=getattr(litellm_params, "fail_on_error", True), extra_headers=getattr(litellm_params, "extra_headers", None), guardrail_name=guardrail.get("guardrail_name", ""), diff --git a/litellm/proxy/guardrails/guardrail_hooks/repelloai/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/repelloai/__init__.py index 37788b35ec7..a20a3d8e1a9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/repelloai/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/repelloai/__init__.py @@ -30,7 +30,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" api_key=litellm_params.api_key, api_base=litellm_params.api_base, asset_id=litellm_params.asset_id, - unreachable_fallback=litellm_params.unreachable_fallback, + unreachable_fallback=litellm_params.unreachable_fallback or "fail_closed", event_hook=_event_hook_from_mode(litellm_params.mode), default_on=litellm_params.default_on or False, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/typesafe/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/typesafe/__init__.py index dcea75d3a98..b521c2e03a3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/typesafe/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/typesafe/__init__.py @@ -55,9 +55,7 @@ def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) -> guardrail_name=guardrail["guardrail_name"], event_hook=_coerce_event_hook(litellm_params.mode), default_on=litellm_params.default_on or False, - unreachable_fallback=( - litellm_params.unreachable_fallback if "unreachable_fallback" in litellm_params.model_fields_set else None - ), + unreachable_fallback=litellm_params.unreachable_fallback, ) litellm.logging_callback_manager.add_litellm_callback( # pyright: ignore[reportUnknownMemberType] # callback manager is untyped _callback diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 579a3f6322f..4a79f41b030 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -1055,12 +1055,13 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up description="Additional provider-specific parameters for generic guardrail APIs", ) - unreachable_fallback: Literal["fail_closed", "fail_open"] = Field( - default="fail_closed", + unreachable_fallback: Literal["fail_closed", "fail_open"] | None = Field( + default=None, description=( "Behavior when a guardrail endpoint is unreachable due to network errors. " "Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. " - "'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed." + "'fail_closed' raises an error. 'fail_open' logs a critical error and allows the request to proceed. " + "Unset applies the guardrail's own default: 'agent_365' and 'typesafe' fail open, the others fail closed." ), ) @@ -1204,6 +1205,13 @@ class LitellmParams( # pyright: ignore[reportIncompatibleVariableOverride] # o mode: str | list[str] | Mode = Field( description="When to apply the guardrail (pre_call, post_call, during_call, logging_only)" ) + unreachable_fallback: Literal["fail_closed", "fail_open"] | None = Field( # pyright: ignore[reportIncompatibleVariableOverride] # mixins pin a default; unset defers to the guardrail's own + default=None, + description=( + "Behavior when the guardrail endpoint is unreachable. 'fail_closed' blocks, 'fail_open' allows and logs. " + "Unset applies the guardrail's own default: 'agent_365' and 'typesafe' fail open, the others fail closed." + ), + ) @field_validator("timeout", mode="before", check_fields=False) @classmethod diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/agent_365.py b/litellm/types/proxy/guardrails/guardrail_hooks/agent_365.py index dd3d7fe5f74..83754df9c59 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/agent_365.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/agent_365.py @@ -1,4 +1,4 @@ -from typing import Final +from typing import Final, Literal from pydantic import Field @@ -61,6 +61,16 @@ class Agent365GuardrailConfigModel(GuardrailConfigModel): ), ) + unreachable_fallback: Literal["fail_closed", "fail_open"] = Field( + default="fail_open", + description=( + "Behavior when Agent 365 or Entra is unreachable, times out, returns 5xx, or skips the evaluation. " + "'fail_open' (default) allows the tool call and records it as Unscanned in the logs, OpenTelemetry " + "and the litellm_guardrail_errors_total Prometheus counter. 'fail_closed' blocks it with HTTP 503. " + "Blocks, 4xx rejections, throttling and Entra token failures always block." + ), + ) + @staticmethod def ui_friendly_name() -> str: return "Microsoft Agent 365" diff --git a/tests/test_litellm/integrations/test_prometheus_queue_guardrail_metrics.py b/tests/test_litellm/integrations/test_prometheus_queue_guardrail_metrics.py index f04e8d0d2c7..9d608594047 100644 --- a/tests/test_litellm/integrations/test_prometheus_queue_guardrail_metrics.py +++ b/tests/test_litellm/integrations/test_prometheus_queue_guardrail_metrics.py @@ -554,3 +554,24 @@ class TestPrometheusGuardrailMetrics: mock_latency_metric.labels.assert_called_once() call_kwargs = mock_latency_metric.labels.call_args[1] assert call_kwargs["guardrail_name"] == guardrail_name + + def test_record_guardrail_fail_open_increments_errors_total(self): + prometheus_logger = PrometheusLogger() + + prometheus_logger.record_guardrail_fail_open(guardrail_name="a365", hook_type="pre_call") + prometheus_logger.record_guardrail_fail_open(guardrail_name="a365", hook_type="pre_call") + + assert ( + REGISTRY.get_sample_value( + "litellm_guardrail_errors_total", + {"guardrail_name": "a365", "error_type": "fail_open", "hook_type": "pre_call"}, + ) + == 2.0 + ) + assert ( + REGISTRY.get_sample_value( + "litellm_guardrail_requests_total", + {"guardrail_name": "a365", "status": "success", "hook_type": "pre_call"}, + ) + is None + ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_agent_365.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_agent_365.py index f9b7561b9d3..7ccb30b960c 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_agent_365.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_agent_365.py @@ -11,6 +11,7 @@ import litellm from litellm.caching.caching import DualCache from litellm.exceptions import Timeout as LitellmTimeout from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.prometheus import PrometheusLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.proxy._types import UserAPIKeyAuth @@ -20,6 +21,7 @@ from litellm.proxy.guardrails.guardrail_hooks.agent_365 import ( guardrail_initializer_registry, initialize_guardrail, ) +from litellm.proxy.guardrails.guardrail_hooks.agent_365.agent_365 import registered_prometheus_logger from litellm.proxy.utils import ProxyLogging from litellm.types.guardrails import ( GuardrailEventHooks, @@ -117,12 +119,21 @@ class FakeHandler: return item +class FakePrometheus: + def __init__(self) -> None: + self.fail_opens: list[tuple[str, str]] = [] + + def record_guardrail_fail_open(self, guardrail_name: str, hook_type: str) -> None: + self.fail_opens.append((guardrail_name, hook_type)) + + def _make_guardrail( handler: FakeHandler, *, unreachable_fallback: str = "fail_closed", agent_id: str | None = None, api_base: str = AGENT_365_PROD_API_BASE, + prometheus: FakePrometheus | None = None, ) -> Agent365Guardrail: return Agent365Guardrail( guardrail_name="agent-365-guard", @@ -133,6 +144,20 @@ def _make_guardrail( agent_id=agent_id, unreachable_fallback=unreachable_fallback, async_handler=handler, + prometheus_logger_lookup=lambda: prometheus, + event_hook="pre_mcp_call", + default_on=True, + ) + + +def _default_fallback_guardrail(handler: FakeHandler, prometheus: FakePrometheus | None = None) -> Agent365Guardrail: + return Agent365Guardrail( + guardrail_name="agent-365-guard", + tenant_id="tenant-abc", + client_id="client-xyz", + client_secret="secret-123", + async_handler=handler, + prometheus_logger_lookup=lambda: prometheus, event_hook="pre_mcp_call", default_on=True, ) @@ -218,8 +243,32 @@ class TestInitializeGuardrail: assert guardrail.client_secret == "env-secret" assert guardrail.api_base == "https://env.example.test" assert guardrail.resource_app_id == AGENT_365_PROD_RESOURCE_APP_ID + assert guardrail.unreachable_fallback == "fail_open" + + def test_unset_fallback_survives_a_model_dump_round_trip(self): + stored: Final = LitellmParams( + guardrail="agent_365", mode="pre_mcp_call", tenant_id="t", client_id="c", client_secret="s" + ).model_dump() + assert stored["unreachable_fallback"] is None + guardrail: Final = initialize_guardrail(LitellmParams(**stored), {"guardrail_name": "a365-db"}) + assert guardrail.unreachable_fallback == "fail_open" + + def test_explicit_fail_closed_is_kept(self): + params: Final = LitellmParams( + guardrail="agent_365", + mode="pre_mcp_call", + tenant_id="t", + client_id="c", + client_secret="s", + unreachable_fallback="fail_closed", + ) + guardrail: Final = initialize_guardrail(params, {"guardrail_name": "a365-closed"}) assert guardrail.unreachable_fallback == "fail_closed" + def test_agent_365_default_leaves_other_guardrails_unset(self): + assert LitellmParams(guardrail="generic_guardrail_api", mode="pre_call").unreachable_fallback is None + assert Agent365GuardrailConfigModel.model_fields["unreachable_fallback"].default == "fail_open" + def test_explicit_params_win(self, monkeypatch): monkeypatch.setenv("AGENT365_TENANT_ID", "env-tenant") params: Final = LitellmParams( @@ -471,6 +520,70 @@ class TestDefenderNotEvaluated: assert "rejected" in exc_info.value.detail["error"] +class TestFailOpenDefault: + @pytest.mark.asyncio + async def test_constructor_default_lets_timed_out_evaluation_through_unscanned(self): + handler: Final = FakeHandler([_token_response(), httpx.ReadTimeout("timed out")]) + guardrail: Final = _default_fallback_guardrail(handler) + assert guardrail.unreachable_fallback == "fail_open" + data: Final = _mcp_data() + assert await _run(guardrail, data) is data + info: Final = _guardrail_info(data) + assert info["guardrail_status"] == "guardrail_failed_to_respond" + assert info["guardrail_response"]["verdict"] == "Unscanned" + + @pytest.mark.asyncio + async def test_constructor_default_still_blocks_a_policy_block(self): + handler: Final = FakeHandler([_token_response(), _block_response()]) + guardrail: Final = _default_fallback_guardrail(handler) + with pytest.raises(HTTPException) as exc_info: + await _run(guardrail, _mcp_data()) + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "responses", + [ + [_token_response(), httpx.ReadTimeout("timed out")], + [_token_response(), _response(502, text="bad gateway")], + [_token_response(), _not_evaluated_response("Skipped")], + [_response(503, text="entra down")], + ], + ) + async def test_each_fail_open_counts_once_in_prometheus(self, responses): + prometheus: Final = FakePrometheus() + guardrail: Final = _default_fallback_guardrail(FakeHandler(responses), prometheus) + data: Final = _mcp_data() + assert await _run(guardrail, data) is data + assert prometheus.fail_opens == [("agent-365-guard", "pre_call")] + + @pytest.mark.asyncio + async def test_allowed_and_blocked_calls_do_not_count_as_fail_open(self): + prometheus: Final = FakePrometheus() + handler: Final = FakeHandler([_token_response(), _allow_response(), _block_response()]) + guardrail: Final = _default_fallback_guardrail(handler, prometheus) + await _run(guardrail, _mcp_data()) + with pytest.raises(HTTPException): + await _run(guardrail, _mcp_data()) + assert prometheus.fail_opens == [] + + @pytest.mark.asyncio + async def test_fail_closed_does_not_count_as_fail_open(self): + prometheus: Final = FakePrometheus() + handler: Final = FakeHandler([_token_response(), httpx.ReadTimeout("timed out")]) + guardrail: Final = _make_guardrail(handler, unreachable_fallback="fail_closed", prometheus=prometheus) + with pytest.raises(HTTPException): + await _run(guardrail, _mcp_data()) + assert prometheus.fail_opens == [] + + def test_registered_prometheus_logger_reads_litellm_callbacks(self, monkeypatch): + monkeypatch.setattr(litellm, "callbacks", []) + assert registered_prometheus_logger() is None + logger: Final = PrometheusLogger.__new__(PrometheusLogger) + monkeypatch.setattr(litellm, "callbacks", ["langfuse", logger]) + assert registered_prometheus_logger() is logger + + class TestUnreachableFallback: @pytest.mark.asyncio async def test_evaluate_litellm_timeout_fail_closed(self): diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx index d45cfc3fe7d..614fdaefade 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx @@ -210,6 +210,14 @@ const TEAM_COLORS: Record = { Finance: "bg-success/15 text-success", }; +const FAIL_OPEN_BY_DEFAULT_GUARDRAILS: ReadonlySet = new Set(["agent_365", "typesafe"]); + +function defaultUnreachableFallback(guardrailType: string | undefined): "fail_open" | "fail_closed" { + return guardrailType !== undefined && FAIL_OPEN_BY_DEFAULT_GUARDRAILS.has(guardrailType) + ? "fail_open" + : "fail_closed"; +} + function buildEquivalentConfigYaml(g: TeamGuardrail): string { const lines: string[] = [ "litellm_settings:", @@ -220,7 +228,7 @@ function buildEquivalentConfigYaml(g: TeamGuardrail): string { ` mode: ${g.mode ?? "pre_call"} # or post_call, during_call`, ` api_base: ${g.endpoint || "https://your-guardrail-api.com"}`, " api_key: os.environ/YOUR_GUARDRAIL_API_KEY # optional", - ` unreachable_fallback: ${g.unreachable_fallback ?? "fail_closed"} # default: fail_closed. Set to fail_open to proceed if the guardrail endpoint is unreachable.`, + ` unreachable_fallback: ${g.unreachable_fallback ?? defaultUnreachableFallback(g.guardrailType)} # fail_closed blocks, fail_open proceeds when the guardrail endpoint is unreachable. Shown value is this guardrail's default.`, ` forward_api_key: ${g.forwardKey}`, ]; if (g.model && g.model !== "—") { diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bba67bcf6c2..cff334981ca 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25570,11 +25570,9 @@ export interface components { timeout?: number | null; /** * Unreachable Fallback - * @description Behavior when a guardrail endpoint is unreachable due to network errors. Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed. - * @default fail_closed - * @enum {string} + * @description Behavior when a guardrail endpoint is unreachable due to network errors. Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. 'fail_closed' raises an error. 'fail_open' logs a critical error and allows the request to proceed. Unset applies the guardrail's own default: 'agent_365' and 'typesafe' fail open, the others fail closed. */ - unreachable_fallback: "fail_closed" | "fail_open"; + unreachable_fallback?: ("fail_closed" | "fail_open") | null; /** * Violation Message Template * @description Custom message when a guardrail blocks an action. Supports placeholders like {tool_name}, {rule_id}, and {default_message}. @@ -34786,11 +34784,9 @@ export interface components { tracker_api_key?: string | null; /** * Unreachable Fallback - * @description Behavior when the headroom compression service is unreachable or errors. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and forwards the request uncompressed instead of blocking it. - * @default fail_closed - * @enum {string} + * @description Behavior when the guardrail endpoint is unreachable. 'fail_closed' blocks, 'fail_open' allows and logs. Unset applies the guardrail's own default: 'agent_365' and 'typesafe' fail open, the others fail closed. */ - unreachable_fallback: "fail_closed" | "fail_open"; + unreachable_fallback?: ("fail_closed" | "fail_open") | null; /** * Use V2 * @description If True and guardrail='noma', route to the new Noma v2 implementation instead of the legacy implementation.