diff --git a/litellm/proxy/guardrails/guardrail_hooks/grayswan/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/grayswan/__init__.py index 99f58f654a7..3aca9078c0a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/grayswan/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/grayswan/__init__.py @@ -46,6 +46,10 @@ def initialize_guardrail( streaming_sampling_rate=_get_config_value( litellm_params, optional_params, "streaming_sampling_rate" ) or 5, + fail_open=_get_config_value(litellm_params, optional_params, "fail_open"), + guardrail_timeout=_get_config_value( + litellm_params, optional_params, "guardrail_timeout" + ), event_hook=litellm_params.mode, default_on=litellm_params.default_on, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py index 59e737f7d21..2a852cbda08 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py +++ b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py @@ -1,6 +1,7 @@ """Gray Swan Cygnal guardrail integration.""" import os +import time from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional from fastapi import HTTPException @@ -28,6 +29,10 @@ class GraySwanGuardrailMissingSecrets(Exception): class GraySwanGuardrailAPIError(Exception): """Raised when the Gray Swan API returns an error.""" + def __init__(self, message: str, status_code: Optional[int] = None) -> None: + super().__init__(message) + self.status_code = status_code + class GraySwanGuardrail(CustomGuardrail): """ @@ -63,6 +68,8 @@ class GraySwanGuardrail(CustomGuardrail): policy_id: Optional[str] = None, streaming_end_of_stream_only: bool = False, streaming_sampling_rate: int = 5, + fail_open: Optional[bool] = True, + guardrail_timeout: Optional[float] = 30.0, **kwargs: Any, ) -> None: self.async_handler = get_async_httpx_client( @@ -96,6 +103,8 @@ class GraySwanGuardrail(CustomGuardrail): self.reasoning_mode = self._resolve_reasoning_mode(reasoning_mode) self.categories = categories self.policy_id = policy_id + self.fail_open = True if fail_open is None else bool(fail_open) + self.guardrail_timeout = 30.0 if guardrail_timeout is None else float(guardrail_timeout) # Streaming configuration self.streaming_end_of_stream_only = streaming_end_of_stream_only @@ -202,18 +211,36 @@ class GraySwanGuardrail(CustomGuardrail): if payload is None: return inputs - # Call GraySwan API - response_json = await self._call_grayswan_api(payload) - # Process response - is_output = input_type == "response" - result = self._process_response_internal( - response_json=response_json, - request_data=request_data, - inputs=inputs, - is_output=is_output, - ) - - return result + start_time = time.time() + try: + response_json = await self._call_grayswan_api(payload) + is_output = input_type == "response" + result = self._process_response_internal( + response_json=response_json, + request_data=request_data, + inputs=inputs, + is_output=is_output, + ) + return result + except Exception as exc: + end_time = time.time() + status_code = getattr(exc, "status_code", None) or getattr( + exc, "exception_status_code", None + ) + self._log_guardrail_failure( + exc=exc, + request_data=request_data or {}, + start_time=start_time, + end_time=end_time, + status_code=status_code, + ) + if self.fail_open: + verbose_proxy_logger.warning( + "Gray Swan Guardrail: fail_open=True. Allowing request to proceed despite error: %s", + exc, + ) + return inputs + raise GraySwanGuardrailAPIError(str(exc), status_code=status_code) from exc # ------------------------------------------------------------------ # Legacy Test Interface (for backward compatibility) @@ -348,7 +375,7 @@ class GraySwanGuardrail(CustomGuardrail): url=self.monitor_url, headers=headers, json=payload, - timeout=30.0, + timeout=self.guardrail_timeout, ) response.raise_for_status() result = response.json() @@ -356,13 +383,11 @@ class GraySwanGuardrail(CustomGuardrail): "Gray Swan Guardrail: monitor response %s", safe_dumps(result) ) return result - except HTTPException: - raise except Exception as exc: - verbose_proxy_logger.exception( - "Gray Swan Guardrail: API request failed: %s", exc + status_code = getattr(exc, "status_code", None) or getattr( + exc, "exception_status_code", None ) - raise GraySwanGuardrailAPIError(str(exc)) from exc + raise GraySwanGuardrailAPIError(str(exc), status_code=status_code) from exc def _process_response_internal( self, @@ -579,3 +604,33 @@ class GraySwanGuardrail(CustomGuardrail): if env_val and env_val.lower() in self.SUPPORTED_REASONING_MODES: return env_val.lower() return None + + def _log_guardrail_failure( + self, + exc: Exception, + request_data: dict, + start_time: float, + end_time: float, + status_code: Optional[int] = None, + ) -> None: + """Log guardrail failure and attach standard logging metadata.""" + try: + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=str(exc), + request_data=request_data, + guardrail_status="guardrail_failed_to_respond", + start_time=start_time, + end_time=end_time, + duration=end_time - start_time, + guardrail_provider="grayswan", + ) + except Exception: + verbose_proxy_logger.exception( + "Gray Swan Guardrail: failed to log guardrail failure for error: %s", + exc, + ) + verbose_proxy_logger.error( + "Gray Swan Guardrail: API request failed%s: %s", + f" (status_code={status_code})" if status_code else "", + exc, + ) diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/grayswan.py b/litellm/types/proxy/guardrails/guardrail_hooks/grayswan.py index 79239e52621..24d8cd85a0b 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/grayswan.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/grayswan.py @@ -32,6 +32,14 @@ class GraySwanGuardrailConfigModelOptionalParams(BaseModel): default=None, description="Default Gray Swan category definitions to send with each request.", ) + fail_open: Optional[bool] = Field( + default=True, + description="If true (default), errors contacting Gray Swan are logged and the request proceeds. If false, errors propagate and block the request.", + ) + guardrail_timeout: Optional[float] = Field( + default=30.0, + description="Timeout in seconds for calling the Gray Swan guardrail service.", + ) class GraySwanGuardrailConfigModel( diff --git a/scripts/mock_grayswan_timeout_server.py b/scripts/mock_grayswan_timeout_server.py new file mode 100644 index 00000000000..bc40e13dfa5 --- /dev/null +++ b/scripts/mock_grayswan_timeout_server.py @@ -0,0 +1,85 @@ +""" +Minimal mock GraySwan monitor server that intentionally responds slowly. + +Usage: + python scripts/mock_grayswan_timeout_server.py --port 8787 --delay 35 + +Point GRAYSWAN_API_BASE at http://127.0.0.1:8787 so the guardrail hits this +endpoint and times out (the guardrail client has a 30s timeout). +""" + +from __future__ import annotations + +import argparse +import json +import logging +import time +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import Optional + +LOG = logging.getLogger("mock_grayswan") +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + + +class SlowHandler(BaseHTTPRequestHandler): + delay_seconds: float = 35.0 + + def log_message(self, fmt: str, *args) -> None: # noqa: D401 + """Route handler logs through the logging module.""" + LOG.info("%s - %s", self.address_string(), fmt % args) + + def _read_body(self) -> Optional[bytes]: + content_length = self.headers.get("content-length") + if content_length is None: + return None + try: + length = int(content_length) + except ValueError: + return None + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 + if self.path != "/cygnal/monitor": + self.send_error(404, "Not Found") + return + + body = self._read_body() + LOG.info("Received POST %s body=%s", self.path, body) + + LOG.info("Sleeping for %.1fs to trigger client timeout", self.delay_seconds) + time.sleep(self.delay_seconds) + + response = {"status": "ok", "delayed": self.delay_seconds} + response_bytes = json.dumps(response).encode("utf-8") + + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(response_bytes))) + self.end_headers() + self.wfile.write(response_bytes) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Mock GraySwan monitor server") + parser.add_argument("--port", type=int, default=8787, help="Port to listen on") + parser.add_argument( + "--delay", + type=float, + default=35.0, + help="Seconds to delay before responding (must exceed guardrail timeout)", + ) + args = parser.parse_args() + + SlowHandler.delay_seconds = args.delay + server = HTTPServer(("0.0.0.0", args.port), SlowHandler) + LOG.info("Starting mock server on port %d with delay %.1fs", args.port, args.delay) + try: + server.serve_forever() + except KeyboardInterrupt: + LOG.info("Shutting down mock server") + finally: + server.server_close() + + +if __name__ == "__main__": + main()