diff --git a/litellm/exceptions.py b/litellm/exceptions.py index abdba09dd8d..88fd6721858 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -9,7 +9,7 @@ ## LiteLLM versions of the OpenAI Exception Types -from typing import Optional +from typing import Any, Dict, Optional import httpx import openai @@ -1016,6 +1016,35 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore return self.__str__() +class ModifyResponseException(Exception): + """ + Exception raised when a guardrail wants to modify the response. + + This exception carries the synthetic response that should be returned + to the user instead of calling the LLM or instead of the LLM's response. + It should be caught by the proxy and returned with a 200 status code. + + This is a base exception that all guardrails can use to replace responses, + allowing violation messages to be returned as successful responses + rather than errors. + """ + + def __init__( + self, + message: str, + model: str, + request_data: Dict[str, Any], + guardrail_name: Optional[str] = None, + detection_info: Optional[Dict[str, Any]] = None, + ): + self.message = message + self.model = model + self.request_data = request_data + self.guardrail_name = guardrail_name + self.detection_info = detection_info or {} + super().__init__(message) + + class GuardrailInterventionNormalStringError( Exception ): # custom exception to raise when a guardrail intervenes, but we want to return a normal string to the user diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 6046f1bb581..42c1028688a 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -41,43 +41,7 @@ if TYPE_CHECKING: dc = DualCache() -class ModifyResponseException(Exception): - """ - Exception raised when a guardrail wants to modify the response. - - This exception carries the synthetic response that should be returned - to the user instead of calling the LLM or instead of the LLM's response. - It should be caught by the proxy and returned with a 200 status code. - - This is a base exception that all guardrails can use to replace responses, - allowing violation messages to be returned as successful responses - rather than errors. - """ - - def __init__( - self, - message: str, - model: str, - request_data: Dict[str, Any], - guardrail_name: Optional[str] = None, - detection_info: Optional[Dict[str, Any]] = None, - ): - """ - Initialize the modify response exception. - - Args: - message: The violation message to return to the user - model: The model that was being called - request_data: The original request data - guardrail_name: Name of the guardrail that raised this exception - detection_info: Additional detection metadata (scores, rules, etc.) - """ - self.message = message - self.model = model - self.request_data = request_data - self.guardrail_name = guardrail_name - self.detection_info = detection_info or {} - super().__init__(message) +from litellm.exceptions import ModifyResponseException as ModifyResponseException class CustomGuardrail(CustomLogger): @@ -417,7 +381,9 @@ class CustomGuardrail(CustomLogger): """ requested_guardrails = self.get_guardrail_from_metadata(data) disable_global_guardrail = self.get_disable_global_guardrail(data) - opted_out_global_guardrails = self.get_opted_out_global_guardrails_from_metadata(data) + opted_out_global_guardrails = ( + self.get_opted_out_global_guardrails_from_metadata(data) + ) verbose_logger.debug( "inside should_run_guardrail for guardrail=%s event_type= %s guardrail_supported_event_hooks= %s requested_guardrails= %s self.default_on= %s", self.guardrail_name, @@ -426,7 +392,10 @@ class CustomGuardrail(CustomLogger): requested_guardrails, self.default_on, ) - if self.default_on is True and self.guardrail_name in opted_out_global_guardrails: + if ( + self.default_on is True + and self.guardrail_name in opted_out_global_guardrails + ): return False if self.default_on is True and disable_global_guardrail is not True: diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index a1623121da5..f29eab47cf9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -227,6 +227,16 @@ class UnifiedLLMGuardrails(CustomLogger): if call_type is None: call_type = _infer_call_type(call_type=None, completion_response=response) # type: ignore + # Fallback: resolve call_type from logging_obj for pass-through endpoints + if call_type is None: + litellm_logging_obj = data.get("litellm_logging_obj") + if ( + litellm_logging_obj is not None + and getattr(litellm_logging_obj, "call_type", None) + == CallTypes.pass_through.value + ): + call_type = CallTypes.pass_through.value + if call_type is None: return response diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index d582240395f..8104c587f65 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -635,6 +635,7 @@ async def pass_through_request( # noqa: PLR0915 custom_llm_provider: Optional field - custom LLM provider for the endpoint guardrails_config: Optional field - guardrails configuration for passthrough endpoint """ + from litellm.exceptions import ModifyResponseException from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.pass_through_endpoints.passthrough_guardrails import ( PassthroughGuardrailHandler, @@ -915,8 +916,41 @@ async def pass_through_request( # noqa: PLR0915 content = await response.aread() - ## LOG SUCCESS + ## POST-CALL GUARDRAILS ## + _content_modified = False response_body: Optional[dict] = get_response_body(response) + if response_body is not None and guardrails_to_run: + # Build an enriched data dict: _parsed_body has been stripped of + # `metadata` by both pre_call_hook and _init_kwargs_for_pass_through_endpoint, + # so we re-attach the configured guardrails here so should_run_guardrail + # sees them. + hook_data = dict(_parsed_body or {}) + existing_metadata = hook_data.get("metadata") + if not isinstance(existing_metadata, dict): + existing_metadata = {} + hook_data["metadata"] = { + **existing_metadata, + "guardrails": guardrails_to_run, + } + response_body = await proxy_logging_obj.post_call_success_hook( + data=hook_data, + user_api_key_dict=user_api_key_dict, + response=response_body, # type: ignore[arg-type] + ) + if isinstance(response_body, dict): + content = json.dumps(response_body).encode("utf-8") + _content_modified = True + else: + verbose_proxy_logger.debug( + "pass_through_endpoint: post_call_success_hook returned %s, expected dict — using original response", + type(response_body).__name__, + ) + elif response_body is None: + verbose_proxy_logger.debug( + "pass_through_endpoint: response body not JSON-parseable, skipping post-call guardrails" + ) + + ## LOG SUCCESS passthrough_logging_payload["response_body"] = response_body end_time = datetime.now() asyncio.create_task( @@ -944,13 +978,47 @@ async def pass_through_request( # noqa: PLR0915 api_base=str(url._uri_reference), ) + response_headers = HttpPassThroughEndpointHelpers.get_response_headers( + headers=response.headers, + custom_headers=custom_headers, + ) + if _content_modified: + response_headers.pop("content-length", None) + return Response( content=content, status_code=response.status_code, - headers=HttpPassThroughEndpointHelpers.get_response_headers( - headers=response.headers, - custom_headers=custom_headers, - ), + headers=response_headers, + ) + except ModifyResponseException as e: + verbose_proxy_logger.info( + "pass_through_endpoint: Guardrail %s modified response: %s", + e.guardrail_name, + str(e.message or "")[:200], + ) + try: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=e.request_data, + ) + except Exception: + verbose_proxy_logger.warning( + "pass_through_endpoint: post_call_failure_hook raised during guardrail block", + exc_info=True, + ) + error_body = { + "error": { + "message": e.message or "Response blocked by guardrail", + "type": "content_filter", + "guardrail_name": e.guardrail_name, + "model": e.model, + } + } + return Response( + content=json.dumps(error_body), + status_code=200, + media_type="application/json", ) except Exception as e: custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers(