From 1276c10338b8246bfd0576945d54bb8ae6d352e9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 10:12:23 -0700 Subject: [PATCH 1/9] migrate presidio to new guardrails --- .../guardrails/guardrail_hooks/presidio.py | 363 ++++++++++++++++++ litellm/proxy/guardrails/init_guardrails.py | 10 + litellm/proxy/proxy_config.yaml | 6 + 3 files changed, 379 insertions(+) create mode 100644 litellm/proxy/guardrails/guardrail_hooks/presidio.py diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py new file mode 100644 index 00000000000..165257ffaef --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -0,0 +1,363 @@ +# +-----------------------------------------------+ +# | | +# | PII Masking | +# | with Microsoft Presidio | +# | https://github.com/BerriAI/litellm/issues/ | +# +-----------------------------------------------+ +# +# Tell us how we can improve! - Krrish & Ishaan + + +import asyncio +import json +import traceback +import uuid +from typing import Any, List, Optional, Tuple, Union + +import aiohttp +from fastapi import HTTPException + +import litellm # noqa: E401 +from litellm._logging import verbose_proxy_logger +from litellm.caching import DualCache +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.custom_httpx.http_handler import _get_async_httpx_client +from litellm.proxy._types import UserAPIKeyAuth +from litellm.utils import ( + EmbeddingResponse, + ImageResponse, + ModelResponse, + StreamingChoices, + get_formatted_prompt, +) + + +class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): + user_api_key_cache = None + ad_hoc_recognizers = None + + # Class variables or attributes + def __init__( + self, + logging_only: Optional[bool] = None, + mock_testing: bool = False, + mock_redacted_text: Optional[dict] = None, + presidio_analyzer_api_base: Optional[str] = None, + presidio_anonymizer_api_base: Optional[str] = None, + **kwargs, + ): + self.pii_tokens: dict = ( + {} + ) # mapping of PII token to original text - only used with Presidio `replace` operation + + self.mock_redacted_text = mock_redacted_text + self.logging_only = logging_only + if mock_testing is True: # for testing purposes only + return + + ad_hoc_recognizers = litellm.presidio_ad_hoc_recognizers + if ad_hoc_recognizers is not None: + try: + with open(ad_hoc_recognizers, "r") as file: + self.ad_hoc_recognizers = json.load(file) + except FileNotFoundError: + raise Exception(f"File not found. file_path={ad_hoc_recognizers}") + except json.JSONDecodeError as e: + raise Exception( + f"Error decoding JSON file: {str(e)}, file_path={ad_hoc_recognizers}" + ) + except Exception as e: + raise Exception( + f"An error occurred: {str(e)}, file_path={ad_hoc_recognizers}" + ) + self.async_http_client = _get_async_httpx_client() + self.validate_environment( + presidio_analyzer_api_base=presidio_analyzer_api_base, + presidio_anonymizer_api_base=presidio_anonymizer_api_base, + ) + + def validate_environment( + self, + presidio_analyzer_api_base: Optional[str] = None, + presidio_anonymizer_api_base: Optional[str] = None, + ): + self.presidio_analyzer_api_base: Optional[str] = ( + presidio_analyzer_api_base + or litellm.get_secret("PRESIDIO_ANALYZER_API_BASE", None) + ) + self.presidio_anonymizer_api_base: Optional[ + str + ] = presidio_anonymizer_api_base or litellm.get_secret( + "PRESIDIO_ANONYMIZER_API_BASE", None + ) # type: ignore + + if self.presidio_analyzer_api_base is None: + raise Exception("Missing `PRESIDIO_ANALYZER_API_BASE` from environment") + if not self.presidio_analyzer_api_base.endswith("/"): + self.presidio_analyzer_api_base += "/" + if not ( + self.presidio_analyzer_api_base.startswith("http://") + or self.presidio_analyzer_api_base.startswith("https://") + ): + # add http:// if unset, assume communicating over private network - e.g. render + self.presidio_analyzer_api_base = ( + "http://" + self.presidio_analyzer_api_base + ) + + if self.presidio_anonymizer_api_base is None: + raise Exception("Missing `PRESIDIO_ANONYMIZER_API_BASE` from environment") + if not self.presidio_anonymizer_api_base.endswith("/"): + self.presidio_anonymizer_api_base += "/" + if not ( + self.presidio_anonymizer_api_base.startswith("http://") + or self.presidio_anonymizer_api_base.startswith("https://") + ): + # add http:// if unset, assume communicating over private network - e.g. render + self.presidio_anonymizer_api_base = ( + "http://" + self.presidio_anonymizer_api_base + ) + + def print_verbose(self, print_statement): + try: + verbose_proxy_logger.debug(print_statement) + if litellm.set_verbose: + print(print_statement) # noqa + except: + pass + + async def check_pii(self, text: str, output_parse_pii: bool) -> str: # type: ignore + """ + [TODO] make this more performant for high-throughput scenario + """ + try: + if self.mock_redacted_text is not None: + redacted_text = self.mock_redacted_text + else: + # Make the first request to /analyze + analyze_url = f"{self.presidio_analyzer_api_base}analyze" + verbose_proxy_logger.debug("Making request to: %s", analyze_url) + analyze_payload = {"text": text, "language": "en"} + if self.ad_hoc_recognizers is not None: + analyze_payload["ad_hoc_recognizers"] = self.ad_hoc_recognizers + redacted_text = None + + reponse = await self.async_http_client.post( + analyze_url, json=analyze_payload + ) + analyze_results = await reponse.json() + + # Make the second request to /anonymize + anonymize_url = f"{self.presidio_anonymizer_api_base}anonymize" + verbose_proxy_logger.debug("Making request to: %s", anonymize_url) + anonymize_payload = { + "text": text, + "analyzer_results": analyze_results, + } + + response_2 = await self.async_http_client.post( + anonymize_url, json=anonymize_payload + ) + redacted_text = await response_2.json() + + new_text = text + if redacted_text is not None: + verbose_proxy_logger.debug("redacted_text: %s", redacted_text) + for item in redacted_text["items"]: + start = item["start"] + end = item["end"] + replacement = item["text"] # replacement token + if item["operator"] == "replace" and output_parse_pii == True: + # check if token in dict + # if exists, add a uuid to the replacement token for swapping back to the original text in llm response output parsing + if replacement in self.pii_tokens: + replacement = replacement + str(uuid.uuid4()) + + self.pii_tokens[replacement] = new_text[ + start:end + ] # get text it'll replace + + new_text = new_text[:start] + replacement + new_text[end:] + return redacted_text["text"] + else: + raise Exception(f"Invalid anonymizer response: {redacted_text}") + except Exception as e: + verbose_proxy_logger.error( + "litellm.proxy.hooks.presidio_pii_masking.py::async_pre_call_hook(): Exception occured - {}".format( + str(e) + ) + ) + verbose_proxy_logger.debug(traceback.format_exc()) + raise e + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict, + call_type: str, + ): + """ + - Check if request turned off pii + - Check if user allowed to turn off pii (key permissions -> 'allow_pii_controls') + + - Take the request data + - Call /analyze -> get the results + - Call /anonymize w/ the analyze results -> get the redacted text + + For multiple messages in /chat/completions, we'll need to call them in parallel. + """ + try: + if ( + self.logging_only is True + ): # only modify the logging obj data (done by async_logging_hook) + return data + permissions = user_api_key_dict.permissions + output_parse_pii = permissions.get( + "output_parse_pii", litellm.output_parse_pii + ) # allow key to turn on/off output parsing for pii + no_pii = permissions.get( + "no-pii", None + ) # allow key to turn on/off pii masking (if user is allowed to set pii controls, then they can override the key defaults) + + if no_pii is None: + # check older way of turning on/off pii + no_pii = not permissions.get("pii", True) + + content_safety = data.get("content_safety", None) + verbose_proxy_logger.debug("content_safety: %s", content_safety) + ## Request-level turn on/off PII controls ## + if content_safety is not None and isinstance(content_safety, dict): + # pii masking ## + if ( + content_safety.get("no-pii", None) is not None + and content_safety.get("no-pii") == True + ): + # check if user allowed to turn this off + if permissions.get("allow_pii_controls", False) == False: + raise HTTPException( + status_code=400, + detail={ + "error": "Not allowed to set PII controls per request" + }, + ) + else: # user allowed to turn off pii masking + no_pii = content_safety.get("no-pii") + if not isinstance(no_pii, bool): + raise HTTPException( + status_code=400, + detail={"error": "no_pii needs to be a boolean value"}, + ) + ## pii output parsing ## + if content_safety.get("output_parse_pii", None) is not None: + # check if user allowed to turn this off + if permissions.get("allow_pii_controls", False) == False: + raise HTTPException( + status_code=400, + detail={ + "error": "Not allowed to set PII controls per request" + }, + ) + else: # user allowed to turn on/off pii output parsing + output_parse_pii = content_safety.get("output_parse_pii") + if not isinstance(output_parse_pii, bool): + raise HTTPException( + status_code=400, + detail={ + "error": "output_parse_pii needs to be a boolean value" + }, + ) + + if no_pii is True: # turn off pii masking + return data + + if call_type == "completion": # /chat/completions requests + messages = data["messages"] + tasks = [] + + for m in messages: + if isinstance(m["content"], str): + tasks.append( + self.check_pii( + text=m["content"], output_parse_pii=output_parse_pii + ) + ) + responses = await asyncio.gather(*tasks) + for index, r in enumerate(responses): + if isinstance(messages[index]["content"], str): + messages[index][ + "content" + ] = r # replace content with redacted string + verbose_proxy_logger.info( + f"Presidio PII Masking: Redacted pii message: {data['messages']}" + ) + return data + except Exception as e: + verbose_proxy_logger.info( + f"An error occurred -", + ) + raise e + + async def async_logging_hook( + self, kwargs: dict, result: Any, call_type: str + ) -> Tuple[dict, Any]: + """ + Masks the input before logging to langfuse, datadog, etc. + """ + if ( + call_type == "completion" or call_type == "acompletion" + ): # /chat/completions requests + messages: Optional[List] = kwargs.get("messages", None) + tasks = [] + + if messages is None: + return kwargs, result + + for m in messages: + text_str = "" + if m["content"] is None: + continue + if isinstance(m["content"], str): + text_str = m["content"] + tasks.append( + self.check_pii(text=text_str, output_parse_pii=False) + ) # need to pass separately b/c presidio has context window limits + responses = await asyncio.gather(*tasks) + for index, r in enumerate(responses): + if isinstance(messages[index]["content"], str): + messages[index][ + "content" + ] = r # replace content with redacted string + verbose_proxy_logger.info( + f"Presidio PII Masking: Redacted pii message: {messages}" + ) + kwargs["messages"] = messages + + return kwargs, responses + + async def async_post_call_success_hook( + self, + data: dict, + user_api_key_dict: UserAPIKeyAuth, + response: Union[ModelResponse, EmbeddingResponse, ImageResponse], + ): + """ + Output parse the response object to replace the masked tokens with user sent values + """ + verbose_proxy_logger.debug( + f"PII Masking Args: litellm.output_parse_pii={litellm.output_parse_pii}; type of response={type(response)}" + ) + if litellm.output_parse_pii == False: + return response + + if isinstance(response, ModelResponse) and not isinstance( + response.choices[0], StreamingChoices + ): # /chat/completions requests + if isinstance(response.choices[0].message.content, str): + verbose_proxy_logger.debug( + f"self.pii_tokens: {self.pii_tokens}; initial response: {response.choices[0].message.content}" + ) + for key, value in self.pii_tokens.items(): + response.choices[0].message.content = response.choices[ + 0 + ].message.content.replace(key, value) + return response diff --git a/litellm/proxy/guardrails/init_guardrails.py b/litellm/proxy/guardrails/init_guardrails.py index 643e135961d..9a43171f478 100644 --- a/litellm/proxy/guardrails/init_guardrails.py +++ b/litellm/proxy/guardrails/init_guardrails.py @@ -165,6 +165,16 @@ def init_guardrails_v2( category_thresholds=litellm_params.get("category_thresholds"), ) litellm.callbacks.append(_lakera_callback) # type: ignore + elif litellm_params["guardrail"] == "presidio": + from litellm.proxy.guardrails.guardrail_hooks.presidio import ( + _OPTIONAL_PresidioPIIMasking, + ) + + _presidio_callback = _OPTIONAL_PresidioPIIMasking( + guardrail_name=guardrail["guardrail_name"], + event_hook=litellm_params["mode"], + ) + litellm.callbacks.append(_presidio_callback) # type: ignore elif ( isinstance(litellm_params["guardrail"], str) and "." in litellm_params["guardrail"] diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 7566f348afc..913c2e94650 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -15,6 +15,12 @@ litellm_settings: success_callback: ["prometheus"] failure_callback: ["prometheus"] +guardrails: + - guardrail_name: "presidio" + litellm_params: + guardrail: presidio # supported values: "aporia", "lakera", "presidio" + mode: "pre_call" # pre_call, during_call, post_call + general_settings: master_key: sk-1234 From d954413b145df2c2386c31a56e5be5b597b8b03e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 12:14:30 -0700 Subject: [PATCH 2/9] fix presidio calling logic --- .../guardrails/guardrail_hooks/presidio.py | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 165257ffaef..cb1d3df1bf7 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -76,6 +76,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): presidio_anonymizer_api_base=presidio_anonymizer_api_base, ) + super().__init__(**kwargs) + def validate_environment( self, presidio_analyzer_api_base: Optional[str] = None, @@ -125,39 +127,41 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): except: pass - async def check_pii(self, text: str, output_parse_pii: bool) -> str: # type: ignore + async def check_pii(self, text: str, output_parse_pii: bool) -> str: """ [TODO] make this more performant for high-throughput scenario """ try: - if self.mock_redacted_text is not None: - redacted_text = self.mock_redacted_text - else: - # Make the first request to /analyze - analyze_url = f"{self.presidio_analyzer_api_base}analyze" - verbose_proxy_logger.debug("Making request to: %s", analyze_url) - analyze_payload = {"text": text, "language": "en"} - if self.ad_hoc_recognizers is not None: - analyze_payload["ad_hoc_recognizers"] = self.ad_hoc_recognizers - redacted_text = None + async with aiohttp.ClientSession() as session: + if self.mock_redacted_text is not None: + redacted_text = self.mock_redacted_text + else: + # Make the first request to /analyze + analyze_url = f"{self.presidio_analyzer_api_base}analyze" + verbose_proxy_logger.debug("Making request to: %s", analyze_url) + analyze_payload = {"text": text, "language": "en"} + if self.ad_hoc_recognizers is not None: + analyze_payload["ad_hoc_recognizers"] = self.ad_hoc_recognizers + redacted_text = None - reponse = await self.async_http_client.post( - analyze_url, json=analyze_payload - ) - analyze_results = await reponse.json() + async with session.post( + analyze_url, json=analyze_payload + ) as response: - # Make the second request to /anonymize - anonymize_url = f"{self.presidio_anonymizer_api_base}anonymize" - verbose_proxy_logger.debug("Making request to: %s", anonymize_url) - anonymize_payload = { - "text": text, - "analyzer_results": analyze_results, - } + analyze_results = await response.json() - response_2 = await self.async_http_client.post( - anonymize_url, json=anonymize_payload - ) - redacted_text = await response_2.json() + # Make the second request to /anonymize + anonymize_url = f"{self.presidio_anonymizer_api_base}anonymize" + verbose_proxy_logger.debug("Making request to: %s", anonymize_url) + anonymize_payload = { + "text": text, + "analyzer_results": analyze_results, + } + + async with session.post( + anonymize_url, json=anonymize_payload + ) as response: + redacted_text = await response.json() new_text = text if redacted_text is not None: @@ -206,6 +210,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): For multiple messages in /chat/completions, we'll need to call them in parallel. """ + try: if ( self.logging_only is True From 9b5164b38d5ebed1fff1febe2f78fd84cd7dc66e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 12:46:59 -0700 Subject: [PATCH 3/9] fix allow setting language per call to presidio --- .../guardrails/guardrail_hooks/presidio.py | 70 +++++++++++++++---- litellm/proxy/litellm_pre_call_utils.py | 4 ++ 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index cb1d3df1bf7..a44bdaa9f41 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -16,6 +16,7 @@ from typing import Any, List, Optional, Tuple, Union import aiohttp from fastapi import HTTPException +from pydantic import BaseModel import litellm # noqa: E401 from litellm._logging import verbose_proxy_logger @@ -32,6 +33,14 @@ from litellm.utils import ( ) +class PresidioPerRequestConfig(BaseModel): + """ + presdio params that can be controlled per request, api key + """ + + language: Optional[str] = None + + class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): user_api_key_cache = None ad_hoc_recognizers = None @@ -70,7 +79,6 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): raise Exception( f"An error occurred: {str(e)}, file_path={ad_hoc_recognizers}" ) - self.async_http_client = _get_async_httpx_client() self.validate_environment( presidio_analyzer_api_base=presidio_analyzer_api_base, presidio_anonymizer_api_base=presidio_anonymizer_api_base, @@ -119,15 +127,12 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): "http://" + self.presidio_anonymizer_api_base ) - def print_verbose(self, print_statement): - try: - verbose_proxy_logger.debug(print_statement) - if litellm.set_verbose: - print(print_statement) # noqa - except: - pass - - async def check_pii(self, text: str, output_parse_pii: bool) -> str: + async def check_pii( + self, + text: str, + output_parse_pii: bool, + presidio_config: Optional[PresidioPerRequestConfig], + ) -> str: """ [TODO] make this more performant for high-throughput scenario """ @@ -137,13 +142,21 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): redacted_text = self.mock_redacted_text else: # Make the first request to /analyze + # Construct Request 1 analyze_url = f"{self.presidio_analyzer_api_base}analyze" - verbose_proxy_logger.debug("Making request to: %s", analyze_url) analyze_payload = {"text": text, "language": "en"} + if presidio_config and presidio_config.language: + analyze_payload["language"] = presidio_config.language if self.ad_hoc_recognizers is not None: analyze_payload["ad_hoc_recognizers"] = self.ad_hoc_recognizers - redacted_text = None + # End of constructing Request 1 + redacted_text = None + verbose_proxy_logger.debug( + "Making request to: %s with payload: %s", + analyze_url, + analyze_payload, + ) async with session.post( analyze_url, json=analyze_payload ) as response: @@ -275,6 +288,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if no_pii is True: # turn off pii masking return data + presidio_config = self.get_presidio_settings_from_request_data(data) + if call_type == "completion": # /chat/completions requests messages = data["messages"] tasks = [] @@ -283,7 +298,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if isinstance(m["content"], str): tasks.append( self.check_pii( - text=m["content"], output_parse_pii=output_parse_pii + text=m["content"], + output_parse_pii=output_parse_pii, + presidio_config=presidio_config, ) ) responses = await asyncio.gather(*tasks) @@ -317,6 +334,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if messages is None: return kwargs, result + presidio_config = self.get_presidio_settings_from_request_data(kwargs) + for m in messages: text_str = "" if m["content"] is None: @@ -324,7 +343,11 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if isinstance(m["content"], str): text_str = m["content"] tasks.append( - self.check_pii(text=text_str, output_parse_pii=False) + self.check_pii( + text=text_str, + output_parse_pii=False, + presidio_config=presidio_config, + ) ) # need to pass separately b/c presidio has context window limits responses = await asyncio.gather(*tasks) for index, r in enumerate(responses): @@ -366,3 +389,22 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): 0 ].message.content.replace(key, value) return response + + def get_presidio_settings_from_request_data( + self, data: dict + ) -> Optional[PresidioPerRequestConfig]: + if "metadata" in data: + _metadata = data["metadata"] + _guardrail_config = _metadata.get("guardrail_config") + _presidio_config = PresidioPerRequestConfig(**_guardrail_config) + return _presidio_config + + return None + + def print_verbose(self, print_statement): + try: + verbose_proxy_logger.debug(print_statement) + if litellm.set_verbose: + print(print_statement) # noqa + except: + pass diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 60052bc273a..d41aae50f6e 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -420,6 +420,10 @@ def move_guardrails_to_metadata( data[_metadata_variable_name]["guardrails"] = data["guardrails"] del data["guardrails"] + if "guardrail_config" in data: + data[_metadata_variable_name]["guardrail_config"] = data["guardrail_config"] + del data["guardrail_config"] + def add_provider_specific_headers_to_request( data: dict, From 6c30f18f8cb6f9e6e2bfa16fd68b90c20bd4aeec Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 13:04:19 -0700 Subject: [PATCH 4/9] docs new presidio language controls --- .../docs/proxy/guardrails/pii_masking_v2.md | 129 ++++++++++++++++++ docs/my-website/docs/proxy/pii_masking.md | 10 +- docs/my-website/sidebars.js | 12 +- .../guardrails/guardrail_hooks/presidio.py | 5 +- litellm/proxy/proxy_config.yaml | 13 +- 5 files changed, 154 insertions(+), 15 deletions(-) create mode 100644 docs/my-website/docs/proxy/guardrails/pii_masking_v2.md diff --git a/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md new file mode 100644 index 00000000000..592c0e6bf2f --- /dev/null +++ b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md @@ -0,0 +1,129 @@ +import Image from '@theme/IdealImage'; + +# PII Masking - Presidio + +## Quick Start + +LiteLLM supports [Microsoft Presidio](https://github.com/microsoft/presidio/) for PII masking. + +### 1. Define Guardrails on your LiteLLM config.yaml + +Define your guardrails under the `guardrails` section +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: "presidio-pre-guard" + litellm_params: + guardrail: presidio # supported values: "aporia", "bedrock", "lakera", "presidio" + mode: "pre_call" +``` + +Set the following env vars + +```bash +export PRESIDIO_ANALYZER_API_BASE="http://localhost:5002" +export PRESIDIO_ANONYMIZER_API_BASE="http://localhost:5001" +``` + +#### Supported values for `mode` + +- `pre_call` Run **before** LLM call, on **input** +- `post_call` Run **after** LLM call, on **input & output** + + +### 2. Start LiteLLM Gateway + + +```shell +litellm --config config.yaml --detailed_debug +``` + +### 3. Test request + +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** + + + + +Expect this to mask `Jane Doe` since it's PII + +```shell +curl http://localhost:4000/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "Hello my name is Jane Doe"} + ], + "guardrails": ["presidio-pre-guard"], + }' +``` + +Expected response on failure + +```shell +{ + "id": "chatcmpl-A3qSC39K7imjGbZ8xCDacGJZBoTJQ", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello, ! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + } + } + ], + "created": 1725479980, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_5bd87c427a", + "usage": { + "completion_tokens": 13, + "prompt_tokens": 14, + "total_tokens": 27 + }, + "service_tier": null +} +``` + + + + + +```shell +curl http://localhost:4000/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "Hello good morning"} + ], + "guardrails": ["presidio-pre-guard"], + }' +``` + + + + + + + +## Set `language` per request + +## Output parsing + +## Ad Hoc Recognizers + + + + diff --git a/docs/my-website/docs/proxy/pii_masking.md b/docs/my-website/docs/proxy/pii_masking.md index 8106765f40e..83e4965a495 100644 --- a/docs/my-website/docs/proxy/pii_masking.md +++ b/docs/my-website/docs/proxy/pii_masking.md @@ -1,6 +1,14 @@ import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; -# PII Masking +# PII Masking - LiteLLM Gateway (Deprecated Version) + +:::warning + +This is deprecated, please use [our new Presidio pii masking integration](./guardrails/pii_masking_v2) + +::: LiteLLM supports [Microsoft Presidio](https://github.com/microsoft/presidio/) for PII masking. diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 0abb5144f74..f3780b84eeb 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -67,7 +67,15 @@ const sidebars = { { type: "category", label: "🛡️ [Beta] Guardrails", - items: ["proxy/guardrails/quick_start", "proxy/guardrails/aporia_api", "proxy/guardrails/lakera_ai", "proxy/guardrails/bedrock", "proxy/guardrails/custom_guardrail", "prompt_injection"], + items: [ + "proxy/guardrails/quick_start", + "proxy/guardrails/aporia_api", + "proxy/guardrails/lakera_ai", + "proxy/guardrails/bedrock", + "proxy/guardrails/pii_masking_v2", + "proxy/guardrails/custom_guardrail", + "prompt_injection" + ], }, { type: "category", @@ -101,7 +109,6 @@ const sidebars = { "proxy/model_management", "proxy/health", "proxy/debugging", - "proxy/pii_masking", "proxy/call_hooks", "proxy/rules", "proxy/cli", @@ -291,6 +298,7 @@ const sidebars = { "data_security", "migration_policy", "contributing", + "proxy/pii_masking", "rules", "proxy_server", { diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index a44bdaa9f41..eb894ecbbc4 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -396,8 +396,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if "metadata" in data: _metadata = data["metadata"] _guardrail_config = _metadata.get("guardrail_config") - _presidio_config = PresidioPerRequestConfig(**_guardrail_config) - return _presidio_config + if _guardrail_config: + _presidio_config = PresidioPerRequestConfig(**_guardrail_config) + return _presidio_config return None diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 913c2e94650..0aaadc13fe1 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,22 +1,15 @@ model_list: - - model_name: fake-openai-endpoint + - model_name: openai/* litellm_params: - model: openai/fake - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + model: gpt-3.5-turbo api_key: os.environ/OPENAI_API_KEY - - model_name: gpt-3.5-turbo-end-user-test - litellm_params: - model: azure/chatgpt-v-2 - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - api_version: "2023-05-15" - api_key: os.environ/AZURE_API_KEY litellm_settings: success_callback: ["prometheus"] failure_callback: ["prometheus"] guardrails: - - guardrail_name: "presidio" + - guardrail_name: "presidio-pre-guard" litellm_params: guardrail: presidio # supported values: "aporia", "lakera", "presidio" mode: "pre_call" # pre_call, during_call, post_call From 36505058e0aa72327dfe5d0fc436aa1d846c122e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 13:23:17 -0700 Subject: [PATCH 5/9] doc setting language per request --- .../docs/proxy/guardrails/pii_masking_v2.md | 65 ++++++++++++++++++- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md index 592c0e6bf2f..ff1be9c7f81 100644 --- a/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md +++ b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md @@ -1,4 +1,6 @@ import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; # PII Masking - Presidio @@ -57,7 +59,7 @@ curl http://localhost:4000/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer sk-1234" \ -d '{ - "model": "gpt-4o-mini", + "model": "gpt-3.5-turbo", "messages": [ {"role": "user", "content": "Hello my name is Jane Doe"} ], @@ -83,7 +85,7 @@ Expected response on failure } ], "created": 1725479980, - "model": "gpt-4o-mini-2024-07-18", + "model": "gpt-3.5-turbo-2024-07-18", "object": "chat.completion", "system_fingerprint": "fp_5bd87c427a", "usage": { @@ -104,7 +106,7 @@ curl http://localhost:4000/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer sk-1234" \ -d '{ - "model": "gpt-4o-mini", + "model": "gpt-3.5-turbo", "messages": [ {"role": "user", "content": "Hello good morning"} ], @@ -120,10 +122,67 @@ curl http://localhost:4000/chat/completions \ ## Set `language` per request +The Presidio API [supports passing the `language` param](https://microsoft.github.io/presidio/api-docs/api-docs.html#tag/Analyzer/paths/~1analyze/post). Here is how to set the `language` per request + + + + +```shell +curl http://localhost:4000/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "is this credit card number 9283833 correct?"} + ], + "guardrails": ["presidio-pre-guard"], + "guardrail_config": {"language": "es"} + }' +``` + + + + + + +```python + +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "metadata": { + "guardrails": ["presidio-pre-guard"], + "guardrail_config": {"language": "es"} + } + } +) +print(response) +``` + + + + + + ## Output parsing ## Ad Hoc Recognizers +## Logging Only From 528154764b4ab6f0f4ea29839a0ffd516f6c50ab Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 13:43:14 -0700 Subject: [PATCH 6/9] docs update presidio --- .../docs/proxy/guardrails/pii_masking_v2.md | 162 +++++++++++++++++- 1 file changed, 156 insertions(+), 6 deletions(-) diff --git a/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md index ff1be9c7f81..59690666ee4 100644 --- a/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md +++ b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md @@ -36,6 +36,7 @@ export PRESIDIO_ANONYMIZER_API_BASE="http://localhost:5001" - `pre_call` Run **before** LLM call, on **input** - `post_call` Run **after** LLM call, on **input & output** +- `logging_only` Run **after** LLM call, only apply PII Masking before logging to Langfuse, etc. Not on the actual llm api request / response. ### 2. Start LiteLLM Gateway @@ -119,8 +120,9 @@ curl http://localhost:4000/chat/completions \ +## Advanced -## Set `language` per request +### Set `language` per request The Presidio API [supports passing the `language` param](https://microsoft.github.io/presidio/api-docs/api-docs.html#tag/Analyzer/paths/~1analyze/post). Here is how to set the `language` per request @@ -178,11 +180,159 @@ print(response) -## Output parsing - -## Ad Hoc Recognizers - -## Logging Only +### Output parsing +LLM responses can sometimes contain the masked tokens. + +For presidio 'replace' operations, LiteLLM can check the LLM response and replace the masked token with the user-submitted values. + +Define your guardrails under the `guardrails` section +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: "presidio-pre-guard" + litellm_params: + guardrail: presidio # supported values: "aporia", "bedrock", "lakera", "presidio" + mode: "pre_call" + output_parse_pii: True +``` + +**Expected Flow: ** + +1. User Input: "hello world, my name is Jane Doe. My number is: 034453334" + +2. LLM Input: "hello world, my name is [PERSON]. My number is: [PHONE_NUMBER]" + +3. LLM Response: "Hey [PERSON], nice to meet you!" + +4. User Response: "Hey Jane Doe, nice to meet you!" + +### Ad Hoc Recognizers + + +Send ad-hoc recognizers to presidio `/analyze` by passing a json file to the proxy + +[**Example** ad-hoc recognizer](../../../../litellm/proxy/hooks/example_presidio_ad_hoc_recognize) + +#### Define ad-hoc recognizer on your LiteLLM config.yaml + +Define your guardrails under the `guardrails` section +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: "presidio-pre-guard" + litellm_params: + guardrail: presidio # supported values: "aporia", "bedrock", "lakera", "presidio" + mode: "pre_call" + presidio_ad_hoc_recognizers: "./hooks/example_presidio_ad_hoc_recognizer.json" +``` + +Set the following env vars + +```bash +export PRESIDIO_ANALYZER_API_BASE="http://localhost:5002" +export PRESIDIO_ANONYMIZER_API_BASE="http://localhost:5001" +``` + + +You can see this working, when you run the proxy: + +```bash +litellm --config /path/to/config.yaml --debug +``` + +Make a chat completions request, example: + +``` +{ + "model": "azure-gpt-3.5", + "messages": [{"role": "user", "content": "John Smith AHV number is 756.3026.0705.92. Zip code: 1334023"}] +} +``` + +And search for any log starting with `Presidio PII Masking`, example: +``` +Presidio PII Masking: Redacted pii message: AHV number is . Zip code: +``` + +### Logging Only + + +Only apply PII Masking before logging to Langfuse, etc. + +Not on the actual llm api request / response. + +:::note +This is currently only applied for +- `/chat/completion` requests +- on 'success' logging + +::: + +1. Define mode: `logging_only` on your LiteLLM config.yaml + +Define your guardrails under the `guardrails` section +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: "presidio-pre-guard" + litellm_params: + guardrail: presidio # supported values: "aporia", "bedrock", "lakera", "presidio" + mode: "logging_only" +``` + +Set the following env vars + +```bash +export PRESIDIO_ANALYZER_API_BASE="http://localhost:5002" +export PRESIDIO_ANONYMIZER_API_BASE="http://localhost:5001" +``` + + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "Hi, my name is Jane!" + } + ] + }' +``` + + +**Expected Logged Response** + +``` +Hi, my name is ! +``` + From f1111f9a1bc93ea9c21b0b986947259631320dc7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 13:57:04 -0700 Subject: [PATCH 7/9] handle logging_only logic for guardrails --- litellm/litellm_core_utils/litellm_logging.py | 19 ++++++++++++++++++- litellm/types/guardrails.py | 5 +++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 537ca15a477..eb77a0a198c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -25,6 +25,7 @@ from litellm import ( ) from litellm.caching import DualCache, InMemoryCache, S3Cache from litellm.cost_calculator import _select_model_name_for_cost_calc +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_logging, @@ -1350,7 +1351,23 @@ class Logging: ## LOGGING HOOK ## for callback in callbacks: - if isinstance(callback, CustomLogger): + if isinstance(callback, CustomGuardrail): + from litellm.types.guardrails import GuardrailEventHooks + + if ( + callback.should_run_guardrail( + data=self.model_call_details, + event_type=GuardrailEventHooks.logging_only, + ) + is not True + ): + self.model_call_details, result = await callback.async_logging_hook( + kwargs=self.model_call_details, + result=result, + call_type=self.call_type, + ) + continue + elif isinstance(callback, CustomLogger): self.model_call_details, result = await callback.async_logging_hook( kwargs=self.model_call_details, result=result, diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 10f4be7e1eb..cb70de5052d 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -84,6 +84,10 @@ class LitellmParams(TypedDict, total=False): guardrailIdentifier: Optional[str] guardrailVersion: Optional[str] + # Presidio params + output_parse_pii: Optional[bool] + presidio_ad_hoc_recognizers: Optional[str] + class Guardrail(TypedDict): guardrail_name: str @@ -98,6 +102,7 @@ class GuardrailEventHooks(str, Enum): pre_call = "pre_call" post_call = "post_call" during_call = "during_call" + logging_only = "logging_only" class BedrockTextContent(TypedDict, total=False): From 4ab8e52bfa3e4461b314ae7f523df82a7f03734c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 14:40:35 -0700 Subject: [PATCH 8/9] allow init guardrails with output parsing logic --- .../guardrails/guardrail_hooks/presidio.py | 73 ++----------------- litellm/proxy/guardrails/init_guardrails.py | 22 ++++++ 2 files changed, 29 insertions(+), 66 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index eb894ecbbc4..857704bf2a5 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -48,11 +48,12 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): # Class variables or attributes def __init__( self, - logging_only: Optional[bool] = None, mock_testing: bool = False, mock_redacted_text: Optional[dict] = None, presidio_analyzer_api_base: Optional[str] = None, presidio_anonymizer_api_base: Optional[str] = None, + output_parse_pii: Optional[bool] = False, + presidio_ad_hoc_recognizers: Optional[str] = None, **kwargs, ): self.pii_tokens: dict = ( @@ -60,11 +61,11 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) # mapping of PII token to original text - only used with Presidio `replace` operation self.mock_redacted_text = mock_redacted_text - self.logging_only = logging_only + self.output_parse_pii = output_parse_pii or False if mock_testing is True: # for testing purposes only return - ad_hoc_recognizers = litellm.presidio_ad_hoc_recognizers + ad_hoc_recognizers = presidio_ad_hoc_recognizers if ad_hoc_recognizers is not None: try: with open(ad_hoc_recognizers, "r") as file: @@ -225,69 +226,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): """ try: - if ( - self.logging_only is True - ): # only modify the logging obj data (done by async_logging_hook) - return data - permissions = user_api_key_dict.permissions - output_parse_pii = permissions.get( - "output_parse_pii", litellm.output_parse_pii - ) # allow key to turn on/off output parsing for pii - no_pii = permissions.get( - "no-pii", None - ) # allow key to turn on/off pii masking (if user is allowed to set pii controls, then they can override the key defaults) - - if no_pii is None: - # check older way of turning on/off pii - no_pii = not permissions.get("pii", True) content_safety = data.get("content_safety", None) verbose_proxy_logger.debug("content_safety: %s", content_safety) - ## Request-level turn on/off PII controls ## - if content_safety is not None and isinstance(content_safety, dict): - # pii masking ## - if ( - content_safety.get("no-pii", None) is not None - and content_safety.get("no-pii") == True - ): - # check if user allowed to turn this off - if permissions.get("allow_pii_controls", False) == False: - raise HTTPException( - status_code=400, - detail={ - "error": "Not allowed to set PII controls per request" - }, - ) - else: # user allowed to turn off pii masking - no_pii = content_safety.get("no-pii") - if not isinstance(no_pii, bool): - raise HTTPException( - status_code=400, - detail={"error": "no_pii needs to be a boolean value"}, - ) - ## pii output parsing ## - if content_safety.get("output_parse_pii", None) is not None: - # check if user allowed to turn this off - if permissions.get("allow_pii_controls", False) == False: - raise HTTPException( - status_code=400, - detail={ - "error": "Not allowed to set PII controls per request" - }, - ) - else: # user allowed to turn on/off pii output parsing - output_parse_pii = content_safety.get("output_parse_pii") - if not isinstance(output_parse_pii, bool): - raise HTTPException( - status_code=400, - detail={ - "error": "output_parse_pii needs to be a boolean value" - }, - ) - - if no_pii is True: # turn off pii masking - return data - presidio_config = self.get_presidio_settings_from_request_data(data) if call_type == "completion": # /chat/completions requests @@ -299,7 +240,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): tasks.append( self.check_pii( text=m["content"], - output_parse_pii=output_parse_pii, + output_parse_pii=self.output_parse_pii, presidio_config=presidio_config, ) ) @@ -372,9 +313,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): Output parse the response object to replace the masked tokens with user sent values """ verbose_proxy_logger.debug( - f"PII Masking Args: litellm.output_parse_pii={litellm.output_parse_pii}; type of response={type(response)}" + f"PII Masking Args: self.output_parse_pii={self.output_parse_pii}; type of response={type(response)}" ) - if litellm.output_parse_pii == False: + if self.output_parse_pii == False: return response if isinstance(response, ModelResponse) and not isinstance( diff --git a/litellm/proxy/guardrails/init_guardrails.py b/litellm/proxy/guardrails/init_guardrails.py index 9a43171f478..cff9fca0560 100644 --- a/litellm/proxy/guardrails/init_guardrails.py +++ b/litellm/proxy/guardrails/init_guardrails.py @@ -11,6 +11,7 @@ from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_pr # v2 implementation from litellm.types.guardrails import ( Guardrail, + GuardrailEventHooks, GuardrailItem, GuardrailItemSpec, LakeraCategoryThresholds, @@ -104,6 +105,10 @@ def init_guardrails_v2( api_base=litellm_params_data.get("api_base"), guardrailIdentifier=litellm_params_data.get("guardrailIdentifier"), guardrailVersion=litellm_params_data.get("guardrailVersion"), + output_parse_pii=litellm_params_data.get("output_parse_pii"), + presidio_ad_hoc_recognizers=litellm_params_data.get( + "presidio_ad_hoc_recognizers" + ), ) if ( @@ -173,7 +178,24 @@ def init_guardrails_v2( _presidio_callback = _OPTIONAL_PresidioPIIMasking( guardrail_name=guardrail["guardrail_name"], event_hook=litellm_params["mode"], + output_parse_pii=litellm_params["output_parse_pii"], + presidio_ad_hoc_recognizers=litellm_params[ + "presidio_ad_hoc_recognizers" + ], ) + + if litellm_params["output_parse_pii"] is True: + _success_callback = _OPTIONAL_PresidioPIIMasking( + output_parse_pii=True, + guardrail_name=guardrail["guardrail_name"], + event_hook=GuardrailEventHooks.post_call.value, + presidio_ad_hoc_recognizers=litellm_params[ + "presidio_ad_hoc_recognizers" + ], + ) + + litellm.callbacks.append(_success_callback) # type: ignore + litellm.callbacks.append(_presidio_callback) # type: ignore elif ( isinstance(litellm_params["guardrail"], str) From 7712aa652aeb59153dee07e4765ad9b77f27e2ac Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 15:22:37 -0700 Subject: [PATCH 9/9] fix init presidio guardrail --- litellm/litellm_core_utils/litellm_logging.py | 11 ++++++----- litellm/proxy/proxy_config.yaml | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index eb77a0a198c..4e40af11ca7 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1361,12 +1361,13 @@ class Logging: ) is not True ): - self.model_call_details, result = await callback.async_logging_hook( - kwargs=self.model_call_details, - result=result, - call_type=self.call_type, - ) continue + + self.model_call_details, result = await callback.async_logging_hook( + kwargs=self.model_call_details, + result=result, + call_type=self.call_type, + ) elif isinstance(callback, CustomLogger): self.model_call_details, result = await callback.async_logging_hook( kwargs=self.model_call_details, diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 0aaadc13fe1..f6942dd29eb 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -13,6 +13,7 @@ guardrails: litellm_params: guardrail: presidio # supported values: "aporia", "lakera", "presidio" mode: "pre_call" # pre_call, during_call, post_call + output_parse_pii: True general_settings: master_key: sk-1234