From 0a3284e753bc9fe18443d05d086d1431e2683771 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 17 Feb 2024 09:25:49 -0800 Subject: [PATCH] docs(enterprise.md): add llama guard tutorial to enterprise docs --- .../docs/proxy/{spend.md => enterprise.md} | 35 ++++++++++++++++--- docs/my-website/sidebars.js | 2 +- litellm/proxy/hooks/presidio_pii_masking.py | 3 ++ 3 files changed, 34 insertions(+), 6 deletions(-) rename docs/my-website/docs/proxy/{spend.md => enterprise.md} (76%) diff --git a/docs/my-website/docs/proxy/spend.md b/docs/my-website/docs/proxy/enterprise.md similarity index 76% rename from docs/my-website/docs/proxy/spend.md rename to docs/my-website/docs/proxy/enterprise.md index c76bbc7e98f..f881215e6e1 100644 --- a/docs/my-website/docs/proxy/spend.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -1,21 +1,46 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# 💸 Spend Tracking +# ✨ Enterprise Features + +Features here are behind a commercial license in our `/enterprise` folder. [**See Code**](https://github.com/BerriAI/litellm/tree/main/enterprise) :::info -This is an Enterprise only feature [Get Started with Enterprise here](https://github.com/BerriAI/litellm/tree/main/enterprise) +[Get Started with Enterprise here](https://github.com/BerriAI/litellm/tree/main/enterprise) ::: +Features: +- [ ] Content Moderation with LlamaGuard +- [ ] Tracking Spend for Custom Tags + +## Content Moderation with LlamaGuard + +Currently works with Sagemaker's LlamaGuard endpoint. + +How to enable this in your config.yaml: + +```yaml +litellm_settings: + callbacks: ["llamaguard_moderations"] + llamaguard_model_name: "sagemaker/jumpstart-dft-meta-textgeneration-llama-guard-7b" +``` + +Make sure you have the relevant keys in your environment, eg.: + +``` +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" +``` + +## Tracking Spend for Custom Tags + Requirements: - Virtual Keys & a database should be set up, see [virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys) - -## Tracking Spend per Request Tag - ### Usage - /chat/completions requests with request tags diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index b564befba5c..abdb4bc717d 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -109,10 +109,10 @@ const sidebars = { label: '📖 All Endpoints', href: 'https://litellm-api.up.railway.app/', }, + "proxy/enterprise", "proxy/user_keys", "proxy/virtual_keys", "proxy/users", - "proxy/spend", "proxy/ui", "proxy/model_management", "proxy/health", diff --git a/litellm/proxy/hooks/presidio_pii_masking.py b/litellm/proxy/hooks/presidio_pii_masking.py index 5152046bc5c..6ea32961375 100644 --- a/litellm/proxy/hooks/presidio_pii_masking.py +++ b/litellm/proxy/hooks/presidio_pii_masking.py @@ -69,6 +69,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomLogger): async with aiohttp.ClientSession() as session: # Make the first request to /analyze analyze_url = f"{self.presidio_analyzer_api_base}/analyze" + verbose_proxy_logger.debug(f"Making request to: {analyze_url}") analyze_payload = {"text": text, "language": "en"} redacted_text = None async with session.post(analyze_url, json=analyze_payload) as response: @@ -76,6 +77,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomLogger): # Make the second request to /anonymize anonymize_url = f"{self.presidio_anonymizer_api_base}/anonymize" + verbose_proxy_logger.debug(f"Making request to: {anonymize_url}") anonymize_payload = { "text": "hello world, my name is Jane Doe. My number is: 034453334", "analyzer_results": analyze_results, @@ -88,6 +90,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomLogger): new_text = text if redacted_text is not None: + verbose_proxy_logger.debug(f"redacted_text: {redacted_text}") for item in redacted_text["items"]: start = item["start"] end = item["end"]