diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index c64d475fdaa..db38cf5426b 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -197,6 +197,7 @@ router_settings: | key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) | | disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. | | use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. | +| skip_system_message_in_guardrail | boolean | If true, unified guardrails omit `role: system` from scanned input on **chat completions** and **Anthropic `/v1/messages`** only; the LLM still receives full messages. Per-guardrail override: `litellm_params.skip_system_message_in_guardrail` on each guardrail. [Guardrails quick start](./guardrails/quick_start#skip-system-messages-in-guardrail-evaluation) | | disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). | | enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. | | enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. | diff --git a/docs/my-website/docs/proxy/guardrails/quick_start.md b/docs/my-website/docs/proxy/guardrails/quick_start.md index 5abe499e30b..ed9d2ca128b 100644 --- a/docs/my-website/docs/proxy/guardrails/quick_start.md +++ b/docs/my-website/docs/proxy/guardrails/quick_start.md @@ -9,6 +9,7 @@ Setup Prompt Injection Detection, PII Masking on LiteLLM Proxy (AI Gateway) ## 1. Define guardrails on your LiteLLM config.yaml Set your guardrails under the `guardrails` section + ```yaml model_list: - model_name: gpt-3.5-turbo @@ -82,27 +83,58 @@ For generic guardrail APIs you can also set **static headers** (`headers`: key/v - `during_call` Run **during** LLM call, on **input** Same as `pre_call` but runs in parallel as LLM call. Response not returned until guardrail check completes - A list of the above values to run multiple modes, e.g. `mode: [pre_call, post_call]` +### Skip system messages in guardrail evaluation + +You can stop **unified** guardrails from scanning `role: system` content while still sending the full `messages` list to the model. + +**Global** — in `litellm_settings`: + +```yaml +litellm_settings: + skip_system_message_in_guardrail: true +``` + +**Per guardrail** — under that guardrail’s `litellm_params`: set `skip_system_message_in_guardrail: true` or `false`. If omitted, the global `litellm_settings` value is used; per-guardrail `false` forces system messages to be included even when the global flag is `true`. + +**Via LiteLLM UI** — when **creating** or **editing** a guardrail in the LiteLLM Admin Dashboard, set **Skip system messages in guardrail** (under Basic Info on create, or in the edit / guardrail settings flows): + + +| UI option | Effect | +| ------------------------------------- | -------------------------------------------------------------------------------------- | +| **Use global default** | Uses `litellm_settings.skip_system_message_in_guardrail` from your proxy config | +| **Yes — exclude from guardrail scan** | Sets per-guardrail `skip_system_message_in_guardrail: true` | +| **No — always include in scan** | Sets per-guardrail `skip_system_message_in_guardrail: false` (overrides a global skip) | + + +Create guardrail: Skip system messages in guardrail dropdown with Use global default, Yes exclude from guardrail scan, and No always include in scan + +**Where this applies:** Only the **unified** guardrail path (providers that implement `apply_guardrail` and run through LiteLLM’s message translation layer) on **OpenAI Chat Completions** (`/v1/chat/completions`) and **Anthropic Messages** (`/v1/messages`). Examples include Presidio, Bedrock guardrails, `litellm_content_filter`, OpenAI Moderation, Generic Guardrail API, and custom code guardrails that define `apply_guardrail`. + +**Where this does *not* apply:** Guardrails that run only via direct hooks on the raw request (e.g. Lakera v2, Aporia, DynamoAI, Javelin, Lasso, Pangea, Model Armor, Azure Content Safety hooks, Guardrails AI, AIM, tool permission, MCP security). It also does not apply to other routes until those endpoints use the same translation layer (e.g. Responses API, embeddings, speech). + ### Load Balancing Guardrails Need to distribute guardrail requests across multiple accounts or regions? See [Guardrail Load Balancing](./guardrail_load_balancing.md) for details on: + - Load balancing across multiple AWS Bedrock accounts (useful for rate limit management) - Weighted distribution across guardrail instances - Multi-region guardrail deployments - -## 2. Start LiteLLM Gateway - +## 2. Start LiteLLM Gateway ```shell litellm --config config.yaml --detailed_debug ``` -## 3. Test request +## 3. Test request **[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** - - + Expect this to fail since since `ishaan@berri.ai` in the request is PII @@ -141,9 +173,9 @@ Expected response on failure ``` - - + + ```shell curl -i http://localhost:4000/v1/chat/completions \ @@ -158,10 +190,8 @@ curl -i http://localhost:4000/v1/chat/completions \ }' ``` - - ## **Default On Guardrails** @@ -183,7 +213,6 @@ guardrails: In this request, the guardrail `aporia-pre-guard` will run on every request because `default_on: true` is set. - ```shell curl -i http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ @@ -207,6 +236,7 @@ x-litellm-applied-guardrails: aporia-pre-guard ### Guardrail Policies Need more control? Use [Guardrail Policies](./guardrail_policies.md) to: + - Group guardrails into reusable policies - Enable/disable guardrails for specific teams, keys, or models - Inherit from existing policies and override specific guardrails @@ -217,7 +247,6 @@ Need more control? Use [Guardrail Policies](./guardrail_policies.md) to: Pass `guardrails` to your request body to test it - ```shell curl -i http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ @@ -239,7 +268,6 @@ Follow this simple workflow to implement and tune guardrails: First, check what guardrails are available and their parameters: - Call `/guardrails/list` to view available guardrails and the guardrail info (supported parameters, description, etc) ```shell @@ -271,9 +299,12 @@ Expected response } ``` -> + + This config will return the `/guardrails/list` response above. The `guardrail_info` field is optional and you can add any fields under info for consumers of your guardrail -> + + + ```yaml - guardrail_name: "aporia-post-guard" litellm_params: @@ -291,9 +322,10 @@ This config will return the `/guardrails/list` response above. The `guardrail_in type: "boolean" ``` - ### 2. Apply Guardrails + Add selected guardrails to your chat completion request: + ```shell curl -i http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ @@ -322,7 +354,6 @@ curl -i http://localhost:4000/v1/chat/completions \ }' ``` - ### 4. ✨ Pass Dynamic Parameters to Guardrail :::info @@ -334,9 +365,8 @@ curl -i http://localhost:4000/v1/chat/completions \ Use this to pass additional parameters to the guardrail API call. e.g. things like success threshold. **[See `guardrails` spec for more details](#spec-guardrails-parameter)** - - + Set `guardrails={"aporia-pre-guard": {"extra_body": {"success_threshold": 0.9}}}` to pass additional parameters to the guardrail @@ -371,10 +401,10 @@ response = client.chat.completions.create( print(response) ``` - - + + ```shell curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -396,11 +426,8 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } }' ``` - - - @@ -426,9 +453,6 @@ Monitor which guardrails were executed and whether they passed or failed. e.g. g - - - ### ✨ Control Guardrails per API Key :::info @@ -438,12 +462,12 @@ Monitor which guardrails were executed and whether they passed or failed. e.g. g ::: Use this to control what guardrails run per API Key. In this tutorial we only want the following guardrails to run for 1 API Key + - `guardrails`: ["aporia-pre-guard", "aporia-post-guard"] **Step 1** Create Key with guardrail settings - - + ```shell curl -X POST 'http://0.0.0.0:4000/key/generate' \ @@ -454,8 +478,7 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \ }' ``` - - + ```shell curl --location 'http://0.0.0.0:4000/key/update' \ @@ -467,8 +490,7 @@ curl --location 'http://0.0.0.0:4000/key/update' \ }' ``` - - + **Step 2** Test it with new key @@ -499,8 +521,7 @@ Run guardrails based on the user-agent header. This is useful for running pre-ca Both `default` and tag values can be a single mode string or a list of modes. - - + ```yaml model_list: @@ -522,11 +543,10 @@ guardrails: default_on: true # run on every request ``` - - + ```yaml -model_list: +Per guardrailmodel_list: - model_name: gpt-3.5-turbo litellm_params: model: gpt-3.5-turbo @@ -545,8 +565,7 @@ guardrails: default_on: true ``` - - + ```yaml model_list: @@ -568,8 +587,6 @@ guardrails: default_on: true ``` - - ### ✨ Model-level Guardrails @@ -580,10 +597,8 @@ guardrails: ::: - This is great for cases when you have an on-prem and hosted model, and just want to run prevent sending PII to the hosted model. - ```yaml model_list: - model_name: claude-sonnet-4 @@ -620,8 +635,7 @@ guardrails: ::: - -#### 1. Disable team from modifying guardrails +#### 1. Disable team from modifying guardrails ```bash curl -X POST 'http://0.0.0.0:4000/team/update' \ @@ -633,7 +647,7 @@ curl -X POST 'http://0.0.0.0:4000/team/update' \ }' ``` -#### 2. Try to disable guardrails for a call +#### 2. Try to disable guardrails for a call ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -672,8 +686,7 @@ Expect to NOT see `+1 412-612-9992` in your server logs on your callback. The `pii_masking` guardrail ran on this request because api key=sk-jNm1Zar7XfNdZXp49Z1kSQ has `"permissions": {"pii_masking": true}` ::: - -## Specification +## Specification ### `guardrails` Configuration on YAML @@ -723,6 +736,7 @@ The `guardrails` parameter can be passed to any LiteLLM Proxy endpoint (`/chat/c #### Format Options 1. Simple List Format: + ```python "guardrails": [ "aporia-pre-guard", @@ -730,9 +744,10 @@ The `guardrails` parameter can be passed to any LiteLLM Proxy endpoint (`/chat/c ] ``` -2. Advanced Dictionary Format: +1. Advanced Dictionary Format: In this format the dictionary key is `guardrail_name` you want to run + ```python "guardrails": { "aporia-pre-guard": { @@ -745,6 +760,7 @@ In this format the dictionary key is `guardrail_name` you want to run ``` #### Type Definition + ```python guardrails: Union[ List[str], # Simple list of guardrail names @@ -754,3 +770,4 @@ guardrails: Union[ class DynamicGuardrailParams: extra_body: Dict[str, Any] # Additional parameters for the guardrail ``` + diff --git a/docs/my-website/img/skip_system_message_guardrail_ui.png b/docs/my-website/img/skip_system_message_guardrail_ui.png new file mode 100644 index 00000000000..466ac7daa6e Binary files /dev/null and b/docs/my-website/img/skip_system_message_guardrail_ui.png differ diff --git a/litellm/__init__.py b/litellm/__init__.py index 64c60ca3374..565b86f818d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -203,6 +203,7 @@ add_user_information_to_llm_headers: Optional[ bool ] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers store_audit_logs = False # Enterprise feature, allow users to see audit logs +skip_system_message_in_guardrail: bool = False ### end of callbacks ############# email: Optional[ diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index d31a0a091e9..2d1ca4b6e30 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -21,6 +21,10 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im LiteLLMAnthropicMessagesAdapter, ) from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.llms.base_llm.guardrail_translation.utils import ( + effective_skip_system_message_for_guardrail, + openai_messages_without_system, +) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, ) @@ -29,6 +33,7 @@ from litellm.types.llms.anthropic import ( AnthropicMessagesRequest, ) from litellm.types.llms.openai import ( + AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolParam, ) @@ -75,6 +80,8 @@ class AnthropicMessagesHandler(BaseTranslation): if messages is None: return data + skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply) + ( chat_completion_compatible_request, _tool_name_mapping, @@ -83,7 +90,12 @@ class AnthropicMessagesHandler(BaseTranslation): anthropic_message_request=cast(AnthropicMessagesRequest, data.copy()) ) - structured_messages = chat_completion_compatible_request.get("messages", []) + structured_messages = cast( + List[AllMessageValues], + chat_completion_compatible_request.get("messages", []), + ) + if skip_system: + structured_messages = openai_messages_without_system(structured_messages) texts_to_check: List[str] = [] images_to_check: List[str] = [] @@ -102,6 +114,7 @@ class AnthropicMessagesHandler(BaseTranslation): texts_to_check=texts_to_check, images_to_check=images_to_check, task_mappings=task_mappings, + skip_system_message=skip_system, ) # Step 2: Apply guardrail to all texts in batch @@ -165,12 +178,16 @@ class AnthropicMessagesHandler(BaseTranslation): texts_to_check: List[str], images_to_check: List[str], task_mappings: List[Tuple[int, Optional[int]]], + skip_system_message: bool = False, ) -> None: """ Extract text content and images from a message. Override this method to customize text/image extraction logic. """ + if skip_system_message and str(message.get("role") or "").lower() == "system": + return + content = message.get("content", None) tools = message.get("tools", None) if content is None and tools is None: diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py new file mode 100644 index 00000000000..cc401d07406 --- /dev/null +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Any, List + +from litellm.types.llms.openai import AllMessageValues + + +def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool: + per = getattr(guardrail_to_apply, "skip_system_message_in_guardrail", None) + if per is not None: + return bool(per) + import litellm + + return bool(getattr(litellm, "skip_system_message_in_guardrail", False)) + + +def openai_messages_without_system( + messages: List[AllMessageValues], +) -> List[AllMessageValues]: + return [ + m + for m in messages + if str((m or {}).get("role") or "").lower() != "system" + ] diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index f854cdb13d0..2db19dea0b9 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -19,8 +19,12 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast import litellm from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.llms.base_llm.guardrail_translation.utils import ( + effective_skip_system_message_for_guardrail, + openai_messages_without_system, +) from litellm.main import stream_chunk_builder -from litellm.types.llms.openai import ChatCompletionToolParam +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from litellm.types.utils import ( Choices, GenericGuardrailAPIInputs, @@ -57,6 +61,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if messages is None: return data + skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply) + texts_to_check: List[str] = [] images_to_check: List[str] = [] tool_calls_to_check: List[ChatCompletionToolParam] = [] @@ -76,6 +82,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): tool_calls_to_check=tool_calls_to_check, text_task_mappings=text_task_mappings, tool_call_task_mappings=tool_call_task_mappings, + skip_system_message=skip_system, ) # Step 2: Apply guardrail to all texts and tool calls in batch @@ -86,9 +93,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if tool_calls_to_check: inputs["tool_calls"] = tool_calls_to_check # type: ignore if messages: - inputs[ - "structured_messages" - ] = messages # pass the openai /chat/completions messages to the guardrail, as-is + msg_list = cast(List[AllMessageValues], messages) + inputs["structured_messages"] = ( + openai_messages_without_system(msg_list) + if skip_system + else msg_list + ) # Pass tools (function definitions) to the guardrail tools = data.get("tools") if tools: @@ -157,12 +167,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation): tool_calls_to_check: List[ChatCompletionToolParam], text_task_mappings: List[Tuple[int, Optional[int]]], tool_call_task_mappings: List[Tuple[int, int]], + skip_system_message: bool = False, ) -> None: """ Extract text content, images, and tool calls from a message. Override this method to customize text/image/tool call extraction logic. """ + if skip_system_message and str(message.get("role") or "").lower() == "system": + return + content = message.get("content", None) if content is not None: if isinstance(content, str): diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index d41be370f7b..96175877d65 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -472,6 +472,13 @@ class InMemoryGuardrailHandler: else: raise ValueError(f"Unsupported guardrail: {guardrail_type}") + if custom_guardrail_callback is not None: + setattr( + custom_guardrail_callback, + "skip_system_message_in_guardrail", + getattr(litellm_params, "skip_system_message_in_guardrail", None), + ) + parsed_guardrail = Guardrail( guardrail_id=guardrail.get("guardrail_id"), guardrail_name=guardrail["guardrail_name"], diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index cfec0398c81..ec869d7917d 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -607,6 +607,16 @@ class BaseLitellmParams( description="When True, guardrails only receive the latest message for the relevant role (e.g., newest user input pre-call, newest assistant output post-call)", ) + skip_system_message_in_guardrail: Optional[bool] = Field( + default=None, + description=( + "When True, unified guardrails skip system-role messages when building " + "evaluation inputs (texts and structured_messages). When False, system " + "messages are included even if litellm_settings sets a global skip. When " + "None, use the global litellm.skip_system_message_in_guardrail setting." + ), + ) + # Lakera specific params category_thresholds: Optional[LakeraCategoryThresholds] = Field( default=None, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index bbba8e4d03d..11115f06d8b 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -2,9 +2,17 @@ import pytest +import litellm from litellm.caching import DualCache from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.llms.base_llm.guardrail_translation.utils import ( + effective_skip_system_message_for_guardrail, + openai_messages_without_system, +) +from litellm.llms.openai.chat.guardrail_translation.handler import ( + OpenAIChatCompletionsHandler, +) from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse from litellm.llms.mistral.ocr.guardrail_translation.handler import OCRHandler from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import ( @@ -68,6 +76,109 @@ def _inject_mcp_handler_mapping(): class TestUnifiedLLMGuardrails: + class TestSkipSystemMessageForChatCompletions: + def test_openai_messages_without_system(self): + msgs = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "hi"}, + ] + out = openai_messages_without_system(msgs) + assert len(out) == 1 + assert out[0]["role"] == "user" + assert msgs[0]["content"] == "sys" + + def test_effective_skip_respects_per_guardrail_over_global(self, monkeypatch): + monkeypatch.setattr( + litellm, "skip_system_message_in_guardrail", True, raising=False + ) + + class G: + skip_system_message_in_guardrail = False + + assert effective_skip_system_message_for_guardrail(G()) is False + + class G2: + skip_system_message_in_guardrail = None + + assert effective_skip_system_message_for_guardrail(G2()) is True + + @pytest.mark.asyncio + async def test_openai_handler_skips_system_in_guardrail_inputs( + self, monkeypatch + ): + monkeypatch.setattr( + litellm, "skip_system_message_in_guardrail", True, raising=False + ) + + captured = {} + + class MockGuardrail: + skip_system_message_in_guardrail = None + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + captured["inputs"] = inputs + return inputs + + data = { + "messages": [ + {"role": "system", "content": "secret system"}, + {"role": "user", "content": "hello"}, + ], + "model": "gpt-4o", + } + + handler = OpenAIChatCompletionsHandler() + await handler.process_input_messages( + data=data, + guardrail_to_apply=MockGuardrail(), + litellm_logging_obj=None, + ) + + assert captured["inputs"]["texts"] == ["hello"] + sm = captured["inputs"].get("structured_messages") or [] + assert all(m.get("role") != "system" for m in sm) + assert data["messages"][0]["content"] == "secret system" + + @pytest.mark.asyncio + async def test_openai_handler_per_guardrail_skip_false_overrides_global( + self, monkeypatch + ): + monkeypatch.setattr( + litellm, "skip_system_message_in_guardrail", True, raising=False + ) + + captured = {} + + class MockGuardrail: + skip_system_message_in_guardrail = False + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + captured["inputs"] = inputs + return inputs + + data = { + "messages": [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "u"}, + ], + } + + await OpenAIChatCompletionsHandler().process_input_messages( + data=data, + guardrail_to_apply=MockGuardrail(), + litellm_logging_obj=None, + ) + + assert "sys" in captured["inputs"]["texts"] + roles = { + m.get("role") for m in (captured["inputs"].get("structured_messages") or []) + } + assert "system" in roles + class TestAsyncPreCallHook: @pytest.mark.asyncio async def test_uses_mcp_event_type(self): diff --git a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx index 3bdd18f2650..4cbe5664c6b 100644 --- a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx @@ -4,6 +4,7 @@ import NotificationsManager from "../molecules/notifications_manager"; import { createGuardrailCall, getGuardrailProviderSpecificParams, getGuardrailUISettings } from "../networking"; import ContentFilterConfiguration from "./content_filter/ContentFilterConfiguration"; import { + choiceToSkipSystemForCreate, getGuardrailProviders, guardrail_provider_map, guardrailLogoMap, @@ -179,6 +180,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a guardrail_name: preset.guardrailNameSuggestion, mode: preset.mode, default_on: preset.defaultOn, + skip_system_message_choice: "inherit", }; if (preset.provider === "BlockCodeExecution") { baseValues.confidence_threshold = 0.5; @@ -414,6 +416,11 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a guardrail_info: {}, }; + const skipForCreate = choiceToSkipSystemForCreate(values.skip_system_message_choice); + if (skipForCreate !== undefined) { + guardrailData.litellm_params.skip_system_message_in_guardrail = skipForCreate; + } + // For Presidio PII, add the entity and action configurations if (values.provider === "PresidioPII" && selectedEntities.length > 0) { const piiEntitiesConfig: { [key: string]: string } = {}; @@ -749,6 +756,18 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a + + + + {/* Use the GuardrailProviderFields component to render provider-specific fields */} {!isToolPermissionProvider && !shouldRenderContentFilterConfigSettings(selectedProvider) && ( = ({ visible, onClose, a initialValues={{ mode: "pre_call", default_on: false, + skip_system_message_choice: "inherit", }} > {stepConfigs.map((step, index) => { diff --git a/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx index a2cc3ad41dd..ad823df53fc 100644 --- a/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx @@ -1,7 +1,12 @@ import React, { useState, useEffect } from "react"; import { Form, Typography, Select, Input, Switch, Modal } from "antd"; import { Button, TextInput } from "@tremor/react"; -import { guardrail_provider_map, guardrailLogoMap, getGuardrailProviders } from "./guardrail_info_helpers"; +import { + guardrail_provider_map, + guardrailLogoMap, + getGuardrailProviders, + type SkipSystemMessageChoice, +} from "./guardrail_info_helpers"; import { getGuardrailUISettings, getGlobalLitellmHeaderName } from "../networking"; import PiiConfiguration from "./pii_configuration"; import NotificationsManager from "../molecules/notifications_manager"; @@ -15,12 +20,15 @@ interface EditGuardrailFormProps { accessToken: string | null; onSuccess: () => void; guardrailId: string; + /** Full stored params merged into PUT so optional fields (e.g. content filter) are preserved. */ + fullLitellmParams?: Record | null; initialValues: { guardrail_name: string; provider: string; mode: string; default_on: boolean; pii_entities_config?: { [key: string]: string }; + skip_system_message_choice?: SkipSystemMessageChoice; [key: string]: any; }; } @@ -41,6 +49,7 @@ const EditGuardrailForm: React.FC = ({ accessToken, onSuccess, guardrailId, + fullLitellmParams, initialValues, }) => { const [form] = Form.useForm(); @@ -113,31 +122,23 @@ const EditGuardrailForm: React.FC = ({ // Get the guardrail provider value from the map const guardrailProvider = guardrail_provider_map[values.provider]; - // Prepare the guardrail data with proper types for litellm_params - const guardrailData: { - guardrail_id: string; - guardrail: { - guardrail_name: string; - litellm_params: { - guardrail: string; - mode: string; - default_on: boolean; - [key: string]: any; // Allow dynamic properties - }; - guardrail_info: any; - }; - } = { - guardrail_id: guardrailId, - guardrail: { - guardrail_name: values.guardrail_name, - litellm_params: { - guardrail: guardrailProvider, - mode: values.mode, - default_on: values.default_on, - }, - guardrail_info: {}, - }, - }; + const litellm_params: Record = + fullLitellmParams && typeof fullLitellmParams === "object" ? { ...fullLitellmParams } : {}; + + litellm_params.guardrail = guardrailProvider; + litellm_params.mode = values.mode; + litellm_params.default_on = values.default_on; + + const skipChoice = values.skip_system_message_choice as SkipSystemMessageChoice | undefined; + if (skipChoice === "yes") { + litellm_params.skip_system_message_in_guardrail = true; + } else if (skipChoice === "no") { + litellm_params.skip_system_message_in_guardrail = false; + } else { + delete litellm_params.skip_system_message_in_guardrail; + } + + let guardrail_info: any = {}; // For Presidio PII, add the entity and action configurations if (values.provider === "PresidioPII" && selectedEntities.length > 0) { @@ -146,7 +147,7 @@ const EditGuardrailForm: React.FC = ({ piiEntitiesConfig[entity] = selectedActions[entity] || "MASK"; // Default to MASK if no action selected }); - guardrailData.guardrail.litellm_params.pii_entities_config = piiEntitiesConfig; + litellm_params.pii_entities_config = piiEntitiesConfig; } // Add config values to the guardrail_info if provided else if (values.config) { @@ -156,14 +157,14 @@ const EditGuardrailForm: React.FC = ({ // Especially for providers like Bedrock that need guardrailIdentifier and guardrailVersion if (values.provider === "Bedrock" && configObj) { if (configObj.guardrail_id) { - guardrailData.guardrail.litellm_params.guardrailIdentifier = configObj.guardrail_id; + litellm_params.guardrailIdentifier = configObj.guardrail_id; } if (configObj.guardrail_version) { - guardrailData.guardrail.litellm_params.guardrailVersion = configObj.guardrail_version; + litellm_params.guardrailVersion = configObj.guardrail_version; } } else { // For other providers, add the config to guardrail_info - guardrailData.guardrail.guardrail_info = configObj; + guardrail_info = configObj; } } catch (error) { NotificationsManager.fromBackend("Invalid JSON in configuration"); @@ -172,6 +173,22 @@ const EditGuardrailForm: React.FC = ({ } } + const guardrailData: { + guardrail_id: string; + guardrail: { + guardrail_name: string; + litellm_params: Record; + guardrail_info: any; + }; + } = { + guardrail_id: guardrailId, + guardrail: { + guardrail_name: values.guardrail_name, + litellm_params, + guardrail_info, + }, + }; + if (!accessToken) { throw new Error("No access token available"); } @@ -403,6 +420,18 @@ const EditGuardrailForm: React.FC = ({ + + + + {renderProviderSpecificFields()}
diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx index 2151a91d9d7..60400443d5c 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx @@ -25,7 +25,12 @@ import React, { useCallback, useEffect, useState } from "react"; import NotificationsManager from "../molecules/notifications_manager"; import ContentFilterManager, { formatContentFilterDataForAPI } from "./content_filter/ContentFilterManager"; import CustomCodeModal, { EditGuardrailData } from "./custom_code/CustomCodeModal"; -import { getGuardrailLogoAndName, guardrail_provider_map } from "./guardrail_info_helpers"; +import { + getGuardrailLogoAndName, + guardrail_provider_map, + skipSystemMessageToChoice, + type SkipSystemMessageChoice, +} from "./guardrail_info_helpers"; import GuardrailOptionalParams from "./guardrail_optional_params"; import GuardrailProviderFields from "./guardrail_provider_fields"; import PiiConfiguration from "./pii_configuration"; @@ -207,9 +212,14 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, // Reset form when guardrail data or provider params change useEffect(() => { if (guardrailData && form) { + const lp = { ...(guardrailData.litellm_params || {}) }; + delete lp.skip_system_message_in_guardrail; form.setFieldsValue({ guardrail_name: guardrailData.guardrail_name, - ...guardrailData.litellm_params, + ...lp, + skip_system_message_choice: skipSystemMessageToChoice( + guardrailData.litellm_params?.skip_system_message_in_guardrail, + ), guardrail_info: guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", // Include any optional_params if they exist ...(guardrailData.litellm_params?.optional_params && { @@ -278,6 +288,20 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, updateData.litellm_params.default_on = values.default_on; } + const prevSkipChoice = skipSystemMessageToChoice( + guardrailData.litellm_params?.skip_system_message_in_guardrail, + ); + const nextSkipChoice = values.skip_system_message_choice as SkipSystemMessageChoice | undefined; + if (nextSkipChoice !== undefined && nextSkipChoice !== prevSkipChoice) { + if (nextSkipChoice === "inherit") { + updateData.litellm_params.skip_system_message_in_guardrail = null; + } else if (nextSkipChoice === "yes") { + updateData.litellm_params.skip_system_message_in_guardrail = true; + } else { + updateData.litellm_params.skip_system_message_in_guardrail = false; + } + } + // Only include guardrail_info if it has changed const originalGuardrailInfo = guardrailData.guardrail_info; const newGuardrailInfo = values.guardrail_info ? JSON.parse(values.guardrail_info) : undefined; @@ -647,7 +671,14 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, onFinish={handleGuardrailUpdate} initialValues={{ guardrail_name: guardrailData.guardrail_name, - ...guardrailData.litellm_params, + ...(() => { + const lp = { ...(guardrailData.litellm_params || {}) }; + delete lp.skip_system_message_in_guardrail; + return lp; + })(), + skip_system_message_choice: skipSystemMessageToChoice( + guardrailData.litellm_params?.skip_system_message_in_guardrail, + ), guardrail_info: guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", @@ -673,6 +704,18 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, + + + + {guardrailData.litellm_params?.guardrail === "presidio" && ( <> PII Protection diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx index d9e01acaadf..dfda86c1e4a 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx @@ -10,6 +10,8 @@ import { DynamicGuardrailProviders, guardrail_provider_map, GuardrailProviders, + skipSystemMessageToChoice, + choiceToSkipSystemForCreate, } from "./guardrail_info_helpers"; describe("guardrail_info_helpers", () => { @@ -199,4 +201,18 @@ describe("guardrail_info_helpers", () => { expect(result.logo).toContain("noma_security.png"); }); }); + + describe("skipSystemMessageToChoice / choiceToSkipSystemForCreate", () => { + it("maps API values to form choices and back for create", () => { + expect(skipSystemMessageToChoice(undefined)).toBe("inherit"); + expect(skipSystemMessageToChoice(null)).toBe("inherit"); + expect(skipSystemMessageToChoice(true)).toBe("yes"); + expect(skipSystemMessageToChoice(false)).toBe("no"); + + expect(choiceToSkipSystemForCreate("inherit")).toBeUndefined(); + expect(choiceToSkipSystemForCreate(undefined)).toBeUndefined(); + expect(choiceToSkipSystemForCreate("yes")).toBe(true); + expect(choiceToSkipSystemForCreate("no")).toBe(false); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx index c78835dae04..38b1b952845 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx @@ -149,3 +149,19 @@ export const getGuardrailLogoAndName = (guardrailValue: string): { logo: string; return { logo: logo || "", displayName: displayName || guardrailValue }; }; + +/** Tri-state UI value for `litellm_params.skip_system_message_in_guardrail` (inherit = use global). */ +export type SkipSystemMessageChoice = "inherit" | "yes" | "no"; + +export function skipSystemMessageToChoice(v: boolean | null | undefined): SkipSystemMessageChoice { + if (v === true) return "yes"; + if (v === false) return "no"; + return "inherit"; +} + +/** Create flow: omit key when inheriting global default. */ +export function choiceToSkipSystemForCreate(choice: SkipSystemMessageChoice | undefined): boolean | undefined { + if (choice === "yes") return true; + if (choice === "no") return false; + return undefined; +} diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx index 352f3148e7b..5bb2da78fa2 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx @@ -11,7 +11,7 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; -import { getGuardrailLogoAndName, guardrail_provider_map } from "./guardrail_info_helpers"; +import { getGuardrailLogoAndName, guardrail_provider_map, skipSystemMessageToChoice } from "./guardrail_info_helpers"; import EditGuardrailForm from "./edit_guardrail_form"; import { Guardrail, GuardrailDefinitionLocation } from "./types"; @@ -291,6 +291,7 @@ const GuardrailTable: React.FC = ({ accessToken={accessToken} onSuccess={handleEditSuccess} guardrailId={selectedGuardrail.guardrail_id || ""} + fullLitellmParams={selectedGuardrail.litellm_params} initialValues={{ guardrail_name: selectedGuardrail.guardrail_name || "", provider: @@ -300,6 +301,9 @@ const GuardrailTable: React.FC = ({ mode: selectedGuardrail.litellm_params.mode, default_on: selectedGuardrail.litellm_params.default_on, pii_entities_config: selectedGuardrail.litellm_params.pii_entities_config, + skip_system_message_choice: skipSystemMessageToChoice( + selectedGuardrail.litellm_params?.skip_system_message_in_guardrail, + ), ...selectedGuardrail.guardrail_info, }} />