mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
fix(guardrail): hide-secrets playground redaction and guardrail telemetry (#39398)
* Fix hide-secrets guardrail: playground redaction, UI dropdown entry, spend-log telemetry The hide-secrets guardrail never implemented apply_guardrail, so the UI test playground echoed secrets verbatim; it was missing from the Add Guardrail dropdown; and it recorded no guardrail_information, so Spend Logs could not distinguish a redacted request from a clean one. - implement apply_guardrail (unified interface) with use_native_lifecycle_hooks so proxied traffic stays on async_pre_call_hook (per-key opt-out and data["prompt"] handling live only there) - record standard_logging_guardrail_information (allow/mask + masked_entity_count) via _process_response/_process_error; opted-out keys and legacy nameless callback instances record nothing - advertise hide-secrets in /guardrails/ui/add_guardrail_settings (pre_call only) and /guardrails/ui/provider_specific_params with a config model Resolves LIT-3548 * Fix hide-secrets passthrough telemetry and JSON config input * fix(guardrails): validate hide-secrets object config before submit - apply_guardrail treats empty-string-only texts as no input, so no false allow is recorded - the UI object field keeps raw text while editing and blocks submission until it parses to a JSON object, instead of posting a string to an object-only API - supported_modes_by_provider keeps its dict[str, list[str]] value type * fix(guardrails): record no hide-secrets telemetry when nothing was inspected walk_user_text and the prompt redaction now report how many non-empty strings they visited; when neither inspected anything (image-only content, empty strings), the run records no guardrail entry instead of an 'allow' row that counts a check which never saw any text.
This commit is contained in:
yucheng-berri
GitHub
parent
3cac5e5cd4
commit
ecabfbd5af
9 changed files with 659 additions and 50 deletions
|
|
@ -11,17 +11,35 @@ import sys
|
|||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
) # Adds the parent directory to the system path
|
||||
import functools
|
||||
import tempfile
|
||||
from typing import Optional
|
||||
from contextvars import ContextVar
|
||||
from typing import TYPE_CHECKING, ClassVar, Literal, Optional
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.guardrails._content_utils import walk_user_text
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
GUARDRAIL_NAME = "hide_secrets"
|
||||
|
||||
GUARDRAIL_PROVIDER = "hide-secrets"
|
||||
|
||||
# Per-invocation tally of redacted secrets by detect-secrets plugin type; None
|
||||
# means the guardrail did not run, so _process_response records nothing.
|
||||
_masked_entity_count: ContextVar[Optional[dict]] = ContextVar(
|
||||
"hide_secrets_masked_entity_count", default=None
|
||||
)
|
||||
|
||||
_custom_plugins_path = "file://" + os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "secrets_plugins"
|
||||
)
|
||||
|
|
@ -422,6 +440,10 @@ _default_detect_secrets_config = {
|
|||
|
||||
|
||||
class _ENTERPRISE_SecretDetection(CustomGuardrail):
|
||||
# Keeps proxied traffic on async_pre_call_hook (the unified apply_guardrail
|
||||
# path skips should_run_check and never sees data["prompt"]).
|
||||
use_native_lifecycle_hooks: ClassVar[bool] = True
|
||||
|
||||
def __init__(self, detect_secrets_config: Optional[dict] = None, **kwargs):
|
||||
self.user_defined_detect_secrets_config = detect_secrets_config
|
||||
super().__init__(**kwargs)
|
||||
|
|
@ -455,6 +477,26 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
|
|||
|
||||
return detected_secrets
|
||||
|
||||
def redact_text(self, text: str, source: str = "message") -> str:
|
||||
"""Replace every detected secret in ``text`` with ``[REDACTED]`` and
|
||||
tally the detected types into the per-invocation masked-entity count."""
|
||||
detected_secrets = self.scan_message_for_secrets(text)
|
||||
if not detected_secrets:
|
||||
return text
|
||||
counts = _masked_entity_count.get()
|
||||
if counts is not None:
|
||||
for secret in detected_secrets:
|
||||
counts[secret["type"]] = counts.get(secret["type"], 0) + 1
|
||||
secret_types = [secret["type"] for secret in detected_secrets]
|
||||
verbose_proxy_logger.warning(
|
||||
f"Detected and redacted secrets in {source}: {secret_types}"
|
||||
)
|
||||
return functools.reduce(
|
||||
lambda redacted, secret: redacted.replace(secret["value"], "[REDACTED]"),
|
||||
detected_secrets,
|
||||
text,
|
||||
)
|
||||
|
||||
async def should_run_check(self, user_api_key_dict: UserAPIKeyAuth) -> bool:
|
||||
if user_api_key_dict.permissions is not None:
|
||||
if GUARDRAIL_NAME in user_api_key_dict.permissions:
|
||||
|
|
@ -463,7 +505,45 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
|
|||
|
||||
return True
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
request_data: dict,
|
||||
input_type: Literal["request", "response"],
|
||||
logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
"""Unified-interface entrypoint, used by /guardrails/apply_guardrail
|
||||
(the UI test playground). Proxied traffic keeps using
|
||||
``async_pre_call_hook``, see ``use_native_lifecycle_hooks``."""
|
||||
texts = inputs.get("texts")
|
||||
if not texts or not any(texts):
|
||||
return inputs
|
||||
_masked_entity_count.set({})
|
||||
return {**inputs, "texts": [self.redact_text(text) for text in texts]}
|
||||
|
||||
def _redact_prompt(self, data: dict) -> int:
|
||||
"""Redact ``data["prompt"]`` (the text-completion shape, which
|
||||
``walk_user_text`` does not cover) and return how many non-empty
|
||||
strings were inspected."""
|
||||
prompt = data.get("prompt")
|
||||
if isinstance(prompt, str):
|
||||
if not prompt:
|
||||
return 0
|
||||
data["prompt"] = self.redact_text(prompt, source="prompt")
|
||||
return 1
|
||||
if isinstance(prompt, list):
|
||||
data["prompt"] = [ # mutable-ok: data["prompt"] is a list on the wire
|
||||
self.redact_text(item, source="prompt")
|
||||
if isinstance(item, str) and item
|
||||
else item
|
||||
for item in prompt
|
||||
]
|
||||
return sum(1 for item in prompt if isinstance(item, str) and item)
|
||||
return 0
|
||||
|
||||
#### CALL HOOKS - proxy only ####
|
||||
@log_guardrail_information
|
||||
async def async_pre_call_hook(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
|
|
@ -471,53 +551,84 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
|
|||
data: dict,
|
||||
call_type: str, # "completion", "embeddings", "image_generation", "moderation"
|
||||
):
|
||||
_masked_entity_count.set(None)
|
||||
if await self.should_run_check(user_api_key_dict) is False:
|
||||
return
|
||||
|
||||
_masked_entity_count.set({})
|
||||
|
||||
# Covers multimodal list content + Responses-API input.
|
||||
def _redact_message_text(text: str) -> str:
|
||||
detected_secrets = self.scan_message_for_secrets(text)
|
||||
for secret in detected_secrets:
|
||||
text = text.replace(secret["value"], "[REDACTED]")
|
||||
if detected_secrets:
|
||||
secret_types = [secret["type"] for secret in detected_secrets]
|
||||
verbose_proxy_logger.warning(
|
||||
f"Detected and redacted secrets in message: {secret_types}"
|
||||
)
|
||||
return text
|
||||
inspected = walk_user_text(data, self.redact_text) + self._redact_prompt(data)
|
||||
|
||||
walk_user_text(data, _redact_message_text)
|
||||
if inspected == 0:
|
||||
# Image-only, empty-text, and unsupported payloads inspected
|
||||
# nothing, so recording "allow" would count a run that never
|
||||
# looked at any content.
|
||||
_masked_entity_count.set(None)
|
||||
|
||||
if "prompt" in data:
|
||||
if isinstance(data["prompt"], str):
|
||||
detected_secrets = self.scan_message_for_secrets(data["prompt"])
|
||||
for secret in detected_secrets:
|
||||
data["prompt"] = data["prompt"].replace(
|
||||
secret["value"], "[REDACTED]"
|
||||
)
|
||||
if len(detected_secrets) > 0:
|
||||
secret_types = [secret["type"] for secret in detected_secrets]
|
||||
verbose_proxy_logger.warning(
|
||||
f"Detected and redacted secrets in prompt: {secret_types}"
|
||||
)
|
||||
elif isinstance(data["prompt"], list):
|
||||
# Index back into the list — assigning to ``item`` would only
|
||||
# rebind the loop variable and leave ``data["prompt"]``
|
||||
# carrying the unredacted secret.
|
||||
for idx, item in enumerate(data["prompt"]):
|
||||
if isinstance(item, str):
|
||||
detected_secrets = self.scan_message_for_secrets(item)
|
||||
for secret in detected_secrets:
|
||||
item = item.replace(secret["value"], "[REDACTED]")
|
||||
data["prompt"][idx] = item
|
||||
if len(detected_secrets) > 0:
|
||||
secret_types = [
|
||||
secret["type"] for secret in detected_secrets
|
||||
]
|
||||
verbose_proxy_logger.warning(
|
||||
f"Detected and redacted secrets in prompt: {secret_types}"
|
||||
)
|
||||
|
||||
# ``data["input"]`` (Responses API and embeddings/moderation) is
|
||||
# already covered by ``walk_user_text`` above.
|
||||
return
|
||||
|
||||
def _process_response(
|
||||
self,
|
||||
response: Optional[dict],
|
||||
request_data: dict,
|
||||
start_time: Optional[float] = None,
|
||||
end_time: Optional[float] = None,
|
||||
duration: Optional[float] = None,
|
||||
event_type: Optional[GuardrailEventHooks] = None,
|
||||
original_inputs: Optional[dict] = None,
|
||||
):
|
||||
"""Record allow/mask plus the masked-entity tally for a completed run.
|
||||
|
||||
Records nothing when the guardrail inspected nothing (opted-out key,
|
||||
empty inputs) or when the instance has no guardrail_name (legacy
|
||||
``litellm_settings.callbacks`` deployments, which predate guardrail
|
||||
telemetry and stay without it).
|
||||
"""
|
||||
counts = _masked_entity_count.get()
|
||||
_masked_entity_count.set(None)
|
||||
if counts is None or self.guardrail_name is None:
|
||||
return response
|
||||
self.add_standard_logging_guardrail_information_to_request_data(
|
||||
guardrail_json_response="mask" if counts else "allow",
|
||||
request_data=request_data,
|
||||
guardrail_status="success",
|
||||
duration=duration,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
event_type=event_type,
|
||||
guardrail_provider=GUARDRAIL_PROVIDER,
|
||||
masked_entity_count=counts,
|
||||
)
|
||||
return response
|
||||
|
||||
def _process_error(
|
||||
self,
|
||||
e: Exception,
|
||||
request_data: dict,
|
||||
start_time: Optional[float] = None,
|
||||
end_time: Optional[float] = None,
|
||||
duration: Optional[float] = None,
|
||||
event_type: Optional[GuardrailEventHooks] = None,
|
||||
):
|
||||
"""Label the failed run with this guardrail's provider so error rows
|
||||
group with the successful ones in the monitor. Nameless legacy
|
||||
instances record nothing, matching ``_process_response``."""
|
||||
_masked_entity_count.set(None)
|
||||
if self.guardrail_name is None:
|
||||
raise e
|
||||
self.add_standard_logging_guardrail_information_to_request_data(
|
||||
guardrail_json_response=e,
|
||||
request_data=request_data,
|
||||
guardrail_status=(
|
||||
"guardrail_intervened"
|
||||
if self._is_guardrail_intervention(e)
|
||||
else "guardrail_failed_to_respond"
|
||||
),
|
||||
duration=duration,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
event_type=event_type,
|
||||
guardrail_provider=GUARDRAIL_PROVIDER,
|
||||
)
|
||||
raise e
|
||||
|
|
|
|||
|
|
@ -1485,7 +1485,7 @@ def log_guardrail_information(func):
|
|||
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
|
||||
original_inputs = kwargs.get("inputs")
|
||||
|
||||
logging_obj: Final = kwargs.get("logging_obj")
|
||||
logging_obj: Final = kwargs.get("logging_obj") or request_data.get("litellm_logging_obj")
|
||||
self_recorded_token: Final = _guardrail_self_recorded.set(False)
|
||||
try:
|
||||
response: Final = await func(*args, **kwargs)
|
||||
|
|
@ -1527,7 +1527,7 @@ def log_guardrail_information(func):
|
|||
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
|
||||
original_inputs = kwargs.get("inputs")
|
||||
|
||||
logging_obj: Final = kwargs.get("logging_obj")
|
||||
logging_obj: Final = kwargs.get("logging_obj") or request_data.get("litellm_logging_obj")
|
||||
self_recorded_token: Final = _guardrail_self_recorded.set(False)
|
||||
try:
|
||||
response: Final = func(*args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import json
|
|||
import os
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from types import UnionType
|
||||
from types import MappingProxyType, UnionType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, Union, cast, get_args, get_origin
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
|
@ -51,6 +51,9 @@ from litellm.types.guardrails import (
|
|||
SupportedGuardrailIntegrations,
|
||||
ToolPermissionGuardrailConfigModel,
|
||||
)
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.hide_secrets import (
|
||||
HideSecretsGuardrailConfigModel,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from types import CodeType
|
||||
|
|
@ -1401,7 +1404,11 @@ async def get_guardrail_ui_settings():
|
|||
provider: [hook.value for hook in hooks]
|
||||
for provider, guardrail_class in guardrail_class_registry.items()
|
||||
if (hooks := guardrail_class.get_supported_event_hooks()) is not None
|
||||
}
|
||||
} | MappingProxyType(
|
||||
# hide-secrets lives in the enterprise package, not in the registry
|
||||
# above; it only runs on pre_call.
|
||||
{SupportedGuardrailIntegrations.HIDE_SECRETS.value: [GuardrailEventHooks.pre_call.value]}
|
||||
)
|
||||
|
||||
return GuardrailUIAddGuardrailSettings(
|
||||
supported_entities=[entity.value for entity in PiiEntityType],
|
||||
|
|
@ -1953,12 +1960,18 @@ async def get_provider_specific_params():
|
|||
|
||||
tool_permission_fields["ui_friendly_name"] = ToolPermissionGuardrailConfigModel.ui_friendly_name()
|
||||
|
||||
# hide-secrets lives in the enterprise package, not in the registry loop below.
|
||||
hide_secrets_fields: Final = _get_fields_from_model(HideSecretsGuardrailConfigModel)
|
||||
|
||||
hide_secrets_fields["ui_friendly_name"] = HideSecretsGuardrailConfigModel.ui_friendly_name()
|
||||
|
||||
# Return the provider-specific parameters
|
||||
provider_params: Final = {
|
||||
SupportedGuardrailIntegrations.BEDROCK.value: bedrock_fields,
|
||||
SupportedGuardrailIntegrations.PRESIDIO.value: presidio_fields,
|
||||
SupportedGuardrailIntegrations.LAKERA_V2.value: lakera_v2_fields,
|
||||
SupportedGuardrailIntegrations.TOOL_PERMISSION.value: tool_permission_fields,
|
||||
SupportedGuardrailIntegrations.HIDE_SECRETS.value: hide_secrets_fields,
|
||||
}
|
||||
|
||||
### get the config model for the guardrail - go through the registry and get the config model for the guardrail
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
"""Types for the Hide Secrets guardrail."""
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from .base import GuardrailConfigModel
|
||||
|
||||
|
||||
class HideSecretsGuardrailConfigModel(GuardrailConfigModel):
|
||||
"""Configuration for the Hide Secrets guardrail. Detection runs in-process
|
||||
on the detect-secrets library; ``detect_secrets_config`` overrides the
|
||||
bundled plugin set."""
|
||||
|
||||
detect_secrets_config: dict | None = Field( # mutable-ok: UI type derivation maps dict to "object"
|
||||
default=None,
|
||||
description="Optional detect-secrets configuration (plugins_used, filters_used) overriding the bundled plugin set",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def ui_friendly_name() -> str:
|
||||
return "Hide Secrets"
|
||||
|
|
@ -0,0 +1,274 @@
|
|||
"""Tests for the hide-secrets guardrail (LIT-3548).
|
||||
|
||||
Covers the three defects from the ticket:
|
||||
- ``apply_guardrail`` (the UI test playground path) must redact, not echo.
|
||||
- Guardrail runs must record ``standard_logging_guardrail_information`` so
|
||||
Spend Logs / the guardrails monitor show activity, with hits ("mask" +
|
||||
masked_entity_count) distinguishable from clean requests ("allow").
|
||||
- Defining ``apply_guardrail`` must NOT reroute proxied traffic off the
|
||||
native ``async_pre_call_hook`` (per-key opt-out and ``data["prompt"]``
|
||||
handling live only on the native path).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm_enterprise.enterprise_callbacks.secret_detection import (
|
||||
_ENTERPRISE_SecretDetection,
|
||||
)
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
AWS_KEY = "AKIAIOSFODNN7EXAMPLE"
|
||||
|
||||
|
||||
def _guardrail() -> _ENTERPRISE_SecretDetection:
|
||||
return _ENTERPRISE_SecretDetection(
|
||||
guardrail_name="hide-secrets", event_hook="pre_call", default_on=True
|
||||
)
|
||||
|
||||
|
||||
def _recorded(request_data: dict) -> dict:
|
||||
entries = request_data["metadata"]["standard_logging_guardrail_information"]
|
||||
assert len(entries) == 1
|
||||
return entries[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_guardrail_redacts_secrets():
|
||||
"""Playground path: the returned texts must carry [REDACTED], not the secret."""
|
||||
guardrail = _guardrail()
|
||||
request_data: dict = {"metadata": {}}
|
||||
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [f"my key is {AWS_KEY}, keep it safe"]},
|
||||
request_data=request_data,
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert result["texts"] == ["my key is [REDACTED], keep it safe"]
|
||||
|
||||
recorded = _recorded(request_data)
|
||||
assert recorded["guardrail_status"] == "success"
|
||||
assert recorded["guardrail_response"] == "mask"
|
||||
assert recorded["guardrail_provider"] == "hide-secrets"
|
||||
assert recorded["masked_entity_count"] == {"AWS Access Key": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_guardrail_clean_text_records_allow():
|
||||
guardrail = _guardrail()
|
||||
request_data: dict = {"metadata": {}}
|
||||
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": ["nothing sensitive here"]},
|
||||
request_data=request_data,
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert result["texts"] == ["nothing sensitive here"]
|
||||
|
||||
recorded = _recorded(request_data)
|
||||
assert recorded["guardrail_status"] == "success"
|
||||
assert recorded["guardrail_response"] == "allow"
|
||||
assert recorded["masked_entity_count"] == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_hook_records_mask_with_entity_count():
|
||||
"""Live-traffic path: a redaction must be visible in spend-log telemetry."""
|
||||
guardrail = _guardrail()
|
||||
data = {
|
||||
"messages": [{"role": "user", "content": f"use {AWS_KEY} for auth"}],
|
||||
"metadata": {},
|
||||
}
|
||||
|
||||
await guardrail.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
cache=DualCache(),
|
||||
data=data,
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
assert data["messages"][0]["content"] == "use [REDACTED] for auth"
|
||||
|
||||
recorded = _recorded(data)
|
||||
assert recorded["guardrail_status"] == "success"
|
||||
assert recorded["guardrail_response"] == "mask"
|
||||
assert recorded["guardrail_provider"] == "hide-secrets"
|
||||
assert recorded["masked_entity_count"] == {"AWS Access Key": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_hook_clean_request_records_allow():
|
||||
"""A request with no secrets must be distinguishable from a redacted one."""
|
||||
guardrail = _guardrail()
|
||||
data = {
|
||||
"messages": [{"role": "user", "content": "what's the weather"}],
|
||||
"metadata": {},
|
||||
}
|
||||
|
||||
await guardrail.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
cache=DualCache(),
|
||||
data=data,
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
recorded = _recorded(data)
|
||||
assert recorded["guardrail_status"] == "success"
|
||||
assert recorded["guardrail_response"] == "allow"
|
||||
assert recorded["masked_entity_count"] == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_hook_opt_out_records_nothing():
|
||||
"""A key with permissions={"hide_secrets": False} skips redaction, so no
|
||||
telemetry is recorded: every reader of a recorded entry (guardrail usage
|
||||
tracking, compliance checks, the spend-log viewer) counts it as a run."""
|
||||
guardrail = _guardrail()
|
||||
content = f"my key is {AWS_KEY}"
|
||||
data = {"messages": [{"role": "user", "content": content}], "metadata": {}}
|
||||
|
||||
await guardrail.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(permissions={"hide_secrets": False}),
|
||||
cache=DualCache(),
|
||||
data=data,
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
assert data["messages"][0]["content"] == content # untouched
|
||||
assert "standard_logging_guardrail_information" not in data["metadata"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_hook_still_redacts_text_completion_prompt():
|
||||
"""data["prompt"] (str and list) is a native-hook-only surface; it must
|
||||
keep redacting now that the class also implements apply_guardrail."""
|
||||
guardrail = _guardrail()
|
||||
data = {"prompt": f"key {AWS_KEY} end", "metadata": {}}
|
||||
await guardrail.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
cache=DualCache(),
|
||||
data=data,
|
||||
call_type="completion",
|
||||
)
|
||||
assert data["prompt"] == "key [REDACTED] end"
|
||||
|
||||
guardrail = _guardrail()
|
||||
data = {"prompt": [f"key {AWS_KEY}", "clean"], "metadata": {}}
|
||||
await guardrail.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
cache=DualCache(),
|
||||
data=data,
|
||||
call_type="completion",
|
||||
)
|
||||
assert data["prompt"] == ["key [REDACTED]", "clean"]
|
||||
|
||||
|
||||
def test_proxied_traffic_stays_on_native_hooks():
|
||||
"""Implementing apply_guardrail must not reroute proxied requests onto the
|
||||
unified path: that path skips ``should_run_check`` (per-key opt-out) and
|
||||
never sees ``data["prompt"]``."""
|
||||
guardrail = _guardrail()
|
||||
assert guardrail.uses_apply_guardrail_interface() is True
|
||||
assert guardrail._deployment_pre_call_target() is guardrail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_guardrail_without_texts_records_nothing():
|
||||
"""No inputs means nothing was inspected, so no "allow" row is recorded.
|
||||
Empty strings count as no input: there is no content to inspect."""
|
||||
guardrail = _guardrail()
|
||||
|
||||
empty_variants: list[list[str]] = [[], ["", ""]]
|
||||
for texts in empty_variants:
|
||||
request_data: dict = {"metadata": {}}
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": texts}, request_data=request_data, input_type="request"
|
||||
)
|
||||
assert result == {"texts": texts}
|
||||
assert "standard_logging_guardrail_information" not in request_data["metadata"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"data",
|
||||
[
|
||||
pytest.param(
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": "https://x/y.png"}}
|
||||
],
|
||||
}
|
||||
],
|
||||
"metadata": {},
|
||||
},
|
||||
id="image_only",
|
||||
),
|
||||
pytest.param(
|
||||
{"messages": [{"role": "user", "content": ""}], "metadata": {}},
|
||||
id="empty_message",
|
||||
),
|
||||
pytest.param({"prompt": "", "metadata": {}}, id="empty_prompt"),
|
||||
pytest.param({"prompt": ["", ""], "metadata": {}}, id="empty_prompt_list"),
|
||||
],
|
||||
)
|
||||
async def test_pre_call_hook_without_inspectable_text_records_nothing(data: dict):
|
||||
"""A payload the guardrail could not inspect (image-only content, empty
|
||||
strings) must not record an "allow" run: monitoring would count a check
|
||||
that never looked at any text."""
|
||||
guardrail = _guardrail()
|
||||
|
||||
await guardrail.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
cache=DualCache(),
|
||||
data=data,
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
assert "standard_logging_guardrail_information" not in data["metadata"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_hook_mixed_prompt_list_still_redacts_and_records():
|
||||
"""A prompt list mixing empty and real strings is inspected, so the run is
|
||||
recorded and the non-empty entry is still redacted."""
|
||||
guardrail = _guardrail()
|
||||
data = {"prompt": ["", f"key {AWS_KEY}"], "metadata": {}}
|
||||
|
||||
await guardrail.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
cache=DualCache(),
|
||||
data=data,
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
assert data["prompt"] == ["", "key [REDACTED]"]
|
||||
recorded = _recorded(data)
|
||||
assert recorded["guardrail_response"] == "mask"
|
||||
assert recorded["masked_entity_count"] == {"AWS Access Key": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_legacy_nameless_instance_records_nothing():
|
||||
"""``litellm_settings.callbacks: ["hide_secrets"]`` builds an arg-less
|
||||
instance with no guardrail_name. It still redacts, but recording a nameless
|
||||
entry would flip every spend row's guardrail status with nothing to join on."""
|
||||
guardrail = _ENTERPRISE_SecretDetection()
|
||||
data = {
|
||||
"messages": [{"role": "user", "content": f"use {AWS_KEY} for auth"}],
|
||||
"metadata": {},
|
||||
}
|
||||
|
||||
await guardrail.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
cache=DualCache(),
|
||||
data=data,
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
assert data["messages"][0]["content"] == "use [REDACTED] for auth"
|
||||
assert "standard_logging_guardrail_information" not in data["metadata"]
|
||||
|
|
@ -670,6 +670,37 @@ def test_get_provider_specific_params():
|
|||
) # Literal type should be select
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_provider_specific_params_includes_hide_secrets():
|
||||
"""hide-secrets lives in the enterprise package so it is not in
|
||||
guardrail_class_registry; the endpoint must still advertise it or the
|
||||
Add Guardrail UI dropdown never offers it (LIT-3548)."""
|
||||
from litellm.proxy.guardrails.guardrail_endpoints import (
|
||||
get_provider_specific_params,
|
||||
)
|
||||
|
||||
provider_params = await get_provider_specific_params()
|
||||
|
||||
assert "hide-secrets" in provider_params
|
||||
# populateGuardrailProviders() in the dashboard only lists providers whose
|
||||
# entry carries a ui_friendly_name.
|
||||
assert provider_params["hide-secrets"]["ui_friendly_name"] == "Hide Secrets"
|
||||
assert provider_params["hide-secrets"]["detect_secrets_config"]["required"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_guardrail_settings_restricts_hide_secrets_to_pre_call():
|
||||
"""hide-secrets only implements async_pre_call_hook, so offering the other
|
||||
modes in the UI would create configs that boot clean and never run."""
|
||||
from litellm.proxy.guardrails.guardrail_endpoints import (
|
||||
get_guardrail_ui_settings,
|
||||
)
|
||||
|
||||
settings = await get_guardrail_ui_settings()
|
||||
|
||||
assert settings.supported_modes_by_provider["hide-secrets"] == ["pre_call"]
|
||||
|
||||
|
||||
def test_optional_params_not_returned_when_not_overridden():
|
||||
"""Test that optional_params is not returned when the config model doesn't override it"""
|
||||
from typing import Optional
|
||||
|
|
|
|||
|
|
@ -201,6 +201,7 @@ export const guardrailLogoMap = {
|
|||
XecGuard: xecguardLogo.src,
|
||||
"LiteLLM Content Filter": litellmLogo.src,
|
||||
"LiteLLM LLM as a Judge": litellmLogo.src,
|
||||
"Hide Secrets": litellmLogo.src,
|
||||
Akto: aktoLogo.src,
|
||||
"DeepKeep AI Firewall": deepkeepLogo.src,
|
||||
"Qostodian Nexus": qohashLogo.src,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
import React from "react";
|
||||
import { fireEvent, screen, waitFor } from "@testing-library/react";
|
||||
import { useForm } from "react-hook-form";
|
||||
import { renderWithProviders } from "@/../tests/test-utils";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import GuardrailProviderFields from "./guardrail_provider_fields";
|
||||
import { populateGuardrailProviderMap } from "./guardrail_info_helpers";
|
||||
import type { GuardrailFormValues } from "./GuardrailFormField";
|
||||
|
||||
vi.mock("@/lib/toast", () => ({ toast: { error: vi.fn() } }));
|
||||
|
||||
const HIDE_SECRETS_PARAMS = {
|
||||
"hide-secrets": {
|
||||
ui_friendly_name: "Hide Secrets",
|
||||
detect_secrets_config: {
|
||||
param: "detect_secrets_config",
|
||||
description: "Optional detect-secrets configuration",
|
||||
required: false,
|
||||
type: "object",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const Harness: React.FC<{ onValid: (values: GuardrailFormValues) => void }> = ({ onValid }) => {
|
||||
const form = useForm<GuardrailFormValues>();
|
||||
return (
|
||||
<form onSubmit={form.handleSubmit(onValid)}>
|
||||
<GuardrailProviderFields
|
||||
selectedProvider="Hide-secrets"
|
||||
control={form.control}
|
||||
providerParams={HIDE_SECRETS_PARAMS}
|
||||
/>
|
||||
<button type="submit">save</button>
|
||||
</form>
|
||||
);
|
||||
};
|
||||
|
||||
const renderHarness = () => {
|
||||
populateGuardrailProviderMap(HIDE_SECRETS_PARAMS);
|
||||
const onValid = vi.fn();
|
||||
renderWithProviders(<Harness onValid={onValid} />);
|
||||
const textarea = screen.getByLabelText(/detect_secrets_config/) as HTMLTextAreaElement;
|
||||
return { onValid, textarea };
|
||||
};
|
||||
|
||||
describe("GuardrailProviderFields object field", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("commits a valid JSON object as a parsed dict", async () => {
|
||||
const { onValid, textarea } = renderHarness();
|
||||
|
||||
fireEvent.change(textarea, { target: { value: '{"plugins_used": [{"name": "AWSKeyDetector"}]}' } });
|
||||
fireEvent.blur(textarea);
|
||||
fireEvent.click(screen.getByRole("button", { name: "save" }));
|
||||
|
||||
await waitFor(() => expect(onValid).toHaveBeenCalledTimes(1));
|
||||
expect(onValid.mock.calls[0][0].detect_secrets_config).toEqual({
|
||||
plugins_used: [{ name: "AWSKeyDetector" }],
|
||||
});
|
||||
});
|
||||
|
||||
it("blocks submission while the field holds malformed JSON", async () => {
|
||||
const { onValid, textarea } = renderHarness();
|
||||
|
||||
fireEvent.change(textarea, { target: { value: "{not json" } });
|
||||
fireEvent.blur(textarea);
|
||||
fireEvent.click(screen.getByRole("button", { name: "save" }));
|
||||
|
||||
await screen.findByText("detect_secrets_config must be a valid JSON object");
|
||||
expect(onValid).not.toHaveBeenCalled();
|
||||
expect(textarea.value).toBe("{not json");
|
||||
});
|
||||
|
||||
it.each(['["array"]', '"scalar"', "null", "42"])("blocks non-object JSON %s", async (raw) => {
|
||||
const { onValid, textarea } = renderHarness();
|
||||
|
||||
fireEvent.change(textarea, { target: { value: raw } });
|
||||
fireEvent.blur(textarea);
|
||||
fireEvent.click(screen.getByRole("button", { name: "save" }));
|
||||
|
||||
await screen.findByText("detect_secrets_config must be a valid JSON object");
|
||||
expect(onValid).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("treats a cleared field as unset and submits", async () => {
|
||||
const { onValid, textarea } = renderHarness();
|
||||
|
||||
fireEvent.change(textarea, { target: { value: '{"a": 1}' } });
|
||||
fireEvent.blur(textarea);
|
||||
fireEvent.change(textarea, { target: { value: "" } });
|
||||
fireEvent.blur(textarea);
|
||||
fireEvent.click(screen.getByRole("button", { name: "save" }));
|
||||
|
||||
await waitFor(() => expect(onValid).toHaveBeenCalledTimes(1));
|
||||
expect(onValid.mock.calls[0][0].detect_secrets_config).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
@ -13,6 +13,8 @@ import { FieldGroup } from "@/components/ui/field";
|
|||
import { Input } from "@/components/ui/input";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { Slider } from "@/components/ui/slider";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
|
||||
import {
|
||||
asStringArray,
|
||||
|
|
@ -22,6 +24,7 @@ import {
|
|||
readRecord,
|
||||
requiredRule,
|
||||
type GuardrailFieldControlProps,
|
||||
type GuardrailFieldRules,
|
||||
type GuardrailFormControl,
|
||||
} from "./GuardrailFormField";
|
||||
|
||||
|
|
@ -60,6 +63,44 @@ const BOOLEAN_ITEMS = [
|
|||
const isSecretKey = (fieldKey: string): boolean =>
|
||||
fieldKey.includes("password") || fieldKey.includes("secret") || fieldKey.includes("key");
|
||||
|
||||
const isPlainObject = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
// Object fields hold the raw text while the user types, so submission must be
|
||||
// blocked until the value parses to a plain JSON object (or is cleared).
|
||||
const jsonObjectRule = (fieldKey: string): GuardrailFieldRules => ({
|
||||
validate: (value: unknown) =>
|
||||
value === undefined || isPlainObject(value) ? true : `${fieldKey} must be a valid JSON object`,
|
||||
});
|
||||
|
||||
// Commits a parsed object (or undefined for a cleared field) to the form on
|
||||
// blur; anything else stays as raw text so jsonObjectRule blocks submission.
|
||||
const commitObjectField = (raw: string, onChange: (value: unknown) => void): void => {
|
||||
const next = raw.trim();
|
||||
if (next === "") {
|
||||
onChange(undefined);
|
||||
return;
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(next);
|
||||
} catch {
|
||||
parsed = next;
|
||||
}
|
||||
if (isPlainObject(parsed)) {
|
||||
onChange(parsed);
|
||||
} else {
|
||||
toast.error("Enter a valid JSON object for this configuration");
|
||||
}
|
||||
};
|
||||
|
||||
const fieldRules = (field: ProviderParam, fieldKey: string): GuardrailFieldRules | undefined => {
|
||||
if (field.type === "object") {
|
||||
return jsonObjectRule(fieldKey);
|
||||
}
|
||||
return field.required ? requiredRule(`${fieldKey} is required`) : undefined;
|
||||
};
|
||||
|
||||
interface ProviderFieldInputProps {
|
||||
descriptor: ProviderParam;
|
||||
fieldKey: string;
|
||||
|
|
@ -141,6 +182,25 @@ const ProviderFieldInput: React.FC<ProviderFieldInputProps> = ({ descriptor, fie
|
|||
);
|
||||
}
|
||||
|
||||
if (descriptor.type === "object") {
|
||||
const objectValue = typeof value === "object" && value !== null ? JSON.stringify(value, null, 2) : asText(value);
|
||||
return (
|
||||
<Textarea
|
||||
id={id}
|
||||
name={name}
|
||||
ref={ref}
|
||||
placeholder={descriptor.description}
|
||||
value={objectValue}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
onBlur={(event) => {
|
||||
commitObjectField(event.target.value, onChange);
|
||||
onBlur();
|
||||
}}
|
||||
{...aria}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (descriptor.type === "number") {
|
||||
return (
|
||||
<NumericalInput
|
||||
|
|
@ -316,7 +376,7 @@ const GuardrailProviderFields: React.FC<GuardrailProviderFieldsProps> = ({
|
|||
control={control}
|
||||
name={fullFieldKey}
|
||||
label={labelWithHint(fieldKey, field.description)}
|
||||
rules={field.required ? requiredRule(`${fieldKey} is required`) : undefined}
|
||||
rules={fieldRules(field, fieldKey)}
|
||||
defaultValue={resolvedInitialValue}
|
||||
>
|
||||
{(fieldControl) => <ProviderFieldInput descriptor={field} fieldKey={fieldKey} control={fieldControl} />}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue