Make tone detection selectable as a category in the Content Filter UI

- Add categories/tone_detection.yaml with metadata so it appears in the
  ContentCategoryConfiguration dropdown alongside other categories
- Wire _load_categories() to detect tone_detection category and enable the
  ToneChecker engine (regex-based, response-only) instead of loading keywords
- Add tests: category-based enablement, blocking, and disabled state

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Krrish Dholakia 2026-03-11 22:41:22 -07:00
parent 934b67ae43
commit 8304b3ae75
3 changed files with 64 additions and 0 deletions

View file

@ -0,0 +1,24 @@
# Tone Detection — CPU-only regex detection of inappropriate chatbot tone
#
# Detects 6 categories of inappropriate tone in customer-facing chatbot responses:
# - Dismissive language ("that's not my problem", "just read the FAQ")
# - Blaming the customer ("you should have read the terms", "that's your fault")
# - Refusal to help ("I can't help you", "figure it out yourself")
# - Sarcasm / condescension ("if you'd been paying attention", "let me spell it out")
# - Impatience / frustration ("I've already told you", "are you even listening")
# - Unprofessional casual language ("bruh", "lol", "sounds like a you problem")
#
# This category uses regex-based pattern matching (not keyword lists).
# When selected, it enables the ToneChecker engine which runs on LLM responses only.
#
# Advanced configuration (safe_phrases, blocked_phrases) can be set via
# tone_detection_config in litellm_params for programmatic use.
category_name: "tone_detection"
display_name: "Tone Detection"
description: "Detects dismissive, blaming, condescending, impatient, or unprofessional tone in customer-facing chatbot responses"
default_action: "BLOCK"
# No keywords — tone detection uses the ToneChecker regex engine
keywords: []
exceptions: []

View file

@ -456,6 +456,16 @@ class ContentFilterGuardrail(CustomGuardrail):
)
continue
# Tone detection is a special category — it enables the ToneChecker
# regex engine rather than loading keyword lists.
if category_name == "tone_detection":
if self._tone_checker is None:
self._init_tone_checker({})
verbose_proxy_logger.info(
"Loaded tone_detection category (ToneChecker enabled)"
)
continue
# Load category file (custom or default)
if custom_file:
category_file_path = self._resolve_category_file_path(custom_file)

View file

@ -425,6 +425,36 @@ class TestInit:
g = ContentFilterGuardrail(guardrail_name="test-no-tone")
assert g._tone_checker is None
def test_tone_checker_enabled_via_category(self):
"""Selecting tone_detection as a category should enable the ToneChecker."""
g = ContentFilterGuardrail(
guardrail_name="test-category-tone",
categories=[{"category": "tone_detection", "enabled": True, "action": "BLOCK"}],
)
assert g._tone_checker is not None
@pytest.mark.asyncio
async def test_tone_detection_via_category_blocks(self):
"""Tone detection enabled via category selection should block bad tone."""
g = ContentFilterGuardrail(
guardrail_name="test-category-tone",
categories=[{"category": "tone_detection", "enabled": True, "action": "BLOCK"}],
)
with pytest.raises(HTTPException):
await g.apply_guardrail(
_inputs("That's not my problem."),
{},
"response",
)
def test_tone_detection_category_disabled(self):
"""Disabling the tone_detection category should NOT enable ToneChecker."""
g = ContentFilterGuardrail(
guardrail_name="test-category-disabled",
categories=[{"category": "tone_detection", "enabled": False, "action": "BLOCK"}],
)
assert g._tone_checker is None
def test_invalid_regex_degrades_gracefully(self):
"""Invalid regex in blocked_phrases should degrade gracefully (tone checker disabled)."""
g = _make_guardrail(blocked_phrases=[r"(unclosed"])