From 28157ba59d3ecb40b8378abc693b79860ddf1046 Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Thu, 9 Apr 2026 18:46:14 -0700 Subject: [PATCH] change content filter status code to 400 (#25391) --- .../litellm_content_filter/content_filter.py | 22 ++--- .../guardrail_benchmarks/test_eval.py | 4 +- .../content_filter/test_competitor_intent.py | 92 +++++++++++++------ .../content_filter/test_content_filter.py | 44 ++++----- 4 files changed, 99 insertions(+), 63 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index e4da1c1ae77..5bdf3d8a7e6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -212,17 +212,17 @@ class ContentFilterGuardrail(CustomGuardrail): self.image_model = image_model # Store loaded categories self.loaded_categories: Dict[str, CategoryConfig] = {} - self.category_keywords: Dict[ - str, Tuple[str, str, ContentFilterAction] - ] = {} # keyword -> (category, severity, action) + self.category_keywords: Dict[str, Tuple[str, str, ContentFilterAction]] = ( + {} + ) # keyword -> (category, severity, action) # Always-block keywords are checked after exceptions (exceptions take precedence) self.always_block_category_keywords: Dict[ str, Tuple[str, str, ContentFilterAction] ] = {} # Store conditional categories (identifier_words + block_words) - self.conditional_categories: Dict[ - str, Dict[str, Any] - ] = {} # category_name -> {identifier_words, block_words, action, severity} + self.conditional_categories: Dict[str, Dict[str, Any]] = ( + {} + ) # category_name -> {identifier_words, block_words, action, severity} # Competitor intent checker (optional; airline uses major_airlines.json, generic requires competitors) self._competitor_intent_checker: Optional[BaseCompetitorIntentChecker] = None @@ -1202,7 +1202,7 @@ class ContentFilterGuardrail(CustomGuardrail): ) verbose_proxy_logger.warning(error_msg) raise HTTPException( - status_code=403, + status_code=400, detail={ "error": error_msg, "category": category_name, @@ -1242,7 +1242,7 @@ class ContentFilterGuardrail(CustomGuardrail): ) verbose_proxy_logger.warning(error_msg) raise HTTPException( - status_code=403, + status_code=400, detail={ "error": error_msg, "category": category_name, @@ -1285,7 +1285,7 @@ class ContentFilterGuardrail(CustomGuardrail): error_msg = f"Content blocked: {pattern_name} pattern detected" verbose_proxy_logger.warning(error_msg) raise HTTPException( - status_code=403, + status_code=400, detail={"error": error_msg, "pattern": pattern_name}, ) elif action == ContentFilterAction.MASK: @@ -1325,7 +1325,7 @@ class ContentFilterGuardrail(CustomGuardrail): error_msg += f" ({description})" verbose_proxy_logger.warning(error_msg) raise HTTPException( - status_code=403, + status_code=400, detail={ "error": error_msg, "keyword": keyword, @@ -1677,7 +1677,7 @@ class ContentFilterGuardrail(CustomGuardrail): "ContentFilterGuardrail: competitor intent refuse - %s", intent_val ) raise HTTPException( - status_code=403, + status_code=400, detail={ "error": msg, "intent": intent_val, diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py index 56398739b9b..aedc6acc810 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py @@ -59,7 +59,7 @@ def _run(checker, text: str) -> dict: checker.check(text) return {"decision": "ALLOW", "score": 0.0, "matched_topic": None} except HTTPException as e: - if e.status_code == 403: + if e.status_code == 400: detail: Dict[str, Any] = e.detail if isinstance(e.detail, dict) else {} return { "decision": "BLOCK", @@ -542,7 +542,7 @@ class _LlmJudgeChecker: if "BLOCK" in decision: raise HTTPException( - status_code=403, + status_code=400, detail={ "error": "Content blocked by LLM judge", "topic": "financial_advice", diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_competitor_intent.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_competitor_intent.py index 3f4098ba7e0..723d4b7db75 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_competitor_intent.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_competitor_intent.py @@ -5,7 +5,10 @@ Tests for competitor intent detection (normalize, entity layer, scoring, policy) import pytest from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent import ( - AirlineCompetitorIntentChecker, normalize, text_for_entity_matching) + AirlineCompetitorIntentChecker, + normalize, + text_for_entity_matching, +) class TestNormalize: @@ -70,8 +73,13 @@ class TestAirlineCompetitorIntentChecker: def test_run_competitor_comparison_direct(self, generic_config): checker = AirlineCompetitorIntentChecker(generic_config) result = checker.run("Is Qatar better than Emirates?") - assert result["intent"] in ("competitor_comparison", "possible_competitor_comparison") - assert "competitor_entity" in result.get("signals", []) or "competitors" in str(result.get("entities", {})) + assert result["intent"] in ( + "competitor_comparison", + "possible_competitor_comparison", + ) + assert "competitor_entity" in result.get("signals", []) or "competitors" in str( + result.get("entities", {}) + ) assert result["confidence"] >= 0.45 def test_run_competitor_comparison_as_good_as(self, generic_config): @@ -84,13 +92,20 @@ class TestAirlineCompetitorIntentChecker: checker = AirlineCompetitorIntentChecker(generic_config) result = checker.run("Why is Qatar Airways the best?") assert result["intent"] != "other" - assert "qatar" in str(result.get("entities", {}).get("competitors", [])).lower() or "competitor" in str(result.get("signals", [])) + assert "qatar" in str( + result.get("entities", {}).get("competitors", []) + ).lower() or "competitor" in str(result.get("signals", [])) def test_run_ranking_without_competitor_category_ranking(self, generic_config): checker = AirlineCompetitorIntentChecker(generic_config) result = checker.run("Which Gulf airline is the best?") # domain_words "airline" + ranking "best" + geo "gulf" not in route_geo_cues but "airline" is domain - assert result["intent"] in ("category_ranking", "possible_competitor_comparison", "log_only", "other") + assert result["intent"] in ( + "category_ranking", + "possible_competitor_comparison", + "log_only", + "other", + ) def test_run_evidence_populated(self, generic_config): checker = AirlineCompetitorIntentChecker(generic_config) @@ -118,8 +133,9 @@ class TestContentFilterWithCompetitorIntent: @pytest.mark.asyncio async def test_competitor_intent_type_airline_uses_airline_checker(self): """When competitor_intent_type is airline (default), use AirlineCompetitorIntentChecker.""" - from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import \ - ContentFilterGuardrail + from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, + ) guardrail = ContentFilterGuardrail( guardrail_name="test-airline", @@ -131,17 +147,23 @@ class TestContentFilterWithCompetitorIntent: }, ) assert guardrail._competitor_intent_checker is not None - from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent import \ - AirlineCompetitorIntentChecker - assert isinstance(guardrail._competitor_intent_checker, AirlineCompetitorIntentChecker) + from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent import ( + AirlineCompetitorIntentChecker, + ) + + assert isinstance( + guardrail._competitor_intent_checker, AirlineCompetitorIntentChecker + ) @pytest.mark.asyncio async def test_competitor_intent_type_generic_uses_base_checker(self): """When competitor_intent_type is generic, use BaseCompetitorIntentChecker.""" - from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent import \ - BaseCompetitorIntentChecker - from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import \ - ContentFilterGuardrail + from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent import ( + BaseCompetitorIntentChecker, + ) + from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, + ) guardrail = ContentFilterGuardrail( guardrail_name="test-generic", @@ -153,12 +175,15 @@ class TestContentFilterWithCompetitorIntent: }, ) assert guardrail._competitor_intent_checker is not None - assert isinstance(guardrail._competitor_intent_checker, BaseCompetitorIntentChecker) + assert isinstance( + guardrail._competitor_intent_checker, BaseCompetitorIntentChecker + ) @pytest.mark.asyncio async def test_apply_guardrail_with_competitor_intent_allow(self): - from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import \ - ContentFilterGuardrail + from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, + ) guardrail = ContentFilterGuardrail( guardrail_name="test-competitor", @@ -166,7 +191,10 @@ class TestContentFilterWithCompetitorIntent: "brand_self": ["emirates"], "competitors": ["qatar"], "domain_words": ["airline"], - "policy": {"competitor_comparison": "refuse", "possible_competitor_comparison": "reframe"}, + "policy": { + "competitor_comparison": "refuse", + "possible_competitor_comparison": "reframe", + }, }, ) inputs = {"texts": ["What is the capital of France?"]} @@ -179,8 +207,9 @@ class TestContentFilterWithCompetitorIntent: async def test_apply_guardrail_with_competitor_intent_refuse(self): from fastapi import HTTPException - from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import \ - ContentFilterGuardrail + from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, + ) guardrail = ContentFilterGuardrail( guardrail_name="test-competitor", @@ -197,7 +226,7 @@ class TestContentFilterWithCompetitorIntent: await guardrail.apply_guardrail( inputs, request_data={}, input_type="request" ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Exact config from litellm/proxy/_new_secret_config.yaml (lines 27-53). @@ -260,14 +289,22 @@ AIRLINE_COMPLIANCE_DATASET = [ ("airline_010", "How much connection time do I need in Doha?", "pass"), # Competitor Comparison (should FAIL) ("airline_011", "Is Qatar Airways better than Emirates?", "fail"), - ("airline_012", "Which airline has better business class, Qatar or Singapore Airlines?", "fail"), + ( + "airline_012", + "Which airline has better business class, Qatar or Singapore Airlines?", + "fail", + ), ("airline_013", "Which airline is the best for long haul flights?", "fail"), ("airline_014", "Should I choose Qatar Airways or another airline?", "fail"), ("airline_015", "Compare Turkish Airlines vs Lufthansa.", "fail"), ("airline_016", "Which airline has better lounges?", "fail"), ("airline_017", "Which airline has the best customer satisfaction?", "fail"), ("airline_018", "Is the Doha airline better than other carriers?", "fail"), - ("airline_019", "Should I switch to another airline for a better experience?", "fail"), + ( + "airline_019", + "Should I switch to another airline for a better experience?", + "fail", + ), ("airline_020", "Which airline is ranked number one worldwide?", "fail"), # Ambiguous Entity (should PASS) ("airline_021", "Qatar baggage allowance", "pass"), @@ -303,12 +340,11 @@ class TestAirlineComplianceDataset: f"{prompt_id}: expected pass, got intent={intent!r} action_hint={action_hint!r} for {prompt_text!r}" ) else: - blocked = ( - intent != "other" - and action_hint in ("refuse", "reframe") - ) + blocked = intent != "other" and action_hint in ("refuse", "reframe") if not blocked: failures.append( f"{prompt_id}: expected fail, got intent={intent!r} action_hint={action_hint!r} for {prompt_text!r}" ) - assert not failures, f"Airline compliance dataset failures:\n" + "\n".join(failures) + assert not failures, f"Airline compliance dataset failures:\n" + "\n".join( + failures + ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py index a1d3eb152bb..11bff0c0c04 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py @@ -198,7 +198,7 @@ class TestContentFilterGuardrail: input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "us_ssn" in str(exc_info.value.detail) @pytest.mark.asyncio @@ -498,7 +498,7 @@ class TestContentFilterGuardrail: ): pass - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "us_ssn" in str(exc_info.value.detail) def test_init_with_plain_dicts(self): @@ -666,7 +666,7 @@ class TestContentFilterGuardrail: input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "danger_word" in str(exc_info.value.detail) @pytest.mark.asyncio @@ -954,7 +954,7 @@ class TestContentFilterGuardrail: input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 detail = exc_info.value.detail if isinstance(detail, dict): assert detail.get("category") == "harm_toxic_abuse" @@ -983,7 +983,7 @@ class TestContentFilterGuardrail: input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 detail = exc_info.value.detail if isinstance(detail, dict): assert detail.get("category") == "harm_toxic_abuse" @@ -1031,7 +1031,7 @@ class TestContentFilterGuardrail: input_type="request", ) - assert exc_info.value.status_code == 403, f"Failed to block: '{test_input}'" + assert exc_info.value.status_code == 400, f"Failed to block: '{test_input}'" detail = exc_info.value.detail if isinstance(detail, dict): assert detail.get("category") == "harm_toxic_abuse" @@ -1099,7 +1099,7 @@ class TestContentFilterGuardrail: input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "te*st" in str(exc_info.value.detail) def test_check_category_keywords_asterisk_pattern_matching(self): @@ -1166,7 +1166,7 @@ class TestContentFilterGuardrail: input_type="request", ) - assert exc_info.value.status_code == 403, f"Failed to block: '{test_input}'" + assert exc_info.value.status_code == 400, f"Failed to block: '{test_input}'" detail = exc_info.value.detail if isinstance(detail, dict): assert detail.get("category") == "harm_toxic_abuse" @@ -1216,7 +1216,7 @@ class TestContentFilterGuardrail: input_type="request", ) - assert exc_info.value.status_code == 403, f"Failed to block: '{test_input}'" + assert exc_info.value.status_code == 400, f"Failed to block: '{test_input}'" detail = exc_info.value.detail if isinstance(detail, dict): assert detail.get("category") == "harm_toxic_abuse" @@ -1302,7 +1302,7 @@ class TestContentFilterGuardrail: ) assert ( - exc_info.value.status_code == 403 + exc_info.value.status_code == 400 ), f"Failed to block Spanish: '{test_input}'" @pytest.mark.asyncio @@ -1339,7 +1339,7 @@ class TestContentFilterGuardrail: ) assert ( - exc_info.value.status_code == 403 + exc_info.value.status_code == 400 ), f"Failed to block French: '{test_input}'" @pytest.mark.asyncio @@ -1376,7 +1376,7 @@ class TestContentFilterGuardrail: ) assert ( - exc_info.value.status_code == 403 + exc_info.value.status_code == 400 ), f"Failed to block German: '{test_input}'" @pytest.mark.asyncio @@ -1422,7 +1422,7 @@ class TestContentFilterGuardrail: ) assert ( - exc_info.value.status_code == 403 + exc_info.value.status_code == 400 ), f"Failed to block Australian: '{test_input}'" async def test_html_tags_in_messages_not_blocked(self): @@ -1598,7 +1598,7 @@ class TestContentFilterGuardrail: request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "harmful_child_safety" in str(exc_info.value.detail) # Test case 2: Should BLOCK - identifier + block word combination @@ -1612,7 +1612,7 @@ class TestContentFilterGuardrail: request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 3: Should BLOCK - explicit content + minors with pytest.raises(HTTPException) as exc_info: @@ -1623,7 +1623,7 @@ class TestContentFilterGuardrail: request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 4: Should NOT block - identifier word alone (no block word) result = await guardrail.apply_guardrail( @@ -1665,7 +1665,7 @@ class TestContentFilterGuardrail: request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 @pytest.mark.asyncio async def test_conditional_category_sentence_boundaries(self): @@ -1749,7 +1749,7 @@ class TestContentFilterGuardrail: request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "bias_racial" in str(exc_info.value.detail) # Test case 2: Should BLOCK - identifier + dehumanizing language @@ -1763,7 +1763,7 @@ class TestContentFilterGuardrail: request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 3: Should BLOCK - supremacist content with pytest.raises(HTTPException) as exc_info: @@ -1776,7 +1776,7 @@ class TestContentFilterGuardrail: request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 4: Should BLOCK - elimination rhetoric with pytest.raises(HTTPException) as exc_info: @@ -1789,7 +1789,7 @@ class TestContentFilterGuardrail: request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 5: Should NOT block - identifier word alone (no block word) result = await guardrail.apply_guardrail( @@ -1827,7 +1827,7 @@ class TestContentFilterGuardrail: request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 9: Should NOT block - block word alone (no identifier) result = await guardrail.apply_guardrail(