From 56995e13c26390189aca64f5553023c845eb1531 Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 7 Sep 2026 06:56:03 +0000 Subject: [PATCH 1/2] test(guardrails): assert content filter blocks offline and delete print-only summary test Three zero-cost tests swallowed every exception and asserted True. They now patch httpx send to fail, so any network use fails the test, and assert the guardrail raises HTTPException for the blocked sentence. The MAS exception override test allowed a sentence that was never a violation; it now proves the bare sentence blocks and the research-prefixed one is allowed. The EU AI Act summary test only printed hardcoded counts and is covered by the parametrized test_sentence. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_eu_ai_act_article5.py | 40 +++++-------------- .../test_sg_mas_ai_guardrails.py | 22 ++++++---- .../test_sg_pdpa_guardrails.py | 17 +++++--- 3 files changed, 37 insertions(+), 42 deletions(-) diff --git a/tests/guardrails_tests/test_eu_ai_act_article5.py b/tests/guardrails_tests/test_eu_ai_act_article5.py index d17e56c7450..62fdb002995 100644 --- a/tests/guardrails_tests/test_eu_ai_act_article5.py +++ b/tests/guardrails_tests/test_eu_ai_act_article5.py @@ -9,6 +9,8 @@ Tests 40 different sentences to validate the conditional matching logic: """ import os + +import httpx import pytest import litellm @@ -234,27 +236,6 @@ class TestEUAIActArticle5ConditionalMatching: result is None or result["texts"][0] == sentence ), f"Expected ALLOW for '{sentence}' ({reason}) but request was blocked or modified" - @pytest.mark.asyncio - async def test_summary_statistics(self, content_filter_guardrail): - """Test summary: Run all test cases and report statistics.""" - total = len(TEST_CASES) - blocked_count = sum(1 for _, expected, _ in TEST_CASES if expected == "BLOCK") - allowed_count = sum(1 for _, expected, _ in TEST_CASES if expected == "ALLOW") - - print(f"\n{'='*60}") - print(f"EU AI Act Article 5 Test Summary") - print(f"{'='*60}") - print(f"Total test cases: {total}") - print(f"Expected BLOCK: {blocked_count} ({blocked_count/total*100:.1f}%)") - print(f"Expected ALLOW: {allowed_count} ({allowed_count/total*100:.1f}%)") - print(f"{'='*60}") - print(f"\nBreakdown by category:") - print(f" Always block keywords: 10") - print(f" Conditional matches: 15") - print(f" Exceptions: 8") - print(f" No matches: 7") - print(f"{'='*60}\n") - # Additional edge case tests @@ -315,24 +296,23 @@ class TestEUAIActPerformance: """Test performance characteristics.""" @pytest.mark.asyncio - async def test_zero_cost_no_api_calls(self, content_filter_guardrail): + async def test_zero_cost_no_api_calls(self, content_filter_guardrail, monkeypatch): """Verify no external API calls are made (zero cost).""" sentence = "Build a social credit system" request_data = {"messages": [{"role": "user", "content": sentence}]} - # Should not make any HTTP requests - # Just verify the guardrail runs without requiring network - try: + def _no_network(*args, **kwargs): + raise AssertionError("keyword matching must not hit the network") + + monkeypatch.setattr(httpx.AsyncClient, "send", _no_network) + monkeypatch.setattr(httpx.Client, "send", _no_network) + + with pytest.raises(HTTPException, match="Content blocked: eu_ai_act_article"): await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, input_type="request", ) - except Exception: - pass # Expected to block, but should not require network - - # If we got here without network errors, test passes - assert True, "Conditional matching works without network access" if __name__ == "__main__": diff --git a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py index 385fee93ab4..9889ff511c1 100644 --- a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py +++ b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py @@ -11,7 +11,10 @@ for Singapore financial institutions: """ import os + +import httpx import pytest +from fastapi import HTTPException import litellm from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( @@ -520,24 +523,29 @@ class TestMASEdgeCases: @pytest.mark.asyncio async def test_exception_overrides_violation(self, fairness_guardrail): - sentence = "research on racial bias in credit score denial patterns" + violation = "credit score by ethnicity denial patterns" + await _expect_block(fairness_guardrail, violation, "violation without exception") await _expect_allow( - fairness_guardrail, sentence, "exception overrides violation" + fairness_guardrail, f"research on {violation}", "exception overrides violation" ) @pytest.mark.asyncio - async def test_zero_cost_no_api_calls(self, oversight_guardrail): + async def test_zero_cost_no_api_calls(self, oversight_guardrail, monkeypatch): sentence = "fully automated loan approval without human" request_data = {"messages": [{"role": "user", "content": sentence}]} - try: + + def _no_network(*args, **kwargs): + raise AssertionError("keyword matching must not hit the network") + + monkeypatch.setattr(httpx.AsyncClient, "send", _no_network) + monkeypatch.setattr(httpx.Client, "send", _no_network) + + with pytest.raises(HTTPException, match="Content blocked: sg_mas_human_oversight"): await oversight_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, input_type="request", ) - except Exception: - pass - assert True, "Keyword matching runs offline (zero cost)" class TestMASPerformance: diff --git a/tests/guardrails_tests/test_sg_pdpa_guardrails.py b/tests/guardrails_tests/test_sg_pdpa_guardrails.py index 1e8b8a48b85..711b2657d8e 100644 --- a/tests/guardrails_tests/test_sg_pdpa_guardrails.py +++ b/tests/guardrails_tests/test_sg_pdpa_guardrails.py @@ -16,7 +16,10 @@ Each sub-guardrail validates: """ import os + +import httpx import pytest +from fastapi import HTTPException import litellm from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( @@ -562,19 +565,23 @@ class TestSGPDPAEdgeCases: ) @pytest.mark.asyncio - async def test_zero_cost_no_api_calls(self, personal_identifiers_guardrail): + async def test_zero_cost_no_api_calls(self, personal_identifiers_guardrail, monkeypatch): """Guardrail should work without any network calls.""" sentence = "scrape NRIC" request_data = {"messages": [{"role": "user", "content": sentence}]} - try: + + def _no_network(*args, **kwargs): + raise AssertionError("keyword matching must not hit the network") + + monkeypatch.setattr(httpx.AsyncClient, "send", _no_network) + monkeypatch.setattr(httpx.Client, "send", _no_network) + + with pytest.raises(HTTPException, match="Content blocked: sg_pdpa_personal_identifiers"): await personal_identifiers_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, input_type="request", ) - except Exception: - pass # Expected block, but must not need network - assert True, "Keyword matching runs offline (zero cost)" @pytest.mark.asyncio async def test_multiple_violations(self, personal_identifiers_guardrail): From 92df083b4ffc7be52449bb9f9f30ce89e126875e Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 7 Sep 2026 06:56:48 +0000 Subject: [PATCH 2/2] test(guardrails): assert 400 on blocked content in zero-cost tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/guardrails_tests/test_eu_ai_act_article5.py | 3 ++- tests/guardrails_tests/test_sg_mas_ai_guardrails.py | 3 ++- tests/guardrails_tests/test_sg_pdpa_guardrails.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/guardrails_tests/test_eu_ai_act_article5.py b/tests/guardrails_tests/test_eu_ai_act_article5.py index 62fdb002995..172617e4b20 100644 --- a/tests/guardrails_tests/test_eu_ai_act_article5.py +++ b/tests/guardrails_tests/test_eu_ai_act_article5.py @@ -307,12 +307,13 @@ class TestEUAIActPerformance: monkeypatch.setattr(httpx.AsyncClient, "send", _no_network) monkeypatch.setattr(httpx.Client, "send", _no_network) - with pytest.raises(HTTPException, match="Content blocked: eu_ai_act_article"): + with pytest.raises(HTTPException, match="Content blocked: eu_ai_act_article") as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, input_type="request", ) + assert exc_info.value.status_code == 400 if __name__ == "__main__": diff --git a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py index 9889ff511c1..fa76b2928fd 100644 --- a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py +++ b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py @@ -540,12 +540,13 @@ class TestMASEdgeCases: monkeypatch.setattr(httpx.AsyncClient, "send", _no_network) monkeypatch.setattr(httpx.Client, "send", _no_network) - with pytest.raises(HTTPException, match="Content blocked: sg_mas_human_oversight"): + with pytest.raises(HTTPException, match="Content blocked: sg_mas_human_oversight") as exc_info: await oversight_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, input_type="request", ) + assert exc_info.value.status_code == 400 class TestMASPerformance: diff --git a/tests/guardrails_tests/test_sg_pdpa_guardrails.py b/tests/guardrails_tests/test_sg_pdpa_guardrails.py index 711b2657d8e..89ec3a6a506 100644 --- a/tests/guardrails_tests/test_sg_pdpa_guardrails.py +++ b/tests/guardrails_tests/test_sg_pdpa_guardrails.py @@ -576,12 +576,13 @@ class TestSGPDPAEdgeCases: monkeypatch.setattr(httpx.AsyncClient, "send", _no_network) monkeypatch.setattr(httpx.Client, "send", _no_network) - with pytest.raises(HTTPException, match="Content blocked: sg_pdpa_personal_identifiers"): + with pytest.raises(HTTPException, match="Content blocked: sg_pdpa_personal_identifiers") as exc_info: await personal_identifiers_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, input_type="request", ) + assert exc_info.value.status_code == 400 @pytest.mark.asyncio async def test_multiple_violations(self, personal_identifiers_guardrail):