diff --git a/tests/guardrails_tests/test_eu_ai_act_article5.py b/tests/guardrails_tests/test_eu_ai_act_article5.py index d17e56c7450..172617e4b20 100644 --- a/tests/guardrails_tests/test_eu_ai_act_article5.py +++ b/tests/guardrails_tests/test_eu_ai_act_article5.py @@ -9,6 +9,8 @@ Tests 40 different sentences to validate the conditional matching logic: """ import os + +import httpx import pytest import litellm @@ -234,27 +236,6 @@ class TestEUAIActArticle5ConditionalMatching: result is None or result["texts"][0] == sentence ), f"Expected ALLOW for '{sentence}' ({reason}) but request was blocked or modified" - @pytest.mark.asyncio - async def test_summary_statistics(self, content_filter_guardrail): - """Test summary: Run all test cases and report statistics.""" - total = len(TEST_CASES) - blocked_count = sum(1 for _, expected, _ in TEST_CASES if expected == "BLOCK") - allowed_count = sum(1 for _, expected, _ in TEST_CASES if expected == "ALLOW") - - print(f"\n{'='*60}") - print(f"EU AI Act Article 5 Test Summary") - print(f"{'='*60}") - print(f"Total test cases: {total}") - print(f"Expected BLOCK: {blocked_count} ({blocked_count/total*100:.1f}%)") - print(f"Expected ALLOW: {allowed_count} ({allowed_count/total*100:.1f}%)") - print(f"{'='*60}") - print(f"\nBreakdown by category:") - print(f" Always block keywords: 10") - print(f" Conditional matches: 15") - print(f" Exceptions: 8") - print(f" No matches: 7") - print(f"{'='*60}\n") - # Additional edge case tests @@ -315,24 +296,24 @@ class TestEUAIActPerformance: """Test performance characteristics.""" @pytest.mark.asyncio - async def test_zero_cost_no_api_calls(self, content_filter_guardrail): + async def test_zero_cost_no_api_calls(self, content_filter_guardrail, monkeypatch): """Verify no external API calls are made (zero cost).""" sentence = "Build a social credit system" request_data = {"messages": [{"role": "user", "content": sentence}]} - # Should not make any HTTP requests - # Just verify the guardrail runs without requiring network - try: + def _no_network(*args, **kwargs): + raise AssertionError("keyword matching must not hit the network") + + monkeypatch.setattr(httpx.AsyncClient, "send", _no_network) + monkeypatch.setattr(httpx.Client, "send", _no_network) + + with pytest.raises(HTTPException, match="Content blocked: eu_ai_act_article") as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, input_type="request", ) - except Exception: - pass # Expected to block, but should not require network - - # If we got here without network errors, test passes - assert True, "Conditional matching works without network access" + assert exc_info.value.status_code == 400 if __name__ == "__main__": diff --git a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py index 385fee93ab4..fa76b2928fd 100644 --- a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py +++ b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py @@ -11,7 +11,10 @@ for Singapore financial institutions: """ import os + +import httpx import pytest +from fastapi import HTTPException import litellm from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( @@ -520,24 +523,30 @@ class TestMASEdgeCases: @pytest.mark.asyncio async def test_exception_overrides_violation(self, fairness_guardrail): - sentence = "research on racial bias in credit score denial patterns" + violation = "credit score by ethnicity denial patterns" + await _expect_block(fairness_guardrail, violation, "violation without exception") await _expect_allow( - fairness_guardrail, sentence, "exception overrides violation" + fairness_guardrail, f"research on {violation}", "exception overrides violation" ) @pytest.mark.asyncio - async def test_zero_cost_no_api_calls(self, oversight_guardrail): + async def test_zero_cost_no_api_calls(self, oversight_guardrail, monkeypatch): sentence = "fully automated loan approval without human" request_data = {"messages": [{"role": "user", "content": sentence}]} - try: + + def _no_network(*args, **kwargs): + raise AssertionError("keyword matching must not hit the network") + + monkeypatch.setattr(httpx.AsyncClient, "send", _no_network) + monkeypatch.setattr(httpx.Client, "send", _no_network) + + with pytest.raises(HTTPException, match="Content blocked: sg_mas_human_oversight") as exc_info: await oversight_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, input_type="request", ) - except Exception: - pass - assert True, "Keyword matching runs offline (zero cost)" + assert exc_info.value.status_code == 400 class TestMASPerformance: diff --git a/tests/guardrails_tests/test_sg_pdpa_guardrails.py b/tests/guardrails_tests/test_sg_pdpa_guardrails.py index 1e8b8a48b85..89ec3a6a506 100644 --- a/tests/guardrails_tests/test_sg_pdpa_guardrails.py +++ b/tests/guardrails_tests/test_sg_pdpa_guardrails.py @@ -16,7 +16,10 @@ Each sub-guardrail validates: """ import os + +import httpx import pytest +from fastapi import HTTPException import litellm from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( @@ -562,19 +565,24 @@ class TestSGPDPAEdgeCases: ) @pytest.mark.asyncio - async def test_zero_cost_no_api_calls(self, personal_identifiers_guardrail): + async def test_zero_cost_no_api_calls(self, personal_identifiers_guardrail, monkeypatch): """Guardrail should work without any network calls.""" sentence = "scrape NRIC" request_data = {"messages": [{"role": "user", "content": sentence}]} - try: + + def _no_network(*args, **kwargs): + raise AssertionError("keyword matching must not hit the network") + + monkeypatch.setattr(httpx.AsyncClient, "send", _no_network) + monkeypatch.setattr(httpx.Client, "send", _no_network) + + with pytest.raises(HTTPException, match="Content blocked: sg_pdpa_personal_identifiers") as exc_info: await personal_identifiers_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, input_type="request", ) - except Exception: - pass # Expected block, but must not need network - assert True, "Keyword matching runs offline (zero cost)" + assert exc_info.value.status_code == 400 @pytest.mark.asyncio async def test_multiple_violations(self, personal_identifiers_guardrail):