mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-13 03:59:12 +00:00
Revert "test(guardrails): re-apply vacuous content filter test fixes to consolidated file"
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
This reverts commit 88d69acb19.
This commit is contained in:
parent
88d69acb19
commit
110e26dd4b
1 changed files with 42 additions and 33 deletions
|
|
@ -11,7 +11,6 @@ Tests 40 different sentences to validate the conditional matching logic:
|
|||
import importlib
|
||||
import os
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
|
@ -231,6 +230,27 @@ class TestEUAIActArticle5ConditionalMatching:
|
|||
f"Expected ALLOW for '{sentence}' ({reason}) but request was blocked or modified"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summary_statistics(self, content_filter_guardrail):
|
||||
"""Test summary: Run all test cases and report statistics."""
|
||||
total = len(TEST_CASES)
|
||||
blocked_count = sum(1 for _, expected, _ in TEST_CASES if expected == "BLOCK")
|
||||
allowed_count = sum(1 for _, expected, _ in TEST_CASES if expected == "ALLOW")
|
||||
|
||||
print(f"EU AI Act Article 5 Test Summary")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"Total test cases: {total}")
|
||||
print(f"Expected BLOCK: {blocked_count} ({blocked_count / total * 100:.1f}%)")
|
||||
print(f"Expected ALLOW: {allowed_count} ({allowed_count / total * 100:.1f}%)")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"\nBreakdown by category:")
|
||||
print(f" Always block keywords: 10")
|
||||
print(f" Conditional matches: 15")
|
||||
print(f" Exceptions: 8")
|
||||
print(f" No matches: 7")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
|
||||
# Additional edge case tests
|
||||
|
||||
|
||||
|
|
@ -292,24 +312,24 @@ class TestEUAIActPerformance:
|
|||
"""Test performance characteristics."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zero_cost_no_api_calls(self, content_filter_guardrail, monkeypatch):
|
||||
async def test_zero_cost_no_api_calls(self, content_filter_guardrail):
|
||||
"""Verify no external API calls are made (zero cost)."""
|
||||
sentence = "Build a social credit system"
|
||||
request_data = {"messages": [{"role": "user", "content": sentence}]}
|
||||
|
||||
def _no_network(*args, **kwargs):
|
||||
raise AssertionError("keyword matching must not hit the network")
|
||||
|
||||
monkeypatch.setattr(httpx.AsyncClient, "send", _no_network)
|
||||
monkeypatch.setattr(httpx.Client, "send", _no_network)
|
||||
|
||||
with pytest.raises(HTTPException, match="Content blocked: eu_ai_act_article") as exc_info:
|
||||
# Should not make any HTTP requests
|
||||
# Just verify the guardrail runs without requiring network
|
||||
try:
|
||||
await content_filter_guardrail.apply_guardrail(
|
||||
inputs={"texts": [sentence]},
|
||||
request_data=request_data,
|
||||
input_type="request",
|
||||
)
|
||||
assert exc_info.value.status_code == 400
|
||||
except Exception:
|
||||
pass # Expected to block, but should not require network
|
||||
|
||||
# If we got here without network errors, test passes
|
||||
assert True, "Conditional matching works without network access"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
@ -1138,28 +1158,22 @@ class TestMASEdgeCases:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exception_overrides_violation(self, fairness_guardrail):
|
||||
violation = "credit score by ethnicity denial patterns"
|
||||
await _expect_block(fairness_guardrail, violation, "violation without exception")
|
||||
await _expect_allow(fairness_guardrail, f"research on {violation}", "exception overrides violation")
|
||||
sentence = "research on racial bias in credit score denial patterns"
|
||||
await _expect_allow(fairness_guardrail, sentence, "exception overrides violation")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zero_cost_no_api_calls(self, oversight_guardrail, monkeypatch):
|
||||
async def test_zero_cost_no_api_calls(self, oversight_guardrail):
|
||||
sentence = "fully automated loan approval without human"
|
||||
request_data = {"messages": [{"role": "user", "content": sentence}]}
|
||||
|
||||
def _no_network(*args, **kwargs):
|
||||
raise AssertionError("keyword matching must not hit the network")
|
||||
|
||||
monkeypatch.setattr(httpx.AsyncClient, "send", _no_network)
|
||||
monkeypatch.setattr(httpx.Client, "send", _no_network)
|
||||
|
||||
with pytest.raises(HTTPException, match="Content blocked: sg_mas_human_oversight") as exc_info:
|
||||
try:
|
||||
await oversight_guardrail.apply_guardrail(
|
||||
inputs={"texts": [sentence]},
|
||||
request_data=request_data,
|
||||
input_type="request",
|
||||
)
|
||||
assert exc_info.value.status_code == 400
|
||||
except Exception:
|
||||
pass
|
||||
assert True, "Keyword matching runs offline (zero cost)"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
|
|
@ -1724,24 +1738,19 @@ class TestSGPDPAEdgeCases:
|
|||
await _expect_allow_sg_pdpa(personal_identifiers_guardrail, sentence, "exception overrides violation")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zero_cost_no_api_calls(self, personal_identifiers_guardrail, monkeypatch):
|
||||
async def test_zero_cost_no_api_calls(self, personal_identifiers_guardrail):
|
||||
"""Guardrail should work without any network calls."""
|
||||
sentence = "scrape NRIC"
|
||||
request_data = {"messages": [{"role": "user", "content": sentence}]}
|
||||
|
||||
def _no_network(*args, **kwargs):
|
||||
raise AssertionError("keyword matching must not hit the network")
|
||||
|
||||
monkeypatch.setattr(httpx.AsyncClient, "send", _no_network)
|
||||
monkeypatch.setattr(httpx.Client, "send", _no_network)
|
||||
|
||||
with pytest.raises(HTTPException, match="Content blocked: sg_pdpa_personal_identifiers") as exc_info:
|
||||
try:
|
||||
await personal_identifiers_guardrail.apply_guardrail(
|
||||
inputs={"texts": [sentence]},
|
||||
request_data=request_data,
|
||||
input_type="request",
|
||||
)
|
||||
assert exc_info.value.status_code == 400
|
||||
except Exception:
|
||||
pass # Expected block, but must not need network
|
||||
assert True, "Keyword matching runs offline (zero cost)"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_violations(self, personal_identifiers_guardrail):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue