mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
fix: Tighten patterns to reduce false positives and add missing test coverage
- ca_sin: reject leading-zero SINs ([1-9]\d{2}), set allow_word_numbers to false
- ca_immigration_doc: require separators in UCI pattern (prevent bare \d{10} match)
- uoft_utorid: qualify generic keywords (acorn -> acorn login, quercus -> quercus login)
- uoft_tcard: remove generic keywords (student card, id card, library card) that
overlap with credit card contexts; keep only UofT-specific terms (tcard, campus card)
- Add visa/mastercard/amex/iban patterns to test_ca_policy_e2e.py setup to match
deployed template; add Visa card masking test
- Add test verifying "student card" no longer triggers TCard redaction
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Krrish Dholakia
parent
ce96d78203
commit
5d0ea2e1e9
5 changed files with 64 additions and 14 deletions
|
|
@ -547,11 +547,11 @@
|
|||
{
|
||||
"name": "ca_sin",
|
||||
"display_name": "SIN (Canadian Social Insurance Number)",
|
||||
"pattern": "\\b\\d{3}[\\-\\s]\\d{3}[\\-\\s]\\d{3}\\b",
|
||||
"pattern": "\\b[1-9]\\d{2}[\\-\\s]\\d{3}[\\-\\s]\\d{3}\\b",
|
||||
"category": "Canadian PII Patterns",
|
||||
"description": "Detects Canadian Social Insurance Numbers (9-digit federal identifier, dashed or spaced format)",
|
||||
"description": "Detects Canadian Social Insurance Numbers (9-digit federal identifier, dashed or spaced format, first digit 1-9)",
|
||||
"keyword_pattern": "\\b(?:social\\s*insurance\\s*number|numéro\\s*d'assurance\\s*sociale|social\\s*insurance|SIN|NAS)\\b",
|
||||
"allow_word_numbers": true
|
||||
"allow_word_numbers": false
|
||||
},
|
||||
{
|
||||
"name": "ca_ohip",
|
||||
|
|
@ -574,7 +574,7 @@
|
|||
{
|
||||
"name": "ca_immigration_doc",
|
||||
"display_name": "Immigration Document (Canada IRCC)",
|
||||
"pattern": "\\b(?:\\d{4}[\\-\\s]?\\d{4}[\\-\\s]?\\d{2}|[TUFW]\\d{8,10}|IMM[\\-\\s]?\\d{4,5})\\b",
|
||||
"pattern": "\\b(?:\\d{4}[\\-\\s]\\d{4}[\\-\\s]\\d{2}|[TUFW]\\d{8,10}|IMM[\\-\\s]?\\d{4,5})\\b",
|
||||
"category": "Canadian PII Patterns",
|
||||
"description": "Detects Canadian immigration document numbers including UCI (10-digit), work/study permits, and IMM document references",
|
||||
"keyword_pattern": "\\b(?:unique\\s*client\\s*identifier|study\\s*permit\\s*number|work\\s*permit\\s*number|study\\s*permit|work\\s*permit|UCI|immigration|IRCC|permanent\\s*resident|PR\\s*card|permis\\s*d'études|permis\\s*de\\s*travail|landed\\s*immigrant|temporary\\s*resident)\\b",
|
||||
|
|
@ -613,7 +613,7 @@
|
|||
"pattern": "\\b[a-z]{2,6}\\d{1,4}\\b",
|
||||
"category": "Canadian Institutional Identifiers (FIPPA)",
|
||||
"description": "Detects University of Toronto UTORid login identifiers (e.g. smithj12). Context keywords are essential due to the generic format.",
|
||||
"keyword_pattern": "\\b(?:utorid|utoid|utor\\s*id|acorn|quercus|uoft\\s*login|university\\s*of\\s*toronto\\s*login|portal\\s*login|weblogin|shibboleth)\\b",
|
||||
"keyword_pattern": "\\b(?:utorid|utoid|utor\\s*id|uoft\\s*login|university\\s*of\\s*toronto\\s*login|acorn\\s*login|acorn\\s*portal|quercus\\s*login|quercus\\s*course|portal\\s*login|weblogin|shibboleth)\\b",
|
||||
"allow_word_numbers": false
|
||||
},
|
||||
{
|
||||
|
|
@ -622,7 +622,7 @@
|
|||
"pattern": "\\b\\d{16}\\b",
|
||||
"category": "Canadian Institutional Identifiers (FIPPA)",
|
||||
"description": "Detects University of Toronto TCard campus ID card numbers (16-digit). Context keywords are critical to distinguish from payment card numbers.",
|
||||
"keyword_pattern": "\\b(?:tcard\\s*number|t-card\\s*number|campus\\s*card\\s*number|campus\\s*id\\s*number|library\\s*card\\s*number|student\\s*card\\s*number|id\\s*card\\s*number|tcard|t-card|campus\\s*card|campus\\s*id|library\\s*card|student\\s*card|id\\s*card|university\\s*card)\\b",
|
||||
"keyword_pattern": "\\b(?:tcard\\s*number|t-card\\s*number|tcard|t-card|campus\\s*card|campus\\s*id|university\\s*card)\\b",
|
||||
"allow_word_numbers": false
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -123,9 +123,10 @@ class TestCanadianImmigrationDoc:
|
|||
pattern = get_compiled_pattern("ca_immigration_doc")
|
||||
assert pattern.search("1234-5678-90") is not None
|
||||
|
||||
def test_uci_compact(self):
|
||||
def test_uci_compact_rejected(self):
|
||||
"""Compact 10-digit UCI without separators should NOT match (too broad)"""
|
||||
pattern = get_compiled_pattern("ca_immigration_doc")
|
||||
assert pattern.search("1234567890") is not None
|
||||
assert pattern.search("1234567890") is None
|
||||
|
||||
def test_imm_in_sentence(self):
|
||||
pattern = get_compiled_pattern("ca_immigration_doc")
|
||||
|
|
|
|||
|
|
@ -62,6 +62,26 @@ class TestCanadianPIIPolicyE2E:
|
|||
pattern_name="credit_card",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="visa",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="mastercard",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="amex",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="iban",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
# Contact info
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
|
|
@ -333,6 +353,22 @@ class TestCanadianPIIPolicyE2E:
|
|||
assert "[CA_BANK_ACCOUNT_REDACTED]" in output
|
||||
assert "12345-003-1234567" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_visa_card_masked(self):
|
||||
"""Visa card number is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "My Visa card number is 4111111111111111."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "REDACTED" in output
|
||||
assert "4111111111111111" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bank_question_passes(self):
|
||||
"""Question about banking without actual numbers passes through"""
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ class TestUofTPolicyE2E:
|
|||
@pytest.mark.asyncio
|
||||
async def test_utorid_quercus_context_masked(self):
|
||||
guardrail = self.setup_uoft_guardrail()
|
||||
text = "Quercus login: kcheng42"
|
||||
text = "Quercus login kcheng42"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
|
|
@ -132,7 +132,7 @@ class TestUofTPolicyE2E:
|
|||
@pytest.mark.asyncio
|
||||
async def test_utorid_acorn_context_masked(self):
|
||||
guardrail = self.setup_uoft_guardrail()
|
||||
text = "Log in to ACORN with li5"
|
||||
text = "ACORN login li5"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
|
|
@ -185,9 +185,9 @@ class TestUofTPolicyE2E:
|
|||
assert "9876543210987654" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_student_card_masked(self):
|
||||
async def test_university_card_masked(self):
|
||||
guardrail = self.setup_uoft_guardrail()
|
||||
text = "Lost my student card 1111222233334444 yesterday"
|
||||
text = "Lost my university card 1111222233334444 yesterday"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
|
|
@ -197,6 +197,19 @@ class TestUofTPolicyE2E:
|
|||
assert "[UOFT_TCARD_REDACTED]" in output
|
||||
assert "1111222233334444" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_student_card_no_longer_triggers_tcard(self):
|
||||
"""Generic 'student card' keyword should NOT trigger TCard (avoids credit card collision)"""
|
||||
guardrail = self.setup_uoft_guardrail()
|
||||
text = "My student card number is 4111111111111111"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
assert "UOFT_TCARD_REDACTED" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tcard_clean_passes(self):
|
||||
guardrail = self.setup_uoft_guardrail()
|
||||
|
|
|
|||
|
|
@ -297,7 +297,7 @@ export const canadianPiiCompliancePrompts: CompliancePrompt[] = [
|
|||
categoryDescription:
|
||||
"Detects University of Toronto UTORid login identifiers — protected under Ontario FIPPA.",
|
||||
prompt:
|
||||
"My UTORid is smithj12 for ACORN login.",
|
||||
"My UTORid is smithj12.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
|
|
@ -308,7 +308,7 @@ export const canadianPiiCompliancePrompts: CompliancePrompt[] = [
|
|||
categoryDescription:
|
||||
"Detects University of Toronto UTORid login identifiers — protected under Ontario FIPPA.",
|
||||
prompt:
|
||||
"Quercus login: kcheng42 needs password reset.",
|
||||
"Quercus login kcheng42 needs password reset.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue