mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
feat: Add Canadian PII compliance dataset and tests (57 tests)
Adds:
- test_ca_patterns.py: 30 unit tests for regex pattern matching (SIN, OHIP,
driver's licence, immigration docs, bank account, postal code)
- test_ca_policy_e2e.py: 27 end-to-end tests running the full
ContentFilterGuardrail pipeline with MASK action — validates detection
of real PII and pass-through of clean prompts
- canadianPiiCompliancePrompts.ts: 21-prompt compliance dataset for UI
evaluation, wired into the main compliancePrompts framework
Fixes keyword_pattern alternation ordering in patterns.json — longer
alternatives (e.g. "social insurance number") now precede shorter ones
("social insurance") to avoid excessive gap-word count when the regex
engine selects the shorter match first.
Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
635ad6d898
commit
f55fda3be7
5 changed files with 926 additions and 4 deletions
|
|
@ -550,7 +550,7 @@
|
|||
"pattern": "\\b\\d{3}[\\-\\s]\\d{3}[\\-\\s]\\d{3}\\b",
|
||||
"category": "Canadian PII Patterns",
|
||||
"description": "Detects Canadian Social Insurance Numbers (9-digit federal identifier, dashed or spaced format)",
|
||||
"keyword_pattern": "\\b(?:SIN|social\\s*insurance|social\\s*insurance\\s*number|numéro\\s*d'assurance\\s*sociale|NAS)\\b",
|
||||
"keyword_pattern": "\\b(?:social\\s*insurance\\s*number|numéro\\s*d'assurance\\s*sociale|social\\s*insurance|SIN|NAS)\\b",
|
||||
"allow_word_numbers": true
|
||||
},
|
||||
{
|
||||
|
|
@ -559,7 +559,7 @@
|
|||
"pattern": "\\b\\d{4}[\\-\\s]?\\d{3}[\\-\\s]?\\d{3}[\\-\\s]?[A-Z]{2}\\b",
|
||||
"category": "Canadian PII Patterns",
|
||||
"description": "Detects Ontario Health Insurance Plan numbers (10 digits + 2-letter version code)",
|
||||
"keyword_pattern": "\\b(?:OHIP|health\\s*card|health\\s*insurance|ontario\\s*health|health\\s*number|carte\\s*santé)\\b",
|
||||
"keyword_pattern": "\\b(?:OHIP\\s*number|health\\s*card\\s*number|health\\s*insurance\\s*number|OHIP|health\\s*card|health\\s*insurance|ontario\\s*health|health\\s*number|carte\\s*santé)\\b",
|
||||
"allow_word_numbers": false
|
||||
},
|
||||
{
|
||||
|
|
@ -568,7 +568,7 @@
|
|||
"pattern": "\\b[A-Z]\\d{4}[\\-\\s]\\d{5}[\\-\\s]\\d{5}\\b",
|
||||
"category": "Canadian PII Patterns",
|
||||
"description": "Detects Ontario driver's licence numbers (1 letter + 14 digits, dashed format)",
|
||||
"keyword_pattern": "\\b(?:driver'?s?\\s*licen[cs]e|licen[cs]e\\s*number|DL\\s*number|ontario\\s*licen[cs]e|permis\\s*de\\s*conduire|numéro\\s*de\\s*permis)\\b",
|
||||
"keyword_pattern": "\\b(?:driver'?s?\\s*licen[cs]e\\s*number|ontario\\s*licen[cs]e\\s*number|driver'?s?\\s*licen[cs]e|licen[cs]e\\s*number|DL\\s*number|ontario\\s*licen[cs]e|permis\\s*de\\s*conduire|numéro\\s*de\\s*permis)\\b",
|
||||
"allow_word_numbers": false
|
||||
},
|
||||
{
|
||||
|
|
@ -577,7 +577,7 @@
|
|||
"pattern": "\\b(?:\\d{4}[\\-\\s]?\\d{4}[\\-\\s]?\\d{2}|[TUFW]\\d{8,10}|IMM[\\-\\s]?\\d{4,5})\\b",
|
||||
"category": "Canadian PII Patterns",
|
||||
"description": "Detects Canadian immigration document numbers including UCI (10-digit), work/study permits, and IMM document references",
|
||||
"keyword_pattern": "\\b(?:UCI|unique\\s*client\\s*identifier|study\\s*permit|work\\s*permit|immigration|IRCC|permanent\\s*resident|PR\\s*card|permis\\s*d'études|permis\\s*de\\s*travail|landed\\s*immigrant|temporary\\s*resident)\\b",
|
||||
"keyword_pattern": "\\b(?:unique\\s*client\\s*identifier|study\\s*permit\\s*number|work\\s*permit\\s*number|study\\s*permit|work\\s*permit|UCI|immigration|IRCC|permanent\\s*resident|PR\\s*card|permis\\s*d'études|permis\\s*de\\s*travail|landed\\s*immigrant|temporary\\s*resident)\\b",
|
||||
"allow_word_numbers": false
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,208 @@
|
|||
"""
|
||||
Test Canadian PII regex patterns added for PIPEDA compliance.
|
||||
|
||||
Tests SIN, OHIP, Ontario driver's licence, immigration documents,
|
||||
bank account, and postal code detection patterns.
|
||||
"""
|
||||
|
||||
from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.patterns import (
|
||||
get_compiled_pattern,
|
||||
)
|
||||
|
||||
|
||||
class TestCanadianSIN:
|
||||
"""Test Canadian Social Insurance Number detection"""
|
||||
|
||||
def test_dashed_format(self):
|
||||
pattern = get_compiled_pattern("ca_sin")
|
||||
assert pattern.search("123-456-789") is not None
|
||||
|
||||
def test_spaced_format(self):
|
||||
pattern = get_compiled_pattern("ca_sin")
|
||||
assert pattern.search("123 456 789") is not None
|
||||
|
||||
def test_sin_in_sentence(self):
|
||||
pattern = get_compiled_pattern("ca_sin")
|
||||
assert pattern.search("My SIN is 987-654-321 for tax") is not None
|
||||
|
||||
def test_compact_nine_digits_not_matched(self):
|
||||
"""Compact 9-digit format (no dashes/spaces) should NOT match the dashed pattern"""
|
||||
pattern = get_compiled_pattern("ca_sin")
|
||||
assert pattern.search("123456789") is None
|
||||
|
||||
def test_too_few_digits_rejected(self):
|
||||
pattern = get_compiled_pattern("ca_sin")
|
||||
assert pattern.search("12-345-678") is None
|
||||
|
||||
def test_too_many_digits_rejected(self):
|
||||
pattern = get_compiled_pattern("ca_sin")
|
||||
assert pattern.search("1234-567-890") is None
|
||||
|
||||
|
||||
class TestCanadianOHIP:
|
||||
"""Test Ontario Health Insurance Plan Number detection"""
|
||||
|
||||
def test_full_format_with_version_code(self):
|
||||
pattern = get_compiled_pattern("ca_ohip")
|
||||
assert pattern.search("1234-567-890-AB") is not None
|
||||
|
||||
def test_compact_with_version_code(self):
|
||||
pattern = get_compiled_pattern("ca_ohip")
|
||||
assert pattern.search("1234567890AB") is not None
|
||||
|
||||
def test_spaced_format(self):
|
||||
pattern = get_compiled_pattern("ca_ohip")
|
||||
assert pattern.search("1234 567 890 XY") is not None
|
||||
|
||||
def test_ohip_in_sentence(self):
|
||||
pattern = get_compiled_pattern("ca_ohip")
|
||||
assert (
|
||||
pattern.search("My OHIP number is 9876543210ZZ for my appointment")
|
||||
is not None
|
||||
)
|
||||
|
||||
def test_without_version_code_not_matched(self):
|
||||
"""OHIP pattern requires the 2-letter version code"""
|
||||
pattern = get_compiled_pattern("ca_ohip")
|
||||
# 10 digits alone without letters should not match the full OHIP pattern
|
||||
assert pattern.search("1234567890") is None
|
||||
|
||||
def test_lowercase_version_code_detected(self):
|
||||
"""Compiled with IGNORECASE"""
|
||||
pattern = get_compiled_pattern("ca_ohip")
|
||||
assert pattern.search("1234567890ab") is not None
|
||||
|
||||
|
||||
class TestCanadianOntarioDriversLicence:
|
||||
"""Test Ontario Driver's Licence detection"""
|
||||
|
||||
def test_dashed_format(self):
|
||||
pattern = get_compiled_pattern("ca_on_drivers_licence")
|
||||
assert pattern.search("A1234-56789-01234") is not None
|
||||
|
||||
def test_spaced_format(self):
|
||||
pattern = get_compiled_pattern("ca_on_drivers_licence")
|
||||
assert pattern.search("B9876 54321 09876") is not None
|
||||
|
||||
def test_in_sentence(self):
|
||||
pattern = get_compiled_pattern("ca_on_drivers_licence")
|
||||
assert (
|
||||
pattern.search("Driver's licence C1111-22222-33333 on file") is not None
|
||||
)
|
||||
|
||||
def test_compact_format_not_matched(self):
|
||||
"""Compact format (no dashes/spaces) uses a separate pattern"""
|
||||
pattern = get_compiled_pattern("ca_on_drivers_licence")
|
||||
assert pattern.search("A12345678901234") is None
|
||||
|
||||
def test_missing_letter_prefix_rejected(self):
|
||||
pattern = get_compiled_pattern("ca_on_drivers_licence")
|
||||
assert pattern.search("11234-56789-01234") is None
|
||||
|
||||
def test_wrong_digit_groups_rejected(self):
|
||||
pattern = get_compiled_pattern("ca_on_drivers_licence")
|
||||
assert pattern.search("A123-456789-01234") is None
|
||||
|
||||
|
||||
class TestCanadianImmigrationDoc:
|
||||
"""Test Canadian IRCC Immigration Document detection"""
|
||||
|
||||
def test_imm_document_reference(self):
|
||||
pattern = get_compiled_pattern("ca_immigration_doc")
|
||||
assert pattern.search("IMM-5257") is not None
|
||||
assert pattern.search("IMM 1234") is not None
|
||||
assert pattern.search("IMM5257") is not None
|
||||
|
||||
def test_work_study_permit(self):
|
||||
pattern = get_compiled_pattern("ca_immigration_doc")
|
||||
assert pattern.search("T123456789") is not None
|
||||
assert pattern.search("F1234567890") is not None
|
||||
assert pattern.search("W12345678") is not None
|
||||
|
||||
def test_uci_dashed(self):
|
||||
pattern = get_compiled_pattern("ca_immigration_doc")
|
||||
assert pattern.search("1234-5678-90") is not None
|
||||
|
||||
def test_uci_compact(self):
|
||||
pattern = get_compiled_pattern("ca_immigration_doc")
|
||||
assert pattern.search("1234567890") is not None
|
||||
|
||||
def test_imm_in_sentence(self):
|
||||
pattern = get_compiled_pattern("ca_immigration_doc")
|
||||
assert (
|
||||
pattern.search("Submit immigration form IMM-5645 with your application")
|
||||
is not None
|
||||
)
|
||||
|
||||
def test_too_short_imm_rejected(self):
|
||||
pattern = get_compiled_pattern("ca_immigration_doc")
|
||||
assert pattern.search("IMM-12") is None # Less than 4 digits
|
||||
|
||||
|
||||
class TestCanadianBankAccount:
|
||||
"""Test Canadian Bank Account routing detection"""
|
||||
|
||||
def test_standard_format_dashed(self):
|
||||
pattern = get_compiled_pattern("ca_bank_account")
|
||||
assert pattern.search("12345-003-1234567") is not None
|
||||
|
||||
def test_spaced_format(self):
|
||||
pattern = get_compiled_pattern("ca_bank_account")
|
||||
assert pattern.search("00456 001 9876543210") is not None
|
||||
|
||||
def test_longer_account_number(self):
|
||||
pattern = get_compiled_pattern("ca_bank_account")
|
||||
assert pattern.search("12345-003-123456789012") is not None
|
||||
|
||||
def test_in_sentence(self):
|
||||
pattern = get_compiled_pattern("ca_bank_account")
|
||||
assert (
|
||||
pattern.search("Direct deposit to bank account 12345-003-1234567 please")
|
||||
is not None
|
||||
)
|
||||
|
||||
def test_without_separators_rejected(self):
|
||||
pattern = get_compiled_pattern("ca_bank_account")
|
||||
assert pattern.search("123450031234567") is None
|
||||
|
||||
|
||||
class TestCanadianPostalCode:
|
||||
"""Test Canadian Postal Code detection"""
|
||||
|
||||
def test_spaced_format(self):
|
||||
pattern = get_compiled_pattern("ca_postal_code")
|
||||
assert pattern.search("M5V 2T6") is not None
|
||||
assert pattern.search("K1A 0B1") is not None
|
||||
assert pattern.search("V6B 3K9") is not None
|
||||
|
||||
def test_compact_format(self):
|
||||
pattern = get_compiled_pattern("ca_postal_code")
|
||||
assert pattern.search("M5V2T6") is not None
|
||||
assert pattern.search("K1A0B1") is not None
|
||||
|
||||
def test_dashed_format(self):
|
||||
pattern = get_compiled_pattern("ca_postal_code")
|
||||
assert pattern.search("M5V-2T6") is not None
|
||||
|
||||
def test_in_sentence(self):
|
||||
pattern = get_compiled_pattern("ca_postal_code")
|
||||
assert pattern.search("Ship to postal code M5V 2T6 in Toronto") is not None
|
||||
|
||||
def test_invalid_first_letter_rejected(self):
|
||||
"""Letters D, F, I, O, Q, U are not valid as first character"""
|
||||
pattern = get_compiled_pattern("ca_postal_code")
|
||||
assert pattern.search("D5V 2T6") is None
|
||||
assert pattern.search("F1A 0B1") is None
|
||||
assert pattern.search("I5V 2T6") is None
|
||||
assert pattern.search("O1A 0B1") is None
|
||||
assert pattern.search("Q5V 2T6") is None
|
||||
assert pattern.search("U1A 0B1") is None
|
||||
|
||||
def test_lowercase_detected(self):
|
||||
"""Compiled with IGNORECASE"""
|
||||
pattern = get_compiled_pattern("ca_postal_code")
|
||||
assert pattern.search("m5v 2t6") is not None
|
||||
|
||||
def test_all_digits_rejected(self):
|
||||
pattern = get_compiled_pattern("ca_postal_code")
|
||||
assert pattern.search("123 456") is None
|
||||
|
|
@ -0,0 +1,456 @@
|
|||
"""
|
||||
End-to-end tests for Canadian PII Protection (PIPEDA) policy template.
|
||||
|
||||
Tests the complete policy with all Canadian PII patterns — validates that
|
||||
PII-containing prompts are detected/masked and that clean prompts pass through.
|
||||
These tests mirror the canadianPiiCompliancePrompts.ts dataset.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import (
|
||||
ContentFilterGuardrail,
|
||||
)
|
||||
from litellm.types.guardrails import ContentFilterAction, ContentFilterPattern
|
||||
|
||||
|
||||
class TestCanadianPIIPolicyE2E:
|
||||
"""End-to-end tests for Canadian PII policy template"""
|
||||
|
||||
def setup_canadian_guardrail(self):
|
||||
"""
|
||||
Setup guardrail with all Canadian PII patterns (mimics the policy template)
|
||||
"""
|
||||
patterns = [
|
||||
# Government identifiers
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="ca_sin",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="passport_canada",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
# Health & drivers
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="ca_ohip",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="ca_on_drivers_licence",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
# Immigration
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="ca_immigration_doc",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
# Financial
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="ca_bank_account",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="credit_card",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
# Contact info
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="email",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt",
|
||||
pattern_name="ca_postal_code",
|
||||
action=ContentFilterAction.MASK,
|
||||
),
|
||||
]
|
||||
|
||||
return ContentFilterGuardrail(
|
||||
guardrail_name="canadian-pii-protection",
|
||||
patterns=patterns,
|
||||
)
|
||||
|
||||
# =====================
|
||||
# SIN tests
|
||||
# =====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sin_dashed_masked(self):
|
||||
"""SIN in dashed format is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "My SIN is 123-456-789, please update my tax records."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "[CA_SIN_REDACTED]" in output
|
||||
assert "123-456-789" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sin_spaced_masked(self):
|
||||
"""SIN in spaced format is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "The employee's social insurance number is 987 654 321."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "[CA_SIN_REDACTED]" in output
|
||||
assert "987 654 321" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sin_question_passes(self):
|
||||
"""Question about SIN without actual number passes through"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "What is a Social Insurance Number and how do I apply for one?"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "REDACTED" not in output
|
||||
assert output == text
|
||||
|
||||
# =====================
|
||||
# OHIP tests
|
||||
# =====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ohip_dashed_masked(self):
|
||||
"""OHIP number with version code is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "My OHIP number is 1234-567-890-AB, can you verify my coverage?"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "[CA_OHIP_REDACTED]" in output
|
||||
assert "1234-567-890-AB" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ohip_compact_masked(self):
|
||||
"""OHIP number in compact format is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "The health card number 9876543210XY needs to be updated in the system."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "[CA_OHIP_REDACTED]" in output
|
||||
assert "9876543210XY" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ohip_question_passes(self):
|
||||
"""Question about OHIP without actual number passes through"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "How do I renew my Ontario health card?"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "REDACTED" not in output
|
||||
assert output == text
|
||||
|
||||
# =====================
|
||||
# Ontario Driver's Licence tests
|
||||
# =====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drivers_licence_masked(self):
|
||||
"""Ontario driver's licence is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "My driver's licence number is A1234-56789-01234."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "[CA_ON_DRIVERS_LICENCE_REDACTED]" in output
|
||||
assert "A1234-56789-01234" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drivers_licence_question_passes(self):
|
||||
"""Question about driver's licence without actual number passes through"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "How do I renew my Ontario driver's licence?"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "REDACTED" not in output
|
||||
assert output == text
|
||||
|
||||
# =====================
|
||||
# Canadian Passport tests
|
||||
# =====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_passport_masked(self):
|
||||
"""Canadian passport number is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "My Canadian passport number is AB123456."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "[PASSPORT_CANADA_REDACTED]" in output
|
||||
assert "AB123456" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_passport_question_passes(self):
|
||||
"""Question about passports without actual number passes through"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "How long does it take to renew a Canadian passport?"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "REDACTED" not in output
|
||||
assert output == text
|
||||
|
||||
# =====================
|
||||
# Immigration Document tests
|
||||
# =====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_imm_form_masked(self):
|
||||
"""IRCC IMM form reference is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "Please reference immigration form IMM-5257 for the application."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "[CA_IMMIGRATION_DOC_REDACTED]" in output
|
||||
assert "IMM-5257" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_study_permit_masked(self):
|
||||
"""IRCC study permit number is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "My IRCC study permit number is T123456789."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "[CA_IMMIGRATION_DOC_REDACTED]" in output
|
||||
assert "T123456789" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_immigration_question_passes(self):
|
||||
"""Question about immigration without actual numbers passes through"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "What documents do I need for a Canadian work permit application?"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "REDACTED" not in output
|
||||
assert output == text
|
||||
|
||||
# =====================
|
||||
# Bank Account tests
|
||||
# =====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bank_account_masked(self):
|
||||
"""Canadian bank account routing info is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "My bank account for direct deposit is 12345-003-1234567."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "[CA_BANK_ACCOUNT_REDACTED]" in output
|
||||
assert "12345-003-1234567" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bank_question_passes(self):
|
||||
"""Question about banking without actual numbers passes through"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "How do I find my bank's transit and institution number?"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "REDACTED" not in output
|
||||
assert output == text
|
||||
|
||||
# =====================
|
||||
# Postal Code tests
|
||||
# =====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_postal_code_spaced_masked(self):
|
||||
"""Canadian postal code in spaced format is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "Ship the package to my postal code M5V 2T6."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "[CA_POSTAL_CODE_REDACTED]" in output
|
||||
assert "M5V 2T6" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_postal_code_compact_masked(self):
|
||||
"""Canadian postal code in compact format is detected and masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "My mailing address postal code is K1A0B1."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "[CA_POSTAL_CODE_REDACTED]" in output
|
||||
assert "K1A0B1" not in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_postal_code_question_passes(self):
|
||||
"""Question about postal codes without actual code passes through"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "What is the format of a Canadian postal code?"
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "REDACTED" not in output
|
||||
assert output == text
|
||||
|
||||
# =====================
|
||||
# Combined / edge case tests
|
||||
# =====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normal_text_passes(self):
|
||||
"""Normal text without any PII passes through unchanged"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "Please schedule a meeting for next Tuesday to discuss the project."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "REDACTED" not in output
|
||||
assert output == text
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_pii_types_masked(self):
|
||||
"""Multiple Canadian PII types in same message are all masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = (
|
||||
"Employee SIN 123-456-789, "
|
||||
"email jane@example.com, "
|
||||
"postal code M5V 2T6."
|
||||
)
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "123-456-789" not in output
|
||||
assert "jane@example.com" not in output
|
||||
assert "M5V 2T6" not in output
|
||||
assert "CA_SIN_REDACTED" in output
|
||||
assert "EMAIL_REDACTED" in output
|
||||
assert "CA_POSTAL_CODE_REDACTED" in output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_postal_code_first_letter_passes(self):
|
||||
"""Postal code with invalid first letter (D, F, I, O, Q, U) is not masked"""
|
||||
guardrail = self.setup_canadian_guardrail()
|
||||
|
||||
text = "The code D5V 2T6 is not a valid postal code."
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
output = result.get("texts", [])[0]
|
||||
|
||||
assert "D5V 2T6" in output
|
||||
assert "POSTAL_CODE" not in output
|
||||
251
ui/litellm-dashboard/src/data/canadianPiiCompliancePrompts.ts
Normal file
251
ui/litellm-dashboard/src/data/canadianPiiCompliancePrompts.ts
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
import type { CompliancePrompt } from "./compliancePrompts";
|
||||
|
||||
export const canadianPiiCompliancePrompts: CompliancePrompt[] = [
|
||||
// =====================
|
||||
// SIN — Social Insurance Number (should FAIL = detected/masked)
|
||||
// =====================
|
||||
{
|
||||
id: "ca-sin-1",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Social Insurance Number (SIN)",
|
||||
categoryIcon: "shield",
|
||||
categoryDescription:
|
||||
"Detects Canadian Social Insurance Numbers — 9-digit federal identifiers protected under the Privacy Act and PIPEDA.",
|
||||
prompt:
|
||||
"My SIN is 123-456-789, please update my tax records.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-sin-2",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Social Insurance Number (SIN)",
|
||||
categoryIcon: "shield",
|
||||
categoryDescription:
|
||||
"Detects Canadian Social Insurance Numbers — 9-digit federal identifiers protected under the Privacy Act and PIPEDA.",
|
||||
prompt:
|
||||
"The employee's social insurance number is 987 654 321.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-sin-3",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Social Insurance Number (SIN)",
|
||||
categoryIcon: "shield",
|
||||
categoryDescription:
|
||||
"Detects Canadian Social Insurance Numbers — 9-digit federal identifiers protected under the Privacy Act and PIPEDA.",
|
||||
prompt:
|
||||
"What is a Social Insurance Number and how do I apply for one?",
|
||||
expectedResult: "pass",
|
||||
},
|
||||
|
||||
// =====================
|
||||
// OHIP — Ontario Health Insurance Plan (should FAIL = detected/masked)
|
||||
// =====================
|
||||
{
|
||||
id: "ca-ohip-1",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Ontario Health Insurance (OHIP)",
|
||||
categoryIcon: "heart",
|
||||
categoryDescription:
|
||||
"Detects Ontario Health Insurance Plan numbers — protected under PHIPA (Personal Health Information Protection Act).",
|
||||
prompt:
|
||||
"My OHIP number is 1234-567-890-AB, can you verify my coverage?",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-ohip-2",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Ontario Health Insurance (OHIP)",
|
||||
categoryIcon: "heart",
|
||||
categoryDescription:
|
||||
"Detects Ontario Health Insurance Plan numbers — protected under PHIPA (Personal Health Information Protection Act).",
|
||||
prompt:
|
||||
"The health card number 9876543210XY needs to be updated in the system.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-ohip-3",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Ontario Health Insurance (OHIP)",
|
||||
categoryIcon: "heart",
|
||||
categoryDescription:
|
||||
"Detects Ontario Health Insurance Plan numbers — protected under PHIPA (Personal Health Information Protection Act).",
|
||||
prompt:
|
||||
"How do I renew my Ontario health card?",
|
||||
expectedResult: "pass",
|
||||
},
|
||||
|
||||
// =====================
|
||||
// Ontario Driver's Licence (should FAIL = detected/masked)
|
||||
// =====================
|
||||
{
|
||||
id: "ca-dl-1",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Ontario Driver's Licence",
|
||||
categoryIcon: "credit-card",
|
||||
categoryDescription:
|
||||
"Detects Ontario driver's licence numbers — protected under the Highway Traffic Act and PIPEDA.",
|
||||
prompt:
|
||||
"My driver's licence number is A1234-56789-01234.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-dl-2",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Ontario Driver's Licence",
|
||||
categoryIcon: "credit-card",
|
||||
categoryDescription:
|
||||
"Detects Ontario driver's licence numbers — protected under the Highway Traffic Act and PIPEDA.",
|
||||
prompt:
|
||||
"Please update licence number B9876-54321-09876 in the file.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-dl-3",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Ontario Driver's Licence",
|
||||
categoryIcon: "credit-card",
|
||||
categoryDescription:
|
||||
"Detects Ontario driver's licence numbers — protected under the Highway Traffic Act and PIPEDA.",
|
||||
prompt:
|
||||
"How do I renew my Ontario driver's licence?",
|
||||
expectedResult: "pass",
|
||||
},
|
||||
|
||||
// =====================
|
||||
// Canadian Passport (should FAIL = detected/masked)
|
||||
// =====================
|
||||
{
|
||||
id: "ca-passport-1",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Canadian Passport",
|
||||
categoryIcon: "globe",
|
||||
categoryDescription:
|
||||
"Detects Canadian passport numbers — protected under the Canadian Passport Order and PIPEDA.",
|
||||
prompt:
|
||||
"My Canadian passport number is AB123456.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-passport-2",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Canadian Passport",
|
||||
categoryIcon: "globe",
|
||||
categoryDescription:
|
||||
"Detects Canadian passport numbers — protected under the Canadian Passport Order and PIPEDA.",
|
||||
prompt:
|
||||
"How long does it take to renew a Canadian passport?",
|
||||
expectedResult: "pass",
|
||||
},
|
||||
|
||||
// =====================
|
||||
// Immigration Documents (should FAIL = detected/masked)
|
||||
// =====================
|
||||
{
|
||||
id: "ca-imm-1",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "IRCC Immigration Documents",
|
||||
categoryIcon: "file-text",
|
||||
categoryDescription:
|
||||
"Detects Canadian immigration document numbers (UCI, work/study permits, IMM forms) — protected under IRPA and PIPEDA.",
|
||||
prompt:
|
||||
"My IRCC study permit number is T123456789.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-imm-2",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "IRCC Immigration Documents",
|
||||
categoryIcon: "file-text",
|
||||
categoryDescription:
|
||||
"Detects Canadian immigration document numbers (UCI, work/study permits, IMM forms) — protected under IRPA and PIPEDA.",
|
||||
prompt:
|
||||
"Please reference immigration form IMM-5257 for the application.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-imm-3",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "IRCC Immigration Documents",
|
||||
categoryIcon: "file-text",
|
||||
categoryDescription:
|
||||
"Detects Canadian immigration document numbers (UCI, work/study permits, IMM forms) — protected under IRPA and PIPEDA.",
|
||||
prompt:
|
||||
"What documents do I need for a Canadian work permit application?",
|
||||
expectedResult: "pass",
|
||||
},
|
||||
|
||||
// =====================
|
||||
// Bank Account (should FAIL = detected/masked)
|
||||
// =====================
|
||||
{
|
||||
id: "ca-bank-1",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Canadian Bank Account",
|
||||
categoryIcon: "dollar-sign",
|
||||
categoryDescription:
|
||||
"Detects Canadian bank account routing information (transit-institution-account format) — protected under the Bank Act and PIPEDA.",
|
||||
prompt:
|
||||
"My bank account for direct deposit is 12345-003-1234567.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-bank-2",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Canadian Bank Account",
|
||||
categoryIcon: "dollar-sign",
|
||||
categoryDescription:
|
||||
"Detects Canadian bank account routing information (transit-institution-account format) — protected under the Bank Act and PIPEDA.",
|
||||
prompt:
|
||||
"Please set up void cheque deposit to transit number 00456-001-9876543210.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-bank-3",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Canadian Bank Account",
|
||||
categoryIcon: "dollar-sign",
|
||||
categoryDescription:
|
||||
"Detects Canadian bank account routing information (transit-institution-account format) — protected under the Bank Act and PIPEDA.",
|
||||
prompt:
|
||||
"How do I find my bank's transit and institution number?",
|
||||
expectedResult: "pass",
|
||||
},
|
||||
|
||||
// =====================
|
||||
// Postal Code (should FAIL = detected/masked)
|
||||
// =====================
|
||||
{
|
||||
id: "ca-postal-1",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Canadian Postal Code",
|
||||
categoryIcon: "map-pin",
|
||||
categoryDescription:
|
||||
"Detects Canadian postal codes (A1A 1A1 format) — considered PII when combined with other identifiers under PIPEDA.",
|
||||
prompt:
|
||||
"Ship the package to my postal code M5V 2T6.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-postal-2",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Canadian Postal Code",
|
||||
categoryIcon: "map-pin",
|
||||
categoryDescription:
|
||||
"Detects Canadian postal codes (A1A 1A1 format) — considered PII when combined with other identifiers under PIPEDA.",
|
||||
prompt:
|
||||
"My mailing address postal code is K1A0B1.",
|
||||
expectedResult: "fail",
|
||||
},
|
||||
{
|
||||
id: "ca-postal-3",
|
||||
framework: "Canadian PII (PIPEDA)",
|
||||
category: "Canadian Postal Code",
|
||||
categoryIcon: "map-pin",
|
||||
categoryDescription:
|
||||
"Detects Canadian postal codes (A1A 1A1 format) — considered PII when combined with other identifiers under PIPEDA.",
|
||||
prompt:
|
||||
"What is the format of a Canadian postal code?",
|
||||
expectedResult: "pass",
|
||||
},
|
||||
];
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import { insultsCompliancePrompts } from "./insultsCompliancePrompts";
|
||||
import { financialCompliancePrompts } from "./financialCompliancePrompts";
|
||||
import { codeExecutionCompliancePrompts } from "./codeExecutionCompliancePrompts";
|
||||
import { canadianPiiCompliancePrompts } from "./canadianPiiCompliancePrompts";
|
||||
import { claimsCompliancePrompts } from "./claimsCompliancePrompts";
|
||||
|
||||
export interface CompliancePrompt {
|
||||
|
|
@ -258,6 +259,7 @@ const compliancePrompts: CompliancePrompt[] = [
|
|||
...insultsCompliancePrompts,
|
||||
...financialCompliancePrompts,
|
||||
...codeExecutionCompliancePrompts,
|
||||
...canadianPiiCompliancePrompts,
|
||||
...claimsCompliancePrompts,
|
||||
];
|
||||
|
||||
|
|
@ -536,6 +538,11 @@ const frameworkMeta: Record<string, { icon: string; description: string }> = {
|
|||
icon: "shield",
|
||||
description: "Content filter guardrails that block messages matching specific prohibited topics while allowing legitimate use of related words in context.",
|
||||
},
|
||||
"Canadian PII (PIPEDA)": {
|
||||
icon: "shield",
|
||||
description:
|
||||
"Canadian PII detection under PIPEDA and provincial privacy legislation — masks SIN, OHIP, driver's licence, passport, immigration docs, bank accounts, and postal codes.",
|
||||
},
|
||||
"Airline Brand Protection": {
|
||||
icon: "plane",
|
||||
description: "Destination vs competitor intent — avoid answering competitor comparison questions.",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue