From f55fda3be770451a06a0c7e46a10a66b9452ef4d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 5 Mar 2026 19:39:42 -0800 Subject: [PATCH] feat: Add Canadian PII compliance dataset and tests (57 tests) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds: - test_ca_patterns.py: 30 unit tests for regex pattern matching (SIN, OHIP, driver's licence, immigration docs, bank account, postal code) - test_ca_policy_e2e.py: 27 end-to-end tests running the full ContentFilterGuardrail pipeline with MASK action — validates detection of real PII and pass-through of clean prompts - canadianPiiCompliancePrompts.ts: 21-prompt compliance dataset for UI evaluation, wired into the main compliancePrompts framework Fixes keyword_pattern alternation ordering in patterns.json — longer alternatives (e.g. "social insurance number") now precede shorter ones ("social insurance") to avoid excessive gap-word count when the regex engine selects the shorter match first. Co-Authored-By: Claude Haiku 4.5 --- .../litellm_content_filter/patterns.json | 8 +- .../content_filter/test_ca_patterns.py | 208 ++++++++ .../content_filter/test_ca_policy_e2e.py | 456 ++++++++++++++++++ .../src/data/canadianPiiCompliancePrompts.ts | 251 ++++++++++ .../src/data/compliancePrompts.ts | 7 + 5 files changed, 926 insertions(+), 4 deletions(-) create mode 100644 tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_ca_patterns.py create mode 100644 tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_ca_policy_e2e.py create mode 100644 ui/litellm-dashboard/src/data/canadianPiiCompliancePrompts.ts diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json index 0dc4eda8b5f..25d83fb1bd6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json @@ -550,7 +550,7 @@ "pattern": "\\b\\d{3}[\\-\\s]\\d{3}[\\-\\s]\\d{3}\\b", "category": "Canadian PII Patterns", "description": "Detects Canadian Social Insurance Numbers (9-digit federal identifier, dashed or spaced format)", - "keyword_pattern": "\\b(?:SIN|social\\s*insurance|social\\s*insurance\\s*number|numéro\\s*d'assurance\\s*sociale|NAS)\\b", + "keyword_pattern": "\\b(?:social\\s*insurance\\s*number|numéro\\s*d'assurance\\s*sociale|social\\s*insurance|SIN|NAS)\\b", "allow_word_numbers": true }, { @@ -559,7 +559,7 @@ "pattern": "\\b\\d{4}[\\-\\s]?\\d{3}[\\-\\s]?\\d{3}[\\-\\s]?[A-Z]{2}\\b", "category": "Canadian PII Patterns", "description": "Detects Ontario Health Insurance Plan numbers (10 digits + 2-letter version code)", - "keyword_pattern": "\\b(?:OHIP|health\\s*card|health\\s*insurance|ontario\\s*health|health\\s*number|carte\\s*santé)\\b", + "keyword_pattern": "\\b(?:OHIP\\s*number|health\\s*card\\s*number|health\\s*insurance\\s*number|OHIP|health\\s*card|health\\s*insurance|ontario\\s*health|health\\s*number|carte\\s*santé)\\b", "allow_word_numbers": false }, { @@ -568,7 +568,7 @@ "pattern": "\\b[A-Z]\\d{4}[\\-\\s]\\d{5}[\\-\\s]\\d{5}\\b", "category": "Canadian PII Patterns", "description": "Detects Ontario driver's licence numbers (1 letter + 14 digits, dashed format)", - "keyword_pattern": "\\b(?:driver'?s?\\s*licen[cs]e|licen[cs]e\\s*number|DL\\s*number|ontario\\s*licen[cs]e|permis\\s*de\\s*conduire|numéro\\s*de\\s*permis)\\b", + "keyword_pattern": "\\b(?:driver'?s?\\s*licen[cs]e\\s*number|ontario\\s*licen[cs]e\\s*number|driver'?s?\\s*licen[cs]e|licen[cs]e\\s*number|DL\\s*number|ontario\\s*licen[cs]e|permis\\s*de\\s*conduire|numéro\\s*de\\s*permis)\\b", "allow_word_numbers": false }, { @@ -577,7 +577,7 @@ "pattern": "\\b(?:\\d{4}[\\-\\s]?\\d{4}[\\-\\s]?\\d{2}|[TUFW]\\d{8,10}|IMM[\\-\\s]?\\d{4,5})\\b", "category": "Canadian PII Patterns", "description": "Detects Canadian immigration document numbers including UCI (10-digit), work/study permits, and IMM document references", - "keyword_pattern": "\\b(?:UCI|unique\\s*client\\s*identifier|study\\s*permit|work\\s*permit|immigration|IRCC|permanent\\s*resident|PR\\s*card|permis\\s*d'études|permis\\s*de\\s*travail|landed\\s*immigrant|temporary\\s*resident)\\b", + "keyword_pattern": "\\b(?:unique\\s*client\\s*identifier|study\\s*permit\\s*number|work\\s*permit\\s*number|study\\s*permit|work\\s*permit|UCI|immigration|IRCC|permanent\\s*resident|PR\\s*card|permis\\s*d'études|permis\\s*de\\s*travail|landed\\s*immigrant|temporary\\s*resident)\\b", "allow_word_numbers": false }, { diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_ca_patterns.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_ca_patterns.py new file mode 100644 index 00000000000..cc927ea1c57 --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_ca_patterns.py @@ -0,0 +1,208 @@ +""" +Test Canadian PII regex patterns added for PIPEDA compliance. + +Tests SIN, OHIP, Ontario driver's licence, immigration documents, +bank account, and postal code detection patterns. +""" + +from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.patterns import ( + get_compiled_pattern, +) + + +class TestCanadianSIN: + """Test Canadian Social Insurance Number detection""" + + def test_dashed_format(self): + pattern = get_compiled_pattern("ca_sin") + assert pattern.search("123-456-789") is not None + + def test_spaced_format(self): + pattern = get_compiled_pattern("ca_sin") + assert pattern.search("123 456 789") is not None + + def test_sin_in_sentence(self): + pattern = get_compiled_pattern("ca_sin") + assert pattern.search("My SIN is 987-654-321 for tax") is not None + + def test_compact_nine_digits_not_matched(self): + """Compact 9-digit format (no dashes/spaces) should NOT match the dashed pattern""" + pattern = get_compiled_pattern("ca_sin") + assert pattern.search("123456789") is None + + def test_too_few_digits_rejected(self): + pattern = get_compiled_pattern("ca_sin") + assert pattern.search("12-345-678") is None + + def test_too_many_digits_rejected(self): + pattern = get_compiled_pattern("ca_sin") + assert pattern.search("1234-567-890") is None + + +class TestCanadianOHIP: + """Test Ontario Health Insurance Plan Number detection""" + + def test_full_format_with_version_code(self): + pattern = get_compiled_pattern("ca_ohip") + assert pattern.search("1234-567-890-AB") is not None + + def test_compact_with_version_code(self): + pattern = get_compiled_pattern("ca_ohip") + assert pattern.search("1234567890AB") is not None + + def test_spaced_format(self): + pattern = get_compiled_pattern("ca_ohip") + assert pattern.search("1234 567 890 XY") is not None + + def test_ohip_in_sentence(self): + pattern = get_compiled_pattern("ca_ohip") + assert ( + pattern.search("My OHIP number is 9876543210ZZ for my appointment") + is not None + ) + + def test_without_version_code_not_matched(self): + """OHIP pattern requires the 2-letter version code""" + pattern = get_compiled_pattern("ca_ohip") + # 10 digits alone without letters should not match the full OHIP pattern + assert pattern.search("1234567890") is None + + def test_lowercase_version_code_detected(self): + """Compiled with IGNORECASE""" + pattern = get_compiled_pattern("ca_ohip") + assert pattern.search("1234567890ab") is not None + + +class TestCanadianOntarioDriversLicence: + """Test Ontario Driver's Licence detection""" + + def test_dashed_format(self): + pattern = get_compiled_pattern("ca_on_drivers_licence") + assert pattern.search("A1234-56789-01234") is not None + + def test_spaced_format(self): + pattern = get_compiled_pattern("ca_on_drivers_licence") + assert pattern.search("B9876 54321 09876") is not None + + def test_in_sentence(self): + pattern = get_compiled_pattern("ca_on_drivers_licence") + assert ( + pattern.search("Driver's licence C1111-22222-33333 on file") is not None + ) + + def test_compact_format_not_matched(self): + """Compact format (no dashes/spaces) uses a separate pattern""" + pattern = get_compiled_pattern("ca_on_drivers_licence") + assert pattern.search("A12345678901234") is None + + def test_missing_letter_prefix_rejected(self): + pattern = get_compiled_pattern("ca_on_drivers_licence") + assert pattern.search("11234-56789-01234") is None + + def test_wrong_digit_groups_rejected(self): + pattern = get_compiled_pattern("ca_on_drivers_licence") + assert pattern.search("A123-456789-01234") is None + + +class TestCanadianImmigrationDoc: + """Test Canadian IRCC Immigration Document detection""" + + def test_imm_document_reference(self): + pattern = get_compiled_pattern("ca_immigration_doc") + assert pattern.search("IMM-5257") is not None + assert pattern.search("IMM 1234") is not None + assert pattern.search("IMM5257") is not None + + def test_work_study_permit(self): + pattern = get_compiled_pattern("ca_immigration_doc") + assert pattern.search("T123456789") is not None + assert pattern.search("F1234567890") is not None + assert pattern.search("W12345678") is not None + + def test_uci_dashed(self): + pattern = get_compiled_pattern("ca_immigration_doc") + assert pattern.search("1234-5678-90") is not None + + def test_uci_compact(self): + pattern = get_compiled_pattern("ca_immigration_doc") + assert pattern.search("1234567890") is not None + + def test_imm_in_sentence(self): + pattern = get_compiled_pattern("ca_immigration_doc") + assert ( + pattern.search("Submit immigration form IMM-5645 with your application") + is not None + ) + + def test_too_short_imm_rejected(self): + pattern = get_compiled_pattern("ca_immigration_doc") + assert pattern.search("IMM-12") is None # Less than 4 digits + + +class TestCanadianBankAccount: + """Test Canadian Bank Account routing detection""" + + def test_standard_format_dashed(self): + pattern = get_compiled_pattern("ca_bank_account") + assert pattern.search("12345-003-1234567") is not None + + def test_spaced_format(self): + pattern = get_compiled_pattern("ca_bank_account") + assert pattern.search("00456 001 9876543210") is not None + + def test_longer_account_number(self): + pattern = get_compiled_pattern("ca_bank_account") + assert pattern.search("12345-003-123456789012") is not None + + def test_in_sentence(self): + pattern = get_compiled_pattern("ca_bank_account") + assert ( + pattern.search("Direct deposit to bank account 12345-003-1234567 please") + is not None + ) + + def test_without_separators_rejected(self): + pattern = get_compiled_pattern("ca_bank_account") + assert pattern.search("123450031234567") is None + + +class TestCanadianPostalCode: + """Test Canadian Postal Code detection""" + + def test_spaced_format(self): + pattern = get_compiled_pattern("ca_postal_code") + assert pattern.search("M5V 2T6") is not None + assert pattern.search("K1A 0B1") is not None + assert pattern.search("V6B 3K9") is not None + + def test_compact_format(self): + pattern = get_compiled_pattern("ca_postal_code") + assert pattern.search("M5V2T6") is not None + assert pattern.search("K1A0B1") is not None + + def test_dashed_format(self): + pattern = get_compiled_pattern("ca_postal_code") + assert pattern.search("M5V-2T6") is not None + + def test_in_sentence(self): + pattern = get_compiled_pattern("ca_postal_code") + assert pattern.search("Ship to postal code M5V 2T6 in Toronto") is not None + + def test_invalid_first_letter_rejected(self): + """Letters D, F, I, O, Q, U are not valid as first character""" + pattern = get_compiled_pattern("ca_postal_code") + assert pattern.search("D5V 2T6") is None + assert pattern.search("F1A 0B1") is None + assert pattern.search("I5V 2T6") is None + assert pattern.search("O1A 0B1") is None + assert pattern.search("Q5V 2T6") is None + assert pattern.search("U1A 0B1") is None + + def test_lowercase_detected(self): + """Compiled with IGNORECASE""" + pattern = get_compiled_pattern("ca_postal_code") + assert pattern.search("m5v 2t6") is not None + + def test_all_digits_rejected(self): + pattern = get_compiled_pattern("ca_postal_code") + assert pattern.search("123 456") is None diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_ca_policy_e2e.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_ca_policy_e2e.py new file mode 100644 index 00000000000..37c1dcee438 --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_ca_policy_e2e.py @@ -0,0 +1,456 @@ +""" +End-to-end tests for Canadian PII Protection (PIPEDA) policy template. + +Tests the complete policy with all Canadian PII patterns — validates that +PII-containing prompts are detected/masked and that clean prompts pass through. +These tests mirror the canadianPiiCompliancePrompts.ts dataset. +""" + +import pytest + +from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, +) +from litellm.types.guardrails import ContentFilterAction, ContentFilterPattern + + +class TestCanadianPIIPolicyE2E: + """End-to-end tests for Canadian PII policy template""" + + def setup_canadian_guardrail(self): + """ + Setup guardrail with all Canadian PII patterns (mimics the policy template) + """ + patterns = [ + # Government identifiers + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="ca_sin", + action=ContentFilterAction.MASK, + ), + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="passport_canada", + action=ContentFilterAction.MASK, + ), + # Health & drivers + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="ca_ohip", + action=ContentFilterAction.MASK, + ), + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="ca_on_drivers_licence", + action=ContentFilterAction.MASK, + ), + # Immigration + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="ca_immigration_doc", + action=ContentFilterAction.MASK, + ), + # Financial + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="ca_bank_account", + action=ContentFilterAction.MASK, + ), + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="credit_card", + action=ContentFilterAction.MASK, + ), + # Contact info + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="email", + action=ContentFilterAction.MASK, + ), + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="ca_postal_code", + action=ContentFilterAction.MASK, + ), + ] + + return ContentFilterGuardrail( + guardrail_name="canadian-pii-protection", + patterns=patterns, + ) + + # ===================== + # SIN tests + # ===================== + + @pytest.mark.asyncio + async def test_sin_dashed_masked(self): + """SIN in dashed format is detected and masked""" + guardrail = self.setup_canadian_guardrail() + + text = "My SIN is 123-456-789, please update my tax records." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "[CA_SIN_REDACTED]" in output + assert "123-456-789" not in output + + @pytest.mark.asyncio + async def test_sin_spaced_masked(self): + """SIN in spaced format is detected and masked""" + guardrail = self.setup_canadian_guardrail() + + text = "The employee's social insurance number is 987 654 321." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "[CA_SIN_REDACTED]" in output + assert "987 654 321" not in output + + @pytest.mark.asyncio + async def test_sin_question_passes(self): + """Question about SIN without actual number passes through""" + guardrail = self.setup_canadian_guardrail() + + text = "What is a Social Insurance Number and how do I apply for one?" + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "REDACTED" not in output + assert output == text + + # ===================== + # OHIP tests + # ===================== + + @pytest.mark.asyncio + async def test_ohip_dashed_masked(self): + """OHIP number with version code is detected and masked""" + guardrail = self.setup_canadian_guardrail() + + text = "My OHIP number is 1234-567-890-AB, can you verify my coverage?" + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "[CA_OHIP_REDACTED]" in output + assert "1234-567-890-AB" not in output + + @pytest.mark.asyncio + async def test_ohip_compact_masked(self): + """OHIP number in compact format is detected and masked""" + guardrail = self.setup_canadian_guardrail() + + text = "The health card number 9876543210XY needs to be updated in the system." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "[CA_OHIP_REDACTED]" in output + assert "9876543210XY" not in output + + @pytest.mark.asyncio + async def test_ohip_question_passes(self): + """Question about OHIP without actual number passes through""" + guardrail = self.setup_canadian_guardrail() + + text = "How do I renew my Ontario health card?" + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "REDACTED" not in output + assert output == text + + # ===================== + # Ontario Driver's Licence tests + # ===================== + + @pytest.mark.asyncio + async def test_drivers_licence_masked(self): + """Ontario driver's licence is detected and masked""" + guardrail = self.setup_canadian_guardrail() + + text = "My driver's licence number is A1234-56789-01234." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "[CA_ON_DRIVERS_LICENCE_REDACTED]" in output + assert "A1234-56789-01234" not in output + + @pytest.mark.asyncio + async def test_drivers_licence_question_passes(self): + """Question about driver's licence without actual number passes through""" + guardrail = self.setup_canadian_guardrail() + + text = "How do I renew my Ontario driver's licence?" + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "REDACTED" not in output + assert output == text + + # ===================== + # Canadian Passport tests + # ===================== + + @pytest.mark.asyncio + async def test_passport_masked(self): + """Canadian passport number is detected and masked""" + guardrail = self.setup_canadian_guardrail() + + text = "My Canadian passport number is AB123456." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "[PASSPORT_CANADA_REDACTED]" in output + assert "AB123456" not in output + + @pytest.mark.asyncio + async def test_passport_question_passes(self): + """Question about passports without actual number passes through""" + guardrail = self.setup_canadian_guardrail() + + text = "How long does it take to renew a Canadian passport?" + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "REDACTED" not in output + assert output == text + + # ===================== + # Immigration Document tests + # ===================== + + @pytest.mark.asyncio + async def test_imm_form_masked(self): + """IRCC IMM form reference is detected and masked""" + guardrail = self.setup_canadian_guardrail() + + text = "Please reference immigration form IMM-5257 for the application." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "[CA_IMMIGRATION_DOC_REDACTED]" in output + assert "IMM-5257" not in output + + @pytest.mark.asyncio + async def test_study_permit_masked(self): + """IRCC study permit number is detected and masked""" + guardrail = self.setup_canadian_guardrail() + + text = "My IRCC study permit number is T123456789." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "[CA_IMMIGRATION_DOC_REDACTED]" in output + assert "T123456789" not in output + + @pytest.mark.asyncio + async def test_immigration_question_passes(self): + """Question about immigration without actual numbers passes through""" + guardrail = self.setup_canadian_guardrail() + + text = "What documents do I need for a Canadian work permit application?" + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "REDACTED" not in output + assert output == text + + # ===================== + # Bank Account tests + # ===================== + + @pytest.mark.asyncio + async def test_bank_account_masked(self): + """Canadian bank account routing info is detected and masked""" + guardrail = self.setup_canadian_guardrail() + + text = "My bank account for direct deposit is 12345-003-1234567." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "[CA_BANK_ACCOUNT_REDACTED]" in output + assert "12345-003-1234567" not in output + + @pytest.mark.asyncio + async def test_bank_question_passes(self): + """Question about banking without actual numbers passes through""" + guardrail = self.setup_canadian_guardrail() + + text = "How do I find my bank's transit and institution number?" + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "REDACTED" not in output + assert output == text + + # ===================== + # Postal Code tests + # ===================== + + @pytest.mark.asyncio + async def test_postal_code_spaced_masked(self): + """Canadian postal code in spaced format is detected and masked""" + guardrail = self.setup_canadian_guardrail() + + text = "Ship the package to my postal code M5V 2T6." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "[CA_POSTAL_CODE_REDACTED]" in output + assert "M5V 2T6" not in output + + @pytest.mark.asyncio + async def test_postal_code_compact_masked(self): + """Canadian postal code in compact format is detected and masked""" + guardrail = self.setup_canadian_guardrail() + + text = "My mailing address postal code is K1A0B1." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "[CA_POSTAL_CODE_REDACTED]" in output + assert "K1A0B1" not in output + + @pytest.mark.asyncio + async def test_postal_code_question_passes(self): + """Question about postal codes without actual code passes through""" + guardrail = self.setup_canadian_guardrail() + + text = "What is the format of a Canadian postal code?" + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "REDACTED" not in output + assert output == text + + # ===================== + # Combined / edge case tests + # ===================== + + @pytest.mark.asyncio + async def test_normal_text_passes(self): + """Normal text without any PII passes through unchanged""" + guardrail = self.setup_canadian_guardrail() + + text = "Please schedule a meeting for next Tuesday to discuss the project." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "REDACTED" not in output + assert output == text + + @pytest.mark.asyncio + async def test_multiple_pii_types_masked(self): + """Multiple Canadian PII types in same message are all masked""" + guardrail = self.setup_canadian_guardrail() + + text = ( + "Employee SIN 123-456-789, " + "email jane@example.com, " + "postal code M5V 2T6." + ) + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "123-456-789" not in output + assert "jane@example.com" not in output + assert "M5V 2T6" not in output + assert "CA_SIN_REDACTED" in output + assert "EMAIL_REDACTED" in output + assert "CA_POSTAL_CODE_REDACTED" in output + + @pytest.mark.asyncio + async def test_invalid_postal_code_first_letter_passes(self): + """Postal code with invalid first letter (D, F, I, O, Q, U) is not masked""" + guardrail = self.setup_canadian_guardrail() + + text = "The code D5V 2T6 is not a valid postal code." + result = await guardrail.apply_guardrail( + inputs={"texts": [text]}, + request_data={}, + input_type="request", + ) + output = result.get("texts", [])[0] + + assert "D5V 2T6" in output + assert "POSTAL_CODE" not in output diff --git a/ui/litellm-dashboard/src/data/canadianPiiCompliancePrompts.ts b/ui/litellm-dashboard/src/data/canadianPiiCompliancePrompts.ts new file mode 100644 index 00000000000..564b4b3d8b2 --- /dev/null +++ b/ui/litellm-dashboard/src/data/canadianPiiCompliancePrompts.ts @@ -0,0 +1,251 @@ +import type { CompliancePrompt } from "./compliancePrompts"; + +export const canadianPiiCompliancePrompts: CompliancePrompt[] = [ + // ===================== + // SIN — Social Insurance Number (should FAIL = detected/masked) + // ===================== + { + id: "ca-sin-1", + framework: "Canadian PII (PIPEDA)", + category: "Social Insurance Number (SIN)", + categoryIcon: "shield", + categoryDescription: + "Detects Canadian Social Insurance Numbers — 9-digit federal identifiers protected under the Privacy Act and PIPEDA.", + prompt: + "My SIN is 123-456-789, please update my tax records.", + expectedResult: "fail", + }, + { + id: "ca-sin-2", + framework: "Canadian PII (PIPEDA)", + category: "Social Insurance Number (SIN)", + categoryIcon: "shield", + categoryDescription: + "Detects Canadian Social Insurance Numbers — 9-digit federal identifiers protected under the Privacy Act and PIPEDA.", + prompt: + "The employee's social insurance number is 987 654 321.", + expectedResult: "fail", + }, + { + id: "ca-sin-3", + framework: "Canadian PII (PIPEDA)", + category: "Social Insurance Number (SIN)", + categoryIcon: "shield", + categoryDescription: + "Detects Canadian Social Insurance Numbers — 9-digit federal identifiers protected under the Privacy Act and PIPEDA.", + prompt: + "What is a Social Insurance Number and how do I apply for one?", + expectedResult: "pass", + }, + + // ===================== + // OHIP — Ontario Health Insurance Plan (should FAIL = detected/masked) + // ===================== + { + id: "ca-ohip-1", + framework: "Canadian PII (PIPEDA)", + category: "Ontario Health Insurance (OHIP)", + categoryIcon: "heart", + categoryDescription: + "Detects Ontario Health Insurance Plan numbers — protected under PHIPA (Personal Health Information Protection Act).", + prompt: + "My OHIP number is 1234-567-890-AB, can you verify my coverage?", + expectedResult: "fail", + }, + { + id: "ca-ohip-2", + framework: "Canadian PII (PIPEDA)", + category: "Ontario Health Insurance (OHIP)", + categoryIcon: "heart", + categoryDescription: + "Detects Ontario Health Insurance Plan numbers — protected under PHIPA (Personal Health Information Protection Act).", + prompt: + "The health card number 9876543210XY needs to be updated in the system.", + expectedResult: "fail", + }, + { + id: "ca-ohip-3", + framework: "Canadian PII (PIPEDA)", + category: "Ontario Health Insurance (OHIP)", + categoryIcon: "heart", + categoryDescription: + "Detects Ontario Health Insurance Plan numbers — protected under PHIPA (Personal Health Information Protection Act).", + prompt: + "How do I renew my Ontario health card?", + expectedResult: "pass", + }, + + // ===================== + // Ontario Driver's Licence (should FAIL = detected/masked) + // ===================== + { + id: "ca-dl-1", + framework: "Canadian PII (PIPEDA)", + category: "Ontario Driver's Licence", + categoryIcon: "credit-card", + categoryDescription: + "Detects Ontario driver's licence numbers — protected under the Highway Traffic Act and PIPEDA.", + prompt: + "My driver's licence number is A1234-56789-01234.", + expectedResult: "fail", + }, + { + id: "ca-dl-2", + framework: "Canadian PII (PIPEDA)", + category: "Ontario Driver's Licence", + categoryIcon: "credit-card", + categoryDescription: + "Detects Ontario driver's licence numbers — protected under the Highway Traffic Act and PIPEDA.", + prompt: + "Please update licence number B9876-54321-09876 in the file.", + expectedResult: "fail", + }, + { + id: "ca-dl-3", + framework: "Canadian PII (PIPEDA)", + category: "Ontario Driver's Licence", + categoryIcon: "credit-card", + categoryDescription: + "Detects Ontario driver's licence numbers — protected under the Highway Traffic Act and PIPEDA.", + prompt: + "How do I renew my Ontario driver's licence?", + expectedResult: "pass", + }, + + // ===================== + // Canadian Passport (should FAIL = detected/masked) + // ===================== + { + id: "ca-passport-1", + framework: "Canadian PII (PIPEDA)", + category: "Canadian Passport", + categoryIcon: "globe", + categoryDescription: + "Detects Canadian passport numbers — protected under the Canadian Passport Order and PIPEDA.", + prompt: + "My Canadian passport number is AB123456.", + expectedResult: "fail", + }, + { + id: "ca-passport-2", + framework: "Canadian PII (PIPEDA)", + category: "Canadian Passport", + categoryIcon: "globe", + categoryDescription: + "Detects Canadian passport numbers — protected under the Canadian Passport Order and PIPEDA.", + prompt: + "How long does it take to renew a Canadian passport?", + expectedResult: "pass", + }, + + // ===================== + // Immigration Documents (should FAIL = detected/masked) + // ===================== + { + id: "ca-imm-1", + framework: "Canadian PII (PIPEDA)", + category: "IRCC Immigration Documents", + categoryIcon: "file-text", + categoryDescription: + "Detects Canadian immigration document numbers (UCI, work/study permits, IMM forms) — protected under IRPA and PIPEDA.", + prompt: + "My IRCC study permit number is T123456789.", + expectedResult: "fail", + }, + { + id: "ca-imm-2", + framework: "Canadian PII (PIPEDA)", + category: "IRCC Immigration Documents", + categoryIcon: "file-text", + categoryDescription: + "Detects Canadian immigration document numbers (UCI, work/study permits, IMM forms) — protected under IRPA and PIPEDA.", + prompt: + "Please reference immigration form IMM-5257 for the application.", + expectedResult: "fail", + }, + { + id: "ca-imm-3", + framework: "Canadian PII (PIPEDA)", + category: "IRCC Immigration Documents", + categoryIcon: "file-text", + categoryDescription: + "Detects Canadian immigration document numbers (UCI, work/study permits, IMM forms) — protected under IRPA and PIPEDA.", + prompt: + "What documents do I need for a Canadian work permit application?", + expectedResult: "pass", + }, + + // ===================== + // Bank Account (should FAIL = detected/masked) + // ===================== + { + id: "ca-bank-1", + framework: "Canadian PII (PIPEDA)", + category: "Canadian Bank Account", + categoryIcon: "dollar-sign", + categoryDescription: + "Detects Canadian bank account routing information (transit-institution-account format) — protected under the Bank Act and PIPEDA.", + prompt: + "My bank account for direct deposit is 12345-003-1234567.", + expectedResult: "fail", + }, + { + id: "ca-bank-2", + framework: "Canadian PII (PIPEDA)", + category: "Canadian Bank Account", + categoryIcon: "dollar-sign", + categoryDescription: + "Detects Canadian bank account routing information (transit-institution-account format) — protected under the Bank Act and PIPEDA.", + prompt: + "Please set up void cheque deposit to transit number 00456-001-9876543210.", + expectedResult: "fail", + }, + { + id: "ca-bank-3", + framework: "Canadian PII (PIPEDA)", + category: "Canadian Bank Account", + categoryIcon: "dollar-sign", + categoryDescription: + "Detects Canadian bank account routing information (transit-institution-account format) — protected under the Bank Act and PIPEDA.", + prompt: + "How do I find my bank's transit and institution number?", + expectedResult: "pass", + }, + + // ===================== + // Postal Code (should FAIL = detected/masked) + // ===================== + { + id: "ca-postal-1", + framework: "Canadian PII (PIPEDA)", + category: "Canadian Postal Code", + categoryIcon: "map-pin", + categoryDescription: + "Detects Canadian postal codes (A1A 1A1 format) — considered PII when combined with other identifiers under PIPEDA.", + prompt: + "Ship the package to my postal code M5V 2T6.", + expectedResult: "fail", + }, + { + id: "ca-postal-2", + framework: "Canadian PII (PIPEDA)", + category: "Canadian Postal Code", + categoryIcon: "map-pin", + categoryDescription: + "Detects Canadian postal codes (A1A 1A1 format) — considered PII when combined with other identifiers under PIPEDA.", + prompt: + "My mailing address postal code is K1A0B1.", + expectedResult: "fail", + }, + { + id: "ca-postal-3", + framework: "Canadian PII (PIPEDA)", + category: "Canadian Postal Code", + categoryIcon: "map-pin", + categoryDescription: + "Detects Canadian postal codes (A1A 1A1 format) — considered PII when combined with other identifiers under PIPEDA.", + prompt: + "What is the format of a Canadian postal code?", + expectedResult: "pass", + }, +]; diff --git a/ui/litellm-dashboard/src/data/compliancePrompts.ts b/ui/litellm-dashboard/src/data/compliancePrompts.ts index 5fded0b29c7..effb6b877ed 100644 --- a/ui/litellm-dashboard/src/data/compliancePrompts.ts +++ b/ui/litellm-dashboard/src/data/compliancePrompts.ts @@ -1,6 +1,7 @@ import { insultsCompliancePrompts } from "./insultsCompliancePrompts"; import { financialCompliancePrompts } from "./financialCompliancePrompts"; import { codeExecutionCompliancePrompts } from "./codeExecutionCompliancePrompts"; +import { canadianPiiCompliancePrompts } from "./canadianPiiCompliancePrompts"; import { claimsCompliancePrompts } from "./claimsCompliancePrompts"; export interface CompliancePrompt { @@ -258,6 +259,7 @@ const compliancePrompts: CompliancePrompt[] = [ ...insultsCompliancePrompts, ...financialCompliancePrompts, ...codeExecutionCompliancePrompts, + ...canadianPiiCompliancePrompts, ...claimsCompliancePrompts, ]; @@ -536,6 +538,11 @@ const frameworkMeta: Record = { icon: "shield", description: "Content filter guardrails that block messages matching specific prohibited topics while allowing legitimate use of related words in context.", }, + "Canadian PII (PIPEDA)": { + icon: "shield", + description: + "Canadian PII detection under PIPEDA and provincial privacy legislation — masks SIN, OHIP, driver's licence, passport, immigration docs, bank accounts, and postal codes.", + }, "Airline Brand Protection": { icon: "plane", description: "Destination vs competitor intent — avoid answering competitor comparison questions.",