mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge c4ea04e4b8 into a2ef18b257
This commit is contained in:
commit
c2dc10f473
6 changed files with 707 additions and 24 deletions
|
|
@ -496,6 +496,7 @@ GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS: Final = get_env_int(
|
|||
BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS: Final = 25_000
|
||||
CONTENT_FILTER_STREAMING_HOLDBACK_CHARS: Final = 50
|
||||
CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS: Final = 512
|
||||
CONTENT_FILTER_STREAMING_CHECKSUM_CACHE_SIZE: Final = 1024
|
||||
DEFAULT_PRESIDIO_ANALYZE_CHUNK_SIZE_BYTES: Final = 500_000
|
||||
PRESIDIO_ANALYZE_CHUNK_OVERLAP_CHARS: Final = 4096
|
||||
PRESIDIO_ANALYZE_CHUNK_CONCURRENCY: Final = 8
|
||||
|
|
|
|||
|
|
@ -7,12 +7,14 @@ to detect and block/mask sensitive content.
|
|||
|
||||
import asyncio
|
||||
import contextvars
|
||||
import functools
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Coroutine, Mapping, Sequence
|
||||
from bisect import bisect_left, bisect_right
|
||||
from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from re import Pattern
|
||||
|
|
@ -21,10 +23,12 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, cast
|
|||
import yaml
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import ReadOnly
|
||||
|
||||
from litellm import Router
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import (
|
||||
CONTENT_FILTER_STREAMING_CHECKSUM_CACHE_SIZE,
|
||||
CONTENT_FILTER_STREAMING_HOLDBACK_CHARS,
|
||||
CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS,
|
||||
DEFAULT_MAX_RECURSE_DEPTH,
|
||||
|
|
@ -75,6 +79,7 @@ from .competitor_intent import (
|
|||
BaseCompetitorIntentChecker,
|
||||
)
|
||||
from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern
|
||||
from .validators import PATTERN_VALIDATORS, CandidateCheck, run_validator
|
||||
|
||||
MAX_KEYWORD_VALUE_GAP_WORDS: Final = 1
|
||||
_DECODED_ARGUMENTS: Final = TypeAdapter(object)
|
||||
|
|
@ -104,6 +109,12 @@ WORD_NUMBER_SEQUENCE_PATTERN: Final = re.compile(
|
|||
)
|
||||
WORD_NUMBER_TOKEN_FINDER: Final = re.compile(rf"(?:{WORD_NUMBER_TOKEN_REGEX})", re.IGNORECASE)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _overlapping_matches_regex(regex: Pattern[str]) -> Pattern[str]:
|
||||
return re.compile(f"(?=({regex.pattern}))", regex.flags)
|
||||
|
||||
|
||||
_logging_only_scan_guardrail: Final[contextvars.ContextVar["ContentFilterGuardrail | None"]] = contextvars.ContextVar(
|
||||
"litellm_content_filter_logging_only_scan", default=None
|
||||
)
|
||||
|
|
@ -125,11 +136,13 @@ class CompiledPatternEntry(TypedDict):
|
|||
action: ContentFilterAction
|
||||
keyword_regex: Pattern[str] | None
|
||||
allow_word_numbers: bool
|
||||
validator: ReadOnly[Callable[[str], bool] | None]
|
||||
|
||||
|
||||
class _PatternExtraLookup(TypedDict):
|
||||
keyword_pattern: str | None
|
||||
allow_word_numbers: bool
|
||||
validator: ReadOnly[Callable[[str], bool] | None]
|
||||
|
||||
|
||||
class _CategoryConfigView(TypedDict):
|
||||
|
|
@ -148,6 +161,12 @@ class _StreamedChoiceState:
|
|||
next_trim_len: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SpanIndex:
|
||||
spans: tuple[tuple[int, int], ...]
|
||||
furthest_ends: tuple[int, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _StreamedScanPlan:
|
||||
context_chars: int
|
||||
|
|
@ -749,7 +768,11 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
pattern_config: ContentFilterPattern configuration
|
||||
"""
|
||||
try:
|
||||
extra_config: _PatternExtraLookup = {"keyword_pattern": None, "allow_word_numbers": False}
|
||||
extra_config: _PatternExtraLookup = {
|
||||
"keyword_pattern": None,
|
||||
"allow_word_numbers": False,
|
||||
"validator": None,
|
||||
}
|
||||
if pattern_config.pattern_type == "prebuilt":
|
||||
if not pattern_config.pattern_name:
|
||||
raise ValueError("pattern_name is required for prebuilt patterns")
|
||||
|
|
@ -774,6 +797,7 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
"action": pattern_config.action,
|
||||
"keyword_regex": keyword_regex,
|
||||
"allow_word_numbers": extra_config["allow_word_numbers"],
|
||||
"validator": extra_config["validator"],
|
||||
}
|
||||
)
|
||||
verbose_proxy_logger.debug("Added pattern: %s with action %s", pattern_name, pattern_config.action)
|
||||
|
|
@ -784,9 +808,11 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
@staticmethod
|
||||
def _lookup_pattern_extra(pattern_name: str) -> _PatternExtraLookup:
|
||||
extra: Final = PATTERN_EXTRA_CONFIG.get(pattern_name)
|
||||
validator_name: Final = extra.get("validator") if extra is not None else None
|
||||
return {
|
||||
"keyword_pattern": extra.get("keyword_pattern") if extra is not None else None,
|
||||
"allow_word_numbers": bool(extra.get("allow_word_numbers")) if extra is not None else False,
|
||||
"validator": PATTERN_VALIDATORS[validator_name] if validator_name is not None else None,
|
||||
}
|
||||
|
||||
def _load_blocked_words_file(self, file_path: str) -> None:
|
||||
|
|
@ -828,25 +854,34 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
except Exception as e:
|
||||
raise Exception(f"Error loading blocked words file {file_path}: {e}")
|
||||
|
||||
def _find_pattern_spans(self, text: str, pattern_entry: CompiledPatternEntry) -> list[tuple[int, int]]:
|
||||
def _find_pattern_spans(
|
||||
self,
|
||||
text: str,
|
||||
pattern_entry: CompiledPatternEntry,
|
||||
check_candidate: CandidateCheck = run_validator,
|
||||
) -> Sequence[tuple[int, int]]:
|
||||
"""Return all match spans for a pattern, applying contextual rules if required."""
|
||||
|
||||
regex: Final[Pattern[str]] = pattern_entry["regex"]
|
||||
keyword_regex: Final[Pattern[str] | None] = pattern_entry.get("keyword_regex")
|
||||
allow_word_numbers: Final[bool] = pattern_entry.get("allow_word_numbers", False)
|
||||
validator: Final = pattern_entry.get("validator")
|
||||
|
||||
keyword_matches: Final = list(keyword_regex.finditer(text)) if keyword_regex is not None else None
|
||||
if keyword_matches is not None and not keyword_matches:
|
||||
return []
|
||||
|
||||
match_spans: Final[list[tuple[int, int]]] = []
|
||||
candidate_spans: Final = (
|
||||
self._checksum_valid_spans(regex, text, validator, check_candidate)
|
||||
if validator is not None
|
||||
else ((match.start(), match.end()) for match in regex.finditer(text))
|
||||
)
|
||||
|
||||
for match in regex.finditer(text):
|
||||
if keyword_matches is not None and not self._match_near_keyword(
|
||||
match.start(), match.end(), keyword_matches, text
|
||||
):
|
||||
for start, end in candidate_spans:
|
||||
if keyword_matches is not None and not self._match_near_keyword(start, end, keyword_matches, text):
|
||||
continue
|
||||
match_spans.append((match.start(), match.end()))
|
||||
match_spans.append((start, end))
|
||||
|
||||
if allow_word_numbers:
|
||||
for word_match in WORD_NUMBER_SEQUENCE_PATTERN.finditer(text):
|
||||
|
|
@ -861,7 +896,141 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
continue
|
||||
match_spans.append((word_match.start(), word_match.end()))
|
||||
|
||||
return self._merge_spans(match_spans)
|
||||
merged_spans: Final = self._merge_spans(match_spans)
|
||||
if validator is None or not merged_spans:
|
||||
return merged_spans
|
||||
return self._with_overlapping_checksum_matches(text, pattern_entry, merged_spans, check_candidate)
|
||||
|
||||
def _with_overlapping_checksum_matches(
|
||||
self,
|
||||
text: str,
|
||||
pattern_entry: CompiledPatternEntry,
|
||||
spans: Sequence[tuple[int, int]],
|
||||
check_candidate: CandidateCheck,
|
||||
) -> Sequence[tuple[int, int]]:
|
||||
if pattern_entry["action"] == ContentFilterAction.BLOCK:
|
||||
return spans
|
||||
position: Final = next(index for index, entry in enumerate(self.compiled_patterns) if entry is pattern_entry)
|
||||
later_entries: Final = self.compiled_patterns[position + 1 :]
|
||||
later_masking_entries: Final = tuple(
|
||||
entry for entry in later_entries if entry["action"] != ContentFilterAction.BLOCK
|
||||
)
|
||||
checksum_masking: Final = self._span_index(
|
||||
text, (entry for entry in later_masking_entries if entry.get("validator") is not None), check_candidate
|
||||
)
|
||||
other_masking: Final = self._span_index(
|
||||
text, (entry for entry in later_masking_entries if entry.get("validator") is None), check_candidate
|
||||
)
|
||||
blocking: Final = self._span_index(
|
||||
text, (entry for entry in later_entries if entry["action"] == ContentFilterAction.BLOCK), check_candidate
|
||||
)
|
||||
if not checksum_masking.spans and not other_masking.spans and not blocking.spans:
|
||||
return spans
|
||||
leftmost_spans: Final = tuple(match.span() for match in pattern_entry["regex"].finditer(text))
|
||||
union_regions: Final = tuple(self._merge_spans([*spans, *checksum_masking.spans]))
|
||||
resolved: Final = self._merge_spans(
|
||||
[
|
||||
*itertools.chain.from_iterable(
|
||||
self._resolve_component(component, leftmost_spans, checksum_masking, other_masking, union_regions)
|
||||
for component in spans
|
||||
)
|
||||
]
|
||||
)
|
||||
return tuple(
|
||||
itertools.chain.from_iterable(
|
||||
self._spans_overlapping(leftmost_spans, region)
|
||||
if any(self._spans_reaching(region, blocking))
|
||||
else (region,)
|
||||
for region in resolved
|
||||
)
|
||||
)
|
||||
|
||||
def _resolve_component(
|
||||
self,
|
||||
component: tuple[int, int],
|
||||
leftmost_spans: Sequence[tuple[int, int]],
|
||||
checksum_masking: _SpanIndex,
|
||||
other_masking: _SpanIndex,
|
||||
union_regions: Sequence[tuple[int, int]],
|
||||
) -> Sequence[tuple[int, int]]:
|
||||
leftmost_in_component: Final = self._spans_overlapping(leftmost_spans, component)
|
||||
sticking_out: Final = tuple(self._spans_sticking_out(component, checksum_masking))
|
||||
if sticking_out and not any(self._spans_overlapping(leftmost_in_component, span) for span in sticking_out):
|
||||
return leftmost_in_component
|
||||
region: Final = (
|
||||
union_regions[bisect_right(union_regions, component[0], key=lambda region: region[0]) - 1]
|
||||
if sticking_out
|
||||
else component
|
||||
)
|
||||
leftmost_in_region: Final = self._spans_overlapping(leftmost_spans, region)
|
||||
if any(
|
||||
not self._spans_overlapping(leftmost_in_region, span)
|
||||
for span in self._spans_sticking_out(region, other_masking)
|
||||
):
|
||||
return leftmost_in_component
|
||||
return (region,)
|
||||
|
||||
@classmethod
|
||||
def _spans_sticking_out(cls, region: tuple[int, int], index: _SpanIndex) -> Iterator[tuple[int, int]]:
|
||||
return (span for span in cls._spans_reaching(region, index) if span[0] < region[0] or span[1] > region[1])
|
||||
|
||||
def _span_index(
|
||||
self, text: str, entries: Iterable[CompiledPatternEntry], check_candidate: CandidateCheck
|
||||
) -> _SpanIndex:
|
||||
spans: Final = tuple(
|
||||
sorted(itertools.chain.from_iterable(self._entry_spans(entry, text, check_candidate) for entry in entries))
|
||||
)
|
||||
return _SpanIndex(spans=spans, furthest_ends=tuple(itertools.accumulate((end for _, end in spans), max)))
|
||||
|
||||
def _entry_spans(
|
||||
self, entry: CompiledPatternEntry, text: str, check_candidate: CandidateCheck
|
||||
) -> Sequence[tuple[int, int]]:
|
||||
validator: Final = entry.get("validator")
|
||||
if validator is None:
|
||||
return self._find_pattern_spans(text, entry, check_candidate)
|
||||
return self._checksum_valid_spans(entry["regex"], text, validator, check_candidate)
|
||||
|
||||
@staticmethod
|
||||
def _spans_reaching(component: tuple[int, int], index: _SpanIndex) -> Iterator[tuple[int, int]]:
|
||||
later_position: Final = bisect_left(index.spans, (component[1],))
|
||||
positions_reaching_start: Final = itertools.takewhile(
|
||||
lambda position: index.furthest_ends[position] > component[0], range(later_position - 1, -1, -1)
|
||||
)
|
||||
return (
|
||||
index.spans[position] for position in positions_reaching_start if index.spans[position][1] > component[0]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _spans_overlapping(spans: Sequence[tuple[int, int]], component: tuple[int, int]) -> Sequence[tuple[int, int]]:
|
||||
first: Final = bisect_right(spans, component[0], key=lambda span: span[1])
|
||||
return spans[first : bisect_left(spans, component[1], lo=first, key=lambda span: span[0])]
|
||||
|
||||
def _checksum_valid_spans(
|
||||
self,
|
||||
regex: Pattern[str],
|
||||
text: str,
|
||||
validator: Callable[[str], bool],
|
||||
check_candidate: CandidateCheck,
|
||||
) -> tuple[tuple[int, int], ...]:
|
||||
leftmost_spans: Final = tuple(match.span() for match in regex.finditer(text))
|
||||
if not leftmost_spans:
|
||||
return ()
|
||||
candidate_spans: Final = (candidate.span(1) for candidate in _overlapping_matches_regex(regex).finditer(text))
|
||||
valid_spans: Final = tuple(
|
||||
span for span in candidate_spans if check_candidate(validator, text[span[0] : span[1]])
|
||||
)
|
||||
if not valid_spans:
|
||||
return ()
|
||||
valid_starts: Final = tuple(start for start, _ in valid_spans)
|
||||
furthest_valid_ends: Final = tuple(itertools.accumulate((end for _, end in valid_spans), max))
|
||||
return valid_spans + tuple(
|
||||
span for span in leftmost_spans if self._overlaps_any_span(span, valid_starts, furthest_valid_ends)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _overlaps_any_span(span: tuple[int, int], sorted_starts: Sequence[int], furthest_ends: Sequence[int]) -> bool:
|
||||
index: Final = bisect_left(sorted_starts, span[1])
|
||||
return index > 0 and furthest_ends[index - 1] > span[0]
|
||||
|
||||
def _match_near_keyword(
|
||||
self,
|
||||
|
|
@ -915,7 +1084,7 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
merged.append((start, end))
|
||||
return merged
|
||||
|
||||
def _mask_spans(self, text: str, spans: list[tuple[int, int]], redaction: str) -> str:
|
||||
def _mask_spans(self, text: str, spans: Sequence[tuple[int, int]], redaction: str) -> str:
|
||||
"""Apply masking for the provided spans using the given redaction tag."""
|
||||
|
||||
if not spans:
|
||||
|
|
@ -1321,7 +1490,7 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
pattern_name: str,
|
||||
action: ContentFilterAction,
|
||||
text: str,
|
||||
spans: list[tuple[int, int]],
|
||||
spans: Sequence[tuple[int, int]],
|
||||
detections: list[ContentFilterDetection] | None,
|
||||
) -> str:
|
||||
"""Handle regex pattern match detection and action."""
|
||||
|
|
@ -1392,7 +1561,12 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
|
||||
return text
|
||||
|
||||
def _filter_single_text(self, text: str, detections: list[ContentFilterDetection] | None = None) -> str:
|
||||
def _filter_single_text(
|
||||
self,
|
||||
text: str,
|
||||
detections: list[ContentFilterDetection] | None = None,
|
||||
check_candidate: CandidateCheck = run_validator,
|
||||
) -> str:
|
||||
"""
|
||||
Apply all content filtering checks to a single text.
|
||||
|
||||
|
|
@ -1404,6 +1578,7 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
Args:
|
||||
text: Text to filter
|
||||
detections: Optional list to append detection information
|
||||
check_candidate: Runs a pattern's checksum validator on a candidate match
|
||||
|
||||
Returns:
|
||||
Filtered text (with masking applied if action is MASK)
|
||||
|
|
@ -1435,8 +1610,9 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
text = self._handle_category_keyword_match(keyword, category_name, severity, action, text, detections)
|
||||
|
||||
# Check regex patterns - process ALL patterns, not just first match
|
||||
memoized_check: Final = functools.cache(check_candidate)
|
||||
for pattern_entry in self.compiled_patterns:
|
||||
spans = self._find_pattern_spans(text, pattern_entry)
|
||||
spans = self._find_pattern_spans(text, pattern_entry, memoized_check)
|
||||
if spans:
|
||||
pattern_name = pattern_entry["pattern_name"]
|
||||
action = pattern_entry["action"]
|
||||
|
|
@ -2091,6 +2267,9 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
"""
|
||||
state_by_choice: Final[dict[int, _StreamedChoiceState]] = {}
|
||||
plan: Final = self._streamed_scan_plan()
|
||||
check_candidate: Final = functools.lru_cache(maxsize=CONTENT_FILTER_STREAMING_CHECKSUM_CACHE_SIZE)(
|
||||
run_validator
|
||||
)
|
||||
|
||||
start_time: Final = datetime.now()
|
||||
scan_seconds: float = 0.0 # rebind-ok: accumulates per-chunk scan time across the stream
|
||||
|
|
@ -2131,7 +2310,9 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
scan_started = time.perf_counter()
|
||||
|
||||
try:
|
||||
masked_text = self._filter_single_text(text_to_scan, detections=choice_detections)
|
||||
masked_text = self._filter_single_text(
|
||||
text_to_scan, detections=choice_detections, check_candidate=check_candidate
|
||||
)
|
||||
if is_final and masked_text.endswith(" "):
|
||||
masked_text = masked_text[:-1]
|
||||
latest_detections = tuple(choice_detections)
|
||||
|
|
|
|||
|
|
@ -32,36 +32,41 @@
|
|||
"name": "visa",
|
||||
"display_name": "Credit Card (Visa)",
|
||||
"pattern": "\\b4\\d{3}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b",
|
||||
"validator": "luhn",
|
||||
"category": "Payment Card Patterns",
|
||||
"description": "Detects Visa credit card numbers"
|
||||
"description": "Detects Visa credit card numbers that pass the Luhn checksum"
|
||||
},
|
||||
{
|
||||
"name": "mastercard",
|
||||
"display_name": "Credit Card (Mastercard)",
|
||||
"pattern": "\\b5[1-5]\\d{2}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b",
|
||||
"validator": "luhn",
|
||||
"category": "Payment Card Patterns",
|
||||
"description": "Detects Mastercard credit card numbers"
|
||||
"description": "Detects Mastercard credit card numbers that pass the Luhn checksum"
|
||||
},
|
||||
{
|
||||
"name": "amex",
|
||||
"display_name": "Credit Card (American Express)",
|
||||
"pattern": "\\b3[47]\\d{2}[\\s\\-]?\\d{6}[\\s\\-]?\\d{5}\\b",
|
||||
"validator": "luhn",
|
||||
"category": "Payment Card Patterns",
|
||||
"description": "Detects American Express credit card numbers"
|
||||
"description": "Detects American Express credit card numbers that pass the Luhn checksum"
|
||||
},
|
||||
{
|
||||
"name": "discover",
|
||||
"display_name": "Credit Card (Discover)",
|
||||
"pattern": "\\b6(?:011|5\\d{2})[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b",
|
||||
"validator": "luhn",
|
||||
"category": "Payment Card Patterns",
|
||||
"description": "Detects Discover credit card numbers"
|
||||
"description": "Detects Discover credit card numbers that pass the Luhn checksum"
|
||||
},
|
||||
{
|
||||
"name": "credit_card",
|
||||
"display_name": "Credit Card (Any)",
|
||||
"pattern": "(?:\\b4\\d{3}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b|\\b5[1-5]\\d{2}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b|\\b3[47]\\d{2}[\\s\\-]?\\d{6}[\\s\\-]?\\d{5}\\b|\\b6(?:011|5\\d{2})[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b)",
|
||||
"validator": "luhn",
|
||||
"category": "Payment Card Patterns",
|
||||
"description": "Detects any major credit card number"
|
||||
"description": "Detects any major credit card number that passes the Luhn checksum"
|
||||
},
|
||||
{
|
||||
"name": "aws_access_key",
|
||||
|
|
@ -395,8 +400,9 @@
|
|||
"name": "iban",
|
||||
"display_name": "IBAN (International Bank Account Number)",
|
||||
"pattern": "\\b[A-Z]{2}\\d{2}[A-Z0-9]{11,30}\\b",
|
||||
"validator": "iban_mod97",
|
||||
"category": "Payment Card Patterns",
|
||||
"description": "Detects IBANs (2 letter country code + 2 check digits + 4 char bank code + 7 digit base + optional 0-16 alphanumeric)"
|
||||
"description": "Detects IBANs (2 letter country code + 2 check digits + 4 char bank code + 7 digit base + optional 0-16 alphanumeric) that pass the mod-97 check"
|
||||
},
|
||||
{
|
||||
"name": "fr_nir",
|
||||
|
|
@ -409,8 +415,9 @@
|
|||
"name": "eu_iban_enhanced",
|
||||
"display_name": "IBAN (Enhanced EU Format)",
|
||||
"pattern": "\\b[A-Z]{2}[0-9]{2}[A-Z0-9]{4}[0-9]{7}[A-Z0-9]{0,16}\\b",
|
||||
"validator": "iban_mod97",
|
||||
"category": "EU PII Patterns",
|
||||
"description": "Enhanced IBAN detection with more specific format validation (2 letter country + 2 check digits + 4 char bank code + 7 digit account base + optional 0-16 alphanumeric)"
|
||||
"description": "Enhanced IBAN detection with more specific format validation (2 letter country + 2 check digits + 4 char bank code + 7 digit account base + optional 0-16 alphanumeric) that pass the mod-97 check"
|
||||
},
|
||||
{
|
||||
"name": "fr_phone",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,162 @@
|
|||
import re
|
||||
import string
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
_SEPARATORS: Final = re.compile(r"[\s\-]")
|
||||
_IBAN_CHARACTERS: Final = re.compile(r"[0-9A-Z]{5,34}")
|
||||
_IGNORECASE_ASCII_LETTER_ALIASES: Final = str.maketrans({"\u0130": "I", "\u212a": "K"})
|
||||
_ISO_13616_IBAN_LENGTHS: Final[Mapping[str, int]] = MappingProxyType(
|
||||
{
|
||||
"AD": 24,
|
||||
"AE": 23,
|
||||
"AL": 28,
|
||||
"AT": 20,
|
||||
"AX": 18,
|
||||
"AZ": 28,
|
||||
"BA": 20,
|
||||
"BE": 16,
|
||||
"BG": 22,
|
||||
"BH": 22,
|
||||
"BI": 27,
|
||||
"BL": 27,
|
||||
"BR": 29,
|
||||
"BY": 28,
|
||||
"CH": 21,
|
||||
"CR": 22,
|
||||
"CY": 28,
|
||||
"CZ": 24,
|
||||
"DE": 22,
|
||||
"DJ": 27,
|
||||
"DK": 18,
|
||||
"DO": 28,
|
||||
"EE": 20,
|
||||
"EG": 29,
|
||||
"ES": 24,
|
||||
"FI": 18,
|
||||
"FK": 18,
|
||||
"FO": 18,
|
||||
"FR": 27,
|
||||
"GB": 22,
|
||||
"GE": 22,
|
||||
"GF": 27,
|
||||
"GG": 22,
|
||||
"GI": 23,
|
||||
"GL": 18,
|
||||
"GP": 27,
|
||||
"GR": 27,
|
||||
"GT": 28,
|
||||
"HN": 28,
|
||||
"HR": 21,
|
||||
"HU": 28,
|
||||
"IE": 22,
|
||||
"IL": 23,
|
||||
"IM": 22,
|
||||
"IQ": 23,
|
||||
"IS": 26,
|
||||
"IT": 27,
|
||||
"JE": 22,
|
||||
"JO": 30,
|
||||
"KW": 30,
|
||||
"KZ": 20,
|
||||
"LB": 28,
|
||||
"LC": 32,
|
||||
"LI": 21,
|
||||
"LT": 20,
|
||||
"LU": 20,
|
||||
"LV": 21,
|
||||
"LY": 25,
|
||||
"MC": 27,
|
||||
"MD": 24,
|
||||
"ME": 22,
|
||||
"MF": 27,
|
||||
"MK": 19,
|
||||
"MN": 20,
|
||||
"MQ": 27,
|
||||
"MR": 27,
|
||||
"MT": 31,
|
||||
"MU": 30,
|
||||
"NC": 27,
|
||||
"NI": 28,
|
||||
"NL": 18,
|
||||
"NO": 15,
|
||||
"OM": 23,
|
||||
"PF": 27,
|
||||
"PK": 24,
|
||||
"PL": 28,
|
||||
"PM": 27,
|
||||
"PS": 29,
|
||||
"PT": 25,
|
||||
"QA": 29,
|
||||
"RE": 27,
|
||||
"RO": 24,
|
||||
"RS": 22,
|
||||
"RU": 33,
|
||||
"SA": 24,
|
||||
"SC": 31,
|
||||
"SD": 18,
|
||||
"SE": 24,
|
||||
"SI": 19,
|
||||
"SK": 24,
|
||||
"SM": 27,
|
||||
"SO": 23,
|
||||
"ST": 25,
|
||||
"SV": 28,
|
||||
"TF": 27,
|
||||
"TL": 23,
|
||||
"TN": 24,
|
||||
"TR": 26,
|
||||
"UA": 29,
|
||||
"VA": 22,
|
||||
"VG": 24,
|
||||
"WF": 27,
|
||||
"XK": 20,
|
||||
"YE": 30,
|
||||
"YT": 27,
|
||||
}
|
||||
)
|
||||
_IBAN_LETTER_VALUES: Final = str.maketrans(
|
||||
{letter: str(value) for value, letter in enumerate(string.ascii_uppercase, 10)}
|
||||
)
|
||||
_DIGIT_VALUES: Final = bytes.maketrans(b"0123456789", bytes(range(10)))
|
||||
_DOUBLED_DIGIT_SUMS: Final = bytes.maketrans(b"0123456789", bytes((0, 2, 4, 6, 8, 1, 3, 5, 7, 9)))
|
||||
|
||||
|
||||
def _to_ascii(value: str) -> str:
|
||||
if value.isascii():
|
||||
return value
|
||||
folded: Final = value.translate(_IGNORECASE_ASCII_LETTER_ALIASES)
|
||||
return "".join(str(int(character)) if character.isdecimal() else character for character in folded)
|
||||
|
||||
|
||||
def passes_luhn(value: str) -> bool:
|
||||
digits: Final = _SEPARATORS.sub("", value)
|
||||
if not digits.isdecimal():
|
||||
return False
|
||||
reversed_digits: Final = _to_ascii(digits)[::-1].encode()
|
||||
checksum: Final = sum(reversed_digits[::2].translate(_DIGIT_VALUES)) + sum(
|
||||
reversed_digits[1::2].translate(_DOUBLED_DIGIT_SUMS)
|
||||
)
|
||||
return checksum % 10 == 0
|
||||
|
||||
|
||||
def passes_iban_mod97(value: str) -> bool:
|
||||
compact: Final = _to_ascii(_SEPARATORS.sub("", value)).upper()
|
||||
iban: Final = compact[: _ISO_13616_IBAN_LENGTHS.get(compact[:2], len(compact))]
|
||||
if _IBAN_CHARACTERS.fullmatch(iban) is None:
|
||||
return False
|
||||
rearranged: Final = iban[4:] + iban[:4]
|
||||
return int(rearranged.translate(_IBAN_LETTER_VALUES)) % 97 == 1
|
||||
|
||||
|
||||
CandidateCheck: TypeAlias = Callable[[Callable[[str], bool], str], bool]
|
||||
|
||||
|
||||
def run_validator(validator: Callable[[str], bool], candidate: str) -> bool:
|
||||
return validator(candidate)
|
||||
|
||||
|
||||
PATTERN_VALIDATORS: Final[Mapping[str, Callable[[str], bool]]] = MappingProxyType(
|
||||
{"luhn": passes_luhn, "iban_mod97": passes_iban_mod97}
|
||||
)
|
||||
|
|
@ -20,6 +20,10 @@ from litellm.constants import (
|
|||
from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import (
|
||||
ContentFilterGuardrail,
|
||||
)
|
||||
from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.validators import (
|
||||
CandidateCheck,
|
||||
run_validator,
|
||||
)
|
||||
from litellm.types.guardrails import (
|
||||
BlockedWord,
|
||||
ContentFilterAction,
|
||||
|
|
@ -367,10 +371,283 @@ class TestContentFilterGuardrail:
|
|||
)
|
||||
|
||||
# Test Visa card
|
||||
result = guardrail._check_patterns("My card is 4532-1234-5678-9010")
|
||||
result = guardrail._check_patterns("My card is 4532-0151-1283-0366")
|
||||
assert result is not None
|
||||
assert result[1] == "visa"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("pattern_name", "checksum_valid", "checksum_invalid"),
|
||||
[
|
||||
("credit_card", "4532 0151 1283 0366", "4532 0151 1283 0367"),
|
||||
("visa", "4532-0151-1283-0366", "4532-1234-5678-9010"),
|
||||
("mastercard", "5555555555554444", "5555555555554445"),
|
||||
("amex", "3782 822463 10005", "3782 822463 10006"),
|
||||
("discover", "6011111111111117", "6011111111111118"),
|
||||
("iban", "GB82WEST12345698765432", "GB83WEST12345698765432"),
|
||||
("eu_iban_enhanced", "DE89370400440532013000", "DE88370400440532013000"),
|
||||
],
|
||||
)
|
||||
async def test_checksum_pattern_masks_only_values_that_pass_the_checksum(
|
||||
self, pattern_name: str, checksum_valid: str, checksum_invalid: str
|
||||
):
|
||||
guardrail: Final = ContentFilterGuardrail(
|
||||
guardrail_name="test-checksum",
|
||||
patterns=[
|
||||
ContentFilterPattern(pattern_type="prebuilt", pattern_name=pattern_name, action=ContentFilterAction.MASK)
|
||||
],
|
||||
)
|
||||
|
||||
guardrailed_inputs: Final = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [f"real {checksum_valid} and look-alike {checksum_invalid}"]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert guardrailed_inputs.get("texts") == [
|
||||
f"real [{pattern_name.upper()}_REDACTED] and look-alike {checksum_invalid}"
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
("PIN 4821 4532 0151 1283 0366 thanks", "PIN [CREDIT_CARD_REDACTED] thanks"),
|
||||
("order 4111 4532 0151 1283 0366", "order [CREDIT_CARD_REDACTED]"),
|
||||
("codes 4532 4532 4532 4532 4532 0151 1283 0366 thanks", "codes 4532 4532 4532 4532 [CREDIT_CARD_REDACTED] thanks"),
|
||||
("order 4532 1234 5678 9010 paid by 4532 0151 1283 0366", "order 4532 1234 5678 9010 paid by [CREDIT_CARD_REDACTED]"),
|
||||
("card 4532 4151 4283 4000 4777 4888 4999 ok", "card [CREDIT_CARD_REDACTED] 4777 4888 4999 ok"),
|
||||
("card 4532 4111 1111 1116 2025 ok", "card [CREDIT_CARD_REDACTED] 2025 ok"),
|
||||
],
|
||||
)
|
||||
async def test_checksum_pattern_masks_a_valid_card_with_the_base_matches_overlapping_it(self, text: str, expected: str):
|
||||
guardrail: Final = ContentFilterGuardrail(
|
||||
guardrail_name="test-checksum-overlap",
|
||||
patterns=[
|
||||
ContentFilterPattern(pattern_type="prebuilt", pattern_name="credit_card", action=ContentFilterAction.MASK)
|
||||
],
|
||||
)
|
||||
|
||||
guardrailed_inputs: Final = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert guardrailed_inputs.get("texts") == [expected]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
("ids 4853 4091 4909 4475 5508 5016 6232 6258 end", "ids [VISA_REDACTED] [MASTERCARD_REDACTED] end"),
|
||||
("ids 4336 4262 4755 4637 3753 786027 13221 end", "ids [VISA_REDACTED] [AMEX_REDACTED] end"),
|
||||
("4000 4000 0000 0000\n5555 5555 5555 4444", "[VISA_REDACTED]\n[MASTERCARD_REDACTED]"),
|
||||
("cards 4378 4937 4618 4485 5555 5555 5555 4444 ok", "cards [VISA_REDACTED] [MASTERCARD_REDACTED] ok"),
|
||||
("pay 5472 3038 5280 1841 1485 4534 3853 6336 1330 ok", "pay [VISA_REDACTED] ok"),
|
||||
("cards 4532 0151 1283 0366 5555 5555 5555 4444 ok", "cards [VISA_REDACTED] [MASTERCARD_REDACTED] ok"),
|
||||
],
|
||||
)
|
||||
async def test_checksum_patterns_mask_each_valid_card_where_two_brands_overlap(self, text: str, expected: str):
|
||||
guardrail: Final = ContentFilterGuardrail(
|
||||
guardrail_name="test-checksum-cross-pattern",
|
||||
patterns=[
|
||||
ContentFilterPattern(pattern_type="prebuilt", pattern_name=pattern_name, action=ContentFilterAction.MASK)
|
||||
for pattern_name in ("visa", "mastercard", "amex", "discover", "credit_card", "iban")
|
||||
],
|
||||
)
|
||||
|
||||
guardrailed_inputs: Final = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert guardrailed_inputs.get("texts") == [expected]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("masking_pattern", "blocking_pattern", "text"),
|
||||
[
|
||||
("visa", "mastercard", "4004 5555 5555 5557 5555 5555 5555 4444"),
|
||||
("visa", "mastercard", "4000 4000 0000 0000 5555 5555 5555 4444"),
|
||||
("visa", "mastercard", "ids 4853 4091 4909 4475 5508 5016 6232 6258 end"),
|
||||
("discover", "credit_card", "4629 4611-6260-8050-5592-2700 65405563 93269332"),
|
||||
],
|
||||
)
|
||||
async def test_checksum_mask_leaves_a_later_blocking_pattern_its_valid_card(
|
||||
self, masking_pattern: str, blocking_pattern: str, text: str
|
||||
):
|
||||
guardrail: Final = ContentFilterGuardrail(
|
||||
guardrail_name="test-checksum-mask-then-block",
|
||||
patterns=[
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt", pattern_name=masking_pattern, action=ContentFilterAction.MASK
|
||||
),
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt", pattern_name=blocking_pattern, action=ContentFilterAction.BLOCK
|
||||
),
|
||||
],
|
||||
)
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await guardrail.apply_guardrail(inputs={"texts": [text]}, request_data={}, input_type="request")
|
||||
assert exc_info.value.detail["pattern"] == blocking_pattern
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checksum_mask_before_a_blocking_pattern_still_masks_the_card_both_match(self):
|
||||
guardrail: Final = ContentFilterGuardrail(
|
||||
guardrail_name="test-checksum-mask-before-block",
|
||||
patterns=[
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt", pattern_name="credit_card", action=ContentFilterAction.MASK
|
||||
),
|
||||
ContentFilterPattern(pattern_type="prebuilt", pattern_name="visa", action=ContentFilterAction.BLOCK),
|
||||
],
|
||||
)
|
||||
guardrailed_inputs: Final = await guardrail.apply_guardrail(
|
||||
inputs={"texts": ["card 4111 1111 1111 1111 ok"]}, request_data={}, input_type="request"
|
||||
)
|
||||
assert guardrailed_inputs.get("texts") == ["card [CREDIT_CARD_REDACTED] ok"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("checksum_patterns", "regex", "later_patterns", "text", "expected"),
|
||||
[
|
||||
(
|
||||
("credit_card",),
|
||||
"1111 BSN",
|
||||
("nl_bsn_contextual",),
|
||||
"4111 1111 1111 1111 BSN 123456789",
|
||||
"[CREDIT_CARD_REDACTED] BSN [NL_BSN_CONTEXTUAL_REDACTED]",
|
||||
),
|
||||
(
|
||||
("visa", "mastercard"),
|
||||
"4444 BSN",
|
||||
("nl_bsn_contextual",),
|
||||
"4532 0151 1283 0366 5555 5555 5555 4444 BSN 123456789",
|
||||
"[VISA_REDACTED] [MASTERCARD_REDACTED] BSN [NL_BSN_CONTEXTUAL_REDACTED]",
|
||||
),
|
||||
(
|
||||
("visa", "mastercard", "amex", "discover", "credit_card", "iban"),
|
||||
"1330 BSN",
|
||||
("nl_bsn_contextual",),
|
||||
"pay 5472 3038 5280 1841 1485 4534 3853 6336 1330 BSN 123456789",
|
||||
"pay [VISA_REDACTED] BSN [NL_BSN_CONTEXTUAL_REDACTED]",
|
||||
),
|
||||
(("credit_card",), "PIN 4821", (), "PIN 4821 4532 0151 1283 0366 ok", "PIN [CREDIT_CARD_REDACTED] ok"),
|
||||
],
|
||||
)
|
||||
async def test_checksum_mask_keeps_the_words_a_later_regex_match_shares_with_the_card(
|
||||
self,
|
||||
checksum_patterns: tuple[str, ...],
|
||||
regex: str,
|
||||
later_patterns: tuple[str, ...],
|
||||
text: str,
|
||||
expected: str,
|
||||
):
|
||||
guardrail: Final = ContentFilterGuardrail(
|
||||
guardrail_name="test-checksum-mask-then-regex",
|
||||
patterns=[
|
||||
*(
|
||||
ContentFilterPattern(pattern_type="prebuilt", pattern_name=name, action=ContentFilterAction.MASK)
|
||||
for name in checksum_patterns
|
||||
),
|
||||
ContentFilterPattern(
|
||||
pattern_type="regex", pattern=regex, name="account_note", action=ContentFilterAction.MASK
|
||||
),
|
||||
*(
|
||||
ContentFilterPattern(pattern_type="prebuilt", pattern_name=name, action=ContentFilterAction.MASK)
|
||||
for name in later_patterns
|
||||
),
|
||||
],
|
||||
)
|
||||
guardrailed_inputs: Final = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [text]}, request_data={}, input_type="request"
|
||||
)
|
||||
assert guardrailed_inputs.get("texts") == [expected]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checksum_pattern_streams_the_same_text_as_a_whole_response(self):
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
guardrail: Final = ContentFilterGuardrail(
|
||||
guardrail_name="test-checksum-streaming",
|
||||
patterns=[
|
||||
ContentFilterPattern(pattern_type="prebuilt", pattern_name="credit_card", action=ContentFilterAction.MASK)
|
||||
],
|
||||
)
|
||||
chunks: Final = ("4532 " * 20, "4532 0151 1283 0366 thanks")
|
||||
|
||||
async def mock_stream():
|
||||
for index, content in enumerate(chunks):
|
||||
yield ModelResponseStream(
|
||||
id=f"chunk{index}",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
delta=Delta(content=content),
|
||||
index=0,
|
||||
finish_reason="stop" if index == len(chunks) - 1 else None,
|
||||
)
|
||||
],
|
||||
model="gpt-4",
|
||||
)
|
||||
|
||||
streamed: Final = "".join(
|
||||
[
|
||||
chunk.choices[0].delta.content or ""
|
||||
async for chunk in guardrail.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=MagicMock(), response=mock_stream(), request_data={}
|
||||
)
|
||||
]
|
||||
)
|
||||
whole_response: Final = await guardrail.apply_guardrail(
|
||||
inputs={"texts": ["".join(chunks)]}, request_data={}, input_type="response"
|
||||
)
|
||||
|
||||
assert "0366" not in streamed
|
||||
assert [streamed] == whole_response.get("texts")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"iban",
|
||||
[
|
||||
"GB66\u212aEST12345698765432",
|
||||
"GB73W\u0130ST12345698765432",
|
||||
"GB\u0668\u0662WEST12345698765432",
|
||||
],
|
||||
)
|
||||
async def test_checksum_pattern_masks_a_valid_iban_with_characters_the_regex_case_folds(self, iban: str):
|
||||
guardrail: Final = ContentFilterGuardrail(
|
||||
guardrail_name="test-checksum-folding",
|
||||
patterns=[ContentFilterPattern(pattern_type="prebuilt", pattern_name="iban", action=ContentFilterAction.MASK)],
|
||||
)
|
||||
|
||||
guardrailed_inputs: Final = await guardrail.apply_guardrail(
|
||||
inputs={"texts": [f"send it to {iban}"]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert guardrailed_inputs.get("texts") == ["send it to [IBAN_REDACTED]"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("pattern_name", ["iban", "eu_iban_enhanced"])
|
||||
async def test_checksum_pattern_masks_a_valid_iban_with_characters_attached_to_its_end(self, pattern_name: str):
|
||||
guardrail: Final = ContentFilterGuardrail(
|
||||
guardrail_name="test-checksum-attached",
|
||||
patterns=[
|
||||
ContentFilterPattern(pattern_type="prebuilt", pattern_name=pattern_name, action=ContentFilterAction.MASK)
|
||||
],
|
||||
)
|
||||
|
||||
guardrailed_inputs: Final = await guardrail.apply_guardrail(
|
||||
inputs={"texts": ["pay DE89370400440532013000EUR or DE88370400440532013000EUR"]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert guardrailed_inputs.get("texts") == [f"pay [{pattern_name.upper()}_REDACTED] or DE88370400440532013000EUR"]
|
||||
|
||||
def test_api_key_patterns(self):
|
||||
"""
|
||||
Test API key pattern detection
|
||||
|
|
@ -957,9 +1234,10 @@ class TestContentFilterGuardrail:
|
|||
self,
|
||||
text: str,
|
||||
detections: list[ContentFilterDetection] | None = None,
|
||||
check_candidate: CandidateCheck = run_validator,
|
||||
) -> str:
|
||||
scanned_lengths.append(len(text))
|
||||
return super()._filter_single_text(text, detections=detections)
|
||||
return super()._filter_single_text(text, detections=detections, check_candidate=check_candidate)
|
||||
|
||||
guardrail: Final = RecordingGuardrail(
|
||||
guardrail_name="test-streaming-bounded-scan",
|
||||
|
|
@ -999,9 +1277,10 @@ class TestContentFilterGuardrail:
|
|||
self,
|
||||
text: str,
|
||||
detections: list[ContentFilterDetection] | None = None,
|
||||
check_candidate: CandidateCheck = run_validator,
|
||||
) -> str:
|
||||
scanned_lengths.append(len(text))
|
||||
return super()._filter_single_text(text, detections=detections)
|
||||
return super()._filter_single_text(text, detections=detections, check_candidate=check_candidate)
|
||||
|
||||
guardrail: Final = RecordingGuardrail(
|
||||
guardrail_name="test-streaming-refused-cut-backoff",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
from itertools import product
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.validators import (
|
||||
passes_iban_mod97,
|
||||
passes_luhn,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
("4111 1111 1111 1111", True),
|
||||
("4111-1111-1111-1112", False),
|
||||
("٤" + "١" * 15, True),
|
||||
("4111 1111 1111 111x", False),
|
||||
("", False),
|
||||
],
|
||||
)
|
||||
def test_passes_luhn(value: str, expected: bool):
|
||||
assert passes_luhn(value) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"valid_number",
|
||||
["4111111111111111", "4012888888881881", "5555555555554444", "378282246310005", "6011000990139424"],
|
||||
)
|
||||
def test_passes_luhn_rejects_every_single_digit_change(valid_number: str):
|
||||
single_digit_changes: Final = tuple(
|
||||
valid_number[:position] + replacement + valid_number[position + 1 :]
|
||||
for position, replacement in product(range(len(valid_number)), "0123456789")
|
||||
)
|
||||
|
||||
assert passes_luhn(valid_number)
|
||||
assert not any(passes_luhn(changed) for changed in single_digit_changes if changed != valid_number)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
("GB82 WEST 1234 5698 7654 32", True),
|
||||
("gb82west12345698765432", True),
|
||||
("GB82WEST12345698765433", False),
|
||||
("GB82", False),
|
||||
("GB82WEST1234569876543é", False),
|
||||
("DE89370400440532013000EUR", True),
|
||||
("DE88370400440532013000EUR", False),
|
||||
],
|
||||
)
|
||||
def test_passes_iban_mod97(value: str, expected: bool):
|
||||
assert passes_iban_mod97(value) is expected
|
||||
Loading…
Add table
Reference in a new issue