diff --git a/litellm-rust/crates/request-auth-core/src/patterns.rs b/litellm-rust/crates/request-auth-core/src/patterns.rs index dc378e0f01b..bb97e6841cd 100644 --- a/litellm-rust/crates/request-auth-core/src/patterns.rs +++ b/litellm-rust/crates/request-auth-core/src/patterns.rs @@ -9,6 +9,166 @@ use crate::Error; static EXPRESSIONS: LazyLock>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); +fn python_escape(escaped: char, in_class: bool) -> Option { + let word = r"\p{L}\p{N}_"; + let class = match escaped { + 'w' => format!("[{word}]"), + 'W' => format!("[^{word}]"), + 'b' if !in_class => format!("(?:(?<=[{word}])(?![{word}])|(? { + format!("(?:(?<=[{word}])(?=[{word}])|(? return None, + }; + Some(format!("(?-i:{class})")) +} + +fn python_class(pattern: &str) -> Option<(usize, String)> { + let negated = pattern.starts_with("[^"); + let start = if negated { 2 } else { 1 }; + let mut chars = pattern.char_indices().skip(start).peekable(); + let mut literals = String::new(); + let mut alternatives = Vec::new(); + let mut range = false; + if chars.peek().is_some_and(|(_, ch)| *ch == ']') { + literals.push_str(r"\]"); + chars.next(); + } + while let Some((index, ch)) = chars.next() { + match ch { + ']' => { + if alternatives.is_empty() { + return Some((index + 1, pattern[..index + 1].into())); + } + if !literals.is_empty() { + alternatives.push(format!("[{literals}]")); + } + let union = alternatives.join("|"); + return Some(( + index + 1, + if negated { + format!("(?:(?!(?:{union}))[\\s\\S])") + } else { + format!("(?:{union})") + }, + )); + } + '\\' => { + let (_, escaped) = chars.next()?; + if let Some(class) = python_escape(escaped, true) { + // A category escape cannot be a range endpoint in Python. + if range + || pattern[index + 2..].starts_with('-') + && !pattern[index + 2..].starts_with("-]") + { + return None; + } + alternatives.push(class); + } else { + literals.push(ch); + literals.push(escaped); + } + } + '[' => literals.push_str(r"\["), + '^' => literals.push_str(r"\^"), + _ => literals.push(ch), + } + range = ch == '-' && index > start; + } + None +} + +fn python_pattern(pattern: &str) -> String { + let mut output = String::with_capacity(pattern.len()); + let mut verbose = false; + let mut scopes = Vec::new(); + let mut chars = pattern.char_indices().peekable(); + while let Some((index, ch)) = chars.next() { + let rest = &pattern[index..]; + match ch { + '\\' => { + if let Some((_, escaped)) = chars.next() { + if let Some(replacement) = python_escape(escaped, false) { + output.push_str(&replacement); + } else { + output.push(ch); + output.push(escaped); + } + } else { + output.push(ch); + } + } + '[' => { + let Some((length, class)) = python_class(rest) else { + output.push_str(rest); + break; + }; + output.push_str(&class); + while chars.peek().is_some_and(|(next, _)| *next < index + length) { + chars.next(); + } + } + '#' if verbose => { + output.push(ch); + for (_, next) in chars.by_ref() { + output.push(next); + if next == '\n' { + break; + } + } + } + '(' if rest.starts_with("(?#") => { + output.push(ch); + let mut escaped = false; + for (_, next) in chars.by_ref() { + output.push(next); + if next == ')' && !escaped { + break; + } + escaped = next == '\\' && !escaped; + } + } + '(' => { + if let Some(header) = rest.strip_prefix("(?") { + let length = header + .find(|c: char| !"aiLmsux-".contains(c)) + .unwrap_or(header.len()); + if length > 0 && matches!(header.as_bytes().get(length), Some(b':' | b')')) { + let scoped = header.as_bytes()[length] == b':'; + if scoped { + scopes.push(verbose); + } + let mut disabled = false; + for flag in header[..length].chars() { + match flag { + 'x' => verbose = !disabled, + '-' => disabled = true, + _ => (), + } + } + output.push_str(&rest[..length + 3]); + while chars + .peek() + .is_some_and(|(next, _)| *next < index + length + 3) + { + chars.next(); + } + continue; + } + } + scopes.push(verbose); + output.push(ch); + } + ')' => { + verbose = scopes.pop().unwrap_or_default(); + output.push(ch); + } + _ => output.push(ch), + } + } + output +} + fn expression(pattern: &str) -> Result, Error> { if let Some(compiled) = EXPRESSIONS .lock() @@ -17,7 +177,7 @@ fn expression(pattern: &str) -> Result, Error> { { return Ok(Arc::clone(compiled)); } - let compiled = Arc::new(Regex::new(pattern).map_err(Error::Pattern)?); + let compiled = Arc::new(Regex::new(&python_pattern(pattern)).map_err(Error::Pattern)?); let mut expressions = EXPRESSIONS .lock() .unwrap_or_else(|error| error.into_inner()); @@ -107,6 +267,72 @@ pub fn route_access(route: &str, patterns: &[String]) -> Result { mod tests { use rstest::rstest; + #[rstest] + #[case::combining(r"\w+", "\u{0301}a", false)] + #[case::joiner(r"\w+", "\u{200d}a", false)] + #[case::connector(r"\w+", "\u{203f}a", false)] + #[case::superscript(r"\w+", "²a", true)] + #[case::fraction(r"\w+", "¼", true)] + #[case::letter_number(r"\w+", "Ⅷ", true)] + #[case::underscore(r"\w+", "_", true)] + #[case::letter(r"\w+", "한", true)] + #[case::nonword_mark(r"\W+", "\u{0301}a", true)] + #[case::nonword_number(r"\W+", "²a", false)] + #[case::class_word(r"[\w]+", "²a", true)] + #[case::class_nonword(r"[\W]+", "\u{0301}a", true)] + #[case::negated_word(r"[^\w]+", "²a", false)] + #[case::negated_nonword(r"[^\W]+", "²a", true)] + #[case::negated_repeat(r"[^\w]+$", "!a", false)] + #[case::union(r"[\w\W]+", "\u{0301}a", true)] + #[case::negated_union(r"[^\w\W]+", "a", false)] + #[case::class_caret(r"[\w^]+", "^", true)] + #[case::class_dash(r"[\w-]+", "-", true)] + #[case::class_escaped_dash(r"[\-\w]+", "²", true)] + #[case::class_closing_bracket(r"[]\w]+", "]²", true)] + #[case::class_opening_bracket(r"[[\w]+", "[²", true)] + #[case::class_backspace(r"[\b\w]", "\u{8}", true)] + #[case::literal_escape(r"\\w", r"\w", true)] + #[case::literal_class_escape(r"[\\w]", "²", false)] + #[case::boundary_number(r"\b²", "²", true)] + #[case::boundary_mark(r"\b\u0301", "\u{0301}", false)] + #[case::nonboundary_mark(r"\B\u0301", "\u{0301}", true)] + #[case::nonboundary_number(r"\B²", "²", false)] + #[case::boundary_after_word(r"a\b\u0301", "a\u{0301}", true)] + #[case::boundary_after_nonword(r"!\b²", "!²", true)] + #[case::nonboundary_between_words(r"a\B²", "a²", true)] + #[case::nonboundary_between_nonwords(r"!\B\u0301", "!\u{0301}", true)] + #[case::end_boundary(r"²\b", "²", true)] + #[case::end_nonboundary(r"\u0301\B", "\u{0301}", true)] + #[case::empty_boundary(r"\b", "", false)] + #[case::empty_nonboundary(r"\B", "", false)] + #[case::unicode_flag(r"(?u:\w)", "²", true)] + #[case::case_insensitive_word(r"(?i:\w)", "\u{0345}", false)] + #[case::case_insensitive_class_word(r"(?i:[\w])", "\u{0345}", false)] + #[case::case_insensitive_class_literal(r"(?i:[\wΙ])", "\u{0345}", true)] + #[case::case_insensitive_boundary(r"(?i:\b\u0345)", "\u{0345}", false)] + #[case::case_insensitive_literal(r"(?i:a)\w", "A²", true)] + #[case::ascii_class(r"[a-zA-Z0-9_]", "²", false)] + #[case::comment(r"(?# [\b)\w", "²", true)] + #[case::verbose_comment("(?x:# [\\b)\n\\w)", "²", true)] + #[case::verbose_scope("(?x:# [\n)(?-x:#)\\w", "#²", true)] + #[case::verbose_restored("(?x:# [\n)#\\w", "#²", true)] + #[case::backreference(r"(\w)\1", "²²", true)] + #[case::lookbehind(r".(?<=[\w])a", "²a", true)] + fn python_word_semantics(#[case] pattern: &str, #[case] value: &str, #[case] expected: bool) { + assert_eq!(super::regex_match(pattern, value).unwrap(), expected); + } + + #[rstest] + #[case::ascii_flag(r"(?a:\w)")] + #[case::ascii_unicode_flag(r"(?-u:\w)")] + #[case::word_range_start(r"[\w-a]")] + #[case::word_range_end(r"[a-\w]")] + #[case::unclosed_class(r"[\w")] + #[case::unclosed_group(r"(\w")] + fn invalid_patterns_remain_errors(#[case] pattern: &str) { + assert!(super::regex_match(pattern, "a").is_err()); + } + #[rstest] #[case::segment("/key/k/reset", "/key/{id}/reset", true)] #[case::segment_slash("/key/a/b/reset", "/key/{id}/reset", false)] @@ -128,6 +354,9 @@ mod tests { #[case::regex_dot("aa-1", "a.-*", true)] #[case::newline("a\nb", "*", false)] #[case::final_newline("a\n", "a*", true)] + #[case::word_combining_mark("\u{0301}a", r"\w*", false)] + #[case::word_join_control("\u{200d}a", r"\w*", false)] + #[case::word_other_number("\u{00b2}a", r"\w*", true)] fn model_patterns(#[case] model: &str, #[case] pattern: &str, #[case] expected: bool) { assert_eq!(super::model_pattern(model, pattern).unwrap(), expected); } diff --git a/tests/unit/_auth/test_model_access.py b/tests/unit/_auth/test_model_access.py index 205e10ee2ba..aee39808184 100644 --- a/tests/unit/_auth/test_model_access.py +++ b/tests/unit/_auth/test_model_access.py @@ -37,6 +37,9 @@ _CASES: Final = ( pytest.param(replace(_BASE, grants=("mo*",)), "Allow", id="prefix-wildcard"), pytest.param(replace(_BASE, requested=("model\n",), grants=("mo*",)), "Allow", id="python-final-newline"), pytest.param(replace(_BASE, requested=("mod\nel",), grants=("mo*l",)), "Deny", id="embedded-newline"), + pytest.param(replace(_BASE, requested=("\u0301a",), grants=(r"\w*",)), "Deny", id="word-combining-mark"), + pytest.param(replace(_BASE, requested=("\u200da",), grants=(r"\w*",)), "Deny", id="word-join-control"), + pytest.param(replace(_BASE, requested=("\u00b2a",), grants=(r"\w*",)), "Allow", id="word-other-number"), pytest.param( replace(_BASE, grants=("p/*",), registry=Registry(records={"model": Record(provider=Provider("model", "p"))})), "Allow", @@ -191,6 +194,30 @@ def test_model_policy_case_table( assert decision == reference.model_access(facts) +@pytest.mark.parametrize( + "grant", + ( + r"\w*", r"\W*", r"[\w]*", r"[\W]*", r"[^\w]*", r"[^\W]*", + r"[\w\W]*", r"[^\w\W]*", r"[^\w]+*", r"[\w^]*", r"[\w-]*", + r"[\-\w]*", r"[]\w]*", r"[^]\w]*", r"[[\w]*", r"[\b\w]*", + r"\\w*", r"[\\w]*", r"\b\w*", r"\B\w*", r"\w\b*", r"\w\B*", + r"\W\b*", r"\W\B*", r"(?u:\w)*", r"(?i:\w)*", r"(?i:[\wΙ])*", + r"(?i:[^\wΙ])*", r"[a-zA-Z0-9_]*", r"(?# [\b)\w*", + "(?x:# [\\b)\n\\w)*", "(?x:# [\n)(?-x:#)\\w*", r"(\w)\1*", + ), +) +@pytest.mark.parametrize( + "model", + ( + "\u0301a", "\u200da", "\u203fa", "\u0345a", "²a", "¼a", "Ⅷa", "a²", "a\u0301", + "_", "-", "^", "a", "A", "é", "한글", "αα", "\n", "\b", r"\word", "]a", "[a", "#²", " a", + ), +) +def test_unicode_regex_model_access_matches_python(grant: str, model: str) -> None: + facts: Final = replace(_BASE, requested=(model,), grants=(grant,)) + assert native.model_access(facts) == reference.model_access(facts) + + @pytest.mark.parametrize( ("groups", "models", "merge", "kind"), (