diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 982c473ac1f..e7c916652d5 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1620,6 +1620,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "unicode-normalization", ] [[package]] @@ -3126,6 +3127,15 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-normalization-alignments" version = "0.1.12" diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 2f157b2c20d..ac7f72a5c78 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -19,6 +19,7 @@ indexmap = { version = "2.14.0", features = ["serde"] } # HuggingFace tokenizer for input token counting; without the default features it # pulls no HTTP client or progress bars, only the `onig` regex backend. tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] } +unicode-normalization = "0.1.24" aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true } aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } diff --git a/litellm-rust/crates/core/src/token_counter/byte_level.rs b/litellm-rust/crates/core/src/token_counter/byte_level.rs new file mode 100644 index 00000000000..9a453480a40 --- /dev/null +++ b/litellm-rust/crates/core/src/token_counter/byte_level.rs @@ -0,0 +1,387 @@ +//! Token counting for tokenizers shaped like Anthropic's (optional NFKC +//! normalizer, `ByteLevel` pre-tokenizer with the GPT-2 split regex, no +//! post-processor) without running the regex. Oniguruma spends ~90% of +//! `encode_fast` on `'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+`; +//! a hand-written scanner finds the same pieces and hands them to the model. + +use std::borrow::Cow; +use std::cmp::Ordering; + +use tokenizers::normalizers::NormalizerWrapper; +use tokenizers::pre_tokenizers::PreTokenizerWrapper; +use tokenizers::{Model, Tokenizer}; +use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfkc_quick}; + +use super::unicode_classes::{LETTER_RANGES, NUMBER_RANGES, SPACE_RANGES}; + +const CONTRACTIONS: [&str; 7] = ["'s", "'t", "'re", "'ve", "'m", "'ll", "'d"]; + +pub(super) struct ByteLevelCounter { + nfkc: bool, +} + +impl ByteLevelCounter { + pub(super) fn detect(tokenizer: &Tokenizer) -> Option { + let nfkc = match tokenizer.get_normalizer() { + None => false, + Some(NormalizerWrapper::NFKC(_)) => true, + Some(_) => return None, + }; + let Some(PreTokenizerWrapper::ByteLevel(byte_level)) = tokenizer.get_pre_tokenizer() else { + return None; + }; + let plain = !byte_level.add_prefix_space + && byte_level.use_regex + && tokenizer.get_post_processor().is_none() + && tokenizer.get_truncation().is_none() + && tokenizer.get_padding().is_none(); + plain.then_some(Self { nfkc }) + } + + /// `None` when the text contains an added token or the model rejects a + /// piece; the caller then runs the full encoder. + pub(super) fn count(&self, tokenizer: &Tokenizer, text: &str) -> Option { + let normalized = self.normalize(text); + let added_tokens = tokenizer.get_added_vocabulary().get_vocab(); + if added_tokens + .keys() + .any(|token| text.contains(token.as_str()) || normalized.contains(token.as_str())) + { + return None; + } + let model = tokenizer.get_model(); + let mut mapped = String::new(); + let mut total = 0; + let pieces = Pieces { + rest: normalized.as_ref(), + }; + for piece in pieces { + mapped.clear(); + mapped.extend(piece.bytes().map(|byte| BYTE_CHARS[usize::from(byte)])); + total += model.tokenize(&mapped).ok()?.len(); + } + Some(total) + } + + fn normalize<'a>(&self, text: &'a str) -> Cow<'a, str> { + if !self.nfkc || text.is_ascii() || is_nfkc_quick(text.chars()) == IsNormalized::Yes { + return Cow::Borrowed(text); + } + Cow::Owned(text.nfkc().collect()) + } +} + +/// GPT-2 `bytes_to_unicode`: printable Latin-1 bytes map to themselves, the +/// rest to U+0100 onwards in byte order. +const BYTE_CHARS: [char; 256] = byte_chars(); + +const fn byte_chars() -> [char; 256] { + let mut table = ['\0'; 256]; + let mut next_gap = 0x100u32; + let mut byte = 0usize; + while byte < 256 { + let printable = (byte >= 0x21 && byte <= 0x7E) + || (byte >= 0xA1 && byte <= 0xAC) + || (byte >= 0xAE && byte <= 0xFF); + let code = if printable { + byte as u32 + } else { + next_gap += 1; + next_gap - 1 + }; + table[byte] = match char::from_u32(code) { + Some(mapped) => mapped, + None => unreachable!(), + }; + byte += 1; + } + table +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Class { + Letter, + Number, + Space, + Other, +} + +const ASCII_CLASSES: [Class; 128] = ascii_classes(); + +const fn ascii_classes() -> [Class; 128] { + let mut table = [Class::Other; 128]; + let mut code = 0usize; + while code < 128 { + table[code] = match code as u8 { + b'A'..=b'Z' | b'a'..=b'z' => Class::Letter, + b'0'..=b'9' => Class::Number, + b'\t'..=b'\r' | b' ' => Class::Space, + _ => Class::Other, + }; + code += 1; + } + table +} + +fn class(character: char) -> Class { + if character.is_ascii() { + ASCII_CLASSES[character as usize] + } else if in_ranges(LETTER_RANGES, character) { + Class::Letter + } else if in_ranges(NUMBER_RANGES, character) { + Class::Number + } else if in_ranges(SPACE_RANGES, character) { + Class::Space + } else { + Class::Other + } +} + +fn in_ranges(ranges: &[(u32, u32)], character: char) -> bool { + let code = u32::from(character); + ranges + .binary_search_by(|(low, high)| { + if *high < code { + Ordering::Less + } else if *low > code { + Ordering::Greater + } else { + Ordering::Equal + } + }) + .is_ok() +} + +/// The regex matches every character, so the pieces tile the text. +struct Pieces<'a> { + rest: &'a str, +} + +impl<'a> Iterator for Pieces<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option<&'a str> { + let first = self.rest.chars().next()?; + let (piece, rest) = self.rest.split_at(piece_len(self.rest, first)); + self.rest = rest; + Some(piece) + } +} + +fn piece_len(text: &str, first: char) -> usize { + if let Some(contraction) = CONTRACTIONS.iter().find(|word| text.starts_with(**word)) { + return contraction.len(); + } + let first_class = class(first); + if first_class != Class::Space { + return run_len(text, first_class); + } + if first != ' ' { + return space_run_len(text); + } + let after_space = &text[1..]; + match after_space.chars().next().map(class) { + None | Some(Class::Space) => space_run_len(text), + Some(run_class) => 1 + run_len(after_space, run_class), + } +} + +fn run_len(text: &str, run_class: Class) -> usize { + text.char_indices() + .find(|(_, character)| class(*character) != run_class) + .map_or(text.len(), |(index, _)| index) +} + +/// `\s+(?!\S)|\s+`: whitespace followed by a non-space leaves its last +/// character to start the next piece (` ?` on the following alternatives). +fn space_run_len(text: &str) -> usize { + let run = run_len(text, Class::Space); + if run == text.len() { + return run; + } + let last = text[..run].chars().next_back().map_or(0, char::len_utf8); + match run - last { + 0 => run, + shorter => shorter, + } +} + +#[cfg(test)] +mod tests { + use rand::rngs::StdRng; + use rand::seq::SliceRandom; + use rand::{Rng, SeedableRng}; + use tokenizers::pre_tokenizers::byte_level::ByteLevel; + use tokenizers::utils::SysRegex; + use tokenizers::{OffsetReferential, OffsetType, PreTokenizedString, PreTokenizer}; + + use super::*; + + fn anthropic() -> Tokenizer { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json" + ); + std::fs::read_to_string(path) + .expect("anthropic tokenizer json is in the repo") + .parse() + .expect("anthropic tokenizer loads") + } + + fn reference_count(tokenizer: &Tokenizer, text: &str) -> usize { + tokenizer.encode_fast(text, true).expect("encode").len() + } + + const ALPHABET: &[&str] = &[ + "a", + "Z", + "e", + "s", + "t", + "d", + "m", + "'", + "'s", + "'re", + "'ll", + "'S", + "0", + "9", + " ", + " ", + "\t", + "\n", + "\r\n", + "\u{b}", + ".", + ",", + "!", + "-", + "(", + "\"", + "\u{a0}", + "\u{85}", + "\u{2028}", + "\u{3000}", + "\u{200b}", + "\u{200d}", + "é", + "e\u{301}", + "ß", + "漢", + "字", + "ع", + "३", + "½", + "Ⅳ", + "🙂", + "👍🏽", + "A", + "fi", + "㍿", + "<", + ">", + "EOT", + "", + "", + ]; + + fn random_text(rng: &mut StdRng) -> String { + let pieces = rng.gen_range(0..40); + (0..pieces) + .map(|_| *ALPHABET.choose(rng).expect("alphabet is not empty")) + .collect() + } + + #[test] + fn anthropic_tokenizer_takes_the_fast_path() { + let tokenizer = anthropic(); + let fast = ByteLevelCounter::detect(&tokenizer).expect("anthropic shape is supported"); + assert!(fast.nfkc); + let text = "Hello, how are you today?"; + assert_eq!( + fast.count(&tokenizer, text), + Some(reference_count(&tokenizer, text)) + ); + assert_eq!(fast.count(&tokenizer, "stop here"), None); + assert_eq!(fast.count(&tokenizer, "stop <EOT> here"), None); + } + + #[test] + fn counts_match_the_full_encoder() { + let tokenizer = anthropic(); + let fast = ByteLevelCounter::detect(&tokenizer).expect("supported"); + let mut rng = StdRng::seed_from_u64(2026); + for _ in 0..4000 { + let text = random_text(&mut rng); + let expected = reference_count(&tokenizer, &text); + let counted = fast + .count(&tokenizer, &text) + .unwrap_or_else(|| reference_count(&tokenizer, &text)); + assert_eq!(counted, expected, "text {text:?}"); + } + } + + #[test] + fn pieces_match_the_byte_level_pre_tokenizer() { + let byte_level = ByteLevel::new(false, true, true); + let mut rng = StdRng::seed_from_u64(7); + for _ in 0..4000 { + let text: String = random_text(&mut rng).nfkc().collect(); + let mut pre_tokenized = PreTokenizedString::from(text.as_str()); + byte_level + .pre_tokenize(&mut pre_tokenized) + .expect("pre-tokenize"); + let expected: Vec<(String, (usize, usize))> = pre_tokenized + .get_splits(OffsetReferential::Original, OffsetType::Byte) + .into_iter() + .map(|(mapped, offsets, _)| (mapped.to_string(), offsets)) + .collect(); + let mut offset = 0; + let pieces = Pieces { rest: &text }; + let actual: Vec<(String, (usize, usize))> = pieces + .map(|piece| { + let mapped: String = piece + .bytes() + .map(|byte| BYTE_CHARS[usize::from(byte)]) + .collect(); + let span = (offset, offset + piece.len()); + offset += piece.len(); + (mapped, span) + }) + .collect(); + assert_eq!(actual, expected, "text {text:?}"); + } + } + + #[test] + fn classes_match_oniguruma() { + let letter = SysRegex::new(r"\p{L}").expect("regex"); + let number = SysRegex::new(r"\p{N}").expect("regex"); + let space = SysRegex::new(r"\s").expect("regex"); + let whole = + |regex: &SysRegex, text: &str| regex.find_iter(text).next() == Some((0, text.len())); + let mut text = String::new(); + for character in (0..=0x10FFFFu32).filter_map(char::from_u32) { + text.clear(); + text.push(character); + let expected = if whole(&letter, &text) { + Class::Letter + } else if whole(&number, &text) { + Class::Number + } else if whole(&space, &text) { + Class::Space + } else { + Class::Other + }; + assert_eq!(class(character), expected, "U+{:04X}", u32::from(character)); + } + } + + #[test] + fn other_tokenizer_shapes_are_declined() { + let mut tokenizer = anthropic(); + tokenizer.with_pre_tokenizer(Some(ByteLevel::new(true, true, true))); + assert!(ByteLevelCounter::detect(&tokenizer).is_none()); + } +} diff --git a/litellm-rust/crates/core/src/token_counter/mod.rs b/litellm-rust/crates/core/src/token_counter/mod.rs index 765b272979d..81ae89f29d6 100644 --- a/litellm-rust/crates/core/src/token_counter/mod.rs +++ b/litellm-rust/crates/core/src/token_counter/mod.rs @@ -2,13 +2,17 @@ //! for the shapes it can count exactly. Everything else is declined so the host //! keeps its own counter as the reference. +mod byte_level; mod python_json; mod tools; pub mod types; +mod unicode_classes; use serde::Serialize; use thiserror::Error as ThisError; +use byte_level::ByteLevelCounter; + use crate::constants::{ NAMED_TOOL_CHOICE_TOKENS, REPLY_PRIMING_TOKENS, TOKENS_PER_MESSAGE, TOKENS_PER_NAME, TOOL_CHOICE_NONE_TOKENS, TOOL_DEFINITIONS_TOKENS, TOOLS_WITH_SYSTEM_MESSAGE_DISCOUNT, @@ -41,6 +45,7 @@ pub struct InputTokenCount { /// event loop. pub struct TokenCounter { tokenizer: tokenizers::Tokenizer, + byte_level: Option, } impl TokenCounter { @@ -49,10 +54,21 @@ impl TokenCounter { let tokenizer = tokenizer_json .parse::() .map_err(|error| TokenCountError::Load(error.to_string()))?; - Ok(Self { tokenizer }) + let byte_level = ByteLevelCounter::detect(&tokenizer); + Ok(Self { + tokenizer, + byte_level, + }) } pub fn count_text(&self, text: &str) -> Result { + if let Some(count) = self + .byte_level + .as_ref() + .and_then(|counter| counter.count(&self.tokenizer, text)) + { + return Ok(count); + } self.tokenizer .encode_fast(text, true) .map(|encoding| encoding.len()) diff --git a/litellm-rust/crates/core/src/token_counter/unicode_classes.rs b/litellm-rust/crates/core/src/token_counter/unicode_classes.rs new file mode 100644 index 00000000000..dc98b5431f7 --- /dev/null +++ b/litellm-rust/crates/core/src/token_counter/unicode_classes.rs @@ -0,0 +1,841 @@ +//! `\p{L}`, `\p{N}` and `\s` exactly as oniguruma (the `tokenizers` regex +//! backend) classifies them. `byte_level::tests::classes_match_oniguruma` +//! regenerates these from the regex engine and fails on any drift. + +pub(super) const LETTER_RANGES: &[(u32, u32)] = &[ + (0x41, 0x5A), + (0x61, 0x7A), + (0xAA, 0xAA), + (0xB5, 0xB5), + (0xBA, 0xBA), + (0xC0, 0xD6), + (0xD8, 0xF6), + (0xF8, 0x2C1), + (0x2C6, 0x2D1), + (0x2E0, 0x2E4), + (0x2EC, 0x2EC), + (0x2EE, 0x2EE), + (0x370, 0x374), + (0x376, 0x377), + (0x37A, 0x37D), + (0x37F, 0x37F), + (0x386, 0x386), + (0x388, 0x38A), + (0x38C, 0x38C), + (0x38E, 0x3A1), + (0x3A3, 0x3F5), + (0x3F7, 0x481), + (0x48A, 0x52F), + (0x531, 0x556), + (0x559, 0x559), + (0x560, 0x588), + (0x5D0, 0x5EA), + (0x5EF, 0x5F2), + (0x620, 0x64A), + (0x66E, 0x66F), + (0x671, 0x6D3), + (0x6D5, 0x6D5), + (0x6E5, 0x6E6), + (0x6EE, 0x6EF), + (0x6FA, 0x6FC), + (0x6FF, 0x6FF), + (0x710, 0x710), + (0x712, 0x72F), + (0x74D, 0x7A5), + (0x7B1, 0x7B1), + (0x7CA, 0x7EA), + (0x7F4, 0x7F5), + (0x7FA, 0x7FA), + (0x800, 0x815), + (0x81A, 0x81A), + (0x824, 0x824), + (0x828, 0x828), + (0x840, 0x858), + (0x860, 0x86A), + (0x870, 0x887), + (0x889, 0x88E), + (0x8A0, 0x8C9), + (0x904, 0x939), + (0x93D, 0x93D), + (0x950, 0x950), + (0x958, 0x961), + (0x971, 0x980), + (0x985, 0x98C), + (0x98F, 0x990), + (0x993, 0x9A8), + (0x9AA, 0x9B0), + (0x9B2, 0x9B2), + (0x9B6, 0x9B9), + (0x9BD, 0x9BD), + (0x9CE, 0x9CE), + (0x9DC, 0x9DD), + (0x9DF, 0x9E1), + (0x9F0, 0x9F1), + (0x9FC, 0x9FC), + (0xA05, 0xA0A), + (0xA0F, 0xA10), + (0xA13, 0xA28), + (0xA2A, 0xA30), + (0xA32, 0xA33), + (0xA35, 0xA36), + (0xA38, 0xA39), + (0xA59, 0xA5C), + (0xA5E, 0xA5E), + (0xA72, 0xA74), + (0xA85, 0xA8D), + (0xA8F, 0xA91), + (0xA93, 0xAA8), + (0xAAA, 0xAB0), + (0xAB2, 0xAB3), + (0xAB5, 0xAB9), + (0xABD, 0xABD), + (0xAD0, 0xAD0), + (0xAE0, 0xAE1), + (0xAF9, 0xAF9), + (0xB05, 0xB0C), + (0xB0F, 0xB10), + (0xB13, 0xB28), + (0xB2A, 0xB30), + (0xB32, 0xB33), + (0xB35, 0xB39), + (0xB3D, 0xB3D), + (0xB5C, 0xB5D), + (0xB5F, 0xB61), + (0xB71, 0xB71), + (0xB83, 0xB83), + (0xB85, 0xB8A), + (0xB8E, 0xB90), + (0xB92, 0xB95), + (0xB99, 0xB9A), + (0xB9C, 0xB9C), + (0xB9E, 0xB9F), + (0xBA3, 0xBA4), + (0xBA8, 0xBAA), + (0xBAE, 0xBB9), + (0xBD0, 0xBD0), + (0xC05, 0xC0C), + (0xC0E, 0xC10), + (0xC12, 0xC28), + (0xC2A, 0xC39), + (0xC3D, 0xC3D), + (0xC58, 0xC5A), + (0xC5D, 0xC5D), + (0xC60, 0xC61), + (0xC80, 0xC80), + (0xC85, 0xC8C), + (0xC8E, 0xC90), + (0xC92, 0xCA8), + (0xCAA, 0xCB3), + (0xCB5, 0xCB9), + (0xCBD, 0xCBD), + (0xCDD, 0xCDE), + (0xCE0, 0xCE1), + (0xCF1, 0xCF2), + (0xD04, 0xD0C), + (0xD0E, 0xD10), + (0xD12, 0xD3A), + (0xD3D, 0xD3D), + (0xD4E, 0xD4E), + (0xD54, 0xD56), + (0xD5F, 0xD61), + (0xD7A, 0xD7F), + (0xD85, 0xD96), + (0xD9A, 0xDB1), + (0xDB3, 0xDBB), + (0xDBD, 0xDBD), + (0xDC0, 0xDC6), + (0xE01, 0xE30), + (0xE32, 0xE33), + (0xE40, 0xE46), + (0xE81, 0xE82), + (0xE84, 0xE84), + (0xE86, 0xE8A), + (0xE8C, 0xEA3), + (0xEA5, 0xEA5), + (0xEA7, 0xEB0), + (0xEB2, 0xEB3), + (0xEBD, 0xEBD), + (0xEC0, 0xEC4), + (0xEC6, 0xEC6), + (0xEDC, 0xEDF), + (0xF00, 0xF00), + (0xF40, 0xF47), + (0xF49, 0xF6C), + (0xF88, 0xF8C), + (0x1000, 0x102A), + (0x103F, 0x103F), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16F1, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1950, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BAF), + (0x1BBA, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C4D, 0x1C4F), + (0x1C5A, 0x1C7D), + (0x1C80, 0x1C8A), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2071, 0x2071), + (0x207F, 0x207F), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2183, 0x2184), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3006), + (0x3031, 0x3035), + (0x303B, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA61F), + (0xA62A, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6E5), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7CD), + (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), + (0xA7D5, 0xA7DC), + (0xA7F2, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA90A, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9CF), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9EF), + (0xA9FA, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x10300, 0x1031F), + (0x1032D, 0x10340), + (0x10342, 0x10349), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x10400, 0x1049D), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x105C0, 0x105F3), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10860, 0x10876), + (0x10880, 0x1089E), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x10900, 0x10915), + (0x10920, 0x10939), + (0x10980, 0x109B7), + (0x109BE, 0x109BF), + (0x10A00, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A60, 0x10A7C), + (0x10A80, 0x10A9C), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B60, 0x10B72), + (0x10B80, 0x10B91), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10D00, 0x10D23), + (0x10D4A, 0x10D65), + (0x10D6F, 0x10D85), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10EC2, 0x10EC4), + (0x10F00, 0x10F1C), + (0x10F27, 0x10F27), + (0x10F30, 0x10F45), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FC4), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x11103, 0x11126), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111DA, 0x111DA), + (0x111DC, 0x111DC), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11380, 0x11389), + (0x1138B, 0x1138B), + (0x1138E, 0x1138E), + (0x11390, 0x113B5), + (0x113B7, 0x113B7), + (0x113D1, 0x113D1), + (0x113D3, 0x113D3), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x11700, 0x1171A), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118DF), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11BC0, 0x11BE0), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11FB0, 0x11FB0), + (0x12000, 0x12399), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x13460, 0x143FA), + (0x14400, 0x14646), + (0x16100, 0x1611D), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A70, 0x16ABE), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16D40, 0x16D6C), + (0x16E40, 0x16E7F), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), + (0x18800, 0x18CD5), + (0x18CFF, 0x18D08), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E4D0, 0x1E4EB), + (0x1E5D0, 0x1E5ED), + (0x1E5F0, 0x1E5F0), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x323AF), +]; +pub(super) const NUMBER_RANGES: &[(u32, u32)] = &[ + (0x30, 0x39), + (0xB2, 0xB3), + (0xB9, 0xB9), + (0xBC, 0xBE), + (0x660, 0x669), + (0x6F0, 0x6F9), + (0x7C0, 0x7C9), + (0x966, 0x96F), + (0x9E6, 0x9EF), + (0x9F4, 0x9F9), + (0xA66, 0xA6F), + (0xAE6, 0xAEF), + (0xB66, 0xB6F), + (0xB72, 0xB77), + (0xBE6, 0xBF2), + (0xC66, 0xC6F), + (0xC78, 0xC7E), + (0xCE6, 0xCEF), + (0xD58, 0xD5E), + (0xD66, 0xD78), + (0xDE6, 0xDEF), + (0xE50, 0xE59), + (0xED0, 0xED9), + (0xF20, 0xF33), + (0x1040, 0x1049), + (0x1090, 0x1099), + (0x1369, 0x137C), + (0x16EE, 0x16F0), + (0x17E0, 0x17E9), + (0x17F0, 0x17F9), + (0x1810, 0x1819), + (0x1946, 0x194F), + (0x19D0, 0x19DA), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1B50, 0x1B59), + (0x1BB0, 0x1BB9), + (0x1C40, 0x1C49), + (0x1C50, 0x1C59), + (0x2070, 0x2070), + (0x2074, 0x2079), + (0x2080, 0x2089), + (0x2150, 0x2182), + (0x2185, 0x2189), + (0x2460, 0x249B), + (0x24EA, 0x24FF), + (0x2776, 0x2793), + (0x2CFD, 0x2CFD), + (0x3007, 0x3007), + (0x3021, 0x3029), + (0x3038, 0x303A), + (0x3192, 0x3195), + (0x3220, 0x3229), + (0x3248, 0x324F), + (0x3251, 0x325F), + (0x3280, 0x3289), + (0x32B1, 0x32BF), + (0xA620, 0xA629), + (0xA6E6, 0xA6EF), + (0xA830, 0xA835), + (0xA8D0, 0xA8D9), + (0xA900, 0xA909), + (0xA9D0, 0xA9D9), + (0xA9F0, 0xA9F9), + (0xAA50, 0xAA59), + (0xABF0, 0xABF9), + (0xFF10, 0xFF19), + (0x10107, 0x10133), + (0x10140, 0x10178), + (0x1018A, 0x1018B), + (0x102E1, 0x102FB), + (0x10320, 0x10323), + (0x10341, 0x10341), + (0x1034A, 0x1034A), + (0x103D1, 0x103D5), + (0x104A0, 0x104A9), + (0x10858, 0x1085F), + (0x10879, 0x1087F), + (0x108A7, 0x108AF), + (0x108FB, 0x108FF), + (0x10916, 0x1091B), + (0x109BC, 0x109BD), + (0x109C0, 0x109CF), + (0x109D2, 0x109FF), + (0x10A40, 0x10A48), + (0x10A7D, 0x10A7E), + (0x10A9D, 0x10A9F), + (0x10AEB, 0x10AEF), + (0x10B58, 0x10B5F), + (0x10B78, 0x10B7F), + (0x10BA9, 0x10BAF), + (0x10CFA, 0x10CFF), + (0x10D30, 0x10D39), + (0x10D40, 0x10D49), + (0x10E60, 0x10E7E), + (0x10F1D, 0x10F26), + (0x10F51, 0x10F54), + (0x10FC5, 0x10FCB), + (0x11052, 0x1106F), + (0x110F0, 0x110F9), + (0x11136, 0x1113F), + (0x111D0, 0x111D9), + (0x111E1, 0x111F4), + (0x112F0, 0x112F9), + (0x11450, 0x11459), + (0x114D0, 0x114D9), + (0x11650, 0x11659), + (0x116C0, 0x116C9), + (0x116D0, 0x116E3), + (0x11730, 0x1173B), + (0x118E0, 0x118F2), + (0x11950, 0x11959), + (0x11BF0, 0x11BF9), + (0x11C50, 0x11C6C), + (0x11D50, 0x11D59), + (0x11DA0, 0x11DA9), + (0x11F50, 0x11F59), + (0x11FC0, 0x11FD4), + (0x12400, 0x1246E), + (0x16130, 0x16139), + (0x16A60, 0x16A69), + (0x16AC0, 0x16AC9), + (0x16B50, 0x16B59), + (0x16B5B, 0x16B61), + (0x16D70, 0x16D79), + (0x16E80, 0x16E96), + (0x1CCF0, 0x1CCF9), + (0x1D2C0, 0x1D2D3), + (0x1D2E0, 0x1D2F3), + (0x1D360, 0x1D378), + (0x1D7CE, 0x1D7FF), + (0x1E140, 0x1E149), + (0x1E2F0, 0x1E2F9), + (0x1E4F0, 0x1E4F9), + (0x1E5F1, 0x1E5FA), + (0x1E8C7, 0x1E8CF), + (0x1E950, 0x1E959), + (0x1EC71, 0x1ECAB), + (0x1ECAD, 0x1ECAF), + (0x1ECB1, 0x1ECB4), + (0x1ED01, 0x1ED2D), + (0x1ED2F, 0x1ED3D), + (0x1F100, 0x1F10C), + (0x1FBF0, 0x1FBF9), +]; +pub(super) const SPACE_RANGES: &[(u32, u32)] = &[ + (0x9, 0xD), + (0x20, 0x20), + (0x85, 0x85), + (0xA0, 0xA0), + (0x1680, 0x1680), + (0x2000, 0x200A), + (0x2028, 0x2029), + (0x202F, 0x202F), + (0x205F, 0x205F), + (0x3000, 0x3000), +];