From 5a2fc06d019a16dfae3b79f7e8bd82865ce67b0d Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 10 Sep 2026 08:26:51 +0000 Subject: [PATCH] perf(rust): count byte-level BPE tokens without the GPT-2 split regex The oniguruma run of the ByteLevel pre-tokenizer regex is about 90% of encode_fast on a 100k token body (100 ms of the ~110 ms Rust admission count in the gateway pod). A hand-written scanner that yields the same pieces, then feeds the model directly, counts the same text in 10 ms. It only engages for tokenizers with the Anthropic shape (optional NFKC, ByteLevel without prefix space, no post-processor) and falls back to the full encoder when the text contains an added token. Parity with encode_fast is tested on random texts, the pieces are compared with the real pre-tokenizer, and the \p{L}/\p{N}/\s tables are checked against oniguruma for every code point. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/Cargo.lock | 10 + litellm-rust/crates/core/Cargo.toml | 1 + .../core/src/token_counter/byte_level.rs | 387 ++++++++ .../crates/core/src/token_counter/mod.rs | 18 +- .../core/src/token_counter/unicode_classes.rs | 841 ++++++++++++++++++ 5 files changed, 1256 insertions(+), 1 deletion(-) create mode 100644 litellm-rust/crates/core/src/token_counter/byte_level.rs create mode 100644 litellm-rust/crates/core/src/token_counter/unicode_classes.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 982c473ac1f..e7c916652d5 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1620,6 +1620,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "unicode-normalization", ] [[package]] @@ -3126,6 +3127,15 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-normalization-alignments" version = "0.1.12" diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 2f157b2c20d..ac7f72a5c78 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -19,6 +19,7 @@ indexmap = { version = "2.14.0", features = ["serde"] } # HuggingFace tokenizer for input token counting; without the default features it # pulls no HTTP client or progress bars, only the `onig` regex backend. tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] } +unicode-normalization = "0.1.24" aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true } aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } diff --git a/litellm-rust/crates/core/src/token_counter/byte_level.rs b/litellm-rust/crates/core/src/token_counter/byte_level.rs new file mode 100644 index 00000000000..9a453480a40 --- /dev/null +++ b/litellm-rust/crates/core/src/token_counter/byte_level.rs @@ -0,0 +1,387 @@ +//! Token counting for tokenizers shaped like Anthropic's (optional NFKC +//! normalizer, `ByteLevel` pre-tokenizer with the GPT-2 split regex, no +//! post-processor) without running the regex. Oniguruma spends ~90% of +//! `encode_fast` on `'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+`; +//! a hand-written scanner finds the same pieces and hands them to the model. + +use std::borrow::Cow; +use std::cmp::Ordering; + +use tokenizers::normalizers::NormalizerWrapper; +use tokenizers::pre_tokenizers::PreTokenizerWrapper; +use tokenizers::{Model, Tokenizer}; +use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfkc_quick}; + +use super::unicode_classes::{LETTER_RANGES, NUMBER_RANGES, SPACE_RANGES}; + +const CONTRACTIONS: [&str; 7] = ["'s", "'t", "'re", "'ve", "'m", "'ll", "'d"]; + +pub(super) struct ByteLevelCounter { + nfkc: bool, +} + +impl ByteLevelCounter { + pub(super) fn detect(tokenizer: &Tokenizer) -> Option { + let nfkc = match tokenizer.get_normalizer() { + None => false, + Some(NormalizerWrapper::NFKC(_)) => true, + Some(_) => return None, + }; + let Some(PreTokenizerWrapper::ByteLevel(byte_level)) = tokenizer.get_pre_tokenizer() else { + return None; + }; + let plain = !byte_level.add_prefix_space + && byte_level.use_regex + && tokenizer.get_post_processor().is_none() + && tokenizer.get_truncation().is_none() + && tokenizer.get_padding().is_none(); + plain.then_some(Self { nfkc }) + } + + /// `None` when the text contains an added token or the model rejects a + /// piece; the caller then runs the full encoder. + pub(super) fn count(&self, tokenizer: &Tokenizer, text: &str) -> Option { + let normalized = self.normalize(text); + let added_tokens = tokenizer.get_added_vocabulary().get_vocab(); + if added_tokens + .keys() + .any(|token| text.contains(token.as_str()) || normalized.contains(token.as_str())) + { + return None; + } + let model = tokenizer.get_model(); + let mut mapped = String::new(); + let mut total = 0; + let pieces = Pieces { + rest: normalized.as_ref(), + }; + for piece in pieces { + mapped.clear(); + mapped.extend(piece.bytes().map(|byte| BYTE_CHARS[usize::from(byte)])); + total += model.tokenize(&mapped).ok()?.len(); + } + Some(total) + } + + fn normalize<'a>(&self, text: &'a str) -> Cow<'a, str> { + if !self.nfkc || text.is_ascii() || is_nfkc_quick(text.chars()) == IsNormalized::Yes { + return Cow::Borrowed(text); + } + Cow::Owned(text.nfkc().collect()) + } +} + +/// GPT-2 `bytes_to_unicode`: printable Latin-1 bytes map to themselves, the +/// rest to U+0100 onwards in byte order. +const BYTE_CHARS: [char; 256] = byte_chars(); + +const fn byte_chars() -> [char; 256] { + let mut table = ['\0'; 256]; + let mut next_gap = 0x100u32; + let mut byte = 0usize; + while byte < 256 { + let printable = (byte >= 0x21 && byte <= 0x7E) + || (byte >= 0xA1 && byte <= 0xAC) + || (byte >= 0xAE && byte <= 0xFF); + let code = if printable { + byte as u32 + } else { + next_gap += 1; + next_gap - 1 + }; + table[byte] = match char::from_u32(code) { + Some(mapped) => mapped, + None => unreachable!(), + }; + byte += 1; + } + table +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Class { + Letter, + Number, + Space, + Other, +} + +const ASCII_CLASSES: [Class; 128] = ascii_classes(); + +const fn ascii_classes() -> [Class; 128] { + let mut table = [Class::Other; 128]; + let mut code = 0usize; + while code < 128 { + table[code] = match code as u8 { + b'A'..=b'Z' | b'a'..=b'z' => Class::Letter, + b'0'..=b'9' => Class::Number, + b'\t'..=b'\r' | b' ' => Class::Space, + _ => Class::Other, + }; + code += 1; + } + table +} + +fn class(character: char) -> Class { + if character.is_ascii() { + ASCII_CLASSES[character as usize] + } else if in_ranges(LETTER_RANGES, character) { + Class::Letter + } else if in_ranges(NUMBER_RANGES, character) { + Class::Number + } else if in_ranges(SPACE_RANGES, character) { + Class::Space + } else { + Class::Other + } +} + +fn in_ranges(ranges: &[(u32, u32)], character: char) -> bool { + let code = u32::from(character); + ranges + .binary_search_by(|(low, high)| { + if *high < code { + Ordering::Less + } else if *low > code { + Ordering::Greater + } else { + Ordering::Equal + } + }) + .is_ok() +} + +/// The regex matches every character, so the pieces tile the text. +struct Pieces<'a> { + rest: &'a str, +} + +impl<'a> Iterator for Pieces<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option<&'a str> { + let first = self.rest.chars().next()?; + let (piece, rest) = self.rest.split_at(piece_len(self.rest, first)); + self.rest = rest; + Some(piece) + } +} + +fn piece_len(text: &str, first: char) -> usize { + if let Some(contraction) = CONTRACTIONS.iter().find(|word| text.starts_with(**word)) { + return contraction.len(); + } + let first_class = class(first); + if first_class != Class::Space { + return run_len(text, first_class); + } + if first != ' ' { + return space_run_len(text); + } + let after_space = &text[1..]; + match after_space.chars().next().map(class) { + None | Some(Class::Space) => space_run_len(text), + Some(run_class) => 1 + run_len(after_space, run_class), + } +} + +fn run_len(text: &str, run_class: Class) -> usize { + text.char_indices() + .find(|(_, character)| class(*character) != run_class) + .map_or(text.len(), |(index, _)| index) +} + +/// `\s+(?!\S)|\s+`: whitespace followed by a non-space leaves its last +/// character to start the next piece (` ?` on the following alternatives). +fn space_run_len(text: &str) -> usize { + let run = run_len(text, Class::Space); + if run == text.len() { + return run; + } + let last = text[..run].chars().next_back().map_or(0, char::len_utf8); + match run - last { + 0 => run, + shorter => shorter, + } +} + +#[cfg(test)] +mod tests { + use rand::rngs::StdRng; + use rand::seq::SliceRandom; + use rand::{Rng, SeedableRng}; + use tokenizers::pre_tokenizers::byte_level::ByteLevel; + use tokenizers::utils::SysRegex; + use tokenizers::{OffsetReferential, OffsetType, PreTokenizedString, PreTokenizer}; + + use super::*; + + fn anthropic() -> Tokenizer { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json" + ); + std::fs::read_to_string(path) + .expect("anthropic tokenizer json is in the repo") + .parse() + .expect("anthropic tokenizer loads") + } + + fn reference_count(tokenizer: &Tokenizer, text: &str) -> usize { + tokenizer.encode_fast(text, true).expect("encode").len() + } + + const ALPHABET: &[&str] = &[ + "a", + "Z", + "e", + "s", + "t", + "d", + "m", + "'", + "'s", + "'re", + "'ll", + "'S", + "0", + "9", + " ", + " ", + "\t", + "\n", + "\r\n", + "\u{b}", + ".", + ",", + "!", + "-", + "(", + "\"", + "\u{a0}", + "\u{85}", + "\u{2028}", + "\u{3000}", + "\u{200b}", + "\u{200d}", + "é", + "e\u{301}", + "ß", + "漢", + "字", + "ع", + "३", + "½", + "Ⅳ", + "🙂", + "👍🏽", + "Ａ", + "fi", + "㍿", + "<", + ">", + "EOT", + "", + "", + ]; + + fn random_text(rng: &mut StdRng) -> String { + let pieces = rng.gen_range(0..40); + (0..pieces) + .map(|_| *ALPHABET.choose(rng).expect("alphabet is not empty")) + .collect() + } + + #[test] + fn anthropic_tokenizer_takes_the_fast_path() { + let tokenizer = anthropic(); + let fast = ByteLevelCounter::detect(&tokenizer).expect("anthropic shape is supported"); + assert!(fast.nfkc); + let text = "Hello, how are you today?"; + assert_eq!( + fast.count(&tokenizer, text), + Some(reference_count(&tokenizer, text)) + ); + assert_eq!(fast.count(&tokenizer, "stop here"), None); + assert_eq!(fast.count(&tokenizer, "stop ＜ＥＯＴ＞ here"), None); + } + + #[test] + fn counts_match_the_full_encoder() { + let tokenizer = anthropic(); + let fast = ByteLevelCounter::detect(&tokenizer).expect("supported"); + let mut rng = StdRng::seed_from_u64(2026); + for _ in 0..4000 { + let text = random_text(&mut rng); + let expected = reference_count(&tokenizer, &text); + let counted = fast + .count(&tokenizer, &text) + .unwrap_or_else(|| reference_count(&tokenizer, &text)); + assert_eq!(counted, expected, "text {text:?}"); + } + } + + #[test] + fn pieces_match_the_byte_level_pre_tokenizer() { + let byte_level = ByteLevel::new(false, true, true); + let mut rng = StdRng::seed_from_u64(7); + for _ in 0..4000 { + let text: String = random_text(&mut rng).nfkc().collect(); + let mut pre_tokenized = PreTokenizedString::from(text.as_str()); + byte_level + .pre_tokenize(&mut pre_tokenized) + .expect("pre-tokenize"); + let expected: Vec<(String, (usize, usize))> = pre_tokenized + .get_splits(OffsetReferential::Original, OffsetType::Byte) + .into_iter() + .map(|(mapped, offsets, _)| (mapped.to_string(), offsets)) + .collect(); + let mut offset = 0; + let pieces = Pieces { rest: &text }; + let actual: Vec<(String, (usize, usize))> = pieces + .map(|piece| { + let mapped: String = piece + .bytes() + .map(|byte| BYTE_CHARS[usize::from(byte)]) + .collect(); + let span = (offset, offset + piece.len()); + offset += piece.len(); + (mapped, span) + }) + .collect(); + assert_eq!(actual, expected, "text {text:?}"); + } + } + + #[test] + fn classes_match_oniguruma() { + let letter = SysRegex::new(r"\p{L}").expect("regex"); + let number = SysRegex::new(r"\p{N}").expect("regex"); + let space = SysRegex::new(r"\s").expect("regex"); + let whole = + |regex: &SysRegex, text: &str| regex.find_iter(text).next() == Some((0, text.len())); + let mut text = String::new(); + for character in (0..=0x10FFFFu32).filter_map(char::from_u32) { + text.clear(); + text.push(character); + let expected = if whole(&letter, &text) { + Class::Letter + } else if whole(&number, &text) { + Class::Number + } else if whole(&space, &text) { + Class::Space + } else { + Class::Other + }; + assert_eq!(class(character), expected, "U+{:04X}", u32::from(character)); + } + } + + #[test] + fn other_tokenizer_shapes_are_declined() { + let mut tokenizer = anthropic(); + tokenizer.with_pre_tokenizer(Some(ByteLevel::new(true, true, true))); + assert!(ByteLevelCounter::detect(&tokenizer).is_none()); + } +} diff --git a/litellm-rust/crates/core/src/token_counter/mod.rs b/litellm-rust/crates/core/src/token_counter/mod.rs index 765b272979d..81ae89f29d6 100644 --- a/litellm-rust/crates/core/src/token_counter/mod.rs +++ b/litellm-rust/crates/core/src/token_counter/mod.rs @@ -2,13 +2,17 @@ //! for the shapes it can count exactly. Everything else is declined so the host //! keeps its own counter as the reference. +mod byte_level; mod python_json; mod tools; pub mod types; +mod unicode_classes; use serde::Serialize; use thiserror::Error as ThisError; +use byte_level::ByteLevelCounter; + use crate::constants::{ NAMED_TOOL_CHOICE_TOKENS, REPLY_PRIMING_TOKENS, TOKENS_PER_MESSAGE, TOKENS_PER_NAME, TOOL_CHOICE_NONE_TOKENS, TOOL_DEFINITIONS_TOKENS, TOOLS_WITH_SYSTEM_MESSAGE_DISCOUNT, @@ -41,6 +45,7 @@ pub struct InputTokenCount { /// event loop. pub struct TokenCounter { tokenizer: tokenizers::Tokenizer, + byte_level: Option, } impl TokenCounter { @@ -49,10 +54,21 @@ impl TokenCounter { let tokenizer = tokenizer_json .parse::() .map_err(|error| TokenCountError::Load(error.to_string()))?; - Ok(Self { tokenizer }) + let byte_level = ByteLevelCounter::detect(&tokenizer); + Ok(Self { + tokenizer, + byte_level, + }) } pub fn count_text(&self, text: &str) -> Result { + if let Some(count) = self + .byte_level + .as_ref() + .and_then(|counter| counter.count(&self.tokenizer, text)) + { + return Ok(count); + } self.tokenizer .encode_fast(text, true) .map(|encoding| encoding.len()) diff --git a/litellm-rust/crates/core/src/token_counter/unicode_classes.rs b/litellm-rust/crates/core/src/token_counter/unicode_classes.rs new file mode 100644 index 00000000000..dc98b5431f7 --- /dev/null +++ b/litellm-rust/crates/core/src/token_counter/unicode_classes.rs @@ -0,0 +1,841 @@ +//! `\p{L}`, `\p{N}` and `\s` exactly as oniguruma (the `tokenizers` regex +//! backend) classifies them. `byte_level::tests::classes_match_oniguruma` +//! regenerates these from the regex engine and fails on any drift. + +pub(super) const LETTER_RANGES: &[(u32, u32)] = &[ + (0x41, 0x5A), + (0x61, 0x7A), + (0xAA, 0xAA), + (0xB5, 0xB5), + (0xBA, 0xBA), + (0xC0, 0xD6), + (0xD8, 0xF6), + (0xF8, 0x2C1), + (0x2C6, 0x2D1), + (0x2E0, 0x2E4), + (0x2EC, 0x2EC), + (0x2EE, 0x2EE), + (0x370, 0x374), + (0x376, 0x377), + (0x37A, 0x37D), + (0x37F, 0x37F), + (0x386, 0x386), + (0x388, 0x38A), + (0x38C, 0x38C), + (0x38E, 0x3A1), + (0x3A3, 0x3F5), + (0x3F7, 0x481), + (0x48A, 0x52F), + (0x531, 0x556), + (0x559, 0x559), + (0x560, 0x588), + (0x5D0, 0x5EA), + (0x5EF, 0x5F2), + (0x620, 0x64A), + (0x66E, 0x66F), + (0x671, 0x6D3), + (0x6D5, 0x6D5), + (0x6E5, 0x6E6), + (0x6EE, 0x6EF), + (0x6FA, 0x6FC), + (0x6FF, 0x6FF), + (0x710, 0x710), + (0x712, 0x72F), + (0x74D, 0x7A5), + (0x7B1, 0x7B1), + (0x7CA, 0x7EA), + (0x7F4, 0x7F5), + (0x7FA, 0x7FA), + (0x800, 0x815), + (0x81A, 0x81A), + (0x824, 0x824), + (0x828, 0x828), + (0x840, 0x858), + (0x860, 0x86A), + (0x870, 0x887), + (0x889, 0x88E), + (0x8A0, 0x8C9), + (0x904, 0x939), + (0x93D, 0x93D), + (0x950, 0x950), + (0x958, 0x961), + (0x971, 0x980), + (0x985, 0x98C), + (0x98F, 0x990), + (0x993, 0x9A8), + (0x9AA, 0x9B0), + (0x9B2, 0x9B2), + (0x9B6, 0x9B9), + (0x9BD, 0x9BD), + (0x9CE, 0x9CE), + (0x9DC, 0x9DD), + (0x9DF, 0x9E1), + (0x9F0, 0x9F1), + (0x9FC, 0x9FC), + (0xA05, 0xA0A), + (0xA0F, 0xA10), + (0xA13, 0xA28), + (0xA2A, 0xA30), + (0xA32, 0xA33), + (0xA35, 0xA36), + (0xA38, 0xA39), + (0xA59, 0xA5C), + (0xA5E, 0xA5E), + (0xA72, 0xA74), + (0xA85, 0xA8D), + (0xA8F, 0xA91), + (0xA93, 0xAA8), + (0xAAA, 0xAB0), + (0xAB2, 0xAB3), + (0xAB5, 0xAB9), + (0xABD, 0xABD), + (0xAD0, 0xAD0), + (0xAE0, 0xAE1), + (0xAF9, 0xAF9), + (0xB05, 0xB0C), + (0xB0F, 0xB10), + (0xB13, 0xB28), + (0xB2A, 0xB30), + (0xB32, 0xB33), + (0xB35, 0xB39), + (0xB3D, 0xB3D), + (0xB5C, 0xB5D), + (0xB5F, 0xB61), + (0xB71, 0xB71), + (0xB83, 0xB83), + (0xB85, 0xB8A), + (0xB8E, 0xB90), + (0xB92, 0xB95), + (0xB99, 0xB9A), + (0xB9C, 0xB9C), + (0xB9E, 0xB9F), + (0xBA3, 0xBA4), + (0xBA8, 0xBAA), + (0xBAE, 0xBB9), + (0xBD0, 0xBD0), + (0xC05, 0xC0C), + (0xC0E, 0xC10), + (0xC12, 0xC28), + (0xC2A, 0xC39), + (0xC3D, 0xC3D), + (0xC58, 0xC5A), + (0xC5D, 0xC5D), + (0xC60, 0xC61), + (0xC80, 0xC80), + (0xC85, 0xC8C), + (0xC8E, 0xC90), + (0xC92, 0xCA8), + (0xCAA, 0xCB3), + (0xCB5, 0xCB9), + (0xCBD, 0xCBD), + (0xCDD, 0xCDE), + (0xCE0, 0xCE1), + (0xCF1, 0xCF2), + (0xD04, 0xD0C), + (0xD0E, 0xD10), + (0xD12, 0xD3A), + (0xD3D, 0xD3D), + (0xD4E, 0xD4E), + (0xD54, 0xD56), + (0xD5F, 0xD61), + (0xD7A, 0xD7F), + (0xD85, 0xD96), + (0xD9A, 0xDB1), + (0xDB3, 0xDBB), + (0xDBD, 0xDBD), + (0xDC0, 0xDC6), + (0xE01, 0xE30), + (0xE32, 0xE33), + (0xE40, 0xE46), + (0xE81, 0xE82), + (0xE84, 0xE84), + (0xE86, 0xE8A), + (0xE8C, 0xEA3), + (0xEA5, 0xEA5), + (0xEA7, 0xEB0), + (0xEB2, 0xEB3), + (0xEBD, 0xEBD), + (0xEC0, 0xEC4), + (0xEC6, 0xEC6), + (0xEDC, 0xEDF), + (0xF00, 0xF00), + (0xF40, 0xF47), + (0xF49, 0xF6C), + (0xF88, 0xF8C), + (0x1000, 0x102A), + (0x103F, 0x103F), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16F1, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1950, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BAF), + (0x1BBA, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C4D, 0x1C4F), + (0x1C5A, 0x1C7D), + (0x1C80, 0x1C8A), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2071, 0x2071), + (0x207F, 0x207F), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2183, 0x2184), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3006), + (0x3031, 0x3035), + (0x303B, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA61F), + (0xA62A, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6E5), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7CD), + (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), + (0xA7D5, 0xA7DC), + (0xA7F2, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA90A, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9CF), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9EF), + (0xA9FA, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x10300, 0x1031F), + (0x1032D, 0x10340), + (0x10342, 0x10349), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x10400, 0x1049D), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x105C0, 0x105F3), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10860, 0x10876), + (0x10880, 0x1089E), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x10900, 0x10915), + (0x10920, 0x10939), + (0x10980, 0x109B7), + (0x109BE, 0x109BF), + (0x10A00, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A60, 0x10A7C), + (0x10A80, 0x10A9C), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B60, 0x10B72), + (0x10B80, 0x10B91), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10D00, 0x10D23), + (0x10D4A, 0x10D65), + (0x10D6F, 0x10D85), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10EC2, 0x10EC4), + (0x10F00, 0x10F1C), + (0x10F27, 0x10F27), + (0x10F30, 0x10F45), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FC4), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x11103, 0x11126), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111DA, 0x111DA), + (0x111DC, 0x111DC), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11380, 0x11389), + (0x1138B, 0x1138B), + (0x1138E, 0x1138E), + (0x11390, 0x113B5), + (0x113B7, 0x113B7), + (0x113D1, 0x113D1), + (0x113D3, 0x113D3), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x11700, 0x1171A), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118DF), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11BC0, 0x11BE0), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11FB0, 0x11FB0), + (0x12000, 0x12399), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x13460, 0x143FA), + (0x14400, 0x14646), + (0x16100, 0x1611D), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A70, 0x16ABE), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16D40, 0x16D6C), + (0x16E40, 0x16E7F), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), + (0x18800, 0x18CD5), + (0x18CFF, 0x18D08), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E4D0, 0x1E4EB), + (0x1E5D0, 0x1E5ED), + (0x1E5F0, 0x1E5F0), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x323AF), +]; +pub(super) const NUMBER_RANGES: &[(u32, u32)] = &[ + (0x30, 0x39), + (0xB2, 0xB3), + (0xB9, 0xB9), + (0xBC, 0xBE), + (0x660, 0x669), + (0x6F0, 0x6F9), + (0x7C0, 0x7C9), + (0x966, 0x96F), + (0x9E6, 0x9EF), + (0x9F4, 0x9F9), + (0xA66, 0xA6F), + (0xAE6, 0xAEF), + (0xB66, 0xB6F), + (0xB72, 0xB77), + (0xBE6, 0xBF2), + (0xC66, 0xC6F), + (0xC78, 0xC7E), + (0xCE6, 0xCEF), + (0xD58, 0xD5E), + (0xD66, 0xD78), + (0xDE6, 0xDEF), + (0xE50, 0xE59), + (0xED0, 0xED9), + (0xF20, 0xF33), + (0x1040, 0x1049), + (0x1090, 0x1099), + (0x1369, 0x137C), + (0x16EE, 0x16F0), + (0x17E0, 0x17E9), + (0x17F0, 0x17F9), + (0x1810, 0x1819), + (0x1946, 0x194F), + (0x19D0, 0x19DA), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1B50, 0x1B59), + (0x1BB0, 0x1BB9), + (0x1C40, 0x1C49), + (0x1C50, 0x1C59), + (0x2070, 0x2070), + (0x2074, 0x2079), + (0x2080, 0x2089), + (0x2150, 0x2182), + (0x2185, 0x2189), + (0x2460, 0x249B), + (0x24EA, 0x24FF), + (0x2776, 0x2793), + (0x2CFD, 0x2CFD), + (0x3007, 0x3007), + (0x3021, 0x3029), + (0x3038, 0x303A), + (0x3192, 0x3195), + (0x3220, 0x3229), + (0x3248, 0x324F), + (0x3251, 0x325F), + (0x3280, 0x3289), + (0x32B1, 0x32BF), + (0xA620, 0xA629), + (0xA6E6, 0xA6EF), + (0xA830, 0xA835), + (0xA8D0, 0xA8D9), + (0xA900, 0xA909), + (0xA9D0, 0xA9D9), + (0xA9F0, 0xA9F9), + (0xAA50, 0xAA59), + (0xABF0, 0xABF9), + (0xFF10, 0xFF19), + (0x10107, 0x10133), + (0x10140, 0x10178), + (0x1018A, 0x1018B), + (0x102E1, 0x102FB), + (0x10320, 0x10323), + (0x10341, 0x10341), + (0x1034A, 0x1034A), + (0x103D1, 0x103D5), + (0x104A0, 0x104A9), + (0x10858, 0x1085F), + (0x10879, 0x1087F), + (0x108A7, 0x108AF), + (0x108FB, 0x108FF), + (0x10916, 0x1091B), + (0x109BC, 0x109BD), + (0x109C0, 0x109CF), + (0x109D2, 0x109FF), + (0x10A40, 0x10A48), + (0x10A7D, 0x10A7E), + (0x10A9D, 0x10A9F), + (0x10AEB, 0x10AEF), + (0x10B58, 0x10B5F), + (0x10B78, 0x10B7F), + (0x10BA9, 0x10BAF), + (0x10CFA, 0x10CFF), + (0x10D30, 0x10D39), + (0x10D40, 0x10D49), + (0x10E60, 0x10E7E), + (0x10F1D, 0x10F26), + (0x10F51, 0x10F54), + (0x10FC5, 0x10FCB), + (0x11052, 0x1106F), + (0x110F0, 0x110F9), + (0x11136, 0x1113F), + (0x111D0, 0x111D9), + (0x111E1, 0x111F4), + (0x112F0, 0x112F9), + (0x11450, 0x11459), + (0x114D0, 0x114D9), + (0x11650, 0x11659), + (0x116C0, 0x116C9), + (0x116D0, 0x116E3), + (0x11730, 0x1173B), + (0x118E0, 0x118F2), + (0x11950, 0x11959), + (0x11BF0, 0x11BF9), + (0x11C50, 0x11C6C), + (0x11D50, 0x11D59), + (0x11DA0, 0x11DA9), + (0x11F50, 0x11F59), + (0x11FC0, 0x11FD4), + (0x12400, 0x1246E), + (0x16130, 0x16139), + (0x16A60, 0x16A69), + (0x16AC0, 0x16AC9), + (0x16B50, 0x16B59), + (0x16B5B, 0x16B61), + (0x16D70, 0x16D79), + (0x16E80, 0x16E96), + (0x1CCF0, 0x1CCF9), + (0x1D2C0, 0x1D2D3), + (0x1D2E0, 0x1D2F3), + (0x1D360, 0x1D378), + (0x1D7CE, 0x1D7FF), + (0x1E140, 0x1E149), + (0x1E2F0, 0x1E2F9), + (0x1E4F0, 0x1E4F9), + (0x1E5F1, 0x1E5FA), + (0x1E8C7, 0x1E8CF), + (0x1E950, 0x1E959), + (0x1EC71, 0x1ECAB), + (0x1ECAD, 0x1ECAF), + (0x1ECB1, 0x1ECB4), + (0x1ED01, 0x1ED2D), + (0x1ED2F, 0x1ED3D), + (0x1F100, 0x1F10C), + (0x1FBF0, 0x1FBF9), +]; +pub(super) const SPACE_RANGES: &[(u32, u32)] = &[ + (0x9, 0xD), + (0x20, 0x20), + (0x85, 0x85), + (0xA0, 0xA0), + (0x1680, 0x1680), + (0x2000, 0x200A), + (0x2028, 0x2029), + (0x202F, 0x202F), + (0x205F, 0x205F), + (0x3000, 0x3000), +];