mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
perf(rust): count byte-level BPE tokens without the GPT-2 split regex
The oniguruma run of the ByteLevel pre-tokenizer regex is about 90% of
encode_fast on a 100k token body (100 ms of the ~110 ms Rust admission
count in the gateway pod). A hand-written scanner that yields the same
pieces, then feeds the model directly, counts the same text in 10 ms.
It only engages for tokenizers with the Anthropic shape (optional NFKC,
ByteLevel without prefix space, no post-processor) and falls back to the
full encoder when the text contains an added token. Parity with
encode_fast is tested on random texts, the pieces are compared with the
real pre-tokenizer, and the \p{L}/\p{N}/\s tables are checked against
oniguruma for every code point.
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin
parent
3fc95c5e1a
commit
5a2fc06d01
5 changed files with 1256 additions and 1 deletions
10
litellm-rust/Cargo.lock
generated
10
litellm-rust/Cargo.lock
generated
|
|
@ -1620,6 +1620,7 @@ dependencies = [
|
|||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3126,6 +3127,15 @@ version = "1.0.24"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization-alignments"
|
||||
version = "0.1.12"
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ indexmap = { version = "2.14.0", features = ["serde"] }
|
|||
# HuggingFace tokenizer for input token counting; without the default features it
|
||||
# pulls no HTTP client or progress bars, only the `onig` regex backend.
|
||||
tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
|
||||
unicode-normalization = "0.1.24"
|
||||
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true }
|
||||
aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true }
|
||||
aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true }
|
||||
|
|
|
|||
387
litellm-rust/crates/core/src/token_counter/byte_level.rs
Normal file
387
litellm-rust/crates/core/src/token_counter/byte_level.rs
Normal file
|
|
@ -0,0 +1,387 @@
|
|||
//! Token counting for tokenizers shaped like Anthropic's (optional NFKC
|
||||
//! normalizer, `ByteLevel` pre-tokenizer with the GPT-2 split regex, no
|
||||
//! post-processor) without running the regex. Oniguruma spends ~90% of
|
||||
//! `encode_fast` on `'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+`;
|
||||
//! a hand-written scanner finds the same pieces and hands them to the model.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use tokenizers::normalizers::NormalizerWrapper;
|
||||
use tokenizers::pre_tokenizers::PreTokenizerWrapper;
|
||||
use tokenizers::{Model, Tokenizer};
|
||||
use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfkc_quick};
|
||||
|
||||
use super::unicode_classes::{LETTER_RANGES, NUMBER_RANGES, SPACE_RANGES};
|
||||
|
||||
const CONTRACTIONS: [&str; 7] = ["'s", "'t", "'re", "'ve", "'m", "'ll", "'d"];
|
||||
|
||||
pub(super) struct ByteLevelCounter {
|
||||
nfkc: bool,
|
||||
}
|
||||
|
||||
impl ByteLevelCounter {
|
||||
pub(super) fn detect(tokenizer: &Tokenizer) -> Option<Self> {
|
||||
let nfkc = match tokenizer.get_normalizer() {
|
||||
None => false,
|
||||
Some(NormalizerWrapper::NFKC(_)) => true,
|
||||
Some(_) => return None,
|
||||
};
|
||||
let Some(PreTokenizerWrapper::ByteLevel(byte_level)) = tokenizer.get_pre_tokenizer() else {
|
||||
return None;
|
||||
};
|
||||
let plain = !byte_level.add_prefix_space
|
||||
&& byte_level.use_regex
|
||||
&& tokenizer.get_post_processor().is_none()
|
||||
&& tokenizer.get_truncation().is_none()
|
||||
&& tokenizer.get_padding().is_none();
|
||||
plain.then_some(Self { nfkc })
|
||||
}
|
||||
|
||||
/// `None` when the text contains an added token or the model rejects a
|
||||
/// piece; the caller then runs the full encoder.
|
||||
pub(super) fn count(&self, tokenizer: &Tokenizer, text: &str) -> Option<usize> {
|
||||
let normalized = self.normalize(text);
|
||||
let added_tokens = tokenizer.get_added_vocabulary().get_vocab();
|
||||
if added_tokens
|
||||
.keys()
|
||||
.any(|token| text.contains(token.as_str()) || normalized.contains(token.as_str()))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let model = tokenizer.get_model();
|
||||
let mut mapped = String::new();
|
||||
let mut total = 0;
|
||||
let pieces = Pieces {
|
||||
rest: normalized.as_ref(),
|
||||
};
|
||||
for piece in pieces {
|
||||
mapped.clear();
|
||||
mapped.extend(piece.bytes().map(|byte| BYTE_CHARS[usize::from(byte)]));
|
||||
total += model.tokenize(&mapped).ok()?.len();
|
||||
}
|
||||
Some(total)
|
||||
}
|
||||
|
||||
fn normalize<'a>(&self, text: &'a str) -> Cow<'a, str> {
|
||||
if !self.nfkc || text.is_ascii() || is_nfkc_quick(text.chars()) == IsNormalized::Yes {
|
||||
return Cow::Borrowed(text);
|
||||
}
|
||||
Cow::Owned(text.nfkc().collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// GPT-2 `bytes_to_unicode`: printable Latin-1 bytes map to themselves, the
|
||||
/// rest to U+0100 onwards in byte order.
|
||||
const BYTE_CHARS: [char; 256] = byte_chars();
|
||||
|
||||
const fn byte_chars() -> [char; 256] {
|
||||
let mut table = ['\0'; 256];
|
||||
let mut next_gap = 0x100u32;
|
||||
let mut byte = 0usize;
|
||||
while byte < 256 {
|
||||
let printable = (byte >= 0x21 && byte <= 0x7E)
|
||||
|| (byte >= 0xA1 && byte <= 0xAC)
|
||||
|| (byte >= 0xAE && byte <= 0xFF);
|
||||
let code = if printable {
|
||||
byte as u32
|
||||
} else {
|
||||
next_gap += 1;
|
||||
next_gap - 1
|
||||
};
|
||||
table[byte] = match char::from_u32(code) {
|
||||
Some(mapped) => mapped,
|
||||
None => unreachable!(),
|
||||
};
|
||||
byte += 1;
|
||||
}
|
||||
table
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Class {
|
||||
Letter,
|
||||
Number,
|
||||
Space,
|
||||
Other,
|
||||
}
|
||||
|
||||
const ASCII_CLASSES: [Class; 128] = ascii_classes();
|
||||
|
||||
const fn ascii_classes() -> [Class; 128] {
|
||||
let mut table = [Class::Other; 128];
|
||||
let mut code = 0usize;
|
||||
while code < 128 {
|
||||
table[code] = match code as u8 {
|
||||
b'A'..=b'Z' | b'a'..=b'z' => Class::Letter,
|
||||
b'0'..=b'9' => Class::Number,
|
||||
b'\t'..=b'\r' | b' ' => Class::Space,
|
||||
_ => Class::Other,
|
||||
};
|
||||
code += 1;
|
||||
}
|
||||
table
|
||||
}
|
||||
|
||||
fn class(character: char) -> Class {
|
||||
if character.is_ascii() {
|
||||
ASCII_CLASSES[character as usize]
|
||||
} else if in_ranges(LETTER_RANGES, character) {
|
||||
Class::Letter
|
||||
} else if in_ranges(NUMBER_RANGES, character) {
|
||||
Class::Number
|
||||
} else if in_ranges(SPACE_RANGES, character) {
|
||||
Class::Space
|
||||
} else {
|
||||
Class::Other
|
||||
}
|
||||
}
|
||||
|
||||
fn in_ranges(ranges: &[(u32, u32)], character: char) -> bool {
|
||||
let code = u32::from(character);
|
||||
ranges
|
||||
.binary_search_by(|(low, high)| {
|
||||
if *high < code {
|
||||
Ordering::Less
|
||||
} else if *low > code {
|
||||
Ordering::Greater
|
||||
} else {
|
||||
Ordering::Equal
|
||||
}
|
||||
})
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
/// The regex matches every character, so the pieces tile the text.
|
||||
struct Pieces<'a> {
|
||||
rest: &'a str,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for Pieces<'a> {
|
||||
type Item = &'a str;
|
||||
|
||||
fn next(&mut self) -> Option<&'a str> {
|
||||
let first = self.rest.chars().next()?;
|
||||
let (piece, rest) = self.rest.split_at(piece_len(self.rest, first));
|
||||
self.rest = rest;
|
||||
Some(piece)
|
||||
}
|
||||
}
|
||||
|
||||
fn piece_len(text: &str, first: char) -> usize {
|
||||
if let Some(contraction) = CONTRACTIONS.iter().find(|word| text.starts_with(**word)) {
|
||||
return contraction.len();
|
||||
}
|
||||
let first_class = class(first);
|
||||
if first_class != Class::Space {
|
||||
return run_len(text, first_class);
|
||||
}
|
||||
if first != ' ' {
|
||||
return space_run_len(text);
|
||||
}
|
||||
let after_space = &text[1..];
|
||||
match after_space.chars().next().map(class) {
|
||||
None | Some(Class::Space) => space_run_len(text),
|
||||
Some(run_class) => 1 + run_len(after_space, run_class),
|
||||
}
|
||||
}
|
||||
|
||||
fn run_len(text: &str, run_class: Class) -> usize {
|
||||
text.char_indices()
|
||||
.find(|(_, character)| class(*character) != run_class)
|
||||
.map_or(text.len(), |(index, _)| index)
|
||||
}
|
||||
|
||||
/// `\s+(?!\S)|\s+`: whitespace followed by a non-space leaves its last
|
||||
/// character to start the next piece (` ?` on the following alternatives).
|
||||
fn space_run_len(text: &str) -> usize {
|
||||
let run = run_len(text, Class::Space);
|
||||
if run == text.len() {
|
||||
return run;
|
||||
}
|
||||
let last = text[..run].chars().next_back().map_or(0, char::len_utf8);
|
||||
match run - last {
|
||||
0 => run,
|
||||
shorter => shorter,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rand::rngs::StdRng;
|
||||
use rand::seq::SliceRandom;
|
||||
use rand::{Rng, SeedableRng};
|
||||
use tokenizers::pre_tokenizers::byte_level::ByteLevel;
|
||||
use tokenizers::utils::SysRegex;
|
||||
use tokenizers::{OffsetReferential, OffsetType, PreTokenizedString, PreTokenizer};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn anthropic() -> Tokenizer {
|
||||
let path = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
|
||||
);
|
||||
std::fs::read_to_string(path)
|
||||
.expect("anthropic tokenizer json is in the repo")
|
||||
.parse()
|
||||
.expect("anthropic tokenizer loads")
|
||||
}
|
||||
|
||||
fn reference_count(tokenizer: &Tokenizer, text: &str) -> usize {
|
||||
tokenizer.encode_fast(text, true).expect("encode").len()
|
||||
}
|
||||
|
||||
const ALPHABET: &[&str] = &[
|
||||
"a",
|
||||
"Z",
|
||||
"e",
|
||||
"s",
|
||||
"t",
|
||||
"d",
|
||||
"m",
|
||||
"'",
|
||||
"'s",
|
||||
"'re",
|
||||
"'ll",
|
||||
"'S",
|
||||
"0",
|
||||
"9",
|
||||
" ",
|
||||
" ",
|
||||
"\t",
|
||||
"\n",
|
||||
"\r\n",
|
||||
"\u{b}",
|
||||
".",
|
||||
",",
|
||||
"!",
|
||||
"-",
|
||||
"(",
|
||||
"\"",
|
||||
"\u{a0}",
|
||||
"\u{85}",
|
||||
"\u{2028}",
|
||||
"\u{3000}",
|
||||
"\u{200b}",
|
||||
"\u{200d}",
|
||||
"é",
|
||||
"e\u{301}",
|
||||
"ß",
|
||||
"漢",
|
||||
"字",
|
||||
"ع",
|
||||
"३",
|
||||
"½",
|
||||
"Ⅳ",
|
||||
"🙂",
|
||||
"👍🏽",
|
||||
"Ａ",
|
||||
"fi",
|
||||
"㍿",
|
||||
"<",
|
||||
">",
|
||||
"EOT",
|
||||
"<EOT>",
|
||||
"<META_START>",
|
||||
];
|
||||
|
||||
fn random_text(rng: &mut StdRng) -> String {
|
||||
let pieces = rng.gen_range(0..40);
|
||||
(0..pieces)
|
||||
.map(|_| *ALPHABET.choose(rng).expect("alphabet is not empty"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anthropic_tokenizer_takes_the_fast_path() {
|
||||
let tokenizer = anthropic();
|
||||
let fast = ByteLevelCounter::detect(&tokenizer).expect("anthropic shape is supported");
|
||||
assert!(fast.nfkc);
|
||||
let text = "Hello, how are you today?";
|
||||
assert_eq!(
|
||||
fast.count(&tokenizer, text),
|
||||
Some(reference_count(&tokenizer, text))
|
||||
);
|
||||
assert_eq!(fast.count(&tokenizer, "stop <EOT> here"), None);
|
||||
assert_eq!(fast.count(&tokenizer, "stop ＜ＥＯＴ＞ here"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counts_match_the_full_encoder() {
|
||||
let tokenizer = anthropic();
|
||||
let fast = ByteLevelCounter::detect(&tokenizer).expect("supported");
|
||||
let mut rng = StdRng::seed_from_u64(2026);
|
||||
for _ in 0..4000 {
|
||||
let text = random_text(&mut rng);
|
||||
let expected = reference_count(&tokenizer, &text);
|
||||
let counted = fast
|
||||
.count(&tokenizer, &text)
|
||||
.unwrap_or_else(|| reference_count(&tokenizer, &text));
|
||||
assert_eq!(counted, expected, "text {text:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pieces_match_the_byte_level_pre_tokenizer() {
|
||||
let byte_level = ByteLevel::new(false, true, true);
|
||||
let mut rng = StdRng::seed_from_u64(7);
|
||||
for _ in 0..4000 {
|
||||
let text: String = random_text(&mut rng).nfkc().collect();
|
||||
let mut pre_tokenized = PreTokenizedString::from(text.as_str());
|
||||
byte_level
|
||||
.pre_tokenize(&mut pre_tokenized)
|
||||
.expect("pre-tokenize");
|
||||
let expected: Vec<(String, (usize, usize))> = pre_tokenized
|
||||
.get_splits(OffsetReferential::Original, OffsetType::Byte)
|
||||
.into_iter()
|
||||
.map(|(mapped, offsets, _)| (mapped.to_string(), offsets))
|
||||
.collect();
|
||||
let mut offset = 0;
|
||||
let pieces = Pieces { rest: &text };
|
||||
let actual: Vec<(String, (usize, usize))> = pieces
|
||||
.map(|piece| {
|
||||
let mapped: String = piece
|
||||
.bytes()
|
||||
.map(|byte| BYTE_CHARS[usize::from(byte)])
|
||||
.collect();
|
||||
let span = (offset, offset + piece.len());
|
||||
offset += piece.len();
|
||||
(mapped, span)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(actual, expected, "text {text:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classes_match_oniguruma() {
|
||||
let letter = SysRegex::new(r"\p{L}").expect("regex");
|
||||
let number = SysRegex::new(r"\p{N}").expect("regex");
|
||||
let space = SysRegex::new(r"\s").expect("regex");
|
||||
let whole =
|
||||
|regex: &SysRegex, text: &str| regex.find_iter(text).next() == Some((0, text.len()));
|
||||
let mut text = String::new();
|
||||
for character in (0..=0x10FFFFu32).filter_map(char::from_u32) {
|
||||
text.clear();
|
||||
text.push(character);
|
||||
let expected = if whole(&letter, &text) {
|
||||
Class::Letter
|
||||
} else if whole(&number, &text) {
|
||||
Class::Number
|
||||
} else if whole(&space, &text) {
|
||||
Class::Space
|
||||
} else {
|
||||
Class::Other
|
||||
};
|
||||
assert_eq!(class(character), expected, "U+{:04X}", u32::from(character));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_tokenizer_shapes_are_declined() {
|
||||
let mut tokenizer = anthropic();
|
||||
tokenizer.with_pre_tokenizer(Some(ByteLevel::new(true, true, true)));
|
||||
assert!(ByteLevelCounter::detect(&tokenizer).is_none());
|
||||
}
|
||||
}
|
||||
|
|
@ -2,13 +2,17 @@
|
|||
//! for the shapes it can count exactly. Everything else is declined so the host
|
||||
//! keeps its own counter as the reference.
|
||||
|
||||
mod byte_level;
|
||||
mod python_json;
|
||||
mod tools;
|
||||
pub mod types;
|
||||
mod unicode_classes;
|
||||
|
||||
use serde::Serialize;
|
||||
use thiserror::Error as ThisError;
|
||||
|
||||
use byte_level::ByteLevelCounter;
|
||||
|
||||
use crate::constants::{
|
||||
NAMED_TOOL_CHOICE_TOKENS, REPLY_PRIMING_TOKENS, TOKENS_PER_MESSAGE, TOKENS_PER_NAME,
|
||||
TOOL_CHOICE_NONE_TOKENS, TOOL_DEFINITIONS_TOKENS, TOOLS_WITH_SYSTEM_MESSAGE_DISCOUNT,
|
||||
|
|
@ -41,6 +45,7 @@ pub struct InputTokenCount {
|
|||
/// event loop.
|
||||
pub struct TokenCounter {
|
||||
tokenizer: tokenizers::Tokenizer,
|
||||
byte_level: Option<ByteLevelCounter>,
|
||||
}
|
||||
|
||||
impl TokenCounter {
|
||||
|
|
@ -49,10 +54,21 @@ impl TokenCounter {
|
|||
let tokenizer = tokenizer_json
|
||||
.parse::<tokenizers::Tokenizer>()
|
||||
.map_err(|error| TokenCountError::Load(error.to_string()))?;
|
||||
Ok(Self { tokenizer })
|
||||
let byte_level = ByteLevelCounter::detect(&tokenizer);
|
||||
Ok(Self {
|
||||
tokenizer,
|
||||
byte_level,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn count_text(&self, text: &str) -> Result<usize, TokenCountError> {
|
||||
if let Some(count) = self
|
||||
.byte_level
|
||||
.as_ref()
|
||||
.and_then(|counter| counter.count(&self.tokenizer, text))
|
||||
{
|
||||
return Ok(count);
|
||||
}
|
||||
self.tokenizer
|
||||
.encode_fast(text, true)
|
||||
.map(|encoding| encoding.len())
|
||||
|
|
|
|||
841
litellm-rust/crates/core/src/token_counter/unicode_classes.rs
Normal file
841
litellm-rust/crates/core/src/token_counter/unicode_classes.rs
Normal file
|
|
@ -0,0 +1,841 @@
|
|||
//! `\p{L}`, `\p{N}` and `\s` exactly as oniguruma (the `tokenizers` regex
|
||||
//! backend) classifies them. `byte_level::tests::classes_match_oniguruma`
|
||||
//! regenerates these from the regex engine and fails on any drift.
|
||||
|
||||
pub(super) const LETTER_RANGES: &[(u32, u32)] = &[
|
||||
(0x41, 0x5A),
|
||||
(0x61, 0x7A),
|
||||
(0xAA, 0xAA),
|
||||
(0xB5, 0xB5),
|
||||
(0xBA, 0xBA),
|
||||
(0xC0, 0xD6),
|
||||
(0xD8, 0xF6),
|
||||
(0xF8, 0x2C1),
|
||||
(0x2C6, 0x2D1),
|
||||
(0x2E0, 0x2E4),
|
||||
(0x2EC, 0x2EC),
|
||||
(0x2EE, 0x2EE),
|
||||
(0x370, 0x374),
|
||||
(0x376, 0x377),
|
||||
(0x37A, 0x37D),
|
||||
(0x37F, 0x37F),
|
||||
(0x386, 0x386),
|
||||
(0x388, 0x38A),
|
||||
(0x38C, 0x38C),
|
||||
(0x38E, 0x3A1),
|
||||
(0x3A3, 0x3F5),
|
||||
(0x3F7, 0x481),
|
||||
(0x48A, 0x52F),
|
||||
(0x531, 0x556),
|
||||
(0x559, 0x559),
|
||||
(0x560, 0x588),
|
||||
(0x5D0, 0x5EA),
|
||||
(0x5EF, 0x5F2),
|
||||
(0x620, 0x64A),
|
||||
(0x66E, 0x66F),
|
||||
(0x671, 0x6D3),
|
||||
(0x6D5, 0x6D5),
|
||||
(0x6E5, 0x6E6),
|
||||
(0x6EE, 0x6EF),
|
||||
(0x6FA, 0x6FC),
|
||||
(0x6FF, 0x6FF),
|
||||
(0x710, 0x710),
|
||||
(0x712, 0x72F),
|
||||
(0x74D, 0x7A5),
|
||||
(0x7B1, 0x7B1),
|
||||
(0x7CA, 0x7EA),
|
||||
(0x7F4, 0x7F5),
|
||||
(0x7FA, 0x7FA),
|
||||
(0x800, 0x815),
|
||||
(0x81A, 0x81A),
|
||||
(0x824, 0x824),
|
||||
(0x828, 0x828),
|
||||
(0x840, 0x858),
|
||||
(0x860, 0x86A),
|
||||
(0x870, 0x887),
|
||||
(0x889, 0x88E),
|
||||
(0x8A0, 0x8C9),
|
||||
(0x904, 0x939),
|
||||
(0x93D, 0x93D),
|
||||
(0x950, 0x950),
|
||||
(0x958, 0x961),
|
||||
(0x971, 0x980),
|
||||
(0x985, 0x98C),
|
||||
(0x98F, 0x990),
|
||||
(0x993, 0x9A8),
|
||||
(0x9AA, 0x9B0),
|
||||
(0x9B2, 0x9B2),
|
||||
(0x9B6, 0x9B9),
|
||||
(0x9BD, 0x9BD),
|
||||
(0x9CE, 0x9CE),
|
||||
(0x9DC, 0x9DD),
|
||||
(0x9DF, 0x9E1),
|
||||
(0x9F0, 0x9F1),
|
||||
(0x9FC, 0x9FC),
|
||||
(0xA05, 0xA0A),
|
||||
(0xA0F, 0xA10),
|
||||
(0xA13, 0xA28),
|
||||
(0xA2A, 0xA30),
|
||||
(0xA32, 0xA33),
|
||||
(0xA35, 0xA36),
|
||||
(0xA38, 0xA39),
|
||||
(0xA59, 0xA5C),
|
||||
(0xA5E, 0xA5E),
|
||||
(0xA72, 0xA74),
|
||||
(0xA85, 0xA8D),
|
||||
(0xA8F, 0xA91),
|
||||
(0xA93, 0xAA8),
|
||||
(0xAAA, 0xAB0),
|
||||
(0xAB2, 0xAB3),
|
||||
(0xAB5, 0xAB9),
|
||||
(0xABD, 0xABD),
|
||||
(0xAD0, 0xAD0),
|
||||
(0xAE0, 0xAE1),
|
||||
(0xAF9, 0xAF9),
|
||||
(0xB05, 0xB0C),
|
||||
(0xB0F, 0xB10),
|
||||
(0xB13, 0xB28),
|
||||
(0xB2A, 0xB30),
|
||||
(0xB32, 0xB33),
|
||||
(0xB35, 0xB39),
|
||||
(0xB3D, 0xB3D),
|
||||
(0xB5C, 0xB5D),
|
||||
(0xB5F, 0xB61),
|
||||
(0xB71, 0xB71),
|
||||
(0xB83, 0xB83),
|
||||
(0xB85, 0xB8A),
|
||||
(0xB8E, 0xB90),
|
||||
(0xB92, 0xB95),
|
||||
(0xB99, 0xB9A),
|
||||
(0xB9C, 0xB9C),
|
||||
(0xB9E, 0xB9F),
|
||||
(0xBA3, 0xBA4),
|
||||
(0xBA8, 0xBAA),
|
||||
(0xBAE, 0xBB9),
|
||||
(0xBD0, 0xBD0),
|
||||
(0xC05, 0xC0C),
|
||||
(0xC0E, 0xC10),
|
||||
(0xC12, 0xC28),
|
||||
(0xC2A, 0xC39),
|
||||
(0xC3D, 0xC3D),
|
||||
(0xC58, 0xC5A),
|
||||
(0xC5D, 0xC5D),
|
||||
(0xC60, 0xC61),
|
||||
(0xC80, 0xC80),
|
||||
(0xC85, 0xC8C),
|
||||
(0xC8E, 0xC90),
|
||||
(0xC92, 0xCA8),
|
||||
(0xCAA, 0xCB3),
|
||||
(0xCB5, 0xCB9),
|
||||
(0xCBD, 0xCBD),
|
||||
(0xCDD, 0xCDE),
|
||||
(0xCE0, 0xCE1),
|
||||
(0xCF1, 0xCF2),
|
||||
(0xD04, 0xD0C),
|
||||
(0xD0E, 0xD10),
|
||||
(0xD12, 0xD3A),
|
||||
(0xD3D, 0xD3D),
|
||||
(0xD4E, 0xD4E),
|
||||
(0xD54, 0xD56),
|
||||
(0xD5F, 0xD61),
|
||||
(0xD7A, 0xD7F),
|
||||
(0xD85, 0xD96),
|
||||
(0xD9A, 0xDB1),
|
||||
(0xDB3, 0xDBB),
|
||||
(0xDBD, 0xDBD),
|
||||
(0xDC0, 0xDC6),
|
||||
(0xE01, 0xE30),
|
||||
(0xE32, 0xE33),
|
||||
(0xE40, 0xE46),
|
||||
(0xE81, 0xE82),
|
||||
(0xE84, 0xE84),
|
||||
(0xE86, 0xE8A),
|
||||
(0xE8C, 0xEA3),
|
||||
(0xEA5, 0xEA5),
|
||||
(0xEA7, 0xEB0),
|
||||
(0xEB2, 0xEB3),
|
||||
(0xEBD, 0xEBD),
|
||||
(0xEC0, 0xEC4),
|
||||
(0xEC6, 0xEC6),
|
||||
(0xEDC, 0xEDF),
|
||||
(0xF00, 0xF00),
|
||||
(0xF40, 0xF47),
|
||||
(0xF49, 0xF6C),
|
||||
(0xF88, 0xF8C),
|
||||
(0x1000, 0x102A),
|
||||
(0x103F, 0x103F),
|
||||
(0x1050, 0x1055),
|
||||
(0x105A, 0x105D),
|
||||
(0x1061, 0x1061),
|
||||
(0x1065, 0x1066),
|
||||
(0x106E, 0x1070),
|
||||
(0x1075, 0x1081),
|
||||
(0x108E, 0x108E),
|
||||
(0x10A0, 0x10C5),
|
||||
(0x10C7, 0x10C7),
|
||||
(0x10CD, 0x10CD),
|
||||
(0x10D0, 0x10FA),
|
||||
(0x10FC, 0x1248),
|
||||
(0x124A, 0x124D),
|
||||
(0x1250, 0x1256),
|
||||
(0x1258, 0x1258),
|
||||
(0x125A, 0x125D),
|
||||
(0x1260, 0x1288),
|
||||
(0x128A, 0x128D),
|
||||
(0x1290, 0x12B0),
|
||||
(0x12B2, 0x12B5),
|
||||
(0x12B8, 0x12BE),
|
||||
(0x12C0, 0x12C0),
|
||||
(0x12C2, 0x12C5),
|
||||
(0x12C8, 0x12D6),
|
||||
(0x12D8, 0x1310),
|
||||
(0x1312, 0x1315),
|
||||
(0x1318, 0x135A),
|
||||
(0x1380, 0x138F),
|
||||
(0x13A0, 0x13F5),
|
||||
(0x13F8, 0x13FD),
|
||||
(0x1401, 0x166C),
|
||||
(0x166F, 0x167F),
|
||||
(0x1681, 0x169A),
|
||||
(0x16A0, 0x16EA),
|
||||
(0x16F1, 0x16F8),
|
||||
(0x1700, 0x1711),
|
||||
(0x171F, 0x1731),
|
||||
(0x1740, 0x1751),
|
||||
(0x1760, 0x176C),
|
||||
(0x176E, 0x1770),
|
||||
(0x1780, 0x17B3),
|
||||
(0x17D7, 0x17D7),
|
||||
(0x17DC, 0x17DC),
|
||||
(0x1820, 0x1878),
|
||||
(0x1880, 0x1884),
|
||||
(0x1887, 0x18A8),
|
||||
(0x18AA, 0x18AA),
|
||||
(0x18B0, 0x18F5),
|
||||
(0x1900, 0x191E),
|
||||
(0x1950, 0x196D),
|
||||
(0x1970, 0x1974),
|
||||
(0x1980, 0x19AB),
|
||||
(0x19B0, 0x19C9),
|
||||
(0x1A00, 0x1A16),
|
||||
(0x1A20, 0x1A54),
|
||||
(0x1AA7, 0x1AA7),
|
||||
(0x1B05, 0x1B33),
|
||||
(0x1B45, 0x1B4C),
|
||||
(0x1B83, 0x1BA0),
|
||||
(0x1BAE, 0x1BAF),
|
||||
(0x1BBA, 0x1BE5),
|
||||
(0x1C00, 0x1C23),
|
||||
(0x1C4D, 0x1C4F),
|
||||
(0x1C5A, 0x1C7D),
|
||||
(0x1C80, 0x1C8A),
|
||||
(0x1C90, 0x1CBA),
|
||||
(0x1CBD, 0x1CBF),
|
||||
(0x1CE9, 0x1CEC),
|
||||
(0x1CEE, 0x1CF3),
|
||||
(0x1CF5, 0x1CF6),
|
||||
(0x1CFA, 0x1CFA),
|
||||
(0x1D00, 0x1DBF),
|
||||
(0x1E00, 0x1F15),
|
||||
(0x1F18, 0x1F1D),
|
||||
(0x1F20, 0x1F45),
|
||||
(0x1F48, 0x1F4D),
|
||||
(0x1F50, 0x1F57),
|
||||
(0x1F59, 0x1F59),
|
||||
(0x1F5B, 0x1F5B),
|
||||
(0x1F5D, 0x1F5D),
|
||||
(0x1F5F, 0x1F7D),
|
||||
(0x1F80, 0x1FB4),
|
||||
(0x1FB6, 0x1FBC),
|
||||
(0x1FBE, 0x1FBE),
|
||||
(0x1FC2, 0x1FC4),
|
||||
(0x1FC6, 0x1FCC),
|
||||
(0x1FD0, 0x1FD3),
|
||||
(0x1FD6, 0x1FDB),
|
||||
(0x1FE0, 0x1FEC),
|
||||
(0x1FF2, 0x1FF4),
|
||||
(0x1FF6, 0x1FFC),
|
||||
(0x2071, 0x2071),
|
||||
(0x207F, 0x207F),
|
||||
(0x2090, 0x209C),
|
||||
(0x2102, 0x2102),
|
||||
(0x2107, 0x2107),
|
||||
(0x210A, 0x2113),
|
||||
(0x2115, 0x2115),
|
||||
(0x2119, 0x211D),
|
||||
(0x2124, 0x2124),
|
||||
(0x2126, 0x2126),
|
||||
(0x2128, 0x2128),
|
||||
(0x212A, 0x212D),
|
||||
(0x212F, 0x2139),
|
||||
(0x213C, 0x213F),
|
||||
(0x2145, 0x2149),
|
||||
(0x214E, 0x214E),
|
||||
(0x2183, 0x2184),
|
||||
(0x2C00, 0x2CE4),
|
||||
(0x2CEB, 0x2CEE),
|
||||
(0x2CF2, 0x2CF3),
|
||||
(0x2D00, 0x2D25),
|
||||
(0x2D27, 0x2D27),
|
||||
(0x2D2D, 0x2D2D),
|
||||
(0x2D30, 0x2D67),
|
||||
(0x2D6F, 0x2D6F),
|
||||
(0x2D80, 0x2D96),
|
||||
(0x2DA0, 0x2DA6),
|
||||
(0x2DA8, 0x2DAE),
|
||||
(0x2DB0, 0x2DB6),
|
||||
(0x2DB8, 0x2DBE),
|
||||
(0x2DC0, 0x2DC6),
|
||||
(0x2DC8, 0x2DCE),
|
||||
(0x2DD0, 0x2DD6),
|
||||
(0x2DD8, 0x2DDE),
|
||||
(0x2E2F, 0x2E2F),
|
||||
(0x3005, 0x3006),
|
||||
(0x3031, 0x3035),
|
||||
(0x303B, 0x303C),
|
||||
(0x3041, 0x3096),
|
||||
(0x309D, 0x309F),
|
||||
(0x30A1, 0x30FA),
|
||||
(0x30FC, 0x30FF),
|
||||
(0x3105, 0x312F),
|
||||
(0x3131, 0x318E),
|
||||
(0x31A0, 0x31BF),
|
||||
(0x31F0, 0x31FF),
|
||||
(0x3400, 0x4DBF),
|
||||
(0x4E00, 0xA48C),
|
||||
(0xA4D0, 0xA4FD),
|
||||
(0xA500, 0xA60C),
|
||||
(0xA610, 0xA61F),
|
||||
(0xA62A, 0xA62B),
|
||||
(0xA640, 0xA66E),
|
||||
(0xA67F, 0xA69D),
|
||||
(0xA6A0, 0xA6E5),
|
||||
(0xA717, 0xA71F),
|
||||
(0xA722, 0xA788),
|
||||
(0xA78B, 0xA7CD),
|
||||
(0xA7D0, 0xA7D1),
|
||||
(0xA7D3, 0xA7D3),
|
||||
(0xA7D5, 0xA7DC),
|
||||
(0xA7F2, 0xA801),
|
||||
(0xA803, 0xA805),
|
||||
(0xA807, 0xA80A),
|
||||
(0xA80C, 0xA822),
|
||||
(0xA840, 0xA873),
|
||||
(0xA882, 0xA8B3),
|
||||
(0xA8F2, 0xA8F7),
|
||||
(0xA8FB, 0xA8FB),
|
||||
(0xA8FD, 0xA8FE),
|
||||
(0xA90A, 0xA925),
|
||||
(0xA930, 0xA946),
|
||||
(0xA960, 0xA97C),
|
||||
(0xA984, 0xA9B2),
|
||||
(0xA9CF, 0xA9CF),
|
||||
(0xA9E0, 0xA9E4),
|
||||
(0xA9E6, 0xA9EF),
|
||||
(0xA9FA, 0xA9FE),
|
||||
(0xAA00, 0xAA28),
|
||||
(0xAA40, 0xAA42),
|
||||
(0xAA44, 0xAA4B),
|
||||
(0xAA60, 0xAA76),
|
||||
(0xAA7A, 0xAA7A),
|
||||
(0xAA7E, 0xAAAF),
|
||||
(0xAAB1, 0xAAB1),
|
||||
(0xAAB5, 0xAAB6),
|
||||
(0xAAB9, 0xAABD),
|
||||
(0xAAC0, 0xAAC0),
|
||||
(0xAAC2, 0xAAC2),
|
||||
(0xAADB, 0xAADD),
|
||||
(0xAAE0, 0xAAEA),
|
||||
(0xAAF2, 0xAAF4),
|
||||
(0xAB01, 0xAB06),
|
||||
(0xAB09, 0xAB0E),
|
||||
(0xAB11, 0xAB16),
|
||||
(0xAB20, 0xAB26),
|
||||
(0xAB28, 0xAB2E),
|
||||
(0xAB30, 0xAB5A),
|
||||
(0xAB5C, 0xAB69),
|
||||
(0xAB70, 0xABE2),
|
||||
(0xAC00, 0xD7A3),
|
||||
(0xD7B0, 0xD7C6),
|
||||
(0xD7CB, 0xD7FB),
|
||||
(0xF900, 0xFA6D),
|
||||
(0xFA70, 0xFAD9),
|
||||
(0xFB00, 0xFB06),
|
||||
(0xFB13, 0xFB17),
|
||||
(0xFB1D, 0xFB1D),
|
||||
(0xFB1F, 0xFB28),
|
||||
(0xFB2A, 0xFB36),
|
||||
(0xFB38, 0xFB3C),
|
||||
(0xFB3E, 0xFB3E),
|
||||
(0xFB40, 0xFB41),
|
||||
(0xFB43, 0xFB44),
|
||||
(0xFB46, 0xFBB1),
|
||||
(0xFBD3, 0xFD3D),
|
||||
(0xFD50, 0xFD8F),
|
||||
(0xFD92, 0xFDC7),
|
||||
(0xFDF0, 0xFDFB),
|
||||
(0xFE70, 0xFE74),
|
||||
(0xFE76, 0xFEFC),
|
||||
(0xFF21, 0xFF3A),
|
||||
(0xFF41, 0xFF5A),
|
||||
(0xFF66, 0xFFBE),
|
||||
(0xFFC2, 0xFFC7),
|
||||
(0xFFCA, 0xFFCF),
|
||||
(0xFFD2, 0xFFD7),
|
||||
(0xFFDA, 0xFFDC),
|
||||
(0x10000, 0x1000B),
|
||||
(0x1000D, 0x10026),
|
||||
(0x10028, 0x1003A),
|
||||
(0x1003C, 0x1003D),
|
||||
(0x1003F, 0x1004D),
|
||||
(0x10050, 0x1005D),
|
||||
(0x10080, 0x100FA),
|
||||
(0x10280, 0x1029C),
|
||||
(0x102A0, 0x102D0),
|
||||
(0x10300, 0x1031F),
|
||||
(0x1032D, 0x10340),
|
||||
(0x10342, 0x10349),
|
||||
(0x10350, 0x10375),
|
||||
(0x10380, 0x1039D),
|
||||
(0x103A0, 0x103C3),
|
||||
(0x103C8, 0x103CF),
|
||||
(0x10400, 0x1049D),
|
||||
(0x104B0, 0x104D3),
|
||||
(0x104D8, 0x104FB),
|
||||
(0x10500, 0x10527),
|
||||
(0x10530, 0x10563),
|
||||
(0x10570, 0x1057A),
|
||||
(0x1057C, 0x1058A),
|
||||
(0x1058C, 0x10592),
|
||||
(0x10594, 0x10595),
|
||||
(0x10597, 0x105A1),
|
||||
(0x105A3, 0x105B1),
|
||||
(0x105B3, 0x105B9),
|
||||
(0x105BB, 0x105BC),
|
||||
(0x105C0, 0x105F3),
|
||||
(0x10600, 0x10736),
|
||||
(0x10740, 0x10755),
|
||||
(0x10760, 0x10767),
|
||||
(0x10780, 0x10785),
|
||||
(0x10787, 0x107B0),
|
||||
(0x107B2, 0x107BA),
|
||||
(0x10800, 0x10805),
|
||||
(0x10808, 0x10808),
|
||||
(0x1080A, 0x10835),
|
||||
(0x10837, 0x10838),
|
||||
(0x1083C, 0x1083C),
|
||||
(0x1083F, 0x10855),
|
||||
(0x10860, 0x10876),
|
||||
(0x10880, 0x1089E),
|
||||
(0x108E0, 0x108F2),
|
||||
(0x108F4, 0x108F5),
|
||||
(0x10900, 0x10915),
|
||||
(0x10920, 0x10939),
|
||||
(0x10980, 0x109B7),
|
||||
(0x109BE, 0x109BF),
|
||||
(0x10A00, 0x10A00),
|
||||
(0x10A10, 0x10A13),
|
||||
(0x10A15, 0x10A17),
|
||||
(0x10A19, 0x10A35),
|
||||
(0x10A60, 0x10A7C),
|
||||
(0x10A80, 0x10A9C),
|
||||
(0x10AC0, 0x10AC7),
|
||||
(0x10AC9, 0x10AE4),
|
||||
(0x10B00, 0x10B35),
|
||||
(0x10B40, 0x10B55),
|
||||
(0x10B60, 0x10B72),
|
||||
(0x10B80, 0x10B91),
|
||||
(0x10C00, 0x10C48),
|
||||
(0x10C80, 0x10CB2),
|
||||
(0x10CC0, 0x10CF2),
|
||||
(0x10D00, 0x10D23),
|
||||
(0x10D4A, 0x10D65),
|
||||
(0x10D6F, 0x10D85),
|
||||
(0x10E80, 0x10EA9),
|
||||
(0x10EB0, 0x10EB1),
|
||||
(0x10EC2, 0x10EC4),
|
||||
(0x10F00, 0x10F1C),
|
||||
(0x10F27, 0x10F27),
|
||||
(0x10F30, 0x10F45),
|
||||
(0x10F70, 0x10F81),
|
||||
(0x10FB0, 0x10FC4),
|
||||
(0x10FE0, 0x10FF6),
|
||||
(0x11003, 0x11037),
|
||||
(0x11071, 0x11072),
|
||||
(0x11075, 0x11075),
|
||||
(0x11083, 0x110AF),
|
||||
(0x110D0, 0x110E8),
|
||||
(0x11103, 0x11126),
|
||||
(0x11144, 0x11144),
|
||||
(0x11147, 0x11147),
|
||||
(0x11150, 0x11172),
|
||||
(0x11176, 0x11176),
|
||||
(0x11183, 0x111B2),
|
||||
(0x111C1, 0x111C4),
|
||||
(0x111DA, 0x111DA),
|
||||
(0x111DC, 0x111DC),
|
||||
(0x11200, 0x11211),
|
||||
(0x11213, 0x1122B),
|
||||
(0x1123F, 0x11240),
|
||||
(0x11280, 0x11286),
|
||||
(0x11288, 0x11288),
|
||||
(0x1128A, 0x1128D),
|
||||
(0x1128F, 0x1129D),
|
||||
(0x1129F, 0x112A8),
|
||||
(0x112B0, 0x112DE),
|
||||
(0x11305, 0x1130C),
|
||||
(0x1130F, 0x11310),
|
||||
(0x11313, 0x11328),
|
||||
(0x1132A, 0x11330),
|
||||
(0x11332, 0x11333),
|
||||
(0x11335, 0x11339),
|
||||
(0x1133D, 0x1133D),
|
||||
(0x11350, 0x11350),
|
||||
(0x1135D, 0x11361),
|
||||
(0x11380, 0x11389),
|
||||
(0x1138B, 0x1138B),
|
||||
(0x1138E, 0x1138E),
|
||||
(0x11390, 0x113B5),
|
||||
(0x113B7, 0x113B7),
|
||||
(0x113D1, 0x113D1),
|
||||
(0x113D3, 0x113D3),
|
||||
(0x11400, 0x11434),
|
||||
(0x11447, 0x1144A),
|
||||
(0x1145F, 0x11461),
|
||||
(0x11480, 0x114AF),
|
||||
(0x114C4, 0x114C5),
|
||||
(0x114C7, 0x114C7),
|
||||
(0x11580, 0x115AE),
|
||||
(0x115D8, 0x115DB),
|
||||
(0x11600, 0x1162F),
|
||||
(0x11644, 0x11644),
|
||||
(0x11680, 0x116AA),
|
||||
(0x116B8, 0x116B8),
|
||||
(0x11700, 0x1171A),
|
||||
(0x11740, 0x11746),
|
||||
(0x11800, 0x1182B),
|
||||
(0x118A0, 0x118DF),
|
||||
(0x118FF, 0x11906),
|
||||
(0x11909, 0x11909),
|
||||
(0x1190C, 0x11913),
|
||||
(0x11915, 0x11916),
|
||||
(0x11918, 0x1192F),
|
||||
(0x1193F, 0x1193F),
|
||||
(0x11941, 0x11941),
|
||||
(0x119A0, 0x119A7),
|
||||
(0x119AA, 0x119D0),
|
||||
(0x119E1, 0x119E1),
|
||||
(0x119E3, 0x119E3),
|
||||
(0x11A00, 0x11A00),
|
||||
(0x11A0B, 0x11A32),
|
||||
(0x11A3A, 0x11A3A),
|
||||
(0x11A50, 0x11A50),
|
||||
(0x11A5C, 0x11A89),
|
||||
(0x11A9D, 0x11A9D),
|
||||
(0x11AB0, 0x11AF8),
|
||||
(0x11BC0, 0x11BE0),
|
||||
(0x11C00, 0x11C08),
|
||||
(0x11C0A, 0x11C2E),
|
||||
(0x11C40, 0x11C40),
|
||||
(0x11C72, 0x11C8F),
|
||||
(0x11D00, 0x11D06),
|
||||
(0x11D08, 0x11D09),
|
||||
(0x11D0B, 0x11D30),
|
||||
(0x11D46, 0x11D46),
|
||||
(0x11D60, 0x11D65),
|
||||
(0x11D67, 0x11D68),
|
||||
(0x11D6A, 0x11D89),
|
||||
(0x11D98, 0x11D98),
|
||||
(0x11EE0, 0x11EF2),
|
||||
(0x11F02, 0x11F02),
|
||||
(0x11F04, 0x11F10),
|
||||
(0x11F12, 0x11F33),
|
||||
(0x11FB0, 0x11FB0),
|
||||
(0x12000, 0x12399),
|
||||
(0x12480, 0x12543),
|
||||
(0x12F90, 0x12FF0),
|
||||
(0x13000, 0x1342F),
|
||||
(0x13441, 0x13446),
|
||||
(0x13460, 0x143FA),
|
||||
(0x14400, 0x14646),
|
||||
(0x16100, 0x1611D),
|
||||
(0x16800, 0x16A38),
|
||||
(0x16A40, 0x16A5E),
|
||||
(0x16A70, 0x16ABE),
|
||||
(0x16AD0, 0x16AED),
|
||||
(0x16B00, 0x16B2F),
|
||||
(0x16B40, 0x16B43),
|
||||
(0x16B63, 0x16B77),
|
||||
(0x16B7D, 0x16B8F),
|
||||
(0x16D40, 0x16D6C),
|
||||
(0x16E40, 0x16E7F),
|
||||
(0x16F00, 0x16F4A),
|
||||
(0x16F50, 0x16F50),
|
||||
(0x16F93, 0x16F9F),
|
||||
(0x16FE0, 0x16FE1),
|
||||
(0x16FE3, 0x16FE3),
|
||||
(0x17000, 0x187F7),
|
||||
(0x18800, 0x18CD5),
|
||||
(0x18CFF, 0x18D08),
|
||||
(0x1AFF0, 0x1AFF3),
|
||||
(0x1AFF5, 0x1AFFB),
|
||||
(0x1AFFD, 0x1AFFE),
|
||||
(0x1B000, 0x1B122),
|
||||
(0x1B132, 0x1B132),
|
||||
(0x1B150, 0x1B152),
|
||||
(0x1B155, 0x1B155),
|
||||
(0x1B164, 0x1B167),
|
||||
(0x1B170, 0x1B2FB),
|
||||
(0x1BC00, 0x1BC6A),
|
||||
(0x1BC70, 0x1BC7C),
|
||||
(0x1BC80, 0x1BC88),
|
||||
(0x1BC90, 0x1BC99),
|
||||
(0x1D400, 0x1D454),
|
||||
(0x1D456, 0x1D49C),
|
||||
(0x1D49E, 0x1D49F),
|
||||
(0x1D4A2, 0x1D4A2),
|
||||
(0x1D4A5, 0x1D4A6),
|
||||
(0x1D4A9, 0x1D4AC),
|
||||
(0x1D4AE, 0x1D4B9),
|
||||
(0x1D4BB, 0x1D4BB),
|
||||
(0x1D4BD, 0x1D4C3),
|
||||
(0x1D4C5, 0x1D505),
|
||||
(0x1D507, 0x1D50A),
|
||||
(0x1D50D, 0x1D514),
|
||||
(0x1D516, 0x1D51C),
|
||||
(0x1D51E, 0x1D539),
|
||||
(0x1D53B, 0x1D53E),
|
||||
(0x1D540, 0x1D544),
|
||||
(0x1D546, 0x1D546),
|
||||
(0x1D54A, 0x1D550),
|
||||
(0x1D552, 0x1D6A5),
|
||||
(0x1D6A8, 0x1D6C0),
|
||||
(0x1D6C2, 0x1D6DA),
|
||||
(0x1D6DC, 0x1D6FA),
|
||||
(0x1D6FC, 0x1D714),
|
||||
(0x1D716, 0x1D734),
|
||||
(0x1D736, 0x1D74E),
|
||||
(0x1D750, 0x1D76E),
|
||||
(0x1D770, 0x1D788),
|
||||
(0x1D78A, 0x1D7A8),
|
||||
(0x1D7AA, 0x1D7C2),
|
||||
(0x1D7C4, 0x1D7CB),
|
||||
(0x1DF00, 0x1DF1E),
|
||||
(0x1DF25, 0x1DF2A),
|
||||
(0x1E030, 0x1E06D),
|
||||
(0x1E100, 0x1E12C),
|
||||
(0x1E137, 0x1E13D),
|
||||
(0x1E14E, 0x1E14E),
|
||||
(0x1E290, 0x1E2AD),
|
||||
(0x1E2C0, 0x1E2EB),
|
||||
(0x1E4D0, 0x1E4EB),
|
||||
(0x1E5D0, 0x1E5ED),
|
||||
(0x1E5F0, 0x1E5F0),
|
||||
(0x1E7E0, 0x1E7E6),
|
||||
(0x1E7E8, 0x1E7EB),
|
||||
(0x1E7ED, 0x1E7EE),
|
||||
(0x1E7F0, 0x1E7FE),
|
||||
(0x1E800, 0x1E8C4),
|
||||
(0x1E900, 0x1E943),
|
||||
(0x1E94B, 0x1E94B),
|
||||
(0x1EE00, 0x1EE03),
|
||||
(0x1EE05, 0x1EE1F),
|
||||
(0x1EE21, 0x1EE22),
|
||||
(0x1EE24, 0x1EE24),
|
||||
(0x1EE27, 0x1EE27),
|
||||
(0x1EE29, 0x1EE32),
|
||||
(0x1EE34, 0x1EE37),
|
||||
(0x1EE39, 0x1EE39),
|
||||
(0x1EE3B, 0x1EE3B),
|
||||
(0x1EE42, 0x1EE42),
|
||||
(0x1EE47, 0x1EE47),
|
||||
(0x1EE49, 0x1EE49),
|
||||
(0x1EE4B, 0x1EE4B),
|
||||
(0x1EE4D, 0x1EE4F),
|
||||
(0x1EE51, 0x1EE52),
|
||||
(0x1EE54, 0x1EE54),
|
||||
(0x1EE57, 0x1EE57),
|
||||
(0x1EE59, 0x1EE59),
|
||||
(0x1EE5B, 0x1EE5B),
|
||||
(0x1EE5D, 0x1EE5D),
|
||||
(0x1EE5F, 0x1EE5F),
|
||||
(0x1EE61, 0x1EE62),
|
||||
(0x1EE64, 0x1EE64),
|
||||
(0x1EE67, 0x1EE6A),
|
||||
(0x1EE6C, 0x1EE72),
|
||||
(0x1EE74, 0x1EE77),
|
||||
(0x1EE79, 0x1EE7C),
|
||||
(0x1EE7E, 0x1EE7E),
|
||||
(0x1EE80, 0x1EE89),
|
||||
(0x1EE8B, 0x1EE9B),
|
||||
(0x1EEA1, 0x1EEA3),
|
||||
(0x1EEA5, 0x1EEA9),
|
||||
(0x1EEAB, 0x1EEBB),
|
||||
(0x20000, 0x2A6DF),
|
||||
(0x2A700, 0x2B739),
|
||||
(0x2B740, 0x2B81D),
|
||||
(0x2B820, 0x2CEA1),
|
||||
(0x2CEB0, 0x2EBE0),
|
||||
(0x2EBF0, 0x2EE5D),
|
||||
(0x2F800, 0x2FA1D),
|
||||
(0x30000, 0x3134A),
|
||||
(0x31350, 0x323AF),
|
||||
];
|
||||
pub(super) const NUMBER_RANGES: &[(u32, u32)] = &[
|
||||
(0x30, 0x39),
|
||||
(0xB2, 0xB3),
|
||||
(0xB9, 0xB9),
|
||||
(0xBC, 0xBE),
|
||||
(0x660, 0x669),
|
||||
(0x6F0, 0x6F9),
|
||||
(0x7C0, 0x7C9),
|
||||
(0x966, 0x96F),
|
||||
(0x9E6, 0x9EF),
|
||||
(0x9F4, 0x9F9),
|
||||
(0xA66, 0xA6F),
|
||||
(0xAE6, 0xAEF),
|
||||
(0xB66, 0xB6F),
|
||||
(0xB72, 0xB77),
|
||||
(0xBE6, 0xBF2),
|
||||
(0xC66, 0xC6F),
|
||||
(0xC78, 0xC7E),
|
||||
(0xCE6, 0xCEF),
|
||||
(0xD58, 0xD5E),
|
||||
(0xD66, 0xD78),
|
||||
(0xDE6, 0xDEF),
|
||||
(0xE50, 0xE59),
|
||||
(0xED0, 0xED9),
|
||||
(0xF20, 0xF33),
|
||||
(0x1040, 0x1049),
|
||||
(0x1090, 0x1099),
|
||||
(0x1369, 0x137C),
|
||||
(0x16EE, 0x16F0),
|
||||
(0x17E0, 0x17E9),
|
||||
(0x17F0, 0x17F9),
|
||||
(0x1810, 0x1819),
|
||||
(0x1946, 0x194F),
|
||||
(0x19D0, 0x19DA),
|
||||
(0x1A80, 0x1A89),
|
||||
(0x1A90, 0x1A99),
|
||||
(0x1B50, 0x1B59),
|
||||
(0x1BB0, 0x1BB9),
|
||||
(0x1C40, 0x1C49),
|
||||
(0x1C50, 0x1C59),
|
||||
(0x2070, 0x2070),
|
||||
(0x2074, 0x2079),
|
||||
(0x2080, 0x2089),
|
||||
(0x2150, 0x2182),
|
||||
(0x2185, 0x2189),
|
||||
(0x2460, 0x249B),
|
||||
(0x24EA, 0x24FF),
|
||||
(0x2776, 0x2793),
|
||||
(0x2CFD, 0x2CFD),
|
||||
(0x3007, 0x3007),
|
||||
(0x3021, 0x3029),
|
||||
(0x3038, 0x303A),
|
||||
(0x3192, 0x3195),
|
||||
(0x3220, 0x3229),
|
||||
(0x3248, 0x324F),
|
||||
(0x3251, 0x325F),
|
||||
(0x3280, 0x3289),
|
||||
(0x32B1, 0x32BF),
|
||||
(0xA620, 0xA629),
|
||||
(0xA6E6, 0xA6EF),
|
||||
(0xA830, 0xA835),
|
||||
(0xA8D0, 0xA8D9),
|
||||
(0xA900, 0xA909),
|
||||
(0xA9D0, 0xA9D9),
|
||||
(0xA9F0, 0xA9F9),
|
||||
(0xAA50, 0xAA59),
|
||||
(0xABF0, 0xABF9),
|
||||
(0xFF10, 0xFF19),
|
||||
(0x10107, 0x10133),
|
||||
(0x10140, 0x10178),
|
||||
(0x1018A, 0x1018B),
|
||||
(0x102E1, 0x102FB),
|
||||
(0x10320, 0x10323),
|
||||
(0x10341, 0x10341),
|
||||
(0x1034A, 0x1034A),
|
||||
(0x103D1, 0x103D5),
|
||||
(0x104A0, 0x104A9),
|
||||
(0x10858, 0x1085F),
|
||||
(0x10879, 0x1087F),
|
||||
(0x108A7, 0x108AF),
|
||||
(0x108FB, 0x108FF),
|
||||
(0x10916, 0x1091B),
|
||||
(0x109BC, 0x109BD),
|
||||
(0x109C0, 0x109CF),
|
||||
(0x109D2, 0x109FF),
|
||||
(0x10A40, 0x10A48),
|
||||
(0x10A7D, 0x10A7E),
|
||||
(0x10A9D, 0x10A9F),
|
||||
(0x10AEB, 0x10AEF),
|
||||
(0x10B58, 0x10B5F),
|
||||
(0x10B78, 0x10B7F),
|
||||
(0x10BA9, 0x10BAF),
|
||||
(0x10CFA, 0x10CFF),
|
||||
(0x10D30, 0x10D39),
|
||||
(0x10D40, 0x10D49),
|
||||
(0x10E60, 0x10E7E),
|
||||
(0x10F1D, 0x10F26),
|
||||
(0x10F51, 0x10F54),
|
||||
(0x10FC5, 0x10FCB),
|
||||
(0x11052, 0x1106F),
|
||||
(0x110F0, 0x110F9),
|
||||
(0x11136, 0x1113F),
|
||||
(0x111D0, 0x111D9),
|
||||
(0x111E1, 0x111F4),
|
||||
(0x112F0, 0x112F9),
|
||||
(0x11450, 0x11459),
|
||||
(0x114D0, 0x114D9),
|
||||
(0x11650, 0x11659),
|
||||
(0x116C0, 0x116C9),
|
||||
(0x116D0, 0x116E3),
|
||||
(0x11730, 0x1173B),
|
||||
(0x118E0, 0x118F2),
|
||||
(0x11950, 0x11959),
|
||||
(0x11BF0, 0x11BF9),
|
||||
(0x11C50, 0x11C6C),
|
||||
(0x11D50, 0x11D59),
|
||||
(0x11DA0, 0x11DA9),
|
||||
(0x11F50, 0x11F59),
|
||||
(0x11FC0, 0x11FD4),
|
||||
(0x12400, 0x1246E),
|
||||
(0x16130, 0x16139),
|
||||
(0x16A60, 0x16A69),
|
||||
(0x16AC0, 0x16AC9),
|
||||
(0x16B50, 0x16B59),
|
||||
(0x16B5B, 0x16B61),
|
||||
(0x16D70, 0x16D79),
|
||||
(0x16E80, 0x16E96),
|
||||
(0x1CCF0, 0x1CCF9),
|
||||
(0x1D2C0, 0x1D2D3),
|
||||
(0x1D2E0, 0x1D2F3),
|
||||
(0x1D360, 0x1D378),
|
||||
(0x1D7CE, 0x1D7FF),
|
||||
(0x1E140, 0x1E149),
|
||||
(0x1E2F0, 0x1E2F9),
|
||||
(0x1E4F0, 0x1E4F9),
|
||||
(0x1E5F1, 0x1E5FA),
|
||||
(0x1E8C7, 0x1E8CF),
|
||||
(0x1E950, 0x1E959),
|
||||
(0x1EC71, 0x1ECAB),
|
||||
(0x1ECAD, 0x1ECAF),
|
||||
(0x1ECB1, 0x1ECB4),
|
||||
(0x1ED01, 0x1ED2D),
|
||||
(0x1ED2F, 0x1ED3D),
|
||||
(0x1F100, 0x1F10C),
|
||||
(0x1FBF0, 0x1FBF9),
|
||||
];
|
||||
pub(super) const SPACE_RANGES: &[(u32, u32)] = &[
|
||||
(0x9, 0xD),
|
||||
(0x20, 0x20),
|
||||
(0x85, 0x85),
|
||||
(0xA0, 0xA0),
|
||||
(0x1680, 0x1680),
|
||||
(0x2000, 0x200A),
|
||||
(0x2028, 0x2029),
|
||||
(0x202F, 0x202F),
|
||||
(0x205F, 0x205F),
|
||||
(0x3000, 0x3000),
|
||||
];
|
||||
Loading…
Add table
Reference in a new issue