diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 19961d020a7..af3a31ddbfa 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -816,19 +816,6 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" -[[package]] -name = "console" -version = "0.15.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" -dependencies = [ - "encode_unicode", - "libc", - "once_cell", - "unicode-width", - "windows-sys 0.59.0", -] - [[package]] name = "const-oid" version = "0.10.2" @@ -1140,27 +1127,6 @@ dependencies = [ "ctutils", ] -[[package]] -name = "dirs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.61.2", -] - [[package]] name = "displaydoc" version = "0.2.6" @@ -1190,12 +1156,6 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" -[[package]] -name = "encode_unicode" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" - [[package]] name = "equivalent" version = "1.0.2" @@ -1553,26 +1513,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hf-hub" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97" -dependencies = [ - "dirs", - "http 1.4.2", - "indicatif", - "libc", - "log", - "rand 0.9.5", - "reqwest 0.12.28", - "serde", - "serde_json", - "thiserror 2.0.19", - "ureq", - "windows-sys 0.60.2", -] - [[package]] name = "hmac" version = "0.13.0" @@ -1732,7 +1672,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tower-service", - "webpki-roots 1.0.9", + "webpki-roots", ] [[package]] @@ -1914,19 +1854,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "indicatif" -version = "0.17.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" -dependencies = [ - "console", - "number_prefix", - "portable-atomic", - "unicode-width", - "web-time", -] - [[package]] name = "ipnet" version = "2.12.0" @@ -2039,15 +1966,6 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "libredox" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a" -dependencies = [ - "libc", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2264,7 +2182,7 @@ dependencies = [ "thiserror 2.0.19", "tokio", "veil", - "webpki-roots 1.0.9", + "webpki-roots", ] [[package]] @@ -2559,12 +2477,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "once_cell" version = "1.21.4" @@ -2605,12 +2517,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "outref" version = "0.5.2" @@ -3136,16 +3042,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_users" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" -dependencies = [ - "libredox", - "thiserror 2.0.19", -] - [[package]] name = "ref-cast" version = "1.0.27" @@ -3247,7 +3143,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams 0.4.2", "web-sys", - "webpki-roots 1.0.9", + "webpki-roots", ] [[package]] @@ -3389,7 +3285,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", - "log", "once_cell", "ring", "rustls-pki-types", @@ -3787,17 +3682,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "socks" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" -dependencies = [ - "byteorder", - "libc", - "winapi", -] - [[package]] name = "spm_precompiled" version = "0.1.4" @@ -4068,7 +3952,6 @@ dependencies = [ "derive_builder", "esaxx-rs", "getrandom 0.3.4", - "hf-hub", "itertools 0.14.0", "log", "macro_rules_attribute", @@ -4402,12 +4285,6 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - [[package]] name = "unicode_categories" version = "0.1.1" @@ -4420,25 +4297,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "ureq" -version = "2.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" -dependencies = [ - "base64 0.22.1", - "flate2", - "log", - "once_cell", - "rustls 0.23.42", - "rustls-pki-types", - "serde", - "serde_json", - "socks", - "url", - "webpki-roots 0.26.11", -] - [[package]] name = "url" version = "2.5.8" @@ -4666,15 +4524,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.9", -] - [[package]] name = "webpki-roots" version = "1.0.9" @@ -4780,25 +4629,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -4816,31 +4647,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -4849,96 +4663,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "1.0.4" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 7ac088838b4..4250fa81d26 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -48,7 +48,7 @@ serde_with = { version = "=3.16.1", default-features = false, features = ["std", sha2 = "0.10" subtle = "2" thiserror = "2.0" -tokenizers = { version = "0.23.1", default-features = false, features = ["onig", "http", "rustls-tls"] } +tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] } tiktoken-rs = "0.12.0" tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] } tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index d39b9545070..e1a6e829363 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -6,6 +6,7 @@ mod marshal; mod python_settings; mod routes; mod token_counter; +mod tokenizer; #[pymodule(gil_used = true)] mod _native { @@ -29,7 +30,12 @@ mod _native { #[pymodule_export] use crate::routes::responses::ResponsesWebSocketConnection; #[pymodule_export] - use crate::token_counter::{TokenCounter, Tokenizer, tiktoken_encoding_for_model}; + use crate::token_counter::TokenCounter; + #[cfg(feature = "huggingface")] + #[pymodule_export] + use crate::tokenizer::HuggingFaceEncoding; + #[pymodule_export] + use crate::tokenizer::{Tokenizer, tiktoken_encoding_for_model}; #[pymodule_export] use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking}; } @@ -71,6 +77,8 @@ mod tests { "process_state_started", "reserve_process_for_forking", ]; + #[cfg(feature = "huggingface")] + expected.push("HuggingFaceEncoding"); expected.sort_unstable(); let mut public_names: Vec = native_module(py) diff --git a/litellm-rust/crates/python-bridge/src/token_counter.rs b/litellm-rust/crates/python-bridge/src/token_counter.rs index b4079edc40b..9f0af284104 100644 --- a/litellm-rust/crates/python-bridge/src/token_counter.rs +++ b/litellm-rust/crates/python-bridge/src/token_counter.rs @@ -3,10 +3,11 @@ use std::sync::Arc; #[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] use std::{num::NonZero, thread::available_parallelism}; +#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] use litellm_host_python::release_gil; use litellm_host_python::run_async; use litellm_token_counter::{ - CountableRequest, Error, InputTokenCount, TextCodec, TokenCounter as CoreTokenCounter, + CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, }; use pyo3::{ exceptions::{PyRuntimeError, PyValueError}, @@ -28,11 +29,6 @@ pub(crate) struct TokenCounter { encode_slots: Arc, } -#[pyclass(frozen, name = "Tokenizer")] -pub(crate) struct Tokenizer { - inner: Arc, -} - #[pymethods] impl TokenCounter { #[new] @@ -119,128 +115,6 @@ impl TokenCounter { } } -#[pymethods] -impl Tokenizer { - #[staticmethod] - fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { - #[cfg(feature = "tiktoken")] - { - let encoding = encoding.to_owned(); - Self::load(py, move || { - litellm_token_counter::tiktoken::TiktokenTokenizer::from_name(&encoding) - .map_err(Error::from) - }) - } - #[cfg(not(feature = "tiktoken"))] - { - let _ = (py, encoding); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the tiktoken feature", - )) - } - } - - #[staticmethod] - fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult { - #[cfg(feature = "huggingface")] - { - let tokenizer_json = tokenizer_json.to_owned(); - Self::load(py, move || { - litellm_token_counter::huggingface::HuggingFaceTokenizer::from_json(&tokenizer_json) - .map_err(Error::from) - }) - } - #[cfg(not(feature = "huggingface"))] - { - let _ = (py, tokenizer_json); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the huggingface feature", - )) - } - } - - #[staticmethod] - #[pyo3(signature = (identifier, revision = "main", token = None))] - fn from_pretrained( - py: Python<'_>, - identifier: &str, - revision: &str, - token: Option<&str>, - ) -> PyResult { - #[cfg(feature = "huggingface")] - { - let identifier = identifier.to_owned(); - let revision = revision.to_owned(); - let token = token.map(str::to_owned); - Self::load(py, move || { - litellm_token_counter::huggingface::HuggingFaceTokenizer::from_pretrained( - &identifier, - &revision, - token.as_deref(), - ) - .map_err(Error::from) - }) - } - #[cfg(not(feature = "huggingface"))] - { - let _ = (py, identifier, revision, token); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the huggingface feature", - )) - } - } - - fn encode(&self, py: Python<'_>, text: &str) -> PyResult> { - let inner = Arc::clone(&self.inner); - let text = text.to_owned(); - release_gil(py, move || inner.encode(&text)).map_err(token_count_error_to_pyerr) - } - - #[pyo3(signature = (ids, skip_special_tokens = true))] - fn decode(&self, py: Python<'_>, ids: Vec, skip_special_tokens: bool) -> PyResult { - let inner = Arc::clone(&self.inner); - release_gil(py, move || inner.decode(&ids, skip_special_tokens)) - .map_err(token_count_error_to_pyerr) - } - - fn count(&self, py: Python<'_>, text: &str) -> PyResult { - let inner = Arc::clone(&self.inner); - let text = text.to_owned(); - release_gil(py, move || inner.count_tokens(&text)).map_err(token_count_error_to_pyerr) - } - - #[getter] - fn name(&self) -> &str { - self.inner.name() - } -} - -impl Tokenizer { - #[cfg(any(feature = "huggingface", feature = "tiktoken"))] - fn load(py: Python<'_>, load: impl FnOnce() -> Result + Send) -> PyResult - where - T: TextCodec + 'static, - { - let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?; - Ok(Self { - inner: Arc::new(inner), - }) - } -} - -#[pyfunction] -pub(crate) fn tiktoken_encoding_for_model(model: &str) -> Option { - #[cfg(feature = "tiktoken")] - { - litellm_token_counter::tiktoken::encoding_for_model(model).map(str::to_owned) - } - #[cfg(not(feature = "tiktoken"))] - { - let _ = model; - None - } -} - impl TokenCounter { #[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] fn load( @@ -265,7 +139,7 @@ fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result PyErr { +pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr { let message = error.to_string(); match error { Error::Load(_) diff --git a/litellm-rust/crates/python-bridge/src/tokenizer.rs b/litellm-rust/crates/python-bridge/src/tokenizer.rs new file mode 100644 index 00000000000..d334131e682 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/tokenizer.rs @@ -0,0 +1,395 @@ +#[cfg(feature = "tiktoken")] +use std::collections::HashSet; +use std::sync::Arc; + +use litellm_host_python::release_gil; +#[cfg(feature = "huggingface")] +use litellm_token_counter::Error; +use litellm_token_counter::TextCodec; +use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString}; + +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +use pyo3::exceptions::PyValueError; +#[cfg(feature = "huggingface")] +use pyo3::{exceptions::PyIOError, types::PyDict}; +#[cfg(feature = "tiktoken")] +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError}, + types::PyBytes, +}; + +#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))] +use crate::errors::RustBridgeDeclined; +use crate::token_counter::token_count_error_to_pyerr; + +#[cfg(feature = "huggingface")] +use litellm_token_counter::huggingface::{ + EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, +}; +#[cfg(feature = "tiktoken")] +use litellm_token_counter::tiktoken::TiktokenTokenizer; + +enum Codec { + #[cfg(feature = "tiktoken")] + Tiktoken(TiktokenTokenizer), + #[cfg(feature = "huggingface")] + HuggingFace(HuggingFaceTokenizer), +} + +impl Codec { + fn codec(&self) -> &dyn TextCodec { + match *self { + #[cfg(feature = "tiktoken")] + Self::Tiktoken(ref tokenizer) => tokenizer, + #[cfg(feature = "huggingface")] + Self::HuggingFace(ref tokenizer) => tokenizer, + } + } +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Tokenizer { + inner: Arc, +} + +#[pymethods] +impl Tokenizer { + #[staticmethod] + fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { + #[cfg(feature = "tiktoken")] + { + let tokenizer = release_gil(py, || TiktokenTokenizer::from_name(encoding)) + .map_err(|error| token_count_error_to_pyerr(error.into()))?; + Ok(Self { + inner: Arc::new(Codec::Tiktoken(tokenizer)), + }) + } + #[cfg(not(feature = "tiktoken"))] + { + let _ = (py, encoding); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the tiktoken feature", + )) + } + } + + #[staticmethod] + fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult { + #[cfg(feature = "huggingface")] + { + let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json)) + .map_err(|error| token_count_error_to_pyerr(error.into()))?; + Ok(Self { + inner: Arc::new(Codec::HuggingFace(tokenizer)), + }) + } + #[cfg(not(feature = "huggingface"))] + { + let _ = (py, tokenizer_json); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the huggingface feature", + )) + } + } + + #[staticmethod] + #[pyo3(signature = (identifier, revision = "main", token = None))] + fn from_pretrained( + py: Python<'_>, + identifier: &str, + revision: &str, + token: Option<&str>, + ) -> PyResult { + #[cfg(feature = "huggingface")] + { + let kwargs = PyDict::new(py); + kwargs.set_item("repo_id", identifier)?; + kwargs.set_item("filename", "tokenizer.json")?; + kwargs.set_item("revision", revision)?; + kwargs.set_item("token", token)?; + let path: String = PyModule::import(py, "huggingface_hub")? + .getattr("hf_hub_download")? + .call((), Some(&kwargs))? + .extract()?; + let json = + release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?; + Self::from_json(py, &json) + } + #[cfg(not(feature = "huggingface"))] + { + let _ = (py, identifier, revision, token); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the huggingface feature", + )) + } + } + + fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult> { + let text = self.text(text)?; + release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr) + } + + #[pyo3(signature = (ids, skip_special_tokens = true))] + fn decode(&self, py: Python<'_>, ids: Vec, skip_special_tokens: bool) -> PyResult { + release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens)) + .map_err(token_count_error_to_pyerr) + } + + fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult { + let text = self.text(text)?; + release_gil(py, || self.inner.codec().count_tokens(&text)) + .map_err(token_count_error_to_pyerr) + } + + #[getter] + fn name(&self) -> &str { + self.inner.codec().name() + } + + #[cfg(feature = "tiktoken")] + fn encode_special( + &self, + py: Python<'_>, + text: &Bound<'_, PyString>, + allowed: Vec, + ) -> PyResult> { + let text = self.text(text)?; + match *self.inner { + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(ref tokenizer) => { + release_gil(py, || tokenizer.encode_special(&text, &allowed)) + .map_err(PyRuntimeError::new_err) + } + #[cfg(feature = "huggingface")] + Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")), + } + } + + #[cfg(feature = "tiktoken")] + fn special_tokens(&self, py: Python<'_>) -> PyResult> { + match *self.inner { + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(ref tokenizer) => Ok(release_gil(py, || tokenizer.special_tokens())), + #[cfg(feature = "huggingface")] + Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")), + } + } + + #[cfg(feature = "tiktoken")] + fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec) -> PyResult> { + match *self.inner { + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(ref tokenizer) => { + let bytes = release_gil(py, || tokenizer.decode_bytes(&ids)) + .map_err(PyKeyError::new_err)?; + Ok(PyBytes::new(py, &bytes)) + } + #[cfg(feature = "huggingface")] + Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")), + } + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))] + fn encode_huggingface( + &self, + py: Python<'_>, + sequence: Sequence, + pair: Option, + is_pretokenized: bool, + add_special_tokens: bool, + fast: bool, + ) -> PyResult { + let sequence = sequence.input(is_pretokenized)?; + let input = match pair { + Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?), + None => EncodeInput::Single(sequence), + }; + match *self.inner { + Codec::HuggingFace(ref tokenizer) => release_gil(py, || { + tokenizer.encode_result(input, add_special_tokens, fast) + }) + .map(|inner| HuggingFaceEncoding { inner }) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))), + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")), + } + } + #[cfg(feature = "huggingface")] + #[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))] + fn encode_batch_huggingface( + &self, + py: Python<'_>, + inputs: Vec<(Sequence, Option)>, + is_pretokenized: bool, + add_special_tokens: bool, + fast: bool, + ) -> PyResult> { + let inputs = inputs + .into_iter() + .map(|(sequence, pair)| { + let sequence = sequence.input(is_pretokenized)?; + match pair { + Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)), + None => Ok(EncodeInput::Single(sequence)), + } + }) + .collect::>>()?; + match *self.inner { + Codec::HuggingFace(ref tokenizer) => release_gil(py, || { + tokenizer.encode_batch_result(inputs, add_special_tokens, fast) + }) + .map(|encodings| { + encodings + .into_iter() + .map(|inner| HuggingFaceEncoding { inner }) + .collect() + }) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))), + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")), + } + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (pretty = false))] + fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult { + match *self.inner { + #[cfg(feature = "huggingface")] + Codec::HuggingFace(ref tokenizer) => release_gil(py, || tokenizer.to_json(pretty)) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))), + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")), + } + } +} + +impl Tokenizer { + fn text(&self, text: &Bound<'_, PyString>) -> PyResult { + match text.to_cow() { + Ok(text) => Ok(text.into_owned()), + Err(error) + if self.inner.codec().name() == "huggingface" + || !error.is_instance_of::(text.py()) => + { + Err(error) + } + Err(_) => text + .call_method1("encode", ("utf-16", "surrogatepass"))? + .call_method1("decode", ("utf-16", "replace"))? + .extract(), + } + } +} + +#[cfg(feature = "huggingface")] +#[derive(FromPyObject)] +pub(crate) enum Sequence { + Text(String), + Words(Vec), +} + +#[cfg(feature = "huggingface")] +impl Sequence { + fn input(self, is_pretokenized: bool) -> PyResult> { + match (self, is_pretokenized) { + (Self::Text(text), false) => Ok(text.into()), + (Self::Words(words), true) => Ok(words.into()), + _ => Err(pyo3::exceptions::PyTypeError::new_err( + "input must match is_pretokenized", + )), + } + } +} + +#[cfg(feature = "huggingface")] +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct HuggingFaceEncoding { + inner: Encoding, +} + +#[cfg(feature = "huggingface")] +#[pymethods] +impl HuggingFaceEncoding { + #[new] + #[pyo3(signature = (json = None))] + fn new(json: Option<&str>) -> PyResult { + let inner = match json { + Some(json) => serde_json::from_str(json) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + None => Encoding::default(), + }; + Ok(Self { inner }) + } + + fn __reduce__<'py>( + &self, + py: Python<'py>, + ) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> { + let json = serde_json::to_string(&self.inner) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok((py.get_type::(), (json,))) + } + + fn __len__(&self) -> usize { + self.inner.len() + } + #[getter] + fn ids(&self) -> Vec { + self.inner.get_ids().to_vec() + } + #[getter] + fn tokens(&self) -> Vec { + self.inner.get_tokens().to_vec() + } + #[getter] + fn offsets(&self) -> Vec<(usize, usize)> { + self.inner.get_offsets().to_vec() + } + #[getter] + fn type_ids(&self) -> Vec { + self.inner.get_type_ids().to_vec() + } + #[getter] + fn attention_mask(&self) -> Vec { + self.inner.get_attention_mask().to_vec() + } + #[getter] + fn special_tokens_mask(&self) -> Vec { + self.inner.get_special_tokens_mask().to_vec() + } + #[getter] + fn word_ids(&self) -> Vec> { + self.inner.get_word_ids().to_vec() + } + #[getter] + fn sequence_ids(&self) -> Vec> { + self.inner.get_sequence_ids() + } + #[getter] + fn overflowing(&self) -> Vec { + self.inner + .get_overflowing() + .iter() + .cloned() + .map(|inner| Self { inner }) + .collect() + } + #[getter] + fn n_sequences(&self) -> usize { + self.inner.n_sequences() + } +} + +#[pyfunction] +pub(crate) fn tiktoken_encoding_for_model(model: &str) -> Option { + #[cfg(feature = "tiktoken")] + { + litellm_token_counter::tiktoken::encoding_for_model(model).map(str::to_owned) + } + #[cfg(not(feature = "tiktoken"))] + { + let _ = model; + None + } +} diff --git a/litellm-rust/crates/token-counter-huggingface/src/lib.rs b/litellm-rust/crates/token-counter-huggingface/src/lib.rs index f8c90a917a8..711bd083e6d 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/lib.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/lib.rs @@ -5,6 +5,7 @@ mod error; use std::collections::HashSet; pub use error::Error; +pub use tokenizers::{EncodeInput, Encoding, InputSequence}; pub struct HuggingFaceTokenizer { tokenizer: Box, @@ -18,23 +19,6 @@ impl HuggingFaceTokenizer { .map_err(Error::Load) } - pub fn from_pretrained( - identifier: &str, - revision: &str, - token: Option<&str>, - ) -> Result { - tokenizers::Tokenizer::from_pretrained( - identifier, - Some(tokenizers::FromPretrainedParameters { - revision: revision.to_owned(), - token: token.map(str::to_owned), - ..Default::default() - }), - ) - .map(Self::new) - .map_err(Error::Download) - } - fn new(tokenizer: tokenizers::Tokenizer) -> Self { let special_token_ids: HashSet = tokenizer .get_added_tokens_decoder() @@ -61,6 +45,44 @@ impl HuggingFaceTokenizer { .map_err(Error::Encode) } + pub fn encode_result<'a>( + &self, + input: EncodeInput<'a>, + add_special_tokens: bool, + fast: bool, + ) -> Result { + if fast { + return self + .tokenizer + .encode_fast(input, add_special_tokens) + .map_err(Error::Encode); + } + self.tokenizer + .encode_char_offsets(input, add_special_tokens) + .map_err(Error::Encode) + } + + pub fn encode_batch_result<'a>( + &self, + inputs: Vec>, + add_special_tokens: bool, + fast: bool, + ) -> Result, Error> { + if fast { + return self + .tokenizer + .encode_batch_fast(inputs, add_special_tokens) + .map_err(Error::Encode); + } + self.tokenizer + .encode_batch_char_offsets(inputs, add_special_tokens) + .map_err(Error::Encode) + } + + pub fn to_json(&self, pretty: bool) -> Result { + self.tokenizer.to_string(pretty).map_err(Error::Load) + } + pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result { if !skip_special_tokens { return self.tokenizer.decode(ids, false).map_err(Error::Decode); diff --git a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs index 2c33a0dc714..c96591abf86 100644 --- a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs +++ b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs @@ -2,6 +2,8 @@ mod error; +use std::collections::HashSet; + pub use error::UnsupportedTokenizer; pub struct TiktokenTokenizer { @@ -35,8 +37,33 @@ impl TiktokenTokenizer { self.encoder.encode_ordinary(text) } + pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result, String> { + let allowed = allowed.iter().map(String::as_str).collect(); + self.encoder + .encode(text, &allowed) + .map(|(ids, _)| ids) + .map_err(|error| error.to_string()) + } + + pub fn special_tokens(&self) -> HashSet { + self.encoder + .special_tokens() + .into_iter() + .map(str::to_owned) + .collect() + } + + pub fn decode_bytes(&self, ids: &[u32]) -> Result, String> { + self.encoder + .decode_bytes(ids) + .map_err(|error| error.to_string()) + } + pub fn decode(&self, ids: &[u32]) -> Result { - self.encoder.decode(ids).map_err(|error| error.to_string()) + self.encoder + .decode_bytes(ids) + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .map_err(|error| error.to_string()) } pub fn name(&self) -> &str { @@ -128,6 +155,21 @@ mod tests { } } + #[test] + fn decoding_token_prefixes_replaces_incomplete_utf8() { + let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap(); + let reference = tiktoken_rs::cl100k_base_singleton(); + let ids = tokenizer.encode("🙂漢字"); + for end in 1..=ids.len() { + let bytes = reference.decode_bytes(&ids[..end]).unwrap(); + assert_eq!( + tokenizer.decode(&ids[..end]).unwrap(), + String::from_utf8_lossy(&bytes), + ); + } + assert!(tokenizer.decode(&[u32::MAX]).is_err()); + } + #[test] fn encoding_for_model_maps_known_models() { assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base")); diff --git a/litellm-rust/crates/token-counter/src/huggingface.rs b/litellm-rust/crates/token-counter/src/huggingface.rs index 1c0780bcf95..662c6a6f2a7 100644 --- a/litellm-rust/crates/token-counter/src/huggingface.rs +++ b/litellm-rust/crates/token-counter/src/huggingface.rs @@ -1,5 +1,7 @@ use litellm_token_counter_huggingface::Error as BackendError; -pub use litellm_token_counter_huggingface::HuggingFaceTokenizer; +pub use litellm_token_counter_huggingface::{ + EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, +}; use crate::{Error, TextCodec, TokenCounter, Tokenizer}; diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 3b5e3c54d2e..99824e52ff7 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -59,7 +59,7 @@ from ._lazy_imports_registry import ( if TYPE_CHECKING: import httpx - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer def get_litellm_globals() -> dict[str, object]: diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index ad493c38dd4..c2658949fce 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -28,6 +28,6 @@ def o200k_base_rank_file() -> str: return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii") -from litellm.rust_bridge._native import Tokenizer +from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer encoding: Final = Tokenizer.from_tiktoken("cl100k_base") diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 442c0c9ff1f..ded2d652e35 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -29,9 +29,12 @@ from litellm.constants import ( TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer as Tokenizer +from litellm.litellm_core_utils.tokenizer import OpenAIEncoding from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import _get_httpx_client -from litellm.rust_bridge._native import Tokenizer, tiktoken_encoding_for_model +from litellm.rust_bridge._native import Tokenizer as NativeTokenizer +from litellm.rust_bridge._native import tiktoken_encoding_for_model from litellm.types.llms.anthropic import ( AnthropicContentParamSource, AnthropicContentParamSourceFileId, @@ -620,10 +623,14 @@ def _get_exact_count_function( if model is not None or custom_tokenizer is not None: tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model) if tokenizer_json["type"] == "huggingface_tokenizer": - tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"] + tokenizer: Final[Tokenizer | NativeTokenizer] = tokenizer_json["tokenizer"] def count_tokens(text: str) -> int: - return tokenizer.count(text) + return ( + tokenizer.count(text) + if isinstance(tokenizer, NativeTokenizer) + else len(tokenizer.encode_batch_fast([text])[0]) + ) return count_tokens elif tokenizer_json["type"] == "openai_tokenizer": @@ -644,11 +651,11 @@ def _get_exact_count_function( @lru_cache(maxsize=8) -def _native_tokenizer_for_encoding(name: str) -> Tokenizer: - return Tokenizer.from_tiktoken(name) +def _native_tokenizer_for_encoding(name: str) -> OpenAIEncoding: + return OpenAIEncoding.from_tiktoken(name) -def openai_tokenizer_encoding(model: str) -> Tokenizer: +def openai_tokenizer_encoding(model: str) -> OpenAIEncoding: """The native encoding `token_counter` uses for a model on the `openai_tokenizer` path.""" from litellm.utils import print_verbose diff --git a/litellm/litellm_core_utils/tokenizer.py b/litellm/litellm_core_utils/tokenizer.py new file mode 100644 index 00000000000..9b3b026e3d9 --- /dev/null +++ b/litellm/litellm_core_utils/tokenizer.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping, Sequence, Set +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from typing import Final, Literal, Protocol, TypeAlias, runtime_checkable + +from litellm.rust_bridge._native import HuggingFaceEncoding +from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + +SpecialTokens: TypeAlias = Literal["all"] | Collection[str] +HuggingFaceInput: TypeAlias = str | list[str] | tuple[str, ...] +HuggingFaceBatchInput: TypeAlias = HuggingFaceInput | tuple[HuggingFaceInput, HuggingFaceInput] | list[HuggingFaceInput] + + +@dataclass(frozen=True, slots=True) +class OpenAIEncoding: + name: str + _native: NativeTokenizer + _special_tokens: frozenset[str] + + @staticmethod + def from_tiktoken(encoding: str) -> OpenAIEncoding: + native: Final = NativeTokenizer.from_tiktoken(encoding) + return OpenAIEncoding(encoding, native, frozenset(native.special_tokens())) + + @property + def special_tokens_set( + self, + ) -> set[str]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.special_tokens() + + @property + def eot_token(self) -> int: + return self.encode("<|endoftext|>", allowed_special="all")[0] + + def encode( + self, + text: str, + *, + allowed_special: Literal["all"] | Set[str] = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type + allowed: Final = self._special_tokens if allowed_special == "all" else frozenset(allowed_special) + disallowed: Final = ( + self._special_tokens - allowed if disallowed_special == "all" else frozenset(disallowed_special) + ) + for special in disallowed: + if special in text: + raise ValueError(f"Encountered text corresponding to disallowed special token {special!r}") + if not allowed: + return self.encode_ordinary(text) + return self._native.encode_special(text, tuple(allowed)) + + def encode_ordinary(self, text: str) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.encode(text) + + def count(self, text: str) -> int: + return self._native.count(text) + + def encode_batch( + self, + text: Sequence[str], + *, + num_threads: int = 8, + allowed_special: Literal["all"] | Set[str] = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + encode: Final = partial(self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special) + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(encode, text) + ) + + def encode_ordinary_batch( + self, text: Sequence[str], *, num_threads: int = 8 + ) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(self.encode_ordinary, text) + ) + + def decode_bytes(self, tokens: Sequence[int]) -> bytes: + return self._native.decode_bytes(tokens) + + def decode(self, tokens: Sequence[int], errors: str = "replace") -> str: + return self.decode_bytes(tokens).decode("utf-8", errors=errors) + + def decode_single_token_bytes(self, token: int) -> bytes: + return self.decode_bytes((token,)) + + def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + return [ # mutable-ok: [LIT002] SDK returns a list + self.decode_single_token_bytes(token) for token in tokens + ] + + def decode_batch( + self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8 + ) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(partial(self.decode, errors=errors), batch) + ) + + def decode_bytes_batch( + self, batch: Sequence[Sequence[int]], *, num_threads: int = 8 + ) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(self.decode_bytes, batch) + ) + + def __reduce__(self) -> tuple[Callable[[str], OpenAIEncoding], tuple[str]]: + return (OpenAIEncoding.from_tiktoken, (self.name,)) + + +@dataclass(frozen=True, slots=True) +class HuggingFaceTokenizer: + _native: NativeTokenizer + + @staticmethod + def from_str(json: str) -> HuggingFaceTokenizer: + return HuggingFaceTokenizer(NativeTokenizer.from_json(json)) + + from_json = from_str + + @staticmethod + def from_file(path: str) -> HuggingFaceTokenizer: + return HuggingFaceTokenizer.from_str(Path(path).read_text(encoding="utf-8")) + + @staticmethod + def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFaceTokenizer: + return HuggingFaceTokenizer(NativeTokenizer.from_pretrained(identifier, revision=revision, token=token)) + + def to_str(self, pretty: bool = False) -> str: + return self._native.to_json(pretty) + + def save(self, path: str, pretty: bool = True) -> None: + Path(path).write_text(self.to_str(pretty), encoding="utf-8") + + @property + def name(self) -> str: + return self._native.name + + def encode( + self, + sequence: HuggingFaceInput, + pair: HuggingFaceInput | None = None, + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> HuggingFaceEncoding: + return self._native.encode_huggingface(sequence, pair, is_pretokenized, add_special_tokens) + + def encode_batch( + self, + input: Sequence[HuggingFaceBatchInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=False) + + def encode_batch_fast( + self, + input: Sequence[HuggingFaceBatchInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=True) + + def _encode_batch( + self, input: Sequence[HuggingFaceBatchInput], is_pretokenized: bool, add_special_tokens: bool, fast: bool + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + sequences: Final = tuple(_batch_input(item, is_pretokenized) for item in input) + return self._native.encode_batch_huggingface(sequences, is_pretokenized, add_special_tokens, fast) + + def count(self, text: str) -> int: + return self._native.count(text) + + def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str: + return self._native.decode(ids, skip_special_tokens=skip_special_tokens) + + def decode_batch( + self, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True + ) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type + return [ # mutable-ok: [LIT002] SDK returns a list + self.decode(ids, skip_special_tokens=skip_special_tokens) for ids in sequences + ] + + def __reduce__(self) -> tuple[Callable[[str], HuggingFaceTokenizer], tuple[str]]: + return (HuggingFaceTokenizer.from_str, (self.to_str(),)) + + +def _batch_input( + item: HuggingFaceBatchInput, is_pretokenized: bool +) -> tuple[HuggingFaceInput, HuggingFaceInput | None]: + if isinstance(item, str): + return (item, None) + if is_pretokenized and all(isinstance(word, str) for word in item): + return (tuple(word for word in item if isinstance(word, str)), None) + if len(item) != 2: + raise TypeError("batch input must be a sequence or a pair of sequences") + return (item[0], item[1]) + + +Tokenizer: TypeAlias = OpenAIEncoding | HuggingFaceTokenizer + + +class _AddedToken(Protocol): + @property + def special(self) -> bool: ... + + +@runtime_checkable +class _AddedTokenDecoder(Protocol): + def get_added_tokens_decoder(self) -> Mapping[int, _AddedToken]: ... + + +def strip_special_tokens(tokenizer: object, tokens: Sequence[int]) -> Sequence[int]: + if not isinstance(tokenizer, _AddedTokenDecoder): + return tokens + try: + added: Final = tokenizer.get_added_tokens_decoder() + except Exception: # noqa: BLE001 # optional metadata failures historically fall back to decoding + return tokens + special_ids: Final = frozenset(token_id for token_id, token in added.items() if token.special) + return tuple(token for token in tokens if token not in special_ids) diff --git a/litellm/llms/a2a/chat/transformation.py b/litellm/llms/a2a/chat/transformation.py index a19b09c4cbe..29a592caf91 100644 --- a/litellm/llms/a2a/chat/transformation.py +++ b/litellm/llms/a2a/chat/transformation.py @@ -24,7 +24,7 @@ from .streaming_iterator import A2AModelResponseIterator if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer _REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = ( diff --git a/litellm/llms/aiml/image_generation/transformation.py b/litellm/llms/aiml/image_generation/transformation.py index fa0ee31d8b1..e66bb2c09ff 100644 --- a/litellm/llms/aiml/image_generation/transformation.py +++ b/litellm/llms/aiml/image_generation/transformation.py @@ -15,7 +15,7 @@ from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/aiohttp_openai/chat/transformation.py b/litellm/llms/aiohttp_openai/chat/transformation.py index 29de841e223..19d7bae4987 100644 --- a/litellm/llms/aiohttp_openai/chat/transformation.py +++ b/litellm/llms/aiohttp_openai/chat/transformation.py @@ -17,7 +17,7 @@ from litellm.types.utils import Choices, ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/amazon_nova/chat/transformation.py b/litellm/llms/amazon_nova/chat/transformation.py index fd808e5df61..bdbc634150d 100644 --- a/litellm/llms/amazon_nova/chat/transformation.py +++ b/litellm/llms/amazon_nova/chat/transformation.py @@ -17,7 +17,7 @@ from litellm.types.utils import ModelResponse from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class AmazonNovaChatConfig(OpenAILikeChatConfig): diff --git a/litellm/llms/anthropic/batches/transformation.py b/litellm/llms/anthropic/batches/transformation.py index a9e4db6d17d..7e15edef9e1 100644 --- a/litellm/llms/anthropic/batches/transformation.py +++ b/litellm/llms/anthropic/batches/transformation.py @@ -15,7 +15,7 @@ from litellm.types.utils import LiteLLMBatch, LlmProviders, ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 40bebf70089..84ea79c5d8b 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -101,7 +101,7 @@ from ..common_utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: diff --git a/litellm/llms/anthropic/completion/transformation.py b/litellm/llms/anthropic/completion/transformation.py index 06c98802e09..854ff69b5bd 100644 --- a/litellm/llms/anthropic/completion/transformation.py +++ b/litellm/llms/anthropic/completion/transformation.py @@ -33,7 +33,7 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class AnthropicTextError(BaseLLMException): diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 0997c497a9a..418459bf7d9 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -30,7 +30,7 @@ from ..common_utils import AzureOpenAIError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: diff --git a/litellm/llms/azure_ai/agents/transformation.py b/litellm/llms/azure_ai/agents/transformation.py index ad750b4f569..05c319ecf75 100644 --- a/litellm/llms/azure_ai/agents/transformation.py +++ b/litellm/llms/azure_ai/agents/transformation.py @@ -35,8 +35,8 @@ from litellm.types.utils import ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler - from litellm.rust_bridge._native import Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/azure_ai/azure_model_router/transformation.py b/litellm/llms/azure_ai/azure_model_router/transformation.py index 9428f445479..4d0e26c9c06 100644 --- a/litellm/llms/azure_ai/azure_model_router/transformation.py +++ b/litellm/llms/azure_ai/azure_model_router/transformation.py @@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class AzureModelRouterConfig(AzureAIStudioConfig): diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index d7a14d14cfa..187c41e2370 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -30,7 +30,7 @@ from litellm.types.utils import ModelResponse, ProviderField from litellm.utils import _add_path_to_api_base, supports_tool_choice if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class AzureFoundryErrorStrings(str, enum.Enum): diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 92b1664f1e6..d985572ce6d 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -14,7 +14,7 @@ from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 7c7170b310e..431b71b8d72 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -13,7 +13,7 @@ from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/base_llm/bridges/completion_transformation.py b/litellm/llms/base_llm/bridges/completion_transformation.py index 5d68f4b3d78..4439662ba77 100644 --- a/litellm/llms/base_llm/bridges/completion_transformation.py +++ b/litellm/llms/base_llm/bridges/completion_transformation.py @@ -10,8 +10,8 @@ if TYPE_CHECKING: from pydantic import BaseModel from litellm import LiteLLMLoggingObj, ModelResponse + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator - from litellm.rust_bridge._native import Tokenizer from litellm.types.llms.openai import AllMessageValues diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index df5a4aafe7f..a790b245cb7 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -22,7 +22,7 @@ from litellm.types.llms.openai import ( if TYPE_CHECKING: from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.types.utils import ModelResponse from ..base_utils import ( diff --git a/litellm/llms/base_llm/completion/transformation.py b/litellm/llms/base_llm/completion/transformation.py index fc15564294e..a192dbd7672 100644 --- a/litellm/llms/base_llm/completion/transformation.py +++ b/litellm/llms/base_llm/completion/transformation.py @@ -9,7 +9,7 @@ from litellm.types.utils import ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/base_llm/embedding/transformation.py b/litellm/llms/base_llm/embedding/transformation.py index 01c9d63f515..2fe7261289e 100644 --- a/litellm/llms/base_llm/embedding/transformation.py +++ b/litellm/llms/base_llm/embedding/transformation.py @@ -9,7 +9,7 @@ from litellm.types.utils import EmbeddingResponse, ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/base_llm/files/transformation.py b/litellm/llms/base_llm/files/transformation.py index da7967d3149..714c599e149 100644 --- a/litellm/llms/base_llm/files/transformation.py +++ b/litellm/llms/base_llm/files/transformation.py @@ -22,8 +22,8 @@ from ..chat.transformation import BaseConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.router import Router as _Router - from litellm.rust_bridge._native import Tokenizer from litellm.types.llms.openai import HttpxBinaryResponseContent LiteLLMLoggingObj = _LiteLLMLoggingObj diff --git a/litellm/llms/base_llm/image_generation/transformation.py b/litellm/llms/base_llm/image_generation/transformation.py index 240204424fa..152f0f11ca3 100644 --- a/litellm/llms/base_llm/image_generation/transformation.py +++ b/litellm/llms/base_llm/image_generation/transformation.py @@ -12,7 +12,7 @@ from litellm.types.utils import ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/base_llm/image_variations/transformation.py b/litellm/llms/base_llm/image_variations/transformation.py index 365e0ead0f4..658e486f9a0 100644 --- a/litellm/llms/base_llm/image_variations/transformation.py +++ b/litellm/llms/base_llm/image_variations/transformation.py @@ -18,7 +18,7 @@ from litellm.types.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index 19c8ca881ae..4dada55009d 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -41,8 +41,8 @@ from litellm.types.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler - from litellm.rust_bridge._native import Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index b2be641f279..f7f580b4aca 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -99,7 +99,7 @@ from ..common_utils import ( ) if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer # Computer use tool prefixes supported by Bedrock BEDROCK_COMPUTER_USE_TOOLS: Final = [ diff --git a/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/litellm/llms/bedrock/chat/invoke_agent/transformation.py index 7cdb373ac4b..c52f1530b7e 100644 --- a/litellm/llms/bedrock/chat/invoke_agent/transformation.py +++ b/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -38,7 +38,7 @@ from litellm.types.utils import Choices, Message, ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py index 25c2c56a5b4..ecea1ba6e3e 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -25,7 +25,7 @@ from litellm.types.utils import ( from .amazon_llama_transformation import AmazonLlamaConfig if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class AmazonDeepSeekR1Config(AmazonLlamaConfig): diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py index c7fa5cfc143..d3c8c53af14 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py @@ -22,7 +22,7 @@ from litellm.types.utils import Choices if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.types.utils import ModelResponse LiteLLMLoggingObj = _LiteLLMLoggingObj diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py index 0bde4782c27..e8c9ceb542e 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py @@ -28,7 +28,7 @@ from ..converse_transformation import AmazonConverseConfig from .base_invoke_transformation import AmazonInvokeConfig if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer _CachePointCarrier = TypeVar("_CachePointCarrier", SystemContentBlock, ContentBlock) _INJECTION_POINTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py index 641cae72c32..79c9ac46bc0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py @@ -21,7 +21,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class AmazonQwen2Config(AmazonQwen3Config): diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py index 914b7f54fb9..2324d5bd195 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py @@ -19,7 +19,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py index 59cc246d132..4e254dfc2d3 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -26,7 +26,7 @@ from litellm.utils import get_base64_str if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 57c1df21dec..ebc974cf26e 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -35,7 +35,7 @@ from litellm.types.utils import ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 4f1857dc314..ca785869261 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -35,7 +35,7 @@ from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/black_forest_labs/image_generation/transformation.py b/litellm/llms/black_forest_labs/image_generation/transformation.py index a3f80d62c23..61ac93bc34d 100644 --- a/litellm/llms/black_forest_labs/image_generation/transformation.py +++ b/litellm/llms/black_forest_labs/image_generation/transformation.py @@ -30,7 +30,7 @@ from ..common_utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/brave/search/__init__.py b/litellm/llms/brave/search/__init__.py index cc1168d7ef8..de70c62e040 100644 --- a/litellm/llms/brave/search/__init__.py +++ b/litellm/llms/brave/search/__init__.py @@ -1,7 +1,7 @@ -""" -Brave Search API module. -""" - -from litellm.llms.brave.search.transformation import BraveSearchConfig - -__all__ = ["BraveSearchConfig"] +""" +Brave Search API module. +""" + +from litellm.llms.brave.search.transformation import BraveSearchConfig + +__all__ = ["BraveSearchConfig"] diff --git a/litellm/llms/bytez/chat/transformation.py b/litellm/llms/bytez/chat/transformation.py index df66798e37a..161174123ce 100644 --- a/litellm/llms/bytez/chat/transformation.py +++ b/litellm/llms/bytez/chat/transformation.py @@ -24,7 +24,7 @@ from ..common_utils import API_BASE, BytezError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/clarifai/chat/transformation.py b/litellm/llms/clarifai/chat/transformation.py index 90b887ab30e..95bdca20d5f 100644 --- a/litellm/llms/clarifai/chat/transformation.py +++ b/litellm/llms/clarifai/chat/transformation.py @@ -14,7 +14,7 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/cohere/chat/transformation.py b/litellm/llms/cohere/chat/transformation.py index f55bd0c3e55..edaf7960eae 100644 --- a/litellm/llms/cohere/chat/transformation.py +++ b/litellm/llms/cohere/chat/transformation.py @@ -16,7 +16,7 @@ from ..common_utils import validate_environment as cohere_validate_environment if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/cohere/chat/v2_transformation.py b/litellm/llms/cohere/chat/v2_transformation.py index f22dc4dc9b0..4db921342b1 100644 --- a/litellm/llms/cohere/chat/v2_transformation.py +++ b/litellm/llms/cohere/chat/v2_transformation.py @@ -21,7 +21,7 @@ from ..common_utils import validate_environment as cohere_validate_environment if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/cohere/embed/handler.py b/litellm/llms/cohere/embed/handler.py index 620c9c82717..a09262dff54 100644 --- a/litellm/llms/cohere/embed/handler.py +++ b/litellm/llms/cohere/embed/handler.py @@ -20,7 +20,7 @@ from litellm.types.utils import EmbeddingResponse from .v1_transformation import CohereEmbeddingConfig if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer def validate_environment(api_key, headers: dict): diff --git a/litellm/llms/cometapi/image_generation/transformation.py b/litellm/llms/cometapi/image_generation/transformation.py index b09746f58db..e7463e58044 100644 --- a/litellm/llms/cometapi/image_generation/transformation.py +++ b/litellm/llms/cometapi/image_generation/transformation.py @@ -14,7 +14,7 @@ from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/compactifai/chat/transformation.py b/litellm/llms/compactifai/chat/transformation.py index ea1c168a2b1..ce7184b2424 100644 --- a/litellm/llms/compactifai/chat/transformation.py +++ b/litellm/llms/compactifai/chat/transformation.py @@ -18,7 +18,7 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/custom_httpx/aiohttp_handler.py b/litellm/llms/custom_httpx/aiohttp_handler.py index e2ff109b03a..d2287c06fea 100644 --- a/litellm/llms/custom_httpx/aiohttp_handler.py +++ b/litellm/llms/custom_httpx/aiohttp_handler.py @@ -27,7 +27,7 @@ from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigMana if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ce957108f92..64c2fec44cf 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -196,11 +196,11 @@ if TYPE_CHECKING: from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig - from litellm.rust_bridge._native import Tokenizer from litellm.types.llms.openai_evals import ( CancelEvalResponse, CancelRunResponse, diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index b982af2e78f..d4e8c53ddab 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -39,7 +39,7 @@ from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 140a16b1434..feae74b01e0 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -149,7 +149,7 @@ def _split_parallel_tool_calls(messages: list[AllMessageValues]) -> list[AllMess if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/fal_ai/image_generation/bria_transformation.py b/litellm/llms/fal_ai/image_generation/bria_transformation.py index 5fdf4fd064c..1eebfd12691 100644 --- a/litellm/llms/fal_ai/image_generation/bria_transformation.py +++ b/litellm/llms/fal_ai/image_generation/bria_transformation.py @@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py index 2f8b33c7580..b03b237d638 100644 --- a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py +++ b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py @@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py index 02fb8b063db..d37ed8057b5 100644 --- a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py @@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py index d28805c3603..4795f21c170 100644 --- a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py +++ b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py @@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py index 7cbd46fb7f4..1d55f10ea33 100644 --- a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py @@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py index 5e50e03742b..0a5a7afa55a 100644 --- a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py +++ b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py @@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/fal_ai/image_generation/transformation.py b/litellm/llms/fal_ai/image_generation/transformation.py index 7c71c304fbf..0f1a1c65eeb 100644 --- a/litellm/llms/fal_ai/image_generation/transformation.py +++ b/litellm/llms/fal_ai/image_generation/transformation.py @@ -14,7 +14,7 @@ from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 3e7d3838b37..eb5a9ba360e 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -46,7 +46,7 @@ from ..common_utils import ( ) if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer def _map_reasoning_effort(value: object) -> object: diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index c0cf5e514bb..5f7f0d0c558 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -25,7 +25,7 @@ from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index e552b8f6de7..d62b1390536 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -27,7 +27,7 @@ from ..file_handler import upload_file_sync if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 9dea5ed9f67..f35402ce31c 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -27,7 +27,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, ServerToolUs from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer GROQ_COMPOUND_MODELS: Final = frozenset({"compound", "compound-mini"}) diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index c377d628320..0bc651087b2 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -26,7 +26,7 @@ from ..common_utils import HuggingFaceError, hf_task_list, hf_tasks, output_pars if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: diff --git a/litellm/llms/langflow/chat/transformation.py b/litellm/llms/langflow/chat/transformation.py index 3df0e745705..258b3b49acb 100644 --- a/litellm/llms/langflow/chat/transformation.py +++ b/litellm/llms/langflow/chat/transformation.py @@ -15,8 +15,8 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler - from litellm.rust_bridge._native import Tokenizer from litellm.utils import CustomStreamWrapper LiteLLMLoggingObj = _LiteLLMLoggingObj diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index f5473077826..876855c65b3 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -24,8 +24,8 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler - from litellm.rust_bridge._native import Tokenizer from litellm.utils import CustomStreamWrapper LiteLLMLoggingObj = _LiteLLMLoggingObj diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index b9111882b29..b13ef3cd1df 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -19,7 +19,7 @@ from litellm.types.utils import ModelResponse from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class LemonadeChatConfig(OpenAILikeChatConfig): diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index d6b9f1da516..837c42e96a9 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -32,7 +32,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: str) -> str: diff --git a/litellm/llms/nlp_cloud/chat/transformation.py b/litellm/llms/nlp_cloud/chat/transformation.py index 819ccbf363f..f174866cca2 100644 --- a/litellm/llms/nlp_cloud/chat/transformation.py +++ b/litellm/llms/nlp_cloud/chat/transformation.py @@ -15,7 +15,7 @@ from ..common_utils import NLPCloudError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index e2261a262c3..4db09917473 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -66,7 +66,7 @@ from litellm.utils import supports_reasoning if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index f3e6d93b4e1..eb057f454af 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -32,7 +32,7 @@ from ..common_utils import OllamaError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 631a15b2045..0a9fe4bbbf7 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -36,7 +36,7 @@ from ..common_utils import OllamaError, OllamaModelInfo, _convert_image if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -331,7 +331,7 @@ class OllamaConfig(BaseConfig): tokenizer: Final = encoding if encoding is not None else litellm.encoding prompt_tokens: Final = response_json.get( "prompt_eval_count", - len(tokenizer.encode(_prompt)), + len(tokenizer.encode(_prompt, disallowed_special=())), ) completion_tokens: Final = response_json.get( "eval_count", len(response_json.get("message", dict()).get("content", "")) diff --git a/litellm/llms/oobabooga/chat/transformation.py b/litellm/llms/oobabooga/chat/transformation.py index 3e0cd412f8b..9f1ff2a51ca 100644 --- a/litellm/llms/oobabooga/chat/transformation.py +++ b/litellm/llms/oobabooga/chat/transformation.py @@ -12,7 +12,7 @@ from ..common_utils import OobaboogaError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 87dacfaf5ba..506dead3d4c 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -61,8 +61,8 @@ from ..common_utils import OpenAIError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.llms.base_llm.base_utils import BaseTokenCounter - from litellm.rust_bridge._native import Tokenizer from litellm.types.llms.openai import ChatCompletionToolParam LiteLLMLoggingObj = _LiteLLMLoggingObj diff --git a/litellm/llms/openai/image_generation/dall_e_2_transformation.py b/litellm/llms/openai/image_generation/dall_e_2_transformation.py index 0633abf6af9..2b920015af4 100644 --- a/litellm/llms/openai/image_generation/dall_e_2_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_2_transformation.py @@ -12,7 +12,7 @@ from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class DallE2ImageGenerationConfig(BaseImageGenerationConfig): diff --git a/litellm/llms/openai/image_generation/dall_e_3_transformation.py b/litellm/llms/openai/image_generation/dall_e_3_transformation.py index b5076115aeb..305c6d7a324 100644 --- a/litellm/llms/openai/image_generation/dall_e_3_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_3_transformation.py @@ -12,7 +12,7 @@ from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class DallE3ImageGenerationConfig(BaseImageGenerationConfig): diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py index 6388ab960c2..9f272678468 100644 --- a/litellm/llms/openai/image_generation/gpt_transformation.py +++ b/litellm/llms/openai/image_generation/gpt_transformation.py @@ -12,7 +12,7 @@ from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class GPTImageGenerationConfig(BaseImageGenerationConfig): diff --git a/litellm/llms/openai/image_variations/transformation.py b/litellm/llms/openai/image_variations/transformation.py index a839fb9e17d..8272596cd6a 100644 --- a/litellm/llms/openai/image_variations/transformation.py +++ b/litellm/llms/openai/image_variations/transformation.py @@ -12,7 +12,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ..common_utils import OpenAIError if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class OpenAIImageVariationConfig(BaseImageVariationConfig): diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 7cfa308a35d..9fccab767ab 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -8,7 +8,7 @@ import httpx if TYPE_CHECKING: from aiohttp import ClientSession - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer import openai from openai import AsyncOpenAI, OpenAI diff --git a/litellm/llms/openai_like/chat/transformation.py b/litellm/llms/openai_like/chat/transformation.py index 60dadc280c3..0d0290d417c 100644 --- a/litellm/llms/openai_like/chat/transformation.py +++ b/litellm/llms/openai_like/chat/transformation.py @@ -14,7 +14,7 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index a8fc59772e5..ba9828ead01 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -24,7 +24,7 @@ from ..common_utils import OpenRouterException if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class CacheControlSupportedModels(str, Enum): diff --git a/litellm/llms/openrouter/image_generation/transformation.py b/litellm/llms/openrouter/image_generation/transformation.py index 5de6f77cb2c..c186eb35636 100644 --- a/litellm/llms/openrouter/image_generation/transformation.py +++ b/litellm/llms/openrouter/image_generation/transformation.py @@ -51,7 +51,7 @@ from litellm.types.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer else: LiteLLMLoggingObj = Any diff --git a/litellm/llms/perplexity/chat/transformation.py b/litellm/llms/perplexity/chat/transformation.py index a1fbcefea94..9299a101400 100644 --- a/litellm/llms/perplexity/chat/transformation.py +++ b/litellm/llms/perplexity/chat/transformation.py @@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues, ChatCompletionAnnotation from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class PerplexityChatConfig(OpenAIGPTConfig): diff --git a/litellm/llms/petals/completion/transformation.py b/litellm/llms/petals/completion/transformation.py index bd198bca7c0..e0f6dacad38 100644 --- a/litellm/llms/petals/completion/transformation.py +++ b/litellm/llms/petals/completion/transformation.py @@ -14,7 +14,7 @@ from litellm.types.utils import ModelResponse from ..common_utils import PetalsError if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class PetalsConfig(BaseConfig): diff --git a/litellm/llms/predibase/chat/transformation.py b/litellm/llms/predibase/chat/transformation.py index e7e4aeddac2..3d23dae402e 100644 --- a/litellm/llms/predibase/chat/transformation.py +++ b/litellm/llms/predibase/chat/transformation.py @@ -19,7 +19,7 @@ from ..common_utils import PredibaseError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/recraft/image_generation/transformation.py b/litellm/llms/recraft/image_generation/transformation.py index 7a56f4b68be..ea0939d0553 100644 --- a/litellm/llms/recraft/image_generation/transformation.py +++ b/litellm/llms/recraft/image_generation/transformation.py @@ -15,7 +15,7 @@ from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/replicate/chat/transformation.py b/litellm/llms/replicate/chat/transformation.py index ebd06dd4248..e9f5a66e4dd 100644 --- a/litellm/llms/replicate/chat/transformation.py +++ b/litellm/llms/replicate/chat/transformation.py @@ -20,7 +20,7 @@ from ..common_utils import ReplicateError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: diff --git a/litellm/llms/runwayml/image_generation/transformation.py b/litellm/llms/runwayml/image_generation/transformation.py index e8489021321..e05dcb8b720 100644 --- a/litellm/llms/runwayml/image_generation/transformation.py +++ b/litellm/llms/runwayml/image_generation/transformation.py @@ -23,7 +23,7 @@ from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/sagemaker/completion/transformation.py b/litellm/llms/sagemaker/completion/transformation.py index 78ded4b7568..ffcf6a51759 100644 --- a/litellm/llms/sagemaker/completion/transformation.py +++ b/litellm/llms/sagemaker/completion/transformation.py @@ -25,7 +25,7 @@ from ..common_utils import SagemakerError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index 78dfd57ba18..0fb445c3ad2 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -16,7 +16,7 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/stability/image_generation/transformation.py b/litellm/llms/stability/image_generation/transformation.py index c8cf333c407..565ef26fd86 100644 --- a/litellm/llms/stability/image_generation/transformation.py +++ b/litellm/llms/stability/image_generation/transformation.py @@ -27,7 +27,7 @@ from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/topaz/image_variations/transformation.py b/litellm/llms/topaz/image_variations/transformation.py index 38bd0e0d20c..59967974cbc 100644 --- a/litellm/llms/topaz/image_variations/transformation.py +++ b/litellm/llms/topaz/image_variations/transformation.py @@ -23,7 +23,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ..common_utils import TopazException, TopazModelInfo if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 0b8f7bbecdb..e1d69eb93bf 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -29,7 +29,7 @@ from litellm.types.utils import ( from ..common_utils import TritonError if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class TritonConfig(BaseConfig): diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index 24c1c0e625f..b725f8b3230 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -30,8 +30,8 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler - from litellm.rust_bridge._native import Tokenizer from litellm.utils import CustomStreamWrapper LiteLLMLoggingObj = _LiteLLMLoggingObj diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index 9c814cce3bf..1999efa0d82 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -25,7 +25,7 @@ from litellm.types.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py index 18d4a806ac2..4c8b7fb70c8 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py @@ -21,7 +21,7 @@ from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index d0139739fb2..cf904eb2da9 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -14,7 +14,7 @@ from ....anthropic.chat.transformation import AnthropicConfig from .output_params_utils import sanitize_vertex_anthropic_output_params if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class VertexAIError(Exception): diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py index 1f902d7fab9..6ee5bbeb97d 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py @@ -21,7 +21,7 @@ from litellm.types.utils import ( from ...common_utils import VertexAIError if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class VertexAILlama3Config(OpenAIGPTConfig): diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py index 188a85c184e..ffe9ad15826 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py @@ -25,7 +25,7 @@ from litellm.types.utils import ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer class VertexGemmaConfig(OpenAIGPTConfig): diff --git a/litellm/llms/watsonx/completion/transformation.py b/litellm/llms/watsonx/completion/transformation.py index c16e553a1e6..65dcfcc1431 100644 --- a/litellm/llms/watsonx/completion/transformation.py +++ b/litellm/llms/watsonx/completion/transformation.py @@ -22,7 +22,7 @@ from ..common_utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/main.py b/litellm/main.py index 617b7b908f3..fb7476688e9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -99,6 +99,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) +from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, @@ -113,7 +114,6 @@ from litellm.llms.vertex_ai.common_utils import ( get_vertex_ai_model_route, ) from litellm.realtime_api.main import _realtime_health_check -from litellm.rust_bridge._native import Tokenizer from litellm.secret_managers.main import get_secret_bool, get_secret_str from litellm.types.completion import ( _CompletionDispatchContext, diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 06e157498aa..391f0042ed0 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -19632,7 +19632,7 @@ } } }, - "description": "\n Unified rate-limit error.\n\n Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n The :attr:`category` attribute lets callers distinguish the source. See\n :class:`RateLimitErrorCategory` for the available values.\n " + "description": "\nUnified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values.\n" }, "500": { "content": { diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 9d79800208d..1d3dab66b4d 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -119,17 +119,62 @@ class Tokenizer: def encode(self, text: str) -> list[int]: ... def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str: ... def count(self, text: str) -> int: ... + def encode_special(self, text: str, allowed: Sequence[str]) -> list[int]: ... + def special_tokens(self) -> set[str]: ... + def decode_bytes(self, ids: Sequence[int]) -> bytes: ... + def to_json(self, pretty: bool = False) -> str: ... + def encode_huggingface( + self, + sequence: str | Sequence[str], + pair: str | Sequence[str] | None = None, + is_pretokenized: bool = False, + add_special_tokens: bool = True, + fast: bool = False, + ) -> HuggingFaceEncoding: ... + def encode_batch_huggingface( + self, + inputs: Sequence[tuple[str | Sequence[str], str | Sequence[str] | None]], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + fast: bool = False, + ) -> list[HuggingFaceEncoding]: ... @property def name(self) -> str: ... -def tiktoken_encoding_for_model(model: str) -> str | None: ... +@final +class HuggingFaceEncoding: + def __new__(cls, json: str | None = None) -> HuggingFaceEncoding: ... + def __len__(self) -> int: ... + def __reduce__(self) -> tuple[type[HuggingFaceEncoding], tuple[str]]: ... + @property + def ids(self) -> list[int]: ... + @property + def tokens(self) -> list[str]: ... + @property + def offsets(self) -> list[tuple[int, int]]: ... + @property + def type_ids(self) -> list[int]: ... + @property + def attention_mask(self) -> list[int]: ... + @property + def special_tokens_mask(self) -> list[int]: ... + @property + def word_ids(self) -> list[int | None]: ... + @property + def sequence_ids(self) -> list[int | None]: ... + @property + def overflowing(self) -> list[HuggingFaceEncoding]: ... + @property + def n_sequences(self) -> int: ... +def tiktoken_encoding_for_model(model: str) -> str | None: ... def gil_stats() -> dict[str, int]: ... def process_state_started() -> bool: ... def reserve_process_for_forking() -> None: ... __all__ = [ "ForkedAfterNativeRuntimeStarted", + "HuggingFaceEncoding", "ProcessReservedForForking", "ResponsesWebSocketConnection", "RustBridgeDeclined", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ad37d7e241f..ad12e0e628e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -81,7 +81,7 @@ from .llms.openai import ( from .rerank import RerankResponse as RerankResponse if TYPE_CHECKING: - from litellm.rust_bridge._native import Tokenizer + from litellm.litellm_core_utils.tokenizer import Tokenizer from .vector_stores import VectorStoreSearchResponse else: diff --git a/litellm/utils.py b/litellm/utils.py index d6dd5239e57..a128869e405 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -84,7 +84,8 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_fill_missing_generalizations, ) from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload -from litellm.rust_bridge._native import Tokenizer +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer as Tokenizer +from litellm.litellm_core_utils.tokenizer import OpenAIEncoding, strip_special_tokens _CachingHandlerResponse = None _LLMCachingHandler = None @@ -2327,7 +2328,13 @@ def encode(model="", text="", custom_tokenizer: dict | None = None): enc: The encoded text. """ tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model=model) - return tokenizer_json["tokenizer"].encode(text) + if tokenizer_json["type"] == "openai_tokenizer": + openai_tokenizer: Final = cast( # cast-ok: [LIT006] caller's explicit type tag selects this interface + OpenAIEncoding, tokenizer_json["tokenizer"] + ) + return openai_tokenizer.encode(text, disallowed_special=()) + encoded: Final = tokenizer_json["tokenizer"].encode(text) + return encoded.ids if hasattr(encoded, "ids") else encoded def decode( @@ -2346,7 +2353,11 @@ def decode( """ tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model=model) if tokenizer_json["type"] == "huggingface_tokenizer": - return tokenizer_json["tokenizer"].decode(tokens, skip_special_tokens=skip_special_tokens) + ids: Final = strip_special_tokens(tokenizer_json["tokenizer"], tokens) if skip_special_tokens else tokens + hf_tokenizer: Final = cast( # cast-ok: [LIT006] caller's explicit type tag selects this interface + Tokenizer, tokenizer_json["tokenizer"] + ) + return hf_tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) return tokenizer_json["tokenizer"].decode(tokens) diff --git a/pyproject.toml b/pyproject.toml index 6ccc0505458..48c6f63e8f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "pyyaml>=6.0.3,<7.0", "packaging>=24.0", "importlib-metadata>=8.0.0,<9.0", + "huggingface-hub>=0.34.0,<2.0", "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", @@ -188,6 +189,8 @@ litellm-proxy = "litellm.proxy.client.cli:litellm_proxy_cli" [dependency-groups] dev = [ + "tiktoken==0.12.0", + "tokenizers==0.23.1", "diff-cover==9.7.2", "hypothesis==6.165.10", "reportlab==5.0.1", diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index a138dfed8d8..cae2112a3ab 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -181,14 +181,14 @@ class _SlowEncoder: self.in_flight = 0 self.peak_in_flight = 0 - def count(self, text: str) -> int: + def encode_batch_fast(self, texts: list[str]) -> list[list[int]]: with self._lock: self.in_flight += 1 self.peak_in_flight = max(self.peak_in_flight, self.in_flight) time.sleep(0.1) with self._lock: self.in_flight -= 1 - return len(text) + return [[0] * len(text) for text in texts] @pytest.mark.asyncio diff --git a/tests/test_litellm/litellm_core_utils/test_tokenizer.py b/tests/test_litellm/litellm_core_utils/test_tokenizer.py new file mode 100644 index 00000000000..caeb54fcd2a --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_tokenizer.py @@ -0,0 +1,279 @@ +import copy +import os +import pickle +import subprocess +import sys +from pathlib import Path +from typing import Final, Literal + +import pytest +import tiktoken +from tokenizers import Tokenizer as ReferenceTokenizer + +import litellm +from litellm.caching._embedding_router import truncate_embedding_input +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding +from litellm.utils import claude_json_str +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + + +@pytest.mark.parametrize("name", ("cl100k_base", "o200k_base", "p50k_base", "r50k_base", "o200k_harmony")) +@pytest.mark.parametrize( + "text", ("hello world", "café 漢字 🙂", "", "a\ud800b", "\ud83d\ude42", "🙂\ud83d\ude42\udfff") +) +def test_openai_encoding_matches_python_unicode_and_batches(name: str, text: str) -> None: + reference: Final = tiktoken.get_encoding(name) + encoding: Final = OpenAIEncoding.from_tiktoken(name) + expected: Final = reference.encode(text) + + assert encoding.encode(text) == expected + assert encoding.count(text) == len(expected) + assert encoding.encode_batch([text], num_threads=2) == reference.encode_batch([text], num_threads=2) + assert encoding.encode_ordinary_batch([text]) == reference.encode_ordinary_batch([text]) + assert encoding.decode_batch([expected]) == reference.decode_batch([expected]) + assert encoding.decode_bytes_batch([expected]) == reference.decode_bytes_batch([expected]) + + +@pytest.mark.parametrize("allowed", (frozenset(), frozenset({"<|endoftext|>"}), "all")) +@pytest.mark.parametrize("disallowed", (frozenset(), frozenset({"<|fim_prefix|>"}), "all")) +def test_openai_special_token_options_match_python( + allowed: frozenset[str] | Literal["all"], disallowed: frozenset[str] | Literal["all"] +) -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + encoding: Final = OpenAIEncoding.from_tiktoken(reference.name) + text: Final = "hello<|endoftext|><|fim_prefix|>world" + allowed_set: Final = reference.special_tokens_set if allowed == "all" else allowed + disallowed_set: Final = reference.special_tokens_set - allowed_set if disallowed == "all" else disallowed + if any(token in text for token in disallowed_set): + with pytest.raises(ValueError, match="disallowed special token"): + encoding.encode(text, allowed_special=allowed, disallowed_special=disallowed) + return + assert encoding.encode(text, allowed_special=allowed, disallowed_special=disallowed) == reference.encode( + text, allowed_special=allowed, disallowed_special=disallowed + ) + assert encoding.special_tokens_set == reference.special_tokens_set + assert encoding.eot_token == reference.eot_token + + +@pytest.mark.parametrize("errors", ("replace", "ignore", "backslashreplace", "strict")) +def test_openai_partial_token_decoding_preserves_error_policy(errors: str) -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + encoding: Final = OpenAIEncoding.from_tiktoken(reference.name) + tokens: Final = reference.encode("🙂")[:1] + assert encoding.decode_bytes(tokens) == reference.decode_bytes(tokens) + if errors == "strict": + with pytest.raises(UnicodeDecodeError): + encoding.decode(tokens, errors=errors) + return + assert encoding.decode(tokens, errors=errors) == reference.decode(tokens, errors=errors) + assert encoding.decode_tokens_bytes(tokens) == reference.decode_tokens_bytes(tokens) + + +def test_public_encoding_and_semantic_cache_preserve_truncated_unicode() -> None: + reference: Final = tiktoken.get_encoding(litellm.encoding.name) + text: Final = "🙂" + tokens: Final = reference.encode(text) + + assert litellm.encoding.encode(text, disallowed_special=()) == tokens + assert litellm.encoding.encode_batch([text]) == [tokens] + assert litellm.decode(tokens=tokens[:1]) == reference.decode(tokens[:1]) + assert truncate_embedding_input(text, "", 1) == reference.decode(tokens[:1]) + + +@pytest.mark.parametrize("add_special_tokens", (True, False)) +def test_huggingface_encoding_preserves_result_fields_and_serialization(add_special_tokens: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + tokenizer: Final = HuggingFaceTokenizer.from_str(TOKENIZER_JSON) + expected: Final = reference.encode("Hello World", add_special_tokens=add_special_tokens) + actual: Final = tokenizer.encode("Hello World", add_special_tokens=add_special_tokens) + + assert (actual.ids, actual.tokens, actual.type_ids, actual.offsets, actual.word_ids, actual.sequence_ids) == ( + expected.ids, + expected.tokens, + expected.type_ids, + expected.offsets, + expected.word_ids, + expected.sequence_ids, + ) + assert (actual.attention_mask, actual.special_tokens_mask, actual.n_sequences, len(actual)) == ( + expected.attention_mask, + expected.special_tokens_mask, + expected.n_sequences, + len(expected), + ) + assert copy.deepcopy(actual).ids == expected.ids + assert pickle.loads(pickle.dumps(actual)).offsets == expected.offsets + assert tokenizer.decode(actual.ids, skip_special_tokens=False) == reference.decode( + expected.ids, skip_special_tokens=False + ) + + +def test_huggingface_character_offsets_and_pretokenized_pairs_match_python() -> None: + reference: Final = ReferenceTokenizer.from_str(claude_json_str) + tokenizer: Final = HuggingFaceTokenizer.from_str(claude_json_str) + text: Final = "café 漢字 🙂" + actual: Final = tokenizer.encode(text) + expected: Final = reference.encode(text) + + assert actual.offsets == expected.offsets + assert actual.ids == expected.ids + assert ( + tokenizer.encode(["hello", "world"], ["again"], is_pretokenized=True).ids + == reference.encode(["hello", "world"], ["again"], is_pretokenized=True).ids + ) + + +def test_huggingface_batches_apply_padding_across_inputs() -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + reference.enable_padding(pad_id=0, pad_token="[UNK]") + tokenizer: Final = HuggingFaceTokenizer.from_str(reference.to_str()) + inputs: Final = ["Hello", ("Hello World", "World")] + expected: Final = reference.encode_batch(inputs) + actual: Final = tokenizer.encode_batch(inputs) + fast: Final = tokenizer.encode_batch_fast(inputs) + + assert [(item.ids, item.attention_mask, item.offsets) for item in actual] == [ + (item.ids, item.attention_mask, item.offsets) for item in expected + ] + assert [item.ids for item in fast] == [item.ids for item in expected] + assert tokenizer.decode_batch([item.ids for item in actual]) == reference.decode_batch( + [item.ids for item in expected] + ) + + +def test_caller_supplied_huggingface_tokenizer_preserves_public_encode_and_count() -> None: + tokenizer: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + custom: Final = {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + expected: Final = tokenizer.encode("Hello World").ids + + assert litellm.encode(text="Hello World", custom_tokenizer=custom) == expected + assert litellm.token_counter(text="Hello World", custom_tokenizer=custom) == len(expected) + assert litellm.decode(tokens=expected, custom_tokenizer=custom) == "Hello World" + + +def test_caller_supplied_tiktoken_treats_special_spellings_as_text() -> None: + tokenizer: Final = tiktoken.get_encoding("cl100k_base") + custom: Final = {"type": "openai_tokenizer", "tokenizer": tokenizer} + text: Final = "<|endoftext|>" + + assert litellm.encode(text=text, custom_tokenizer=custom) == tokenizer.encode(text, disallowed_special=()) + + +def test_public_tokenizer_objects_survive_pickle_and_deepcopy(tmp_path: Path) -> None: + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) + tokenizer: Final = custom["tokenizer"] + path: Final = tmp_path / "tokenizer.json" + tokenizer.save(str(path)) + + assert copy.deepcopy(custom)["tokenizer"].encode("Hello World").ids == tokenizer.encode("Hello World").ids + assert ( + pickle.loads(pickle.dumps(custom))["tokenizer"].encode("Hello World").ids == tokenizer.encode("Hello World").ids + ) + assert HuggingFaceTokenizer.from_file(str(path)).encode("Hello World").ids == tokenizer.encode("Hello World").ids + assert copy.deepcopy(litellm.encoding).encode("hello") == litellm.encoding.encode("hello") + assert pickle.loads(pickle.dumps(litellm.encoding)).encode("hello") == litellm.encoding.encode("hello") + + +@pytest.mark.parametrize("offline", ("0", "1")) +def test_hub_loader_preserves_environment_auth_cache_and_offline(tmp_path: Path, offline: str) -> None: + script: Final = """ +import json +import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import httpx +import huggingface_hub +from huggingface_hub.errors import LocalEntryNotFoundError +import litellm +payload = sys.argv[2].encode() +offline = sys.argv[3] == "1" +observed = [] +def handle(request): + assert not offline, "offline loading issued a request" + if request.url.path.endswith("/tokenizer.json"): + observed.append(request.headers.get("authorization")) + if request.headers.get("authorization") != "Bearer audit-fixture-token": + return httpx.Response(401) + return httpx.Response(200, headers={"content-length": str(len(payload)), "etag": '"fixture"', "x-repo-commit": "a" * 40}, content=payload if request.method == "GET" else b"") +if not offline: + huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) +try: + tokenizer = litellm.create_pretrained_tokenizer("test-fixture/tokenizer")["tokenizer"] +except LocalEntryNotFoundError: + assert offline + assert observed == [] +else: + assert not offline + assert "Bearer audit-fixture-token" in observed + assert tokenizer.decode(tokenizer.encode("Hello World").ids) == "Hello World" + assert tuple(Path(sys.argv[4]).rglob("tokenizer.json")) +print("compatible") +""" + result: Final = subprocess.run( + [ + sys.executable, + "-I", + "-c", + script, + str(Path(litellm.__file__).parent.parent), + TOKENIZER_JSON, + offline, + str(tmp_path / "cache"), + ], + capture_output=True, + text=True, + timeout=30, + env={ + **os.environ, + "HF_HOME": str(tmp_path / "home"), + "HF_HUB_CACHE": str(tmp_path / "cache"), + "HF_ENDPOINT": "http://127.0.0.1:9", + "HF_TOKEN": "audit-fixture-token", + "HF_HUB_OFFLINE": offline, + "HF_HUB_DISABLE_IMPLICIT_TOKEN": "0", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + + +def test_installed_tokenization_does_not_import_python_tokenizer_packages() -> None: + script: Final = """ +import importlib.abc +import sys +sys.path.insert(0, sys.argv[1]) +class Block(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split(".")[0] in {"tiktoken", "tokenizers"}: + raise AssertionError("unexpected runtime dependency: " + fullname) +sys.meta_path.insert(0, Block()) +import litellm +ids = litellm.encode(text="hello world") +assert litellm.decode(tokens=ids) == "hello world" +assert litellm.token_counter(model=None, text="hello world") == len(ids) +custom = litellm.create_tokenizer(sys.argv[2]) +assert litellm.decode(tokens=litellm.encode(text="Hello World", custom_tokenizer=custom), custom_tokenizer=custom) == "Hello World" +print("compatible") +""" + result: Final = subprocess.run( + [sys.executable, "-I", "-c", script, str(Path(litellm.__file__).parent.parent), TOKENIZER_JSON], + capture_output=True, + text=True, + timeout=30, + env={**os.environ, "LITELLM_RUST": "0", "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + + +@pytest.mark.parametrize("is_pretokenized", (False, True)) +def test_huggingface_batch_sequence_containers_match_python(is_pretokenized: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + tokenizer: Final = HuggingFaceTokenizer.from_str(TOKENIZER_JSON) + inputs: Final = [["Hello", "World"], ("Hello", "World")] + actual: Final = tokenizer.encode_batch(inputs, is_pretokenized=is_pretokenized) + expected: Final = reference.encode_batch(inputs, is_pretokenized=is_pretokenized) + assert [(item.ids, item.type_ids, item.sequence_ids) for item in actual] == [ + (item.ids, item.type_ids, item.sequence_ids) for item in expected + ] diff --git a/tests/test_litellm_rust/test_tokenizer.py b/tests/test_litellm_rust/test_tokenizer.py index 19640ff440f..d21757ba3f7 100644 --- a/tests/test_litellm_rust/test_tokenizer.py +++ b/tests/test_litellm_rust/test_tokenizer.py @@ -1,6 +1,7 @@ from typing import Final import pytest +import tiktoken from litellm.rust_bridge import _native from litellm.utils import claude_json_str @@ -33,3 +34,13 @@ def test_tiktoken_encoding_for_model() -> None: def test_unknown_tiktoken_encoding_raises_value_error() -> None: with pytest.raises(ValueError, match="unsupported tokenizer"): _native.Tokenizer.from_tiktoken("unknown-encoding") + + +def test_tiktoken_codec_decodes_truncated_unicode_like_python() -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + tokenizer: Final = _native.Tokenizer.from_tiktoken(reference.name) + encoded: Final = reference.encode("🙂漢字") + + assert tuple(tokenizer.decode(encoded[:end]) for end in range(1, len(encoded) + 1)) == tuple( + reference.decode(encoded[:end]) for end in range(1, len(encoded) + 1) + ) diff --git a/uv.lock b/uv.lock index 8af4c3845fc..7b71ae8d834 100644 --- a/uv.lock +++ b/uv.lock @@ -1162,14 +1162,11 @@ wheels = [ [[package]] name = "click" -version = "8.3.3" +version = "8.5.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, ] [[package]] @@ -1987,6 +1984,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/f6/da4db31001e854025ffd26bc9ba0740a9cbba2c3259695f7c5834908b336/fastuuid-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:df61342889d0f5e7a32f7284e55ef95103f2110fee433c2ae7c2c0956d76ac8a", size = 156457, upload-time = "2025-10-19T22:33:44.579Z" }, ] +[[package]] +name = "filelock" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/ac/8c98b17ee3900147b38ef7884a1e590b070b63f0d59fd8b46ef0205f4576/filelock-4.0.0.tar.gz", hash = "sha256:3611eca5d818ca9b00ec3cc7db1dcfe1e2aafc8d44fb4920d8cf60ad1f6bfda6", size = 237935, upload-time = "2026-09-17T03:59:02.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/fb/0e4489505dac46a0487ba925ea4b046612f1a40f432d63a000421de78549/filelock-4.0.0-py3-none-any.whl", hash = "sha256:a850aa9ec2acba8db9ca2e9fcf8a327fbc2f85e432725715b37bd76b7dd1f798", size = 106036, upload-time = "2026-09-17T03:59:01.336Z" }, +] + [[package]] name = "flask" version = "3.1.3" @@ -2214,6 +2220,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] +[[package]] +name = "fsspec" +version = "2026.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, +] + [[package]] name = "gevent" version = "25.9.1" @@ -3126,6 +3141,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" }, ] +[[package]] +name = "hf-xet" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, +] + [[package]] name = "hiredis" version = "3.4.0" @@ -3329,6 +3368,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/34/fae9ac8f1c3a552fd3f7ff652b94c78d219dedc5fce0c0a4232457760a00/huey-2.6.0-py3-none-any.whl", hash = "sha256:1b9df9d370b49c6d5721ba8a01ac9a787cf86b3bdc584e4679de27b920395c3f", size = 76951, upload-time = "2026-01-06T03:01:00.808Z" }, ] +[[package]] +name = "huggingface-hub" +version = "1.32.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/0f/e83fdd856da8fca26bf78d71709ebd120432a0ce535e72b9597cab1eb5bf/huggingface_hub-1.32.0.tar.gz", hash = "sha256:ed70a45498abe86039df7c2f4e5f7575de524be908d3840e8f828d5525eafd6a", size = 1038662, upload-time = "2026-09-17T10:27:48.049Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/cf/d98dd561d6d0d7b7d7a64d1563f8aaaa7c235daee41c1c9bcc3da62420ed/huggingface_hub-1.32.0-py3-none-any.whl", hash = "sha256:b0c7c80561969d9cdacdd55fce67ba9584cca0b9d4ea80957a3a5c1445fac5c8", size = 842906, upload-time = "2026-09-17T10:27:46.102Z" }, +] + [[package]] name = "humanfriendly" version = "10.0" @@ -4447,6 +4507,7 @@ dependencies = [ { name = "click" }, { name = "fastuuid" }, { name = "httpx", extra = ["http2"] }, + { name = "huggingface-hub" }, { name = "importlib-metadata" }, { name = "jinja2" }, { name = "jsonschema" }, @@ -4646,6 +4707,8 @@ dev = [ { name = "responses" }, { name = "respx" }, { name = "ruff" }, + { name = "tiktoken" }, + { name = "tokenizers" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "types-boto3", extra = ["bedrock", "bedrock-agent", "bedrock-runtime", "kms", "s3", "sagemaker-runtime", "sts"] }, { name = "types-pyyaml" }, @@ -4719,6 +4782,7 @@ requires-dist = [ { name = "httpx", extras = ["http2"], specifier = ">=0.28.0,<1.0" }, { name = "httpx2", marker = "extra == 'mcp'", specifier = ">=2.5.0,<3" }, { name = "httpx2", marker = "extra == 'proxy'", specifier = ">=2.5.0,<3" }, + { name = "huggingface-hub", specifier = ">=0.34.0,<2.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" }, { name = "inquirerpy", marker = "extra == 'proxy'", specifier = ">=0.3.4,<1.0" }, @@ -4852,6 +4916,8 @@ dev = [ { name = "responses", specifier = "==0.26.0" }, { name = "respx", specifier = "==0.22.0" }, { name = "ruff", specifier = "==0.15.3" }, + { name = "tiktoken", specifier = "==0.12.0" }, + { name = "tokenizers", specifier = "==0.23.1" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = "==2.4.1" }, { name = "types-boto3", extras = ["bedrock", "bedrock-agent", "bedrock-runtime", "kms", "s3", "sagemaker-runtime", "sts"], specifier = "==1.43.30" }, { name = "types-pyyaml", specifier = "==6.0.12.20250915" }, @@ -9665,6 +9731,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" }, ] +[[package]] +name = "tokenizers" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/60/21f715d9faba5f5407ff759472ade058ec4a507ad62bcea47cb847239a73/tokenizers-0.23.1.tar.gz", hash = "sha256:1feeeadf865a7915adc25445dea30e9933e593c31bb96c277cee36de227c8bfa", size = 365748, upload-time = "2026-04-27T14:43:25.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/39/b87a87d5bb9470610b80a2d31df42fcffeaf35118b8b97952b2aff598cc7/tokenizers-0.23.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e03d6ffcbe0d56ee9c1ccd070e70a13fa750727c0277e138152acbc0252c2224", size = 3146732, upload-time = "2026-04-27T14:43:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6a/068ed9f6e444c9d7e9d55ce134181325700f3d7f30410721bdc8f848d727/tokenizers-0.23.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e0948bbb1ac1d7cdfc9fb6d62c596e3b7550036ad60ecd654a66ad273326324e", size = 3054954, upload-time = "2026-04-27T14:43:13.745Z" }, + { url = "https://files.pythonhosted.org/packages/6c/36/e006edf031154cba92b8416057d92c3abe3635e4c4b0aa0b5b9bb39dde70/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bf13402aff9bc533c89cb849ec3b412dc3fbeacc9744840e423d7bf3f7dc0e3", size = 3374081, upload-time = "2026-04-27T14:43:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/a2/ef/7735d226f9c7f874a6bee5e3f27fb25ecabdf207d37b8cf45286d0795893/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f836ca703b89ae07919a309f9651f7a88fd5a33d5f718ba5ad0870ec0256bad6", size = 3247641, upload-time = "2026-04-27T14:43:03.856Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d9/24827036f6e21297bfffda0768e58eb6096a4f411e932964a01707857931/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae848657742035523fdf261773630cb819a26995fcd3d9ecae0c1daf6e5a4959", size = 3585624, upload-time = "2026-04-27T14:43:10.664Z" }, + { url = "https://files.pythonhosted.org/packages/0c/9a/22f3582b3a4f49358293a5206e25317621ee4526bfe9cdaa0f07a12e770e/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53b09e85775d5187941e7bab30e941b4134ab4a7dd8c68e783d231fb7ca27c51", size = 3844062, upload-time = "2026-04-27T14:43:05.643Z" }, + { url = "https://files.pythonhosted.org/packages/7e/65/b8f8814eef95800f20721384136d9a1d22241d50b2874357cb70542c392f/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea5a0ce170074329faaa8ea3f6400ecde604b6678192688533af80980daae71a", size = 3460098, upload-time = "2026-04-27T14:43:08.854Z" }, + { url = "https://files.pythonhosted.org/packages/0d/d5/1353e5f677ec27c2494fb6a6725e82d56c985f53e90ec511369e7e4f02c6/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5075b405006415ea148a992d093699c66eb01952bf59f4d5727089a98bda45a4", size = 3346235, upload-time = "2026-04-27T14:43:12.377Z" }, + { url = "https://files.pythonhosted.org/packages/71/89/39b6b8fc073fb6d413d0147aa333dc7eff7be65639ac9d19930a0b21bf33/tokenizers-0.23.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:56f3a77de629917652f876294dc9fe6bad4a0c43bc229dc72e59bb23a0f4729a", size = 3426398, upload-time = "2026-04-27T14:43:07.264Z" }, + { url = "https://files.pythonhosted.org/packages/0f/80/127c854da64827e5b79264ce524993a90dddcb320e5cd42412c5c02f9e8a/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9d10a6d957ef01896dc274e890eee27d41bd0e74ef31e60616f0fc311345184e", size = 9823279, upload-time = "2026-04-27T14:43:17.222Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ba/44c2502feb1a058f096ddfb4e0996ef3225a01a388e1a9b094e91689fe93/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1974288a609c343774f1b897c8b482c791ab17b75ab5c8c2b1737565c1d82288", size = 9644986, upload-time = "2026-04-27T14:43:19.45Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c1/464019a9fb059870bfe4eebb4ba12208f3042035e258bf5e782906bd3847/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:120468fb4c24faf0543c835a4fabafa4deb3f20a035c9b6e83d0b553a97615d4", size = 9976181, upload-time = "2026-04-27T14:43:21.463Z" }, + { url = "https://files.pythonhosted.org/packages/79/94/3ac1432bda31626071e9b6a12709b97ae05131c804b94c8f3ac622c5da32/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e3d8f40ea6268047de7046906326abed5134f27d4e8447b23763afe5808c8a96", size = 10113853, upload-time = "2026-04-27T14:43:23.617Z" }, + { url = "https://files.pythonhosted.org/packages/6a/dd/631b21433c771b1382535326f0eca80b9c9cee2e64961dd993bc9ac4669e/tokenizers-0.23.1-cp310-abi3-win32.whl", hash = "sha256:93120a930b919416da7cd10a2f606ac9919cc69cacae7980fa2140e277660948", size = 2536263, upload-time = "2026-04-27T14:43:29.888Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/2553f72aaf65a2797d4229e37fa7fbe38ffbf3e32912d31bdd78b3323e59/tokenizers-0.23.1-cp310-abi3-win_amd64.whl", hash = "sha256:e7bfaf995c1bdbbd21d13539decb6650967013759318627d85daeb7881af16b7", size = 2798223, upload-time = "2026-04-27T14:43:28.51Z" }, + { url = "https://files.pythonhosted.org/packages/cd/2b/2be299bab55fc595e3d38567edb1a87f86e594842968fa9515a07bdcf422/tokenizers-0.23.1-cp310-abi3-win_arm64.whl", hash = "sha256:a26197957d8e4425dfba746315f3c425ea00cfa8367c5fbc4ec73447893dcea9", size = 2664127, upload-time = "2026-04-27T14:43:26.949Z" }, +] + [[package]] name = "tomli" version = "2.4.1"