scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-25 01:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 81

fix(tokenizer): preserve Python SDK behavior with Rust tokenizers

Browse source
This commit is contained in:
Yujong Lee 2026-09-20 16:48:46 -07:00
parent a6bfb917f1
commit 94fe6b3c9a
114 changed files with 1304 additions and 518 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

258
litellm-rust/Cargo.lock generated
View file

@ -816,19 +816,6 @@ version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414"
[[package]]
name = "console"
version = "0.15.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
dependencies = [
"encode_unicode",
"libc",
"once_cell",
"unicode-width",
"windows-sys 0.59.0",
]
[[package]]
name = "const-oid"
version = "0.10.2"
@ -1140,27 +1127,6 @@ dependencies = [
"ctutils",
]
[[package]]
name = "dirs"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.61.2",
]
[[package]]
name = "displaydoc"
version = "0.2.6"
@ -1190,12 +1156,6 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
[[package]]
name = "encode_unicode"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
[[package]]
name = "equivalent"
version = "1.0.2"
@ -1553,26 +1513,6 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hf-hub"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97"
dependencies = [
"dirs",
"http 1.4.2",
"indicatif",
"libc",
"log",
"rand 0.9.5",
"reqwest 0.12.28",
"serde",
"serde_json",
"thiserror 2.0.19",
"ureq",
"windows-sys 0.60.2",
]
[[package]]
name = "hmac"
version = "0.13.0"
@ -1732,7 +1672,7 @@ dependencies = [
"tokio",
"tokio-rustls 0.26.4",
"tower-service",
"webpki-roots 1.0.9",
"webpki-roots",
]
[[package]]
@ -1914,19 +1854,6 @@ dependencies = [
"serde_core",
]
[[package]]
name = "indicatif"
version = "0.17.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
dependencies = [
"console",
"number_prefix",
"portable-atomic",
"unicode-width",
"web-time",
]
[[package]]
name = "ipnet"
version = "2.12.0"
@ -2039,15 +1966,6 @@ version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "libredox"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a"
dependencies = [
"libc",
]
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
@ -2264,7 +2182,7 @@ dependencies = [
"thiserror 2.0.19",
"tokio",
"veil",
"webpki-roots 1.0.9",
"webpki-roots",
]
[[package]]
@ -2559,12 +2477,6 @@ dependencies = [
"autocfg",
]
[[package]]
name = "number_prefix"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "once_cell"
version = "1.21.4"
@ -2605,12 +2517,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
[[package]]
name = "option-ext"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "outref"
version = "0.5.2"
@ -3136,16 +3042,6 @@ dependencies = [
"bitflags",
]
[[package]]
name = "redox_users"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0"
dependencies = [
"libredox",
"thiserror 2.0.19",
]
[[package]]
name = "ref-cast"
version = "1.0.27"
@ -3247,7 +3143,7 @@ dependencies = [
"wasm-bindgen-futures",
"wasm-streams 0.4.2",
"web-sys",
"webpki-roots 1.0.9",
"webpki-roots",
]
[[package]]
@ -3389,7 +3285,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
dependencies = [
"aws-lc-rs",
"log",
"once_cell",
"ring",
"rustls-pki-types",
@ -3787,17 +3682,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "socks"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
dependencies = [
"byteorder",
"libc",
"winapi",
]
[[package]]
name = "spm_precompiled"
version = "0.1.4"
@ -4068,7 +3952,6 @@ dependencies = [
"derive_builder",
"esaxx-rs",
"getrandom 0.3.4",
"hf-hub",
"itertools 0.14.0",
"log",
"macro_rules_attribute",
@ -4402,12 +4285,6 @@ version = "1.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
[[package]]
name = "unicode-width"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
[[package]]
name = "unicode_categories"
version = "0.1.1"
@ -4420,25 +4297,6 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "ureq"
version = "2.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d"
dependencies = [
"base64 0.22.1",
"flate2",
"log",
"once_cell",
"rustls 0.23.42",
"rustls-pki-types",
"serde",
"serde_json",
"socks",
"url",
"webpki-roots 0.26.11",
]
[[package]]
name = "url"
version = "2.5.8"
@ -4666,15 +4524,6 @@ dependencies = [
"rustls-pki-types",
]
[[package]]
name = "webpki-roots"
version = "0.26.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
dependencies = [
"webpki-roots 1.0.9",
]
[[package]]
name = "webpki-roots"
version = "1.0.9"
@ -4780,25 +4629,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.5",
"windows-targets",
]
[[package]]
@ -4816,31 +4647,14 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
@ -4849,96 +4663,48 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "1.0.4"

2
litellm-rust/Cargo.toml
View file

@ -48,7 +48,7 @@ serde_with = { version = "=3.16.1", default-features = false, features = ["std",
sha2 = "0.10"
subtle = "2"
thiserror = "2.0"
tokenizers = { version = "0.23.1", default-features = false, features = ["onig", "http", "rustls-tls"] }
tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
tiktoken-rs = "0.12.0"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] }
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }

10
litellm-rust/crates/python-bridge/src/lib.rs
View file

@ -6,6 +6,7 @@ mod marshal;
mod python_settings;
mod routes;
mod token_counter;
mod tokenizer;
#[pymodule(gil_used = true)]
mod _native {
@ -29,7 +30,12 @@ mod _native {
#[pymodule_export]
use crate::routes::responses::ResponsesWebSocketConnection;
#[pymodule_export]
use crate::token_counter::{TokenCounter, Tokenizer, tiktoken_encoding_for_model};
use crate::token_counter::TokenCounter;
#[cfg(feature = "huggingface")]
#[pymodule_export]
use crate::tokenizer::HuggingFaceEncoding;
#[pymodule_export]
use crate::tokenizer::{Tokenizer, tiktoken_encoding_for_model};
#[pymodule_export]
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
}
@ -71,6 +77,8 @@ mod tests {
"process_state_started",
"reserve_process_for_forking",
];
#[cfg(feature = "huggingface")]
expected.push("HuggingFaceEncoding");
expected.sort_unstable();
let mut public_names: Vec<String> = native_module(py)

132
litellm-rust/crates/python-bridge/src/token_counter.rs
View file

@ -3,10 +3,11 @@ use std::sync::Arc;
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
use std::{num::NonZero, thread::available_parallelism};
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
use litellm_host_python::release_gil;
use litellm_host_python::run_async;
use litellm_token_counter::{
CountableRequest, Error, InputTokenCount, TextCodec, TokenCounter as CoreTokenCounter,
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
};
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
@ -28,11 +29,6 @@ pub(crate) struct TokenCounter {
encode_slots: Arc<Semaphore>,
}
#[pyclass(frozen, name = "Tokenizer")]
pub(crate) struct Tokenizer {
inner: Arc<dyn TextCodec>,
}
#[pymethods]
impl TokenCounter {
#[new]
@ -119,128 +115,6 @@ impl TokenCounter {
}
}
#[pymethods]
impl Tokenizer {
#[staticmethod]
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
#[cfg(feature = "tiktoken")]
{
let encoding = encoding.to_owned();
Self::load(py, move || {
litellm_token_counter::tiktoken::TiktokenTokenizer::from_name(&encoding)
.map_err(Error::from)
})
}
#[cfg(not(feature = "tiktoken"))]
{
let _ = (py, encoding);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the tiktoken feature",
))
}
}
#[staticmethod]
fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
#[cfg(feature = "huggingface")]
{
let tokenizer_json = tokenizer_json.to_owned();
Self::load(py, move || {
litellm_token_counter::huggingface::HuggingFaceTokenizer::from_json(&tokenizer_json)
.map_err(Error::from)
})
}
#[cfg(not(feature = "huggingface"))]
{
let _ = (py, tokenizer_json);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the huggingface feature",
))
}
}
#[staticmethod]
#[pyo3(signature = (identifier, revision = "main", token = None))]
fn from_pretrained(
py: Python<'_>,
identifier: &str,
revision: &str,
token: Option<&str>,
) -> PyResult<Self> {
#[cfg(feature = "huggingface")]
{
let identifier = identifier.to_owned();
let revision = revision.to_owned();
let token = token.map(str::to_owned);
Self::load(py, move || {
litellm_token_counter::huggingface::HuggingFaceTokenizer::from_pretrained(
&identifier,
&revision,
token.as_deref(),
)
.map_err(Error::from)
})
}
#[cfg(not(feature = "huggingface"))]
{
let _ = (py, identifier, revision, token);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the huggingface feature",
))
}
}
fn encode(&self, py: Python<'_>, text: &str) -> PyResult<Vec<u32>> {
let inner = Arc::clone(&self.inner);
let text = text.to_owned();
release_gil(py, move || inner.encode(&text)).map_err(token_count_error_to_pyerr)
}
#[pyo3(signature = (ids, skip_special_tokens = true))]
fn decode(&self, py: Python<'_>, ids: Vec<u32>, skip_special_tokens: bool) -> PyResult<String> {
let inner = Arc::clone(&self.inner);
release_gil(py, move || inner.decode(&ids, skip_special_tokens))
.map_err(token_count_error_to_pyerr)
}
fn count(&self, py: Python<'_>, text: &str) -> PyResult<usize> {
let inner = Arc::clone(&self.inner);
let text = text.to_owned();
release_gil(py, move || inner.count_tokens(&text)).map_err(token_count_error_to_pyerr)
}
#[getter]
fn name(&self) -> &str {
self.inner.name()
}
}
impl Tokenizer {
#[cfg(any(feature = "huggingface", feature = "tiktoken"))]
fn load<T>(py: Python<'_>, load: impl FnOnce() -> Result<T, Error> + Send) -> PyResult<Self>
where
T: TextCodec + 'static,
{
let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?;
Ok(Self {
inner: Arc::new(inner),
})
}
}
#[pyfunction]
pub(crate) fn tiktoken_encoding_for_model(model: &str) -> Option<String> {
#[cfg(feature = "tiktoken")]
{
litellm_token_counter::tiktoken::encoding_for_model(model).map(str::to_owned)
}
#[cfg(not(feature = "tiktoken"))]
{
let _ = model;
None
}
}
impl TokenCounter {
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
fn load(
@ -265,7 +139,7 @@ fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount
counter.count_request(&request)
}
fn token_count_error_to_pyerr(error: Error) -> PyErr {
pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr {
let message = error.to_string();
match error {
Error::Load(_)

395
litellm-rust/crates/python-bridge/src/tokenizer.rs Normal file
View file

@ -0,0 +1,395 @@
#[cfg(feature = "tiktoken")]
use std::collections::HashSet;
use std::sync::Arc;
use litellm_host_python::release_gil;
#[cfg(feature = "huggingface")]
use litellm_token_counter::Error;
use litellm_token_counter::TextCodec;
use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString};
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
use pyo3::exceptions::PyValueError;
#[cfg(feature = "huggingface")]
use pyo3::{exceptions::PyIOError, types::PyDict};
#[cfg(feature = "tiktoken")]
use pyo3::{
exceptions::{PyKeyError, PyRuntimeError},
types::PyBytes,
};
#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))]
use crate::errors::RustBridgeDeclined;
use crate::token_counter::token_count_error_to_pyerr;
#[cfg(feature = "huggingface")]
use litellm_token_counter::huggingface::{
EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence,
};
#[cfg(feature = "tiktoken")]
use litellm_token_counter::tiktoken::TiktokenTokenizer;
enum Codec {
#[cfg(feature = "tiktoken")]
Tiktoken(TiktokenTokenizer),
#[cfg(feature = "huggingface")]
HuggingFace(HuggingFaceTokenizer),
}
impl Codec {
fn codec(&self) -> &dyn TextCodec {
match *self {
#[cfg(feature = "tiktoken")]
Self::Tiktoken(ref tokenizer) => tokenizer,
#[cfg(feature = "huggingface")]
Self::HuggingFace(ref tokenizer) => tokenizer,
}
}
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Tokenizer {
inner: Arc<Codec>,
}
#[pymethods]
impl Tokenizer {
#[staticmethod]
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
#[cfg(feature = "tiktoken")]
{
let tokenizer = release_gil(py, || TiktokenTokenizer::from_name(encoding))
.map_err(|error| token_count_error_to_pyerr(error.into()))?;
Ok(Self {
inner: Arc::new(Codec::Tiktoken(tokenizer)),
})
}
#[cfg(not(feature = "tiktoken"))]
{
let _ = (py, encoding);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the tiktoken feature",
))
}
}
#[staticmethod]
fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
#[cfg(feature = "huggingface")]
{
let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json))
.map_err(|error| token_count_error_to_pyerr(error.into()))?;
Ok(Self {
inner: Arc::new(Codec::HuggingFace(tokenizer)),
})
}
#[cfg(not(feature = "huggingface"))]
{
let _ = (py, tokenizer_json);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the huggingface feature",
))
}
}
#[staticmethod]
#[pyo3(signature = (identifier, revision = "main", token = None))]
fn from_pretrained(
py: Python<'_>,
identifier: &str,
revision: &str,
token: Option<&str>,
) -> PyResult<Self> {
#[cfg(feature = "huggingface")]
{
let kwargs = PyDict::new(py);
kwargs.set_item("repo_id", identifier)?;
kwargs.set_item("filename", "tokenizer.json")?;
kwargs.set_item("revision", revision)?;
kwargs.set_item("token", token)?;
let path: String = PyModule::import(py, "huggingface_hub")?
.getattr("hf_hub_download")?
.call((), Some(&kwargs))?
.extract()?;
let json =
release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?;
Self::from_json(py, &json)
}
#[cfg(not(feature = "huggingface"))]
{
let _ = (py, identifier, revision, token);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the huggingface feature",
))
}
}
fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult<Vec<u32>> {
let text = self.text(text)?;
release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr)
}
#[pyo3(signature = (ids, skip_special_tokens = true))]
fn decode(&self, py: Python<'_>, ids: Vec<u32>, skip_special_tokens: bool) -> PyResult<String> {
release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens))
.map_err(token_count_error_to_pyerr)
}
fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult<usize> {
let text = self.text(text)?;
release_gil(py, || self.inner.codec().count_tokens(&text))
.map_err(token_count_error_to_pyerr)
}
#[getter]
fn name(&self) -> &str {
self.inner.codec().name()
}
#[cfg(feature = "tiktoken")]
fn encode_special(
&self,
py: Python<'_>,
text: &Bound<'_, PyString>,
allowed: Vec<String>,
) -> PyResult<Vec<u32>> {
let text = self.text(text)?;
match *self.inner {
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(ref tokenizer) => {
release_gil(py, || tokenizer.encode_special(&text, &allowed))
.map_err(PyRuntimeError::new_err)
}
#[cfg(feature = "huggingface")]
Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")),
}
}
#[cfg(feature = "tiktoken")]
fn special_tokens(&self, py: Python<'_>) -> PyResult<HashSet<String>> {
match *self.inner {
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(ref tokenizer) => Ok(release_gil(py, || tokenizer.special_tokens())),
#[cfg(feature = "huggingface")]
Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")),
}
}
#[cfg(feature = "tiktoken")]
fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec<u32>) -> PyResult<Bound<'py, PyBytes>> {
match *self.inner {
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(ref tokenizer) => {
let bytes = release_gil(py, || tokenizer.decode_bytes(&ids))
.map_err(PyKeyError::new_err)?;
Ok(PyBytes::new(py, &bytes))
}
#[cfg(feature = "huggingface")]
Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")),
}
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))]
fn encode_huggingface(
&self,
py: Python<'_>,
sequence: Sequence,
pair: Option<Sequence>,
is_pretokenized: bool,
add_special_tokens: bool,
fast: bool,
) -> PyResult<HuggingFaceEncoding> {
let sequence = sequence.input(is_pretokenized)?;
let input = match pair {
Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?),
None => EncodeInput::Single(sequence),
};
match *self.inner {
Codec::HuggingFace(ref tokenizer) => release_gil(py, || {
tokenizer.encode_result(input, add_special_tokens, fast)
})
.map(|inner| HuggingFaceEncoding { inner })
.map_err(|error| token_count_error_to_pyerr(Error::from(error))),
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")),
}
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))]
fn encode_batch_huggingface(
&self,
py: Python<'_>,
inputs: Vec<(Sequence, Option<Sequence>)>,
is_pretokenized: bool,
add_special_tokens: bool,
fast: bool,
) -> PyResult<Vec<HuggingFaceEncoding>> {
let inputs = inputs
.into_iter()
.map(|(sequence, pair)| {
let sequence = sequence.input(is_pretokenized)?;
match pair {
Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)),
None => Ok(EncodeInput::Single(sequence)),
}
})
.collect::<PyResult<Vec<_>>>()?;
match *self.inner {
Codec::HuggingFace(ref tokenizer) => release_gil(py, || {
tokenizer.encode_batch_result(inputs, add_special_tokens, fast)
})
.map(|encodings| {
encodings
.into_iter()
.map(|inner| HuggingFaceEncoding { inner })
.collect()
})
.map_err(|error| token_count_error_to_pyerr(Error::from(error))),
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")),
}
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (pretty = false))]
fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult<String> {
match *self.inner {
#[cfg(feature = "huggingface")]
Codec::HuggingFace(ref tokenizer) => release_gil(py, || tokenizer.to_json(pretty))
.map_err(|error| token_count_error_to_pyerr(Error::from(error))),
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")),
}
}
}
impl Tokenizer {
fn text(&self, text: &Bound<'_, PyString>) -> PyResult<String> {
match text.to_cow() {
Ok(text) => Ok(text.into_owned()),
Err(error)
if self.inner.codec().name() == "huggingface"
|| !error.is_instance_of::<PyUnicodeEncodeError>(text.py()) =>
{
Err(error)
}
Err(_) => text
.call_method1("encode", ("utf-16", "surrogatepass"))?
.call_method1("decode", ("utf-16", "replace"))?
.extract(),
}
}
}
#[cfg(feature = "huggingface")]
#[derive(FromPyObject)]
pub(crate) enum Sequence {
Text(String),
Words(Vec<String>),
}
#[cfg(feature = "huggingface")]
impl Sequence {
fn input(self, is_pretokenized: bool) -> PyResult<InputSequence<'static>> {
match (self, is_pretokenized) {
(Self::Text(text), false) => Ok(text.into()),
(Self::Words(words), true) => Ok(words.into()),
_ => Err(pyo3::exceptions::PyTypeError::new_err(
"input must match is_pretokenized",
)),
}
}
}
#[cfg(feature = "huggingface")]
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct HuggingFaceEncoding {
inner: Encoding,
}
#[cfg(feature = "huggingface")]
#[pymethods]
impl HuggingFaceEncoding {
#[new]
#[pyo3(signature = (json = None))]
fn new(json: Option<&str>) -> PyResult<Self> {
let inner = match json {
Some(json) => serde_json::from_str(json)
.map_err(|error| PyValueError::new_err(error.to_string()))?,
None => Encoding::default(),
};
Ok(Self { inner })
}
fn __reduce__<'py>(
&self,
py: Python<'py>,
) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> {
let json = serde_json::to_string(&self.inner)
.map_err(|error| PyValueError::new_err(error.to_string()))?;
Ok((py.get_type::<Self>(), (json,)))
}
fn __len__(&self) -> usize {
self.inner.len()
}
#[getter]
fn ids(&self) -> Vec<u32> {
self.inner.get_ids().to_vec()
}
#[getter]
fn tokens(&self) -> Vec<String> {
self.inner.get_tokens().to_vec()
}
#[getter]
fn offsets(&self) -> Vec<(usize, usize)> {
self.inner.get_offsets().to_vec()
}
#[getter]
fn type_ids(&self) -> Vec<u32> {
self.inner.get_type_ids().to_vec()
}
#[getter]
fn attention_mask(&self) -> Vec<u32> {
self.inner.get_attention_mask().to_vec()
}
#[getter]
fn special_tokens_mask(&self) -> Vec<u32> {
self.inner.get_special_tokens_mask().to_vec()
}
#[getter]
fn word_ids(&self) -> Vec<Option<u32>> {
self.inner.get_word_ids().to_vec()
}
#[getter]
fn sequence_ids(&self) -> Vec<Option<usize>> {
self.inner.get_sequence_ids()
}
#[getter]
fn overflowing(&self) -> Vec<Self> {
self.inner
.get_overflowing()
.iter()
.cloned()
.map(|inner| Self { inner })
.collect()
}
#[getter]
fn n_sequences(&self) -> usize {
self.inner.n_sequences()
}
}
#[pyfunction]
pub(crate) fn tiktoken_encoding_for_model(model: &str) -> Option<String> {
#[cfg(feature = "tiktoken")]
{
litellm_token_counter::tiktoken::encoding_for_model(model).map(str::to_owned)
}
#[cfg(not(feature = "tiktoken"))]
{
let _ = model;
None
}
}

56
litellm-rust/crates/token-counter-huggingface/src/lib.rs
View file

@ -5,6 +5,7 @@ mod error;
use std::collections::HashSet;
pub use error::Error;
pub use tokenizers::{EncodeInput, Encoding, InputSequence};
pub struct HuggingFaceTokenizer {
tokenizer: Box<tokenizers::Tokenizer>,
@ -18,23 +19,6 @@ impl HuggingFaceTokenizer {
.map_err(Error::Load)
}
pub fn from_pretrained(
identifier: &str,
revision: &str,
token: Option<&str>,
) -> Result<Self, Error> {
tokenizers::Tokenizer::from_pretrained(
identifier,
Some(tokenizers::FromPretrainedParameters {
revision: revision.to_owned(),
token: token.map(str::to_owned),
..Default::default()
}),
)
.map(Self::new)
.map_err(Error::Download)
}
fn new(tokenizer: tokenizers::Tokenizer) -> Self {
let special_token_ids: HashSet<u32> = tokenizer
.get_added_tokens_decoder()
@ -61,6 +45,44 @@ impl HuggingFaceTokenizer {
.map_err(Error::Encode)
}
pub fn encode_result<'a>(
&self,
input: EncodeInput<'a>,
add_special_tokens: bool,
fast: bool,
) -> Result<Encoding, Error> {
if fast {
return self
.tokenizer
.encode_fast(input, add_special_tokens)
.map_err(Error::Encode);
}
self.tokenizer
.encode_char_offsets(input, add_special_tokens)
.map_err(Error::Encode)
}
pub fn encode_batch_result<'a>(
&self,
inputs: Vec<EncodeInput<'a>>,
add_special_tokens: bool,
fast: bool,
) -> Result<Vec<Encoding>, Error> {
if fast {
return self
.tokenizer
.encode_batch_fast(inputs, add_special_tokens)
.map_err(Error::Encode);
}
self.tokenizer
.encode_batch_char_offsets(inputs, add_special_tokens)
.map_err(Error::Encode)
}
pub fn to_json(&self, pretty: bool) -> Result<String, Error> {
self.tokenizer.to_string(pretty).map_err(Error::Load)
}
pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
if !skip_special_tokens {
return self.tokenizer.decode(ids, false).map_err(Error::Decode);

44
litellm-rust/crates/token-counter-tiktoken/src/lib.rs
View file

@ -2,6 +2,8 @@
mod error;
use std::collections::HashSet;
pub use error::UnsupportedTokenizer;
pub struct TiktokenTokenizer {
@ -35,8 +37,33 @@ impl TiktokenTokenizer {
self.encoder.encode_ordinary(text)
}
pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result<Vec<u32>, String> {
let allowed = allowed.iter().map(String::as_str).collect();
self.encoder
.encode(text, &allowed)
.map(|(ids, _)| ids)
.map_err(|error| error.to_string())
}
pub fn special_tokens(&self) -> HashSet<String> {
self.encoder
.special_tokens()
.into_iter()
.map(str::to_owned)
.collect()
}
pub fn decode_bytes(&self, ids: &[u32]) -> Result<Vec<u8>, String> {
self.encoder
.decode_bytes(ids)
.map_err(|error| error.to_string())
}
pub fn decode(&self, ids: &[u32]) -> Result<String, String> {
self.encoder.decode(ids).map_err(|error| error.to_string())
self.encoder
.decode_bytes(ids)
.map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
.map_err(|error| error.to_string())
}
pub fn name(&self) -> &str {
@ -128,6 +155,21 @@ mod tests {
}
}
#[test]
fn decoding_token_prefixes_replaces_incomplete_utf8() {
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
let reference = tiktoken_rs::cl100k_base_singleton();
let ids = tokenizer.encode("🙂漢字");
for end in 1..=ids.len() {
let bytes = reference.decode_bytes(&ids[..end]).unwrap();
assert_eq!(
tokenizer.decode(&ids[..end]).unwrap(),
String::from_utf8_lossy(&bytes),
);
}
assert!(tokenizer.decode(&[u32::MAX]).is_err());
}
#[test]
fn encoding_for_model_maps_known_models() {
assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base"));

4
litellm-rust/crates/token-counter/src/huggingface.rs
View file

@ -1,5 +1,7 @@
use litellm_token_counter_huggingface::Error as BackendError;
pub use litellm_token_counter_huggingface::HuggingFaceTokenizer;
pub use litellm_token_counter_huggingface::{
EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence,
};
use crate::{Error, TextCodec, TokenCounter, Tokenizer};

2
litellm/_lazy_imports.py
View file

@ -59,7 +59,7 @@ from ._lazy_imports_registry import (
if TYPE_CHECKING:
import httpx
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
def get_litellm_globals() -> dict[str, object]:

2
litellm/litellm_core_utils/default_encoding.py
View file

@ -28,6 +28,6 @@ def o200k_base_rank_file() -> str:
return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii")
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
encoding: Final = Tokenizer.from_tiktoken("cl100k_base")

19
litellm/litellm_core_utils/token_counter.py
View file

@ -29,9 +29,12 @@ from litellm.constants import (
TOKEN_COUNTER_MAX_EXACT_CHARS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer as Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding
from litellm.litellm_core_utils.url_utils import safe_get
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.rust_bridge._native import Tokenizer, tiktoken_encoding_for_model
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
from litellm.rust_bridge._native import tiktoken_encoding_for_model
from litellm.types.llms.anthropic import (
AnthropicContentParamSource,
AnthropicContentParamSourceFileId,
@ -620,10 +623,14 @@ def _get_exact_count_function(
if model is not None or custom_tokenizer is not None:
tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model)
if tokenizer_json["type"] == "huggingface_tokenizer":
tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"]
tokenizer: Final[Tokenizer | NativeTokenizer] = tokenizer_json["tokenizer"]
def count_tokens(text: str) -> int:
return tokenizer.count(text)
return (
tokenizer.count(text)
if isinstance(tokenizer, NativeTokenizer)
else len(tokenizer.encode_batch_fast([text])[0])
)
return count_tokens
elif tokenizer_json["type"] == "openai_tokenizer":
@ -644,11 +651,11 @@ def _get_exact_count_function(
@lru_cache(maxsize=8)
def _native_tokenizer_for_encoding(name: str) -> Tokenizer:
return Tokenizer.from_tiktoken(name)
def _native_tokenizer_for_encoding(name: str) -> OpenAIEncoding:
return OpenAIEncoding.from_tiktoken(name)
def openai_tokenizer_encoding(model: str) -> Tokenizer:
def openai_tokenizer_encoding(model: str) -> OpenAIEncoding:
"""The native encoding `token_counter` uses for a model on the `openai_tokenizer` path."""
from litellm.utils import print_verbose

228
litellm/litellm_core_utils/tokenizer.py Normal file
View file

@ -0,0 +1,228 @@
from __future__ import annotations
from collections.abc import Callable, Collection, Mapping, Sequence, Set
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from functools import partial
from pathlib import Path
from typing import Final, Literal, Protocol, TypeAlias, runtime_checkable
from litellm.rust_bridge._native import HuggingFaceEncoding
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
SpecialTokens: TypeAlias = Literal["all"] | Collection[str]
HuggingFaceInput: TypeAlias = str | list[str] | tuple[str, ...]
HuggingFaceBatchInput: TypeAlias = HuggingFaceInput | tuple[HuggingFaceInput, HuggingFaceInput] | list[HuggingFaceInput]
@dataclass(frozen=True, slots=True)
class OpenAIEncoding:
name: str
_native: NativeTokenizer
_special_tokens: frozenset[str]
@staticmethod
def from_tiktoken(encoding: str) -> OpenAIEncoding:
native: Final = NativeTokenizer.from_tiktoken(encoding)
return OpenAIEncoding(encoding, native, frozenset(native.special_tokens()))
@property
def special_tokens_set(
self,
) -> set[str]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.special_tokens()
@property
def eot_token(self) -> int:
return self.encode("<|endoftext|>", allowed_special="all")[0]
def encode(
self,
text: str,
*,
allowed_special: Literal["all"] | Set[str] = frozenset(),
disallowed_special: SpecialTokens = "all",
) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type
allowed: Final = self._special_tokens if allowed_special == "all" else frozenset(allowed_special)
disallowed: Final = (
self._special_tokens - allowed if disallowed_special == "all" else frozenset(disallowed_special)
)
for special in disallowed:
if special in text:
raise ValueError(f"Encountered text corresponding to disallowed special token {special!r}")
if not allowed:
return self.encode_ordinary(text)
return self._native.encode_special(text, tuple(allowed))
def encode_ordinary(self, text: str) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.encode(text)
def count(self, text: str) -> int:
return self._native.count(text)
def encode_batch(
self,
text: Sequence[str],
*,
num_threads: int = 8,
allowed_special: Literal["all"] | Set[str] = frozenset(),
disallowed_special: SpecialTokens = "all",
) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
encode: Final = partial(self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special)
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(encode, text)
)
def encode_ordinary_batch(
self, text: Sequence[str], *, num_threads: int = 8
) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(self.encode_ordinary, text)
)
def decode_bytes(self, tokens: Sequence[int]) -> bytes:
return self._native.decode_bytes(tokens)
def decode(self, tokens: Sequence[int], errors: str = "replace") -> str:
return self.decode_bytes(tokens).decode("utf-8", errors=errors)
def decode_single_token_bytes(self, token: int) -> bytes:
return self.decode_bytes((token,))
def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
return [ # mutable-ok: [LIT002] SDK returns a list
self.decode_single_token_bytes(token) for token in tokens
]
def decode_batch(
self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8
) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(partial(self.decode, errors=errors), batch)
)
def decode_bytes_batch(
self, batch: Sequence[Sequence[int]], *, num_threads: int = 8
) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(self.decode_bytes, batch)
)
def __reduce__(self) -> tuple[Callable[[str], OpenAIEncoding], tuple[str]]:
return (OpenAIEncoding.from_tiktoken, (self.name,))
@dataclass(frozen=True, slots=True)
class HuggingFaceTokenizer:
_native: NativeTokenizer
@staticmethod
def from_str(json: str) -> HuggingFaceTokenizer:
return HuggingFaceTokenizer(NativeTokenizer.from_json(json))
from_json = from_str
@staticmethod
def from_file(path: str) -> HuggingFaceTokenizer:
return HuggingFaceTokenizer.from_str(Path(path).read_text(encoding="utf-8"))
@staticmethod
def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFaceTokenizer:
return HuggingFaceTokenizer(NativeTokenizer.from_pretrained(identifier, revision=revision, token=token))
def to_str(self, pretty: bool = False) -> str:
return self._native.to_json(pretty)
def save(self, path: str, pretty: bool = True) -> None:
Path(path).write_text(self.to_str(pretty), encoding="utf-8")
@property
def name(self) -> str:
return self._native.name
def encode(
self,
sequence: HuggingFaceInput,
pair: HuggingFaceInput | None = None,
is_pretokenized: bool = False,
add_special_tokens: bool = True,
) -> HuggingFaceEncoding:
return self._native.encode_huggingface(sequence, pair, is_pretokenized, add_special_tokens)
def encode_batch(
self,
input: Sequence[HuggingFaceBatchInput],
is_pretokenized: bool = False,
add_special_tokens: bool = True,
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=False)
def encode_batch_fast(
self,
input: Sequence[HuggingFaceBatchInput],
is_pretokenized: bool = False,
add_special_tokens: bool = True,
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=True)
def _encode_batch(
self, input: Sequence[HuggingFaceBatchInput], is_pretokenized: bool, add_special_tokens: bool, fast: bool
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
sequences: Final = tuple(_batch_input(item, is_pretokenized) for item in input)
return self._native.encode_batch_huggingface(sequences, is_pretokenized, add_special_tokens, fast)
def count(self, text: str) -> int:
return self._native.count(text)
def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str:
return self._native.decode(ids, skip_special_tokens=skip_special_tokens)
def decode_batch(
self, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True
) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type
return [ # mutable-ok: [LIT002] SDK returns a list
self.decode(ids, skip_special_tokens=skip_special_tokens) for ids in sequences
]
def __reduce__(self) -> tuple[Callable[[str], HuggingFaceTokenizer], tuple[str]]:
return (HuggingFaceTokenizer.from_str, (self.to_str(),))
def _batch_input(
item: HuggingFaceBatchInput, is_pretokenized: bool
) -> tuple[HuggingFaceInput, HuggingFaceInput | None]:
if isinstance(item, str):
return (item, None)
if is_pretokenized and all(isinstance(word, str) for word in item):
return (tuple(word for word in item if isinstance(word, str)), None)
if len(item) != 2:
raise TypeError("batch input must be a sequence or a pair of sequences")
return (item[0], item[1])
Tokenizer: TypeAlias = OpenAIEncoding | HuggingFaceTokenizer
class _AddedToken(Protocol):
@property
def special(self) -> bool: ...
@runtime_checkable
class _AddedTokenDecoder(Protocol):
def get_added_tokens_decoder(self) -> Mapping[int, _AddedToken]: ...
def strip_special_tokens(tokenizer: object, tokens: Sequence[int]) -> Sequence[int]:
if not isinstance(tokenizer, _AddedTokenDecoder):
return tokens
try:
added: Final = tokenizer.get_added_tokens_decoder()
except Exception: # noqa: BLE001 # optional metadata failures historically fall back to decoding
return tokens
special_ids: Final = frozenset(token_id for token_id, token in added.items() if token.special)
return tuple(token for token in tokens if token not in special_ids)

2
litellm/llms/a2a/chat/transformation.py
View file

@ -24,7 +24,7 @@ from .streaming_iterator import A2AModelResponseIterator
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
_REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = (

2
litellm/llms/aiml/image_generation/transformation.py
View file

@ -15,7 +15,7 @@ from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/aiohttp_openai/chat/transformation.py
View file

@ -17,7 +17,7 @@ from litellm.types.utils import Choices, ModelResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/amazon_nova/chat/transformation.py
View file

@ -17,7 +17,7 @@ from litellm.types.utils import ModelResponse
from ...openai_like.chat.transformation import OpenAILikeChatConfig
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class AmazonNovaChatConfig(OpenAILikeChatConfig):

2
litellm/llms/anthropic/batches/transformation.py
View file

@ -15,7 +15,7 @@ from litellm.types.utils import LiteLLMBatch, LlmProviders, ModelResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:

2
litellm/llms/anthropic/chat/transformation.py
View file

@ -101,7 +101,7 @@ from ..common_utils import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:

2
litellm/llms/anthropic/completion/transformation.py
View file

@ -33,7 +33,7 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class AnthropicTextError(BaseLLMException):

2
litellm/llms/azure/chat/gpt_transformation.py
View file

@ -30,7 +30,7 @@ from ..common_utils import AzureOpenAIError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:

2
litellm/llms/azure_ai/agents/transformation.py
View file

@ -35,8 +35,8 @@ from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/azure_ai/azure_model_router/transformation.py
View file

@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class AzureModelRouterConfig(AzureAIStudioConfig):

2
litellm/llms/azure_ai/chat/transformation.py
View file

@ -30,7 +30,7 @@ from litellm.types.utils import ModelResponse, ProviderField
from litellm.utils import _add_path_to_api_base, supports_tool_choice
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class AzureFoundryErrorStrings(str, enum.Enum):

2
litellm/llms/azure_ai/image_generation/mai_transformation.py
View file

@ -14,7 +14,7 @@ from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):

2
litellm/llms/base_llm/audio_transcription/transformation.py
View file

@ -13,7 +13,7 @@ from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/base_llm/bridges/completion_transformation.py
View file

@ -10,8 +10,8 @@ if TYPE_CHECKING:
from pydantic import BaseModel
from litellm import LiteLLMLoggingObj, ModelResponse
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.rust_bridge._native import Tokenizer
from litellm.types.llms.openai import AllMessageValues

2
litellm/llms/base_llm/chat/transformation.py
View file

@ -22,7 +22,7 @@ from litellm.types.llms.openai import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
from litellm.types.utils import ModelResponse
from ..base_utils import (

2
litellm/llms/base_llm/completion/transformation.py
View file

@ -9,7 +9,7 @@ from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/base_llm/embedding/transformation.py
View file

@ -9,7 +9,7 @@ from litellm.types.utils import EmbeddingResponse, ModelResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/base_llm/files/transformation.py
View file

@ -22,8 +22,8 @@ from ..chat.transformation import BaseConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
from litellm.router import Router as _Router
from litellm.rust_bridge._native import Tokenizer
from litellm.types.llms.openai import HttpxBinaryResponseContent
LiteLLMLoggingObj = _LiteLLMLoggingObj

2
litellm/llms/base_llm/image_generation/transformation.py
View file

@ -12,7 +12,7 @@ from litellm.types.utils import ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/base_llm/image_variations/transformation.py
View file

@ -18,7 +18,7 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/bedrock/chat/agentcore/transformation.py
View file

@ -41,8 +41,8 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -99,7 +99,7 @@ from ..common_utils import (
)
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
# Computer use tool prefixes supported by Bedrock
BEDROCK_COMPUTER_USE_TOOLS: Final = [

2
litellm/llms/bedrock/chat/invoke_agent/transformation.py
View file

@ -38,7 +38,7 @@ from litellm.types.utils import Choices, Message, ModelResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py
View file

@ -25,7 +25,7 @@ from litellm.types.utils import (
from .amazon_llama_transformation import AmazonLlamaConfig
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class AmazonDeepSeekR1Config(AmazonLlamaConfig):

2
litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py
View file

@ -22,7 +22,7 @@ from litellm.types.utils import Choices
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
from litellm.types.utils import ModelResponse
LiteLLMLoggingObj = _LiteLLMLoggingObj

2
litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py
View file

@ -28,7 +28,7 @@ from ..converse_transformation import AmazonConverseConfig
from .base_invoke_transformation import AmazonInvokeConfig
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
_CachePointCarrier = TypeVar("_CachePointCarrier", SystemContentBlock, ContentBlock)
_INJECTION_POINTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])

2
litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py
View file

@ -21,7 +21,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class AmazonQwen2Config(AmazonQwen3Config):

2
litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py
View file

@ -19,7 +19,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):

2
litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py
View file

@ -26,7 +26,7 @@ from litellm.utils import get_base64_str
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -35,7 +35,7 @@ from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -35,7 +35,7 @@ from litellm.utils import CustomStreamWrapper
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/black_forest_labs/image_generation/transformation.py
View file

@ -30,7 +30,7 @@ from ..common_utils import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

14
litellm/llms/brave/search/__init__.py
View file

@ -1,7 +1,7 @@
"""
Brave Search API module.
"""
from litellm.llms.brave.search.transformation import BraveSearchConfig
__all__ = ["BraveSearchConfig"]
"""
Brave Search API module.
"""
from litellm.llms.brave.search.transformation import BraveSearchConfig
__all__ = ["BraveSearchConfig"]

2
litellm/llms/bytez/chat/transformation.py
View file

@ -24,7 +24,7 @@ from ..common_utils import API_BASE, BytezError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/clarifai/chat/transformation.py
View file

@ -14,7 +14,7 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/cohere/chat/transformation.py
View file

@ -16,7 +16,7 @@ from ..common_utils import validate_environment as cohere_validate_environment
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/cohere/chat/v2_transformation.py
View file

@ -21,7 +21,7 @@ from ..common_utils import validate_environment as cohere_validate_environment
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/cohere/embed/handler.py
View file

@ -20,7 +20,7 @@ from litellm.types.utils import EmbeddingResponse
from .v1_transformation import CohereEmbeddingConfig
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
def validate_environment(api_key, headers: dict):

2
litellm/llms/cometapi/image_generation/transformation.py
View file

@ -14,7 +14,7 @@ from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/compactifai/chat/transformation.py
View file

@ -18,7 +18,7 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/custom_httpx/aiohttp_handler.py
View file

@ -27,7 +27,7 @@ from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigMana
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -196,11 +196,11 @@ if TYPE_CHECKING:
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.rust_bridge._native import Tokenizer
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
CancelRunResponse,

2
litellm/llms/dashscope/image_generation/transformation.py
View file

@ -39,7 +39,7 @@ from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/databricks/chat/transformation.py
View file

@ -149,7 +149,7 @@ def _split_parallel_tool_calls(messages: list[AllMessageValues]) -> list[AllMess
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/fal_ai/image_generation/bria_transformation.py
View file

@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py
View file

@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py
View file

@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/fal_ai/image_generation/imagen4_transformation.py
View file

@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py
View file

@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py
View file

@ -9,7 +9,7 @@ from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/fal_ai/image_generation/transformation.py
View file

@ -14,7 +14,7 @@ from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -46,7 +46,7 @@ from ..common_utils import (
)
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
def _map_reasoning_effort(value: object) -> object:

2
litellm/llms/gemini/image_generation/transformation.py
View file

@ -25,7 +25,7 @@ from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/gigachat/chat/transformation.py
View file

@ -27,7 +27,7 @@ from ..file_handler import upload_file_sync
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/groq/chat/transformation.py
View file

@ -27,7 +27,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, ServerToolUs
from ...openai_like.chat.transformation import OpenAILikeChatConfig
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
GROQ_COMPOUND_MODELS: Final = frozenset({"compound", "compound-mini"})

2
litellm/llms/huggingface/embedding/transformation.py
View file

@ -26,7 +26,7 @@ from ..common_utils import HuggingFaceError, hf_task_list, hf_tasks, output_pars
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:

2
litellm/llms/langflow/chat/transformation.py
View file

@ -15,8 +15,8 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.rust_bridge._native import Tokenizer
from litellm.utils import CustomStreamWrapper
LiteLLMLoggingObj = _LiteLLMLoggingObj

2
litellm/llms/langgraph/chat/transformation.py
View file

@ -24,8 +24,8 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.rust_bridge._native import Tokenizer
from litellm.utils import CustomStreamWrapper
LiteLLMLoggingObj = _LiteLLMLoggingObj

2
litellm/llms/lemonade/chat/transformation.py
View file

@ -19,7 +19,7 @@ from litellm.types.utils import ModelResponse
from ...openai_like.chat.transformation import OpenAILikeChatConfig
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class LemonadeChatConfig(OpenAILikeChatConfig):

2
litellm/llms/mistral/chat/transformation.py
View file

@ -32,7 +32,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream
from litellm.utils import convert_to_model_response_object, supports_reasoning
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: str) -> str:

2
litellm/llms/nlp_cloud/chat/transformation.py
View file

@ -15,7 +15,7 @@ from ..common_utils import NLPCloudError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:

2
litellm/llms/oci/chat/transformation.py
View file

@ -66,7 +66,7 @@ from litellm.utils import supports_reasoning
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/ollama/chat/transformation.py
View file

@ -32,7 +32,7 @@ from ..common_utils import OllamaError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

4
litellm/llms/ollama/completion/transformation.py
View file

@ -36,7 +36,7 @@ from ..common_utils import OllamaError, OllamaModelInfo, _convert_image
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -331,7 +331,7 @@ class OllamaConfig(BaseConfig):
tokenizer: Final = encoding if encoding is not None else litellm.encoding
prompt_tokens: Final = response_json.get(
"prompt_eval_count",
len(tokenizer.encode(_prompt)),
len(tokenizer.encode(_prompt, disallowed_special=())),
)
completion_tokens: Final = response_json.get(
"eval_count", len(response_json.get("message", dict()).get("content", ""))

2
litellm/llms/oobabooga/chat/transformation.py
View file

@ -12,7 +12,7 @@ from ..common_utils import OobaboogaError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:

2
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -61,8 +61,8 @@ from ..common_utils import OpenAIError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.rust_bridge._native import Tokenizer
from litellm.types.llms.openai import ChatCompletionToolParam
LiteLLMLoggingObj = _LiteLLMLoggingObj

2
litellm/llms/openai/image_generation/dall_e_2_transformation.py
View file

@ -12,7 +12,7 @@ from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class DallE2ImageGenerationConfig(BaseImageGenerationConfig):

2
litellm/llms/openai/image_generation/dall_e_3_transformation.py
View file

@ -12,7 +12,7 @@ from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class DallE3ImageGenerationConfig(BaseImageGenerationConfig):

2
litellm/llms/openai/image_generation/gpt_transformation.py
View file

@ -12,7 +12,7 @@ from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class GPTImageGenerationConfig(BaseImageGenerationConfig):

2
litellm/llms/openai/image_variations/transformation.py
View file

@ -12,7 +12,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig
from ..common_utils import OpenAIError
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class OpenAIImageVariationConfig(BaseImageVariationConfig):

2
litellm/llms/openai/openai.py
View file

@ -8,7 +8,7 @@ import httpx
if TYPE_CHECKING:
from aiohttp import ClientSession
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
import openai
from openai import AsyncOpenAI, OpenAI

2
litellm/llms/openai_like/chat/transformation.py
View file

@ -14,7 +14,7 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/openrouter/chat/transformation.py
View file

@ -24,7 +24,7 @@ from ..common_utils import OpenRouterException
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class CacheControlSupportedModels(str, Enum):

2
litellm/llms/openrouter/image_generation/transformation.py
View file

@ -51,7 +51,7 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
else:
LiteLLMLoggingObj = Any

2
litellm/llms/perplexity/chat/transformation.py
View file

@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues, ChatCompletionAnnotation
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class PerplexityChatConfig(OpenAIGPTConfig):

2
litellm/llms/petals/completion/transformation.py
View file

@ -14,7 +14,7 @@ from litellm.types.utils import ModelResponse
from ..common_utils import PetalsError
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class PetalsConfig(BaseConfig):

2
litellm/llms/predibase/chat/transformation.py
View file

@ -19,7 +19,7 @@ from ..common_utils import PredibaseError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/recraft/image_generation/transformation.py
View file

@ -15,7 +15,7 @@ from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/replicate/chat/transformation.py
View file

@ -20,7 +20,7 @@ from ..common_utils import ReplicateError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:

2
litellm/llms/runwayml/image_generation/transformation.py
View file

@ -23,7 +23,7 @@ from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/sagemaker/completion/transformation.py
View file

@ -25,7 +25,7 @@ from ..common_utils import SagemakerError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/sap/chat/transformation.py
View file

@ -16,7 +16,7 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/stability/image_generation/transformation.py
View file

@ -27,7 +27,7 @@ from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/topaz/image_variations/transformation.py
View file

@ -23,7 +23,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig
from ..common_utils import TopazException, TopazModelInfo
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig):

2
litellm/llms/triton/completion/transformation.py
View file

@ -29,7 +29,7 @@ from litellm.types.utils import (
from ..common_utils import TritonError
if TYPE_CHECKING:
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
class TritonConfig(BaseConfig):

2
litellm/llms/vertex_ai/agent_engine/transformation.py
View file

@ -30,8 +30,8 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.rust_bridge._native import Tokenizer
from litellm.utils import CustomStreamWrapper
LiteLLMLoggingObj = _LiteLLMLoggingObj

2
litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py
View file

@ -25,7 +25,7 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

2
litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py
View file

@ -21,7 +21,7 @@ from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.litellm_core_utils.tokenizer import OpenAIEncoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:

Some files were not shown because too many files have changed in this diff Show more