diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index af3a31ddbfa..c886fd34711 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2279,6 +2279,7 @@ dependencies = [ name = "litellm-token-counter-huggingface" version = "0.1.0" dependencies = [ + "serde_json", "thiserror 2.0.19", "tokenizers", ] diff --git a/litellm-rust/crates/python-bridge/src/tokenizer.rs b/litellm-rust/crates/python-bridge/src/tokenizer.rs index d334131e682..92a21f7e0e1 100644 --- a/litellm-rust/crates/python-bridge/src/tokenizer.rs +++ b/litellm-rust/crates/python-bridge/src/tokenizer.rs @@ -24,7 +24,8 @@ use crate::token_counter::token_count_error_to_pyerr; #[cfg(feature = "huggingface")] use litellm_token_counter::huggingface::{ - EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, + EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, encoding_from_json, + encoding_to_json, }; #[cfg(feature = "tiktoken")] use litellm_token_counter::tiktoken::TiktokenTokenizer; @@ -315,7 +316,7 @@ impl HuggingFaceEncoding { #[pyo3(signature = (json = None))] fn new(json: Option<&str>) -> PyResult { let inner = match json { - Some(json) => serde_json::from_str(json) + Some(json) => encoding_from_json(json) .map_err(|error| PyValueError::new_err(error.to_string()))?, None => Encoding::default(), }; @@ -326,7 +327,7 @@ impl HuggingFaceEncoding { &self, py: Python<'py>, ) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> { - let json = serde_json::to_string(&self.inner) + let json = encoding_to_json(&self.inner) .map_err(|error| PyValueError::new_err(error.to_string()))?; Ok((py.get_type::(), (json,))) } diff --git a/litellm-rust/crates/token-counter-huggingface/Cargo.toml b/litellm-rust/crates/token-counter-huggingface/Cargo.toml index 6d8cb85e524..a5c2b1bb160 100644 --- a/litellm-rust/crates/token-counter-huggingface/Cargo.toml +++ b/litellm-rust/crates/token-counter-huggingface/Cargo.toml @@ -6,5 +6,6 @@ license.workspace = true repository.workspace = true [dependencies] +serde_json.workspace = true thiserror.workspace = true tokenizers.workspace = true diff --git a/litellm-rust/crates/token-counter-huggingface/src/lib.rs b/litellm-rust/crates/token-counter-huggingface/src/lib.rs index 711bd083e6d..2089dba5331 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/lib.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/lib.rs @@ -7,6 +7,14 @@ use std::collections::HashSet; pub use error::Error; pub use tokenizers::{EncodeInput, Encoding, InputSequence}; +pub fn encoding_from_json(json: &str) -> Result { + serde_json::from_str(json).map_err(|error| Error::Load(error.into())) +} + +pub fn encoding_to_json(encoding: &Encoding) -> Result { + serde_json::to_string(encoding).map_err(|error| Error::Load(error.into())) +} + pub struct HuggingFaceTokenizer { tokenizer: Box, special_token_ids: HashSet, diff --git a/litellm-rust/crates/token-counter/src/huggingface.rs b/litellm-rust/crates/token-counter/src/huggingface.rs index 662c6a6f2a7..4bce9d593d1 100644 --- a/litellm-rust/crates/token-counter/src/huggingface.rs +++ b/litellm-rust/crates/token-counter/src/huggingface.rs @@ -1,6 +1,7 @@ use litellm_token_counter_huggingface::Error as BackendError; pub use litellm_token_counter_huggingface::{ - EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, + EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, encoding_from_json, + encoding_to_json, }; use crate::{Error, TextCodec, TokenCounter, Tokenizer}; diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index ded2d652e35..54b548a8ae0 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -626,11 +626,10 @@ def _get_exact_count_function( tokenizer: Final[Tokenizer | NativeTokenizer] = tokenizer_json["tokenizer"] def count_tokens(text: str) -> int: - return ( - tokenizer.count(text) - if isinstance(tokenizer, NativeTokenizer) - else len(tokenizer.encode_batch_fast([text])[0]) - ) + count: Final = getattr(tokenizer, "count", None) + if callable(count): + return count(text) + return len(tokenizer.encode_batch_fast([text])[0]) return count_tokens elif tokenizer_json["type"] == "openai_tokenizer": diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index 495492f50db..60917c68221 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -213,7 +213,7 @@ class HuggingFaceEmbedding(BaseLLM): model_response.model = model input_tokens = 0 for text in input: - input_tokens += len(encoding.encode(text)) + input_tokens += len(encoding.encode_ordinary(text)) setattr( model_response, diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 391f0042ed0..06e157498aa 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -19632,7 +19632,7 @@ } } }, - "description": "\nUnified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values.\n" + "description": "\n Unified rate-limit error.\n\n Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n The :attr:`category` attribute lets callers distinguish the source. See\n :class:`RateLimitErrorCategory` for the available values.\n " }, "500": { "content": {