fix(tokenizer): restore compatibility paths

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm-rust/Cargo.lock

@@ -2279,6 +2279,7 @@ dependencies = [
 name = "litellm-token-counter-huggingface"
 version = "0.1.0"
 dependencies = [
+ "serde_json",
  "thiserror 2.0.19",
  "tokenizers",
 ]

litellm-rust/crates/python-bridge/src/tokenizer.rs

@@ -24,7 +24,8 @@ use crate::token_counter::token_count_error_to_pyerr;
 
 #[cfg(feature = "huggingface")]
 use litellm_token_counter::huggingface::{
-    EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence,
+    EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, encoding_from_json,
+    encoding_to_json,
 };
 #[cfg(feature = "tiktoken")]
 use litellm_token_counter::tiktoken::TiktokenTokenizer;
@@ -315,7 +316,7 @@ impl HuggingFaceEncoding {
     #[pyo3(signature = (json = None))]
     fn new(json: Option<&str>) -> PyResult<Self> {
         let inner = match json {
-            Some(json) => serde_json::from_str(json)
+            Some(json) => encoding_from_json(json)
                 .map_err(|error| PyValueError::new_err(error.to_string()))?,
             None => Encoding::default(),
         };
@@ -326,7 +327,7 @@ impl HuggingFaceEncoding {
         &self,
         py: Python<'py>,
     ) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> {
-        let json = serde_json::to_string(&self.inner)
+        let json = encoding_to_json(&self.inner)
             .map_err(|error| PyValueError::new_err(error.to_string()))?;
         Ok((py.get_type::<Self>(), (json,)))
     }

litellm-rust/crates/token-counter-huggingface/Cargo.toml

@@ -6,5 +6,6 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+serde_json.workspace = true
 thiserror.workspace = true
 tokenizers.workspace = true

litellm-rust/crates/token-counter-huggingface/src/lib.rs

@@ -7,6 +7,14 @@ use std::collections::HashSet;
 pub use error::Error;
 pub use tokenizers::{EncodeInput, Encoding, InputSequence};
 
+pub fn encoding_from_json(json: &str) -> Result<Encoding, Error> {
+    serde_json::from_str(json).map_err(|error| Error::Load(error.into()))
+}
+
+pub fn encoding_to_json(encoding: &Encoding) -> Result<String, Error> {
+    serde_json::to_string(encoding).map_err(|error| Error::Load(error.into()))
+}
+
 pub struct HuggingFaceTokenizer {
     tokenizer: Box<tokenizers::Tokenizer>,
     special_token_ids: HashSet<u32>,

litellm-rust/crates/token-counter/src/huggingface.rs

@@ -1,6 +1,7 @@
 use litellm_token_counter_huggingface::Error as BackendError;
 pub use litellm_token_counter_huggingface::{
-    EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence,
+    EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, encoding_from_json,
+    encoding_to_json,
 };
 
 use crate::{Error, TextCodec, TokenCounter, Tokenizer};

litellm/litellm_core_utils/token_counter.py

@@ -626,11 +626,10 @@ def _get_exact_count_function(
             tokenizer: Final[Tokenizer | NativeTokenizer] = tokenizer_json["tokenizer"]
 
             def count_tokens(text: str) -> int:
-                return (
-                    tokenizer.count(text)
-                    if isinstance(tokenizer, NativeTokenizer)
-                    else len(tokenizer.encode_batch_fast([text])[0])
-                )
+                count: Final = getattr(tokenizer, "count", None)
+                if callable(count):
+                    return count(text)
+                return len(tokenizer.encode_batch_fast([text])[0])
 
             return count_tokens
         elif tokenizer_json["type"] == "openai_tokenizer":

litellm/llms/huggingface/embedding/handler.py

@@ -213,7 +213,7 @@ class HuggingFaceEmbedding(BaseLLM):
         model_response.model = model
         input_tokens = 0
         for text in input:
-            input_tokens += len(encoding.encode(text))
+            input_tokens += len(encoding.encode_ordinary(text))
 
         setattr(
             model_response,

litellm/proxy/_lazy_openapi_snapshot.json

@@ -19632,7 +19632,7 @@
                   }
                 }
               },
-              "description": "\nUnified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values.\n"
+              "description": "\n    Unified rate-limit error.\n\n    Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n    an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n    proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n    max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n    The :attr:`category` attribute lets callers distinguish the source. See\n    :class:`RateLimitErrorCategory` for the available values.\n    "
             },
             "500": {
               "content": {