From 9b2b3d0b90346c0da91a8abf4b52be7561e1dcfe Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sun, 20 Sep 2026 21:21:52 +0000 Subject: [PATCH] refactor(rust): align tokenizer docs and lint Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/token-counter/README.md | 2 +- litellm/rust_bridge/token_counter.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/litellm-rust/crates/token-counter/README.md b/litellm-rust/crates/token-counter/README.md index c8ed0737a7e..effa5324822 100644 --- a/litellm-rust/crates/token-counter/README.md +++ b/litellm-rust/crates/token-counter/README.md @@ -2,7 +2,7 @@ `Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface -The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast`, `TokenCounter::from_cl100k_ranks`, and `TokenCounter::from_o200k_ranks` use this implementation +The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through the upstream `tokenizers` library. `TokenCounter::from_json` uses this implementation diff --git a/litellm/rust_bridge/token_counter.py b/litellm/rust_bridge/token_counter.py index e816f1ab388..d232db19a4a 100644 --- a/litellm/rust_bridge/token_counter.py +++ b/litellm/rust_bridge/token_counter.py @@ -77,7 +77,14 @@ def rust_tokenizer(model: str) -> RustTokenizer | None: if kind is not None or uses_legacy_message_accounting(model): return None encoding: Final = openai_tokenizer_encoding(model).name - if encoding in {"cl100k_base", "o200k_base", "o200k_harmony", "p50k_base", "p50k_edit", "r50k_base"}: + if encoding in ( + "cl100k_base", + "o200k_base", + "o200k_harmony", + "p50k_base", + "p50k_edit", + "r50k_base", + ): return cast(RustTokenizer, encoding) return None