mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
359b7a8489
* feat(rust): count tiktoken cl100k_base admission tokens in Rust The Rust admission token counter only had the Anthropic tokenizer, so every other model (OpenAI gpt-4 family, Azure, Gemini, Bedrock non-Claude, Mistral) tokenized with tiktoken on the Python inference worker. Add an exact cl100k_base counter to litellm-token-counter: the vendored rank file (base64 token / rank lines, the bytes Python's tiktoken uses) is parsed into a byte-level BPE model and the cl100k split pattern is a handwritten scanner over the shared Unicode classes, so no regex engine runs per request. Both tokenizers share the message, tool and reply-priming accounting. The PyO3 TokenCounter gains a from_cl100k_ranks constructor; Python reads the rank file and passes it in, the way claude_json_str already works. The bridge selects the counter through the same predicates litellm.token_counter uses (huggingface_tokenizer_kind, openai_tokenizer_encoding), declines o200k_base, downloaded HuggingFace and custom tokenizers to Python, and budget reservation counts once per distinct tokenizer a request names. The legacy gpt-3.5-turbo-0301 message accounting (4 per message, -1 per name) stays in Python: the selector declines it through the predicate token_counter itself uses. * feat(rust): count tiktoken o200k_base admission tokens in Rust (#40794) Add a handwritten o200k_base split scanner and TokenCounter::from_o200k_ranks next to the cl100k_base counter, sharing MergeRanks and the request accounting. The Python bridge selects it when openai_tokenizer_encoding names o200k_base, so gpt-4o, gpt-4.1, gpt-5, o1/o3/o4 and chatgpt-4o requests stop tokenizing on the Python worker under LITELLM_RUST=true Co-authored-by: yassin <yassin@berri.ai> --------- Co-authored-by: yassin <yassin@berri.ai> Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
30 lines
663 B
TOML
30 lines
663 B
TOML
[package]
|
|
name = "litellm-token-counter"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
repository.workspace = true
|
|
|
|
[dependencies]
|
|
base64.workspace = true
|
|
indexmap = { version = "2.14.0", features = ["serde"] }
|
|
itoa = "1.0"
|
|
rustc-hash = "2.1.3"
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
thiserror.workspace = true
|
|
tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
|
|
unicode-normalization-alignments = "0.1.12"
|
|
|
|
[dev-dependencies]
|
|
criterion.workspace = true
|
|
rand.workspace = true
|
|
rstest.workspace = true
|
|
|
|
[[bench]]
|
|
name = "token_counter"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "allocations"
|
|
harness = false
|