mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
0abd9267c1
* ci: benchmark and gate an installed release wheel Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: simplify installed-wheel benchmark check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(rust): add native tokenizer codec Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(tokenizer): route Python tokenization through the Rust extension Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(lint): format tokenizer call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(packaging): restore runtime dependencies and native images Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tokenizer): preserve Python SDK behavior with Rust tokenizers * fix(tokenizer): restore compatibility paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(tokenizer): count custom tokenizers directly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tokenizer): preserve caller-supplied Python tokenizer counts * fix(tokenizer): reuse packaged vocabularies in the native wheel * refactor(rust_bridge): route token counting through the catalog as RUST_OPT_IN Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend_tracking): compare tokenizer groups by value Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(deps): re-resolve filelock under the <4.0 pin Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(llms): align transformation override signatures with base configs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * build(rust): use fat LTO to keep the native wheel under the 35 MB limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(tokenizer): preserve Python defaults with opt-in Rust dispatch * test(proxy): tolerate missing litellm.utils.Tokenizer when patching it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): patch the tokenizer dispatch function instead of the removed alias Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(tokenizer): give the Rust wrappers the tiktoken and tokenizers surface Callers of litellm.encoding and litellm.create_tokenizer must see the same read-only API whichever backend the catalog selects. - OpenAIEncoding mirrors tiktoken.Encoding: n_vocab, max_token_value, token_byte_values, encode_single_token, encode_with_unstable, encode_to_numpy, decode_with_offsets, is_special_token, repr; the Rust tiktoken crate keeps a Vocabulary beside each CoreBPE and reports the requested encoding name (gpt2 stays gpt2). - HuggingFaceTokenizer mirrors the read-only tokenizers.Tokenizer surface (token_to_id, id_to_token, get_vocab, get_vocab_size, get_added_tokens_decoder, num_special_tokens_to_add, padding, truncation, encode_special_tokens, from_buffer); HuggingFaceEncoding gains the char/word/token lookups, pad, truncate, set_sequence_id and merge. Mutators stay on the Python tokenizer. - from_json/from_pretrained claim the fork gate only when the huggingface feature is compiled in; the surrogate fallback matches on the Codec. - Tokenizer caching is keyed on the same catalog Context the dispatch runs on; rust_tokenizer reads the encoding name without loading an encoding; LITELLM_RUST parsing is cached. - Drop the unused tiktoken_encoding_for_model export and Error::Download. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * fix(tokenizer): close the exhaustive matches with assert_never CodeQL reads a `match` over a Literal with no default arm as an implicit `None` return. `assert_never` makes the exhaustiveness explicit for both the HuggingFace tokenizer loader and the Rust token-counter factory. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * feat(tokenizer): derive the fast counter from the shared tokenizer The count-only counter (`fast` feature) and the codec each parsed the same artifact: TokenCounter took the Anthropic JSON and the tiktoken rank files from Python while Tokenizer loaded them again. One parse now serves both. - FastTokenizer builds from a model another loader holds: `from_shared` takes the Arc<tokenizers::Tokenizer> the HF codec keeps, and `from_*_pairs` take the ranks the tiktoken vocabulary already parsed. - `FastCounter::fast_counter` in the core crate derives it from either codec; encodings the fast scanner does not reproduce are refused. - Native `Tokenizer.count(text, fast=False)` opts into that counter, built once per tokenizer on first use; `TokenCounter.from_tokenizer(tokenizer, fast=False)` replaces the JSON and rank-file constructors. - The Python route counts over the native tokenizers the codec path shares (`native_encoding`, `native_anthropic`) and no longer reads rank files; the packaged Anthropic tokenizer has one loader, `tokenizer_dispatch.anthropic`. - Public wrappers gain `count(text, fast=False)`. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Yujong Lee <yujong@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
97 lines
4 KiB
TOML
97 lines
4 KiB
TOML
[workspace]
|
|
members = ["crates/*"]
|
|
resolver = "2"
|
|
|
|
[workspace.package]
|
|
edition = "2024"
|
|
rust-version = "1.88"
|
|
license = "MIT"
|
|
repository = "https://github.com/BerriAI/litellm"
|
|
|
|
[workspace.dependencies]
|
|
litellm-core = { path = "crates/core" }
|
|
litellm-host = { path = "crates/host" }
|
|
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
|
|
litellm-framing = { path = "crates/framer" }
|
|
litellm-auth = { path = "crates/auth" }
|
|
litellm-auth-types = { path = "crates/auth-types" }
|
|
litellm-auth-aws = { path = "crates/auth-aws" }
|
|
litellm-auth-azure = { path = "crates/auth-azure" }
|
|
litellm-auth-gcp = { path = "crates/auth-gcp" }
|
|
litellm-secrets = { path = "crates/secrets" }
|
|
litellm-secrets-types = { path = "crates/secrets-types" }
|
|
litellm-secrets-aws = { path = "crates/secrets-aws" }
|
|
litellm-secrets-google = { path = "crates/secrets-google" }
|
|
litellm-secrets-hashicorp = { path = "crates/secrets-hashicorp" }
|
|
litellm-secrets-azure = { path = "crates/secrets-azure" }
|
|
litellm-secrets-cyberark = { path = "crates/secrets-cyberark" }
|
|
litellm-http = { path = "crates/http" }
|
|
litellm-llms = { path = "crates/llms" }
|
|
litellm-types = { path = "crates/types" }
|
|
litellm-core-utils = { path = "crates/core-utils" }
|
|
litellm-cache = { path = "crates/cache" }
|
|
litellm-cache-azure-blob = { path = "crates/cache-azure-blob" }
|
|
litellm-cache-memory = { path = "crates/cache-memory" }
|
|
litellm-cache-redis = { path = "crates/cache-redis" }
|
|
litellm-cache-s3 = { path = "crates/cache-s3" }
|
|
litellm-cache-gcs = { path = "crates/cache-gcs" }
|
|
litellm-cache-disk = { path = "crates/cache-disk" }
|
|
litellm-cache-redis-semantic = { path = "crates/cache-redis-semantic" }
|
|
litellm-cache-response = { path = "crates/cache-response" }
|
|
litellm-cache-qdrant-semantic = { path = "crates/cache-qdrant-semantic" }
|
|
litellm-token-counter = { path = "crates/token-counter" }
|
|
litellm-token-counter-fast = { path = "crates/token-counter-fast" }
|
|
litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface" }
|
|
litellm-token-counter-tiktoken = { path = "crates/token-counter-tiktoken" }
|
|
litellm-host-python = { path = "crates/host-python" }
|
|
|
|
bytes = "1"
|
|
http = "1"
|
|
google-cloud-auth = { version = "1.16.0", default-features = false }
|
|
jsonwebtoken = { version = "11.1.0", default-features = false }
|
|
hyper-util = { version = "0.1.20", default-features = false, features = ["client-proxy"] }
|
|
proptest = "1.7.0"
|
|
pyo3 = "0.29.2"
|
|
pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
|
|
pythonize = "0.29.0"
|
|
rand = "0.8"
|
|
reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "http2", "stream"] }
|
|
qdrant-client = { version = "1.19.0", default-features = false }
|
|
uuid = { version = "1", features = ["v4"] }
|
|
rstest = "0.26.1"
|
|
rstest_reuse = "0.7.0"
|
|
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
|
|
rustify = "=0.7.0"
|
|
rustify_derive = "=0.5.5"
|
|
vaultrs = { version = "=0.8.0", default-features = false, features = ["rustls"] }
|
|
rustls-native-certs = "0.8"
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = { version = "1.0", features = ["float_roundtrip"] }
|
|
serde_with = { version = "=3.16.1", default-features = false, features = ["std", "macros"] }
|
|
sha2 = "0.10"
|
|
subtle = "2"
|
|
thiserror = "2.0"
|
|
tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
|
|
tiktoken-rs = "0.12.0"
|
|
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] }
|
|
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
|
|
futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
|
|
base64 = "0.22"
|
|
moka = { version = "0.12.16", features = ["future"] }
|
|
strum = { version = "0.28.0", features = ["derive"] }
|
|
url = "2.5.8"
|
|
percent-encoding = "2.3"
|
|
webpki-roots = "1"
|
|
time = { version = "0.3.53", features = ["parsing"] }
|
|
criterion = "0.8.2"
|
|
fancy-regex = "0.19.2"
|
|
veil = "0.3.0"
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
lto = "fat"
|
|
codegen-units = 1
|
|
panic = "unwind"
|
|
debug = false
|
|
incremental = false
|
|
strip = "symbols"
|