mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
0abd9267c1
* ci: benchmark and gate an installed release wheel Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: simplify installed-wheel benchmark check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(rust): add native tokenizer codec Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(tokenizer): route Python tokenization through the Rust extension Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(lint): format tokenizer call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(packaging): restore runtime dependencies and native images Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tokenizer): preserve Python SDK behavior with Rust tokenizers * fix(tokenizer): restore compatibility paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(tokenizer): count custom tokenizers directly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tokenizer): preserve caller-supplied Python tokenizer counts * fix(tokenizer): reuse packaged vocabularies in the native wheel * refactor(rust_bridge): route token counting through the catalog as RUST_OPT_IN Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend_tracking): compare tokenizer groups by value Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(deps): re-resolve filelock under the <4.0 pin Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(llms): align transformation override signatures with base configs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * build(rust): use fat LTO to keep the native wheel under the 35 MB limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(tokenizer): preserve Python defaults with opt-in Rust dispatch * test(proxy): tolerate missing litellm.utils.Tokenizer when patching it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): patch the tokenizer dispatch function instead of the removed alias Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(tokenizer): give the Rust wrappers the tiktoken and tokenizers surface Callers of litellm.encoding and litellm.create_tokenizer must see the same read-only API whichever backend the catalog selects. - OpenAIEncoding mirrors tiktoken.Encoding: n_vocab, max_token_value, token_byte_values, encode_single_token, encode_with_unstable, encode_to_numpy, decode_with_offsets, is_special_token, repr; the Rust tiktoken crate keeps a Vocabulary beside each CoreBPE and reports the requested encoding name (gpt2 stays gpt2). - HuggingFaceTokenizer mirrors the read-only tokenizers.Tokenizer surface (token_to_id, id_to_token, get_vocab, get_vocab_size, get_added_tokens_decoder, num_special_tokens_to_add, padding, truncation, encode_special_tokens, from_buffer); HuggingFaceEncoding gains the char/word/token lookups, pad, truncate, set_sequence_id and merge. Mutators stay on the Python tokenizer. - from_json/from_pretrained claim the fork gate only when the huggingface feature is compiled in; the surrogate fallback matches on the Codec. - Tokenizer caching is keyed on the same catalog Context the dispatch runs on; rust_tokenizer reads the encoding name without loading an encoding; LITELLM_RUST parsing is cached. - Drop the unused tiktoken_encoding_for_model export and Error::Download. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * fix(tokenizer): close the exhaustive matches with assert_never CodeQL reads a `match` over a Literal with no default arm as an implicit `None` return. `assert_never` makes the exhaustiveness explicit for both the HuggingFace tokenizer loader and the Rust token-counter factory. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * feat(tokenizer): derive the fast counter from the shared tokenizer The count-only counter (`fast` feature) and the codec each parsed the same artifact: TokenCounter took the Anthropic JSON and the tiktoken rank files from Python while Tokenizer loaded them again. One parse now serves both. - FastTokenizer builds from a model another loader holds: `from_shared` takes the Arc<tokenizers::Tokenizer> the HF codec keeps, and `from_*_pairs` take the ranks the tiktoken vocabulary already parsed. - `FastCounter::fast_counter` in the core crate derives it from either codec; encodings the fast scanner does not reproduce are refused. - Native `Tokenizer.count(text, fast=False)` opts into that counter, built once per tokenizer on first use; `TokenCounter.from_tokenizer(tokenizer, fast=False)` replaces the JSON and rank-file constructors. - The Python route counts over the native tokenizers the codec path shares (`native_encoding`, `native_anthropic`) and no longer reads rank files; the packaged Anthropic tokenizer has one loader, `tokenizer_dispatch.anthropic`. - Public wrappers gain `count(text, fast=False)`. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Yujong Lee <yujong@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
72 lines
2.1 KiB
TOML
72 lines
2.1 KiB
TOML
[package]
|
|
name = "litellm-python-bridge"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
repository.workspace = true
|
|
|
|
[lib]
|
|
name = "_native"
|
|
crate-type = ["cdylib"]
|
|
|
|
[features]
|
|
default = ["abi3", "fast", "huggingface", "tiktoken"]
|
|
abi3 = ["pyo3/abi3-py310"]
|
|
extension-module = ["pyo3/extension-module"]
|
|
panic-test = []
|
|
fast = ["litellm-token-counter/fast"]
|
|
huggingface = ["litellm-token-counter/huggingface"]
|
|
tiktoken = ["litellm-token-counter/tiktoken"]
|
|
|
|
[dependencies]
|
|
bytes.workspace = true
|
|
futures-util.workspace = true
|
|
litellm-cache.workspace = true
|
|
litellm-cache-azure-blob.workspace = true
|
|
litellm-cache-memory.workspace = true
|
|
litellm-cache-redis.workspace = true
|
|
litellm-cache-s3.workspace = true
|
|
litellm-cache-gcs.workspace = true
|
|
litellm-cache-disk.workspace = true
|
|
litellm-cache-redis-semantic.workspace = true
|
|
litellm-cache-response.workspace = true
|
|
litellm-cache-qdrant-semantic.workspace = true
|
|
qdrant-client.workspace = true
|
|
litellm-cache-valkey-semantic = { path = "../cache-valkey-semantic" }
|
|
serde.workspace = true
|
|
litellm-auth.workspace = true
|
|
litellm-auth-aws.workspace = true
|
|
litellm-callbacks-legacy-python.workspace = true
|
|
litellm-core.workspace = true
|
|
litellm-core-utils.workspace = true
|
|
litellm-auth-gcp.workspace = true
|
|
litellm-http.workspace = true
|
|
litellm-llms.workspace = true
|
|
litellm-secrets = { workspace = true, features = ["aws"] }
|
|
litellm-secrets-types.workspace = true
|
|
litellm-types.workspace = true
|
|
litellm-host-python.workspace = true
|
|
litellm-token-counter = { path = "../token-counter", default-features = false }
|
|
pyo3.workspace = true
|
|
pyo3-async-runtimes.workspace = true
|
|
reqwest.workspace = true
|
|
redis = { version = "1.7.0", features = ["tls-rustls"] }
|
|
serde_json.workspace = true
|
|
url.workspace = true
|
|
tokio = { workspace = true, features = ["rt", "sync"] }
|
|
|
|
[dev-dependencies]
|
|
litellm-secrets-aws.workspace = true
|
|
serde.workspace = true
|
|
serde_with.workspace = true
|
|
criterion.workspace = true
|
|
futures-util.workspace = true
|
|
rstest.workspace = true
|
|
sha2.workspace = true
|
|
tokio-tungstenite.workspace = true
|
|
wiremock = "0.6.5"
|
|
aws-sdk-secretsmanager = "1.117.0"
|
|
|
|
[[bench]]
|
|
name = "serialization"
|
|
harness = false
|