mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
4a57012560
Adds the litellm-router-core crate, its step-driven PyO3 bridge (RouterCoreHandle / RouterCall) and RustCore, served behind LITELLM_RUST through the ROUTER route rule. The host call-out is the only boundary: every attempt goes through Host::invoke, there is no native executor path, and RustCore never builds, wraps or forwards to PythonCore, so router_python_core.py stays deletable. Attempt preparation lives in litellm/router_attempt.py so it outlives router_python_core.py. RustCore matches PythonCore's behavior, including its known bugs (fixed in follow-up PRs). Routing decisions (strategy picks, routing groups, PTU, pre-routing, pre-call checks, fallback walk, health routing, access groups, weights, cooldowns, allowed fails, usage) live in the crate; work that needs live Python objects (request shaping, endpoints, streaming, model info facts, callbacks, alerting, prompt management) lives in host-side litellm/router_rust_*.py modules driven by Rust steps. The crate owns the registry. It runs the deployment build loop (secret references, ids, organization fan-out, install order) and asks the host for what needs Python, one deployment at a time so errors surface where Python raised them: secrets, validate, provider and added steps. It holds the model list, the index maps and the pattern routers with the Python core's history rules (host_index.rs), and answers the registry lookups (lookup.rs). The Python views of that state (model_list, index(), names(), patterns()) read the handle, and every write to them becomes a changeset. Model info, deployment model info, configured limits and model group info are merged in the crate (model_info.rs, group_info.rs) from catalog rows and facts the host answers on a model_info step asked at query time, so price reloads and register_model stay visible. The usage cache key format is defined once, in the crate. tests/unit/test_router/core characterizes PythonCore's public behavior and runs every test on both cores; the remaining Rust differences (the router's own outcome callbacks) are listed in rust_parity_pending.txt.
158 lines
6.9 KiB
TOML
158 lines
6.9 KiB
TOML
[workspace]
|
|
members = ["crates/*"]
|
|
resolver = "2"
|
|
|
|
[workspace.package]
|
|
edition = "2024"
|
|
rust-version = "1.88"
|
|
license = "MIT"
|
|
repository = "https://github.com/BerriAI/litellm"
|
|
|
|
[workspace.dependencies]
|
|
litellm-config = { path = "crates/config" }
|
|
litellm-error = { path = "crates/error" }
|
|
litellm-router = { path = "crates/router" }
|
|
litellm-router-types = { path = "crates/router-types" }
|
|
litellm-router-core = { path = "crates/router-core" }
|
|
litellm-tracing = { path = "crates/tracing" }
|
|
litellm-spend-clickhouse = { path = "crates/spend-clickhouse" }
|
|
litellm-storage-clickhouse = { path = "crates/storage-clickhouse" }
|
|
litellm-inference = { path = "crates/inference" }
|
|
litellm-inference-transcription = { path = "crates/inference-transcription" }
|
|
litellm-inference-responses = { path = "crates/inference-responses" }
|
|
litellm-inference-messages = { path = "crates/inference-messages" }
|
|
litellm-inference-chat = { path = "crates/inference-chat" }
|
|
litellm-inference-ocr = { path = "crates/inference-ocr" }
|
|
litellm-inference-testing = { path = "crates/inference-testing" }
|
|
litellm-gateway-mcp = { path = "crates/gateway-mcp" }
|
|
litellm-gateway = { path = "crates/gateway" }
|
|
litellm-gateway-inference = { path = "crates/gateway-inference" }
|
|
litellm-gateway-auth = { path = "crates/gateway-auth" }
|
|
litellm-gateway-management = { path = "crates/gateway-management" }
|
|
litellm-gateway-ui = { path = "crates/gateway-ui" }
|
|
litellm-coroutine = { path = "crates/coroutine" }
|
|
litellm-host = { path = "crates/host" }
|
|
litellm-host-http = { path = "crates/host-http" }
|
|
litellm-host-native = { path = "crates/host-native" }
|
|
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
|
|
litellm-framer = { path = "crates/framer" }
|
|
litellm-auth = { path = "crates/auth" }
|
|
litellm-auth-types = { path = "crates/auth-types" }
|
|
litellm-secret-value = { path = "crates/secret-value" }
|
|
litellm-auth-aws = { path = "crates/auth-aws" }
|
|
litellm-auth-azure = { path = "crates/auth-azure" }
|
|
litellm-auth-gcp = { path = "crates/auth-gcp" }
|
|
litellm-secrets = { path = "crates/secrets" }
|
|
litellm-secrets-types = { path = "crates/secrets-types" }
|
|
litellm-secrets-aws = { path = "crates/secrets-aws" }
|
|
litellm-secrets-google = { path = "crates/secrets-google" }
|
|
litellm-secrets-hashicorp = { path = "crates/secrets-hashicorp" }
|
|
litellm-secrets-azure = { path = "crates/secrets-azure" }
|
|
litellm-secrets-cyberark = { path = "crates/secrets-cyberark" }
|
|
litellm-http = { path = "crates/http" }
|
|
litellm-llms = { path = "crates/llms" }
|
|
litellm-llms-types = { path = "crates/llms-types" }
|
|
litellm-core-utils = { path = "crates/core-utils" }
|
|
litellm-db = { path = "crates/db" }
|
|
litellm-db-testing = { path = "crates/db-testing" }
|
|
litellm-cache = { path = "crates/cache" }
|
|
litellm-cache-azure-blob = { path = "crates/cache-azure-blob" }
|
|
litellm-cache-memory = { path = "crates/cache-memory" }
|
|
litellm-cache-redis = { path = "crates/cache-redis" }
|
|
litellm-cache-s3 = { path = "crates/cache-s3" }
|
|
litellm-cache-gcs = { path = "crates/cache-gcs" }
|
|
litellm-cache-disk = { path = "crates/cache-disk" }
|
|
litellm-cache-redis-semantic = { path = "crates/cache-redis-semantic" }
|
|
litellm-cache-response = { path = "crates/cache-response" }
|
|
litellm-cache-qdrant-semantic = { path = "crates/cache-qdrant-semantic" }
|
|
litellm-cache-testing = { path = "crates/cache-testing" }
|
|
litellm-token-counter = { path = "crates/token-counter" }
|
|
litellm-token-counter-fast = { path = "crates/token-counter-fast" }
|
|
litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface" }
|
|
litellm-token-counter-tiktoken = { path = "crates/token-counter-tiktoken" }
|
|
litellm-host-python = { path = "crates/host-python" }
|
|
litellm-python-compat = { path = "crates/python-compat" }
|
|
litellm-circuit-breaker = { path = "crates/circuit-breaker" }
|
|
litellm-redis-client = { path = "crates/redis-client" }
|
|
litellm-redis-testing = { path = "crates/redis-testing" }
|
|
litellm-cost = { path = "crates/cost" }
|
|
litellm-model-catalog = { path = "crates/model-catalog" }
|
|
litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" }
|
|
|
|
arc-swap = "1.9.2"
|
|
async-trait = "0.1.91"
|
|
askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] }
|
|
tracing = "0.1"
|
|
axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "multipart"] }
|
|
axum-login = "0.18.0"
|
|
tower-sessions = { version = "0.14.0", features = ["memory-store"] }
|
|
bytes = "1"
|
|
http = "1"
|
|
google-cloud-auth = { version = "1.16.0", default-features = false }
|
|
jsonwebtoken = { version = "11.1.0", default-features = false }
|
|
hyper-util = { version = "0.1.20", default-features = false, features = ["client-proxy"] }
|
|
parking_lot = "0.12.5"
|
|
proptest = "1.7.0"
|
|
pyo3 = "0.29.2"
|
|
pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
|
|
rand = "0.8"
|
|
macro_rules_attribute = "0.2.3"
|
|
schemars = "1"
|
|
reqwest = { version = "0.13", default-features = false, features = ["json", "multipart", "rustls", "http2", "stream", "query"] }
|
|
redis = "1.8.0"
|
|
redis-test = "1.0.4"
|
|
testcontainers-modules = "0.15.0"
|
|
qdrant-client = { version = "1.19.0", default-features = false }
|
|
uuid = { version = "1", features = ["v4"] }
|
|
rstest = "0.26.1"
|
|
wiremock = "0.6.5"
|
|
rstest_reuse = "0.7.0"
|
|
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
|
|
rustls-native-certs = "0.8"
|
|
rustify = "=0.7.0"
|
|
rustify_derive = "=0.5.5"
|
|
vaultrs = { version = "=0.8.0", default-features = false, features = ["rustls"] }
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = { version = "1.0", features = ["float_roundtrip"] }
|
|
serde_with = { version = "=3.21.0", default-features = false, features = ["std", "macros"] }
|
|
sha2 = "0.10"
|
|
sha1 = "0.10"
|
|
syn = { version = "2", default-features = false }
|
|
sqlx = { version = "0.9.0", default-features = false, features = ["json", "macros", "postgres", "runtime-tokio", "chrono", "tls-rustls-ring-native-roots"] }
|
|
chrono = { version = "0.4.45", default-features = false, features = ["std"] }
|
|
proc-macro2 = "1"
|
|
quote = "1"
|
|
subtle = "2"
|
|
thiserror = "2.0"
|
|
tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
|
|
tiktoken-rs = "0.12.0"
|
|
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] }
|
|
tokio-tungstenite = { version = "0.24", default-features = false, features = ["handshake"] }
|
|
futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
|
|
base64 = "0.22"
|
|
flate2 = "1"
|
|
semver = "1"
|
|
tar = "0.4"
|
|
target-lexicon = "0.13.5"
|
|
tempfile = "3"
|
|
zip = { version = "2", default-features = false, features = ["deflate"] }
|
|
moka = { version = "0.12.16", features = ["future"] }
|
|
strum = { version = "0.28.0", features = ["derive"] }
|
|
url = "2.5.8"
|
|
percent-encoding = "2.3"
|
|
webpki-roots = "1.0.10"
|
|
time = { version = "0.3.53", features = ["parsing"] }
|
|
criterion = "0.8.2"
|
|
fancy-regex = "0.19.2"
|
|
veil = "0.3.0"
|
|
prost = "0.14.4"
|
|
opentelemetry-proto = "0.33"
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
lto = "fat"
|
|
codegen-units = 1
|
|
panic = "unwind"
|
|
debug = false
|
|
incremental = false
|
|
strip = "symbols"
|