mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-04 07:27:51 +00:00
4a57012560
Adds the litellm-router-core crate, its step-driven PyO3 bridge (RouterCoreHandle / RouterCall) and RustCore, served behind LITELLM_RUST through the ROUTER route rule. The host call-out is the only boundary: every attempt goes through Host::invoke, there is no native executor path, and RustCore never builds, wraps or forwards to PythonCore, so router_python_core.py stays deletable. Attempt preparation lives in litellm/router_attempt.py so it outlives router_python_core.py. RustCore matches PythonCore's behavior, including its known bugs (fixed in follow-up PRs). Routing decisions (strategy picks, routing groups, PTU, pre-routing, pre-call checks, fallback walk, health routing, access groups, weights, cooldowns, allowed fails, usage) live in the crate; work that needs live Python objects (request shaping, endpoints, streaming, model info facts, callbacks, alerting, prompt management) lives in host-side litellm/router_rust_*.py modules driven by Rust steps. The crate owns the registry. It runs the deployment build loop (secret references, ids, organization fan-out, install order) and asks the host for what needs Python, one deployment at a time so errors surface where Python raised them: secrets, validate, provider and added steps. It holds the model list, the index maps and the pattern routers with the Python core's history rules (host_index.rs), and answers the registry lookups (lookup.rs). The Python views of that state (model_list, index(), names(), patterns()) read the handle, and every write to them becomes a changeset. Model info, deployment model info, configured limits and model group info are merged in the crate (model_info.rs, group_info.rs) from catalog rows and facts the host answers on a model_info step asked at query time, so price reloads and register_model stay visible. The usage cache key format is defined once, in the crate. tests/unit/test_router/core characterizes PythonCore's public behavior and runs every test on both cores; the remaining Rust differences (the router's own outcome callbacks) are listed in rust_parity_pending.txt.
85 lines
2.6 KiB
TOML
85 lines
2.6 KiB
TOML
[package]
|
|
name = "litellm-python-bridge"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
repository.workspace = true
|
|
|
|
[lib]
|
|
name = "_native"
|
|
crate-type = ["cdylib"]
|
|
|
|
[features]
|
|
default = ["abi3", "fast", "huggingface", "tiktoken"]
|
|
abi3 = ["pyo3/abi3-py310"]
|
|
extension-module = ["pyo3/extension-module"]
|
|
panic-test = []
|
|
fast = ["litellm-token-counter/fast"]
|
|
huggingface = ["litellm-token-counter/huggingface"]
|
|
tiktoken = ["litellm-token-counter/tiktoken"]
|
|
|
|
[dependencies]
|
|
litellm-secret-value.workspace = true
|
|
fancy-regex.workspace = true
|
|
litellm-tracing.workspace = true
|
|
litellm-spend-clickhouse.workspace = true
|
|
litellm-storage-clickhouse.workspace = true
|
|
litellm-host.workspace = true
|
|
async-trait.workspace = true
|
|
bytes.workspace = true
|
|
futures-util.workspace = true
|
|
litellm-cache.workspace = true
|
|
litellm-error.workspace = true
|
|
litellm-cache-memory.workspace = true
|
|
litellm-cache-redis.workspace = true
|
|
litellm-cache-response.workspace = true
|
|
serde.workspace = true
|
|
litellm-auth.workspace = true
|
|
litellm-auth-aws.workspace = true
|
|
litellm-callbacks-legacy-python.workspace = true
|
|
litellm-inference.workspace = true
|
|
litellm-inference-transcription.workspace = true
|
|
litellm-inference-responses.workspace = true
|
|
litellm-inference-messages.workspace = true
|
|
litellm-inference-chat.workspace = true
|
|
litellm-inference-ocr.workspace = true
|
|
litellm-core-utils.workspace = true
|
|
litellm-http.workspace = true
|
|
litellm-llms.workspace = true
|
|
litellm-secrets = { workspace = true, features = ["aws", "azure", "google", "hashicorp", "cyberark"] }
|
|
litellm-secrets-types.workspace = true
|
|
litellm-llms-types.workspace = true
|
|
litellm-router-types.workspace = true
|
|
litellm-router-core.workspace = true
|
|
litellm-host-python.workspace = true
|
|
litellm-model-catalog.workspace = true
|
|
litellm-token-counter = { path = "../token-counter", default-features = false }
|
|
pyo3.workspace = true
|
|
prost.workspace = true
|
|
pyo3-async-runtimes.workspace = true
|
|
reqwest.workspace = true
|
|
redis = { workspace = true, features = ["tls-rustls"] }
|
|
serde_json.workspace = true
|
|
strum.workspace = true
|
|
veil.workspace = true
|
|
thiserror.workspace = true
|
|
tokio = { workspace = true, features = ["rt", "sync"] }
|
|
|
|
[dev-dependencies]
|
|
tracing.workspace = true
|
|
litellm-http = { workspace = true, features = ["test-support"] }
|
|
litellm-secrets-aws.workspace = true
|
|
serde.workspace = true
|
|
serde_with.workspace = true
|
|
criterion.workspace = true
|
|
futures-util.workspace = true
|
|
rstest.workspace = true
|
|
sqlx = { workspace = true, features = ["migrate"] }
|
|
tempfile.workspace = true
|
|
tokio-tungstenite.workspace = true
|
|
wiremock.workspace = true
|
|
aws-sdk-secretsmanager = "1.117.0"
|
|
|
|
[[bench]]
|
|
name = "serialization"
|
|
harness = false
|