mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
17bcc04800
The four shared reqwest clients (core messages/chat_completions/ audio_transcription, ai-gateway) now set pool_max_idle_per_host(64) and tcp_keepalive(60s): - reqwest defaults pool_max_idle_per_host to usize::MAX, so a burst of N concurrent calls to one provider host leaves N idle TLS connections in the pool after completion, each held until its idle timeout; under sustained bursts this grows file descriptors without bound. 64 keeps a warm set large enough for handshake-free reuse. - tcp_keepalive lets silent half-open peers (NAT/LB reaping, vanished upstreams) be detected during long streaming responses instead of blocking until the 600s full-request timeout. The python-bridge cdylib now uses mimalloc as its Rust global allocator (the pydantic-core/orjson pattern). It only governs allocations made by Rust code in the extension - CPython's pymalloc heap is untouched - but Rust-side request/response buffering in a long-lived interpreter process gets mimalloc's sharded free lists and eager page purging instead of glibc malloc, which favors RSS growth under churn. No behavior change beyond the socket/allocator characteristics; timeouts and request paths are untouched.
47 lines
1.4 KiB
TOML
47 lines
1.4 KiB
TOML
[workspace]
|
|
members = [
|
|
"crates/core",
|
|
"crates/ai-gateway",
|
|
"crates/python-interop",
|
|
"crates/python-bridge",
|
|
]
|
|
resolver = "2"
|
|
|
|
[workspace.package]
|
|
edition = "2024"
|
|
rust-version = "1.88"
|
|
license = "MIT"
|
|
repository = "https://github.com/BerriAI/litellm"
|
|
|
|
[workspace.dependencies]
|
|
tracing = "0.1"
|
|
tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] }
|
|
litellm-core = { path = "crates/core" }
|
|
litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false }
|
|
litellm-python-interop = { path = "crates/python-interop" }
|
|
axum = "0.7"
|
|
pyo3 = "0.29.2"
|
|
pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
|
|
pythonize = "0.29.0"
|
|
rand = "0.8"
|
|
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] }
|
|
rstest = "0.26.1"
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = "1.0"
|
|
sha2 = "0.10"
|
|
subtle = "2"
|
|
thiserror = "2.0"
|
|
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] }
|
|
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
|
|
futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
|
|
base64 = "0.22"
|
|
mimalloc = "0.1"
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
lto = "thin"
|
|
codegen-units = 1
|
|
panic = "unwind"
|
|
debug = false
|
|
incremental = false
|
|
strip = "symbols"
|