mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
refactor(rust): extract config crate (#39706)
* refactor(rust): extract config crate * refactor(config): split crate modules * refactor(gateway): remove gil health counter
This commit is contained in:
yujonglee
GitHub
parent
c8635ecc67
commit
7276caecd4
24 changed files with 173 additions and 200 deletions
|
|
@ -1,17 +1,18 @@
|
|||
# AGENTS.md
|
||||
|
||||
litellm-rust has four crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers.
|
||||
litellm-rust has five crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers.
|
||||
|
||||
## Crates
|
||||
|
||||
| Crate | Role |
|
||||
|-------|------|
|
||||
| litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. |
|
||||
| litellm-config | Config-loading boundary. Returns resolved core deployment data and optionally delegates loading to Python. |
|
||||
| litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. |
|
||||
| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. |
|
||||
|
||||
Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
|
||||
Dependency direction is acyclic: `litellm-config` depends on `litellm-core`, the gateway depends on both, and `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`. The interop foundation depends on no LiteLLM domain crate.
|
||||
|
||||
## Where a route lives
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ the base when behavior is genuinely different, and say so explicitly in the PR.
|
|||
## Crates (see AGENTS.md)
|
||||
|
||||
`litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call.
|
||||
`litellm-config` is the config-loading boundary and returns resolved core types.
|
||||
`litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and
|
||||
`litellm-python-bridge` exposes it to the Python SDK. `litellm-python-interop`
|
||||
holds domain-neutral PyO3 primitives shared by Python-facing Rust code. A crate
|
||||
|
|
|
|||
12
litellm-rust/Cargo.lock
generated
12
litellm-rust/Cargo.lock
generated
|
|
@ -1412,8 +1412,8 @@ dependencies = [
|
|||
"base64",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"litellm-config",
|
||||
"litellm-core",
|
||||
"pyo3",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -1425,6 +1425,16 @@ dependencies = [
|
|||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-config"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-core",
|
||||
"pyo3",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-core"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
[workspace]
|
||||
members = [
|
||||
"crates/core",
|
||||
"crates/config",
|
||||
"crates/ai-gateway",
|
||||
"crates/python-interop",
|
||||
"crates/python-bridge",
|
||||
|
|
@ -17,6 +18,7 @@ repository = "https://github.com/BerriAI/litellm"
|
|||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] }
|
||||
litellm-core = { path = "crates/core" }
|
||||
litellm-config = { path = "crates/config" }
|
||||
litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false }
|
||||
litellm-python-interop = { path = "crates/python-interop" }
|
||||
axum = "0.7"
|
||||
|
|
|
|||
|
|
@ -25,11 +25,12 @@ coverage and production evidence.
|
|||
| Crate | Role |
|
||||
|-------|------|
|
||||
| litellm-core | The SDK. Per-route entrypoints (`messages::messages()`), types, provider transforms (modules under `providers/`), provider resolution, auth, the provider HTTP call, and the router. |
|
||||
| litellm-config | Config-loading boundary. Returns resolved deployments and optionally delegates loading to Python. |
|
||||
| litellm-ai-gateway | The axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
|
||||
| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. |
|
||||
|
||||
Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
|
||||
Dependency direction is acyclic: config depends on core, the gateway depends on config and core, and the Python bridge depends on the domain layers and Python interop.
|
||||
|
||||
## Layout
|
||||
|
||||
|
|
@ -38,6 +39,7 @@ crates/
|
|||
core/ The SDK: route modules + provider transforms.
|
||||
src/messages/ mod.rs (entrypoint), types, transformation, prepare, handler, client
|
||||
src/providers/anthropic/messages/transformation.rs
|
||||
config/ Config loading and resolved deployments.
|
||||
ai-gateway/ Axum server + WebSocket hosts; calls core entrypoints.
|
||||
python-interop/ Domain-neutral PyO3 conversion and GIL primitives.
|
||||
python-bridge/ PyO3 API adapter for Python LiteLLM.
|
||||
|
|
|
|||
|
|
@ -9,19 +9,15 @@ such as `litellm_core::messages::messages`. No provider handler lives here.
|
|||
src/
|
||||
main.rs # entrypoint: build AppState (router + master key), bind, serve
|
||||
state.rs # AppState — shared Arc<Router> + master_key
|
||||
gil.rs # GIL-activity tracker (records Python acquisitions)
|
||||
auth/ # authentication as an axum extractor — added to handler args
|
||||
mod.rs # RequireMasterKey: FromRequestParts, single master key (LITELLM_MASTER_KEY)
|
||||
routes/ # one module per route, all matching the same template
|
||||
AGENTS.md # ← the route template (read this before adding a route)
|
||||
mod.rs # app(): merges every module's router()
|
||||
health.rs # simple route (one file): router() + liveness/readiness
|
||||
gil.rs # simple route (one file): router() + GET /health/gil
|
||||
realtime/ # route with logic → axum surface + a no-axum service:
|
||||
mod.rs # router() + handler + WS<->events adapter (the axum surface)
|
||||
service.rs # business logic (select deployment, call provider) — no axum, testable
|
||||
python/ # Python interop (feature: python-config) — load-time only
|
||||
mod.rs, config.rs, AGENTS.md
|
||||
```
|
||||
|
||||
## Rules
|
||||
|
|
@ -53,5 +49,6 @@ proxy in a later phase. Health routes don't add the extractor (unauthenticated).
|
|||
|
||||
## Python interop
|
||||
|
||||
Anything that calls into Python lives in `python/` and is **load-time only** — see
|
||||
`python/AGENTS.md`. The realtime data path never takes the GIL.
|
||||
Python-backed loading lives in `litellm-config` and is **load-time only**. The
|
||||
gateway's `python-config` feature forwards to that crate. The realtime data path
|
||||
never takes the GIL.
|
||||
|
|
|
|||
|
|
@ -9,4 +9,6 @@ flowchart LR
|
|||
C[client] <--> G[Rust ai-gateway<br/>LLM inference]
|
||||
G <--> O[OpenAI realtime]
|
||||
G -. spend tracking callback .-> P[litellm proxy]
|
||||
F[litellm-config<br/>load-time only] --> G
|
||||
F -. Python backend .-> P
|
||||
```
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ required-features = ["server"]
|
|||
[dependencies]
|
||||
tracing.workspace = true
|
||||
litellm-core = { workspace = true, features = ["bedrock-auth"] }
|
||||
litellm-config.workspace = true
|
||||
# reqwest (rustls + json) is used by io/ocr and ships realtime logs to the
|
||||
# Python proxy callbacks API.
|
||||
reqwest.workspace = true
|
||||
|
|
@ -31,7 +32,6 @@ subtle = { workspace = true, optional = true }
|
|||
# sha2 hashes the master key into user_api_key_hash (matches the proxy's
|
||||
# SHA-256 hash_token) so the plaintext credential never enters a log payload.
|
||||
sha2 = { workspace = true, optional = true }
|
||||
pyo3 = { workspace = true, features = ["auto-initialize"], optional = true }
|
||||
tower = { version = "0.5.3", features = ["util"], optional = true }
|
||||
|
||||
[features]
|
||||
|
|
@ -39,7 +39,7 @@ default = []
|
|||
server = ["dep:axum", "dep:subtle", "dep:sha2"]
|
||||
# Build the gateway's config from the proxy YAML via an embedded Python
|
||||
# interpreter (links libpython; requires `litellm` importable at runtime).
|
||||
python-config = ["dep:pyo3"]
|
||||
python-config = ["litellm-config/python"]
|
||||
trace-parity = ["server", "dep:tower", "litellm-core/observability"]
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -6,25 +6,30 @@ dials OpenAI upstream, and splices the two sockets frame-by-frame.
|
|||
|
||||
## Crates
|
||||
|
||||
`litellm-rust` has four crates. A crate is a layer or shared foundation, not a route:
|
||||
`litellm-rust` has five crates. A crate is a layer or shared foundation, not a route:
|
||||
|
||||
| Crate | Role |
|
||||
|-------|------|
|
||||
| litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. |
|
||||
| litellm-config | Config-loading boundary. Returns resolved deployments and optionally delegates loading to Python. |
|
||||
| litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
|
||||
| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. |
|
||||
|
||||
Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
|
||||
Dependency direction is acyclic: config depends on core, the gateway depends on config and core, and the Python bridge depends on the domain layers and Python interop.
|
||||
|
||||
- **Client endpoint:** `wss://<host>/v1/realtime?model=<model>` (WebSocket)
|
||||
- **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset)
|
||||
- **Health:** `GET /health/readiness`, `GET /health/liveness`, `GET /health/gil`
|
||||
- **Health:** `GET /health/readiness`, `GET /health/liveness`
|
||||
- **Request logs:** POSTed to a LiteLLM proxy at `/v1/rust_control_plane/logs` (see [Request logging](#request-logging))
|
||||
|
||||
> **Realtime serving is pure Rust.** Python is used at **load time only** — to
|
||||
> read the config once at boot. The realtime hot path never touches Python.
|
||||
|
||||
The former `/health/gil` route and its acquisition counter were removed. They
|
||||
only observed the single startup config load and did not prove that every GIL
|
||||
acquisition was instrumented
|
||||
|
||||
## Configuration (config.yaml)
|
||||
|
||||
The gateway loads its `model_list` from a **config.yaml**, the same as the
|
||||
|
|
@ -43,9 +48,10 @@ model_list:
|
|||
LITELLM_CONFIG_PATH=./config.yaml ./litellm-ai-gateway
|
||||
```
|
||||
|
||||
At boot the gateway calls into `litellm.proxy.read_model_list`, which reuses the
|
||||
**real proxy config reader** (`ProxyConfig.get_config`). That means everything
|
||||
the proxy supports in config.yaml works here too:
|
||||
At boot `litellm-config` calls into `litellm.proxy.read_model_list` and returns
|
||||
resolved deployments to the gateway, which constructs the router. The Python
|
||||
backend still reuses the **real proxy config reader** (`ProxyConfig.get_config`),
|
||||
so everything the proxy supports in config.yaml works here too:
|
||||
|
||||
- `include:` to merge in other config files,
|
||||
- `os.environ/VAR` secret references (resolved via the secret manager, never
|
||||
|
|
@ -82,8 +88,8 @@ stand-in built from the environment:
|
|||
|---|---|---|
|
||||
| `OPENAI_REALTIME_MODEL` | `gpt-realtime` | The single deployment's model name (also the `?model=` clients pass). |
|
||||
|
||||
This mode links no libpython and needs no config file, but it only supports one
|
||||
hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the
|
||||
The default workspace build links no libpython and needs no config file. This
|
||||
fallback mode only supports one hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the
|
||||
stand-in only for the leanest possible build.
|
||||
|
||||
## Request logging
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# Sample realtime config for the LiteLLM Rust AI Gateway.
|
||||
#
|
||||
# The gateway loads this model_list at boot via the embedded python config
|
||||
# reader (litellm.proxy.read_model_list), which reuses the proxy's own reader —
|
||||
# so include:, os.environ/ secrets, and DB-stored models all work here too.
|
||||
# litellm-config resolves this model_list at boot through the Python config
|
||||
# reader (litellm.proxy.read_model_list), then the gateway builds its router.
|
||||
# Includes, environment secrets, and database-stored models still work.
|
||||
#
|
||||
# Secrets are referenced (never inlined) via os.environ/. A real deploy can
|
||||
# override this file (e.g. mount a Render secret file at LITELLM_CONFIG_PATH).
|
||||
|
|
|
|||
|
|
@ -1,58 +0,0 @@
|
|||
//! GIL-activity tracking.
|
||||
//!
|
||||
//! Every acquisition of the Python GIL is recorded here so the `/health/gil`
|
||||
//! endpoint can report whether Python was touched recently. The design goal is
|
||||
//! that the GIL is acquired **only at load time** (config read) and never on the
|
||||
//! realtime hot path — polling this endpoint during traffic should show the
|
||||
//! count holding steady and `acquired_last_30s` falling to `false`.
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// Window (seconds) for the "recently acquired" signal.
|
||||
pub const RECENT_WINDOW_SECS: u64 = 30;
|
||||
|
||||
static GIL_ACQUISITIONS: AtomicU64 = AtomicU64::new(0);
|
||||
/// Unix seconds of the last acquisition; `0` means "never".
|
||||
static LAST_GIL_UNIX_SECS: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn now_unix_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Record that the GIL was just acquired. Call immediately before taking the GIL.
|
||||
///
|
||||
/// Only invoked under the `python-config` feature; without it the gateway never
|
||||
/// touches Python, so the recorder is unused (and the endpoint reports zero).
|
||||
#[cfg_attr(not(feature = "python-config"), allow(dead_code))]
|
||||
pub fn record_acquisition() {
|
||||
GIL_ACQUISITIONS.fetch_add(1, Ordering::Relaxed);
|
||||
LAST_GIL_UNIX_SECS.store(now_unix_secs(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Point-in-time view of GIL activity.
|
||||
pub struct GilSnapshot {
|
||||
pub total_acquisitions: u64,
|
||||
pub seconds_since_last: Option<u64>,
|
||||
pub acquired_last_30s: bool,
|
||||
}
|
||||
|
||||
/// Read the current GIL-activity snapshot.
|
||||
pub fn snapshot() -> GilSnapshot {
|
||||
let total = GIL_ACQUISITIONS.load(Ordering::Relaxed);
|
||||
let last = LAST_GIL_UNIX_SECS.load(Ordering::Relaxed);
|
||||
let seconds_since_last = if last == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(now_unix_secs().saturating_sub(last))
|
||||
};
|
||||
let acquired_last_30s = seconds_since_last.is_some_and(|secs| secs <= RECENT_WINDOW_SECS);
|
||||
GilSnapshot {
|
||||
total_acquisitions: total,
|
||||
seconds_since_last,
|
||||
acquired_last_30s,
|
||||
}
|
||||
}
|
||||
|
|
@ -10,18 +10,13 @@
|
|||
//! - [`io`]: compatibility exports and realtime WebSocket splice helpers.
|
||||
//! - The server modules ([`auth`], [`routes`], [`state`]) and anything pulling
|
||||
//! `axum` are gated behind the `server` feature, which the `litellm-ai-gateway`
|
||||
//! binary turns on. The `python-config` feature additionally pulls in [`python`]
|
||||
//! for the load-time config reader.
|
||||
//! binary turns on.
|
||||
|
||||
pub mod audio_transcription;
|
||||
mod client;
|
||||
pub mod io;
|
||||
pub mod ocr;
|
||||
|
||||
/// GIL-activity tracking. Pure (atomics only); shared by the `server` routes and
|
||||
/// the `python-config` reader, so it is available without either feature.
|
||||
pub mod gil;
|
||||
|
||||
#[cfg(feature = "server")]
|
||||
pub mod auth;
|
||||
#[cfg(feature = "server")]
|
||||
|
|
@ -35,6 +30,3 @@ mod constants;
|
|||
pub mod integrations;
|
||||
#[cfg(feature = "server")]
|
||||
mod realtime;
|
||||
|
||||
#[cfg(feature = "python-config")]
|
||||
pub mod python;
|
||||
|
|
|
|||
|
|
@ -14,12 +14,12 @@ use std::sync::Arc;
|
|||
use litellm_ai_gateway::io::realtime_pool::{PoolConfig, RealtimePool, upstream_key};
|
||||
use litellm_ai_gateway::routes;
|
||||
use litellm_ai_gateway::state::AppState;
|
||||
#[cfg(feature = "python-config")]
|
||||
use litellm_config::load_model_list;
|
||||
use litellm_core::router::{Deployment, LiteLLMParams, Router};
|
||||
|
||||
use litellm_ai_gateway::integrations::custom_logger::CustomLogger;
|
||||
use litellm_ai_gateway::integrations::litellm_python_proxy_api::LiteLLMPythonProxyAPILogger;
|
||||
#[cfg(feature = "python-config")]
|
||||
use litellm_ai_gateway::python;
|
||||
|
||||
/// Bind to localhost by default so the gateway is not a public, unauthenticated
|
||||
/// provider proxy out of the box. Override with `HOST` (e.g. `0.0.0.0`).
|
||||
|
|
@ -124,10 +124,10 @@ fn resolve_port() -> u16 {
|
|||
fn build_router() -> Router {
|
||||
#[cfg(feature = "python-config")]
|
||||
if let Ok(config_path) = std::env::var("LITELLM_CONFIG_PATH") {
|
||||
match python::config::load_router_from_config(&config_path) {
|
||||
Ok(router) => {
|
||||
match load_model_list(std::path::Path::new(&config_path)) {
|
||||
Ok(deployments) => {
|
||||
eprintln!("loaded model_list from {config_path} via python config reader");
|
||||
return router;
|
||||
return Router::new(deployments);
|
||||
}
|
||||
Err(err) => {
|
||||
eprintln!("config load failed ({err}); falling back to env deployment");
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
# ai-gateway/src/python — Python interop (load-time only)
|
||||
|
||||
Functions here embed the Python interpreter (pyo3) and take the GIL to call into
|
||||
`litellm` (e.g. read the proxy `model_list`). Compiled only under the
|
||||
`python-config` feature.
|
||||
|
||||
## Hard rule: non-hot-path functions only
|
||||
|
||||
Everything in this folder MUST run **at most once per process lifetime — at
|
||||
startup / load time** (config read, warm-up). NEVER call into Python on the
|
||||
request path:
|
||||
|
||||
- No GIL acquisition per request, per connection, or per realtime event.
|
||||
- No Python call inside a route handler, the router's hot path, or any loop that
|
||||
scales with traffic.
|
||||
|
||||
**Why:** the GIL serializes execution and would cap throughput; the realtime data
|
||||
path must stay pure Rust. Every acquisition is recorded by `crate::gil` — poll
|
||||
`GET /health/gil`, and `total_acquisitions` MUST stay flat under load.
|
||||
|
||||
## How to add one
|
||||
|
||||
Resolve whatever Python-derived data you need **once at boot** and hand the rest
|
||||
of the gateway an owned, plain-Rust value (e.g. build a `Router` from the
|
||||
resolved `model_list`). Record the acquisition via `crate::gil::record_acquisition()`
|
||||
immediately before taking the GIL. If a function would need to run per request,
|
||||
it does not belong here — move the work to Rust, or pre-resolve it at startup.
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
//! Build the router by calling the Python proxy config reader (load time only).
|
||||
//!
|
||||
//! Embeds the interpreter via pyo3 and calls
|
||||
//! `litellm.proxy.read_model_list.read_model_list`, which reuses the proxy's
|
||||
//! `os.environ/` + secret-manager resolution. The GIL is taken **once at boot**
|
||||
//! (and recorded in [`crate::gil`]); the realtime hot path never touches Python.
|
||||
//!
|
||||
//! Compiled only under the `python-config` feature.
|
||||
use litellm_core::error::Error;
|
||||
use litellm_core::router::{Deployment, Router};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use crate::gil;
|
||||
|
||||
/// Load the router's `model_list` from `config_path` via the Python reader.
|
||||
pub fn load_router_from_config(config_path: &str) -> Result<Router, Error> {
|
||||
gil::record_acquisition();
|
||||
Python::attach(|py| {
|
||||
let model_list = py
|
||||
.import("litellm.proxy.read_model_list")
|
||||
.and_then(|module| module.getattr("read_model_list"))
|
||||
.and_then(|reader| reader.call1((config_path,)))
|
||||
.map_err(|err| Error::Routing(format!("read_model_list failed: {err}")))?;
|
||||
|
||||
let model_list_json: String = py
|
||||
.import("json")
|
||||
.and_then(|json| json.getattr("dumps"))
|
||||
.and_then(|dumps| dumps.call1((model_list,)))
|
||||
.and_then(|encoded| encoded.extract())
|
||||
.map_err(|err| Error::Routing(format!("serializing model_list failed: {err}")))?;
|
||||
|
||||
let deployments: Vec<Deployment> = serde_json::from_str(&model_list_json)
|
||||
.map_err(|err| Error::Routing(format!("parsing model_list failed: {err}")))?;
|
||||
|
||||
Ok(Router::new(deployments))
|
||||
})
|
||||
}
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
//! Python interop for the gateway. See `AGENTS.md`: **load-time / non-hot-path
|
||||
//! only.** Compiled only under the `python-config` feature.
|
||||
|
||||
pub mod config;
|
||||
|
|
@ -13,7 +13,7 @@ private). This is the norm — don't split until it hurts.
|
|||
pub fn router() -> Router<AppState> { Router::new().route(PATH, get(handle)) }
|
||||
async fn handle(...) -> impl IntoResponse { ... }
|
||||
```
|
||||
`health.rs` and `gil.rs` are examples.
|
||||
`health.rs` is the example.
|
||||
|
||||
## Split out `service` when there's real logic
|
||||
When a route has business logic worth testing without axum, put it in a sibling
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
//! `GET /health/gil` — poll to confirm Python is only touched at load time.
|
||||
//! Simple-route template: a `router()` plus its handler, in one file.
|
||||
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::gil;
|
||||
use crate::state::AppState;
|
||||
|
||||
/// This route's contribution to the app router.
|
||||
pub fn router() -> Router<AppState> {
|
||||
Router::new().route("/health/gil", get(status))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct GilStatusResponse {
|
||||
gil_acquired_last_30s: bool,
|
||||
total_acquisitions: u64,
|
||||
seconds_since_last: Option<u64>,
|
||||
}
|
||||
|
||||
async fn status() -> Json<GilStatusResponse> {
|
||||
let snapshot = gil::snapshot();
|
||||
Json(GilStatusResponse {
|
||||
gil_acquired_last_30s: snapshot.acquired_last_30s,
|
||||
total_acquisitions: snapshot.total_acquisitions,
|
||||
seconds_since_last: snapshot.seconds_since_last,
|
||||
})
|
||||
}
|
||||
|
|
@ -2,10 +2,9 @@
|
|||
//!
|
||||
//! **Template:** every route module exposes `pub fn router() -> Router<AppState>`
|
||||
//! that mounts its own paths; [`app`] merges them. A trivial route is a single
|
||||
//! file (`health.rs`, `gil.rs`); a non-trivial one is a folder (`realtime/`) with
|
||||
//! file (`health.rs`); a non-trivial one is a folder (`realtime/`) with
|
||||
//! `handler` (entry) + `service` (logic) + `transport` (adapters). See AGENTS.md.
|
||||
|
||||
pub mod gil;
|
||||
pub mod health;
|
||||
pub mod messages;
|
||||
pub mod realtime;
|
||||
|
|
@ -19,7 +18,6 @@ use crate::state::AppState;
|
|||
pub fn app(state: AppState) -> Router {
|
||||
Router::new()
|
||||
.merge(health::router())
|
||||
.merge(gil::router())
|
||||
.merge(messages::router())
|
||||
.merge(realtime::router())
|
||||
.merge(responses::router())
|
||||
|
|
|
|||
16
litellm-rust/crates/config/Cargo.toml
Normal file
16
litellm-rust/crates/config/Cargo.toml
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
[package]
|
||||
name = "litellm-config"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-core.workspace = true
|
||||
pyo3 = { workspace = true, features = ["auto-initialize"], optional = true }
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
python = ["dep:pyo3"]
|
||||
11
litellm-rust/crates/config/src/error.rs
Normal file
11
litellm-rust/crates/config/src/error.rs
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
use thiserror::Error as ThisError;
|
||||
|
||||
#[derive(Debug, ThisError)]
|
||||
pub enum Error {
|
||||
#[error("read_model_list failed: {0}")]
|
||||
PythonLoading(String),
|
||||
#[error("serializing model_list failed: {0}")]
|
||||
Serialization(String),
|
||||
#[error("parsing model_list failed: {0}")]
|
||||
ModelListParsing(#[source] serde_json::Error),
|
||||
}
|
||||
7
litellm-rust/crates/config/src/lib.rs
Normal file
7
litellm-rust/crates/config/src/lib.rs
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
mod error;
|
||||
#[cfg(feature = "python")]
|
||||
mod python;
|
||||
|
||||
pub use error::Error;
|
||||
#[cfg(feature = "python")]
|
||||
pub use python::load_model_list;
|
||||
76
litellm-rust/crates/config/src/python.rs
Normal file
76
litellm-rust/crates/config/src/python.rs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
use std::path::Path;
|
||||
|
||||
use litellm_core::router::Deployment;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use crate::Error;
|
||||
|
||||
pub fn load_model_list(config_path: &Path) -> Result<Vec<Deployment>, Error> {
|
||||
Python::attach(|python| {
|
||||
let model_list = python
|
||||
.import("litellm.proxy.read_model_list")
|
||||
.and_then(|module| module.getattr("read_model_list"))
|
||||
.and_then(|reader| reader.call1((config_path.to_string_lossy().as_ref(),)))
|
||||
.map_err(|error| Error::PythonLoading(error.to_string()))?;
|
||||
|
||||
let model_list_json = python
|
||||
.import("json")
|
||||
.and_then(|json| json.getattr("dumps"))
|
||||
.and_then(|dumps| dumps.call1((model_list,)))
|
||||
.and_then(|encoded| encoded.extract::<String>())
|
||||
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||||
|
||||
parse_model_list(&model_list_json)
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_model_list(model_list_json: &str) -> Result<Vec<Deployment>, Error> {
|
||||
serde_json::from_str(model_list_json).map_err(Error::ModelListParsing)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_resolved_model_list() {
|
||||
let deployments = parse_model_list(
|
||||
r#"[
|
||||
{
|
||||
"model_name": "realtime",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-realtime",
|
||||
"api_key": "resolved-secret",
|
||||
"api_base": "https://api.example.test/v1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"model_name": "without-optional-values",
|
||||
"litellm_params": {"model": "openai/gpt-4.1"}
|
||||
}
|
||||
]"#,
|
||||
)
|
||||
.expect("resolved model list should parse");
|
||||
|
||||
assert_eq!(deployments.len(), 2);
|
||||
assert_eq!(deployments[0].model_name, "realtime");
|
||||
assert_eq!(
|
||||
deployments[0].litellm_params.api_key.as_deref(),
|
||||
Some("resolved-secret")
|
||||
);
|
||||
assert_eq!(
|
||||
deployments[0].litellm_params.api_base.as_deref(),
|
||||
Some("https://api.example.test/v1")
|
||||
);
|
||||
assert_eq!(deployments[1].litellm_params.api_key, None);
|
||||
assert_eq!(deployments[1].litellm_params.api_base, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_model_list_returns_parsing_error() {
|
||||
let error = parse_model_list(r#"[{"model_name":"missing-params"}]"#)
|
||||
.expect_err("missing litellm_params should fail");
|
||||
|
||||
assert!(matches!(error, Error::ModelListParsing(_)));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
//! Enforcement: the litellm-rust workspace has exactly four crates.
|
||||
//! Enforcement: the litellm-rust workspace has exactly five crates.
|
||||
//!
|
||||
//! `core` (the Rust SDK), `ai-gateway` (the HTTP/WebSocket host),
|
||||
//! `core` (the Rust SDK), `config` (the config-loading boundary),
|
||||
//! `ai-gateway` (the HTTP/WebSocket host),
|
||||
//! `python-interop` (domain-neutral PyO3 primitives), and `python-bridge` (the
|
||||
//! PyO3 cdylib). Adding or removing a crate must be a
|
||||
//! deliberate act: this test fails until the allowlist here is updated, forcing
|
||||
|
|
@ -19,13 +20,20 @@ use std::path::{Path, PathBuf};
|
|||
/// workspace legitimately gains or loses a crate.
|
||||
const EXPECTED_MEMBERS: &[&str] = &[
|
||||
"crates/core",
|
||||
"crates/config",
|
||||
"crates/ai-gateway",
|
||||
"crates/python-interop",
|
||||
"crates/python-bridge",
|
||||
];
|
||||
|
||||
/// The crate subdirectory names that must exist under `crates/`.
|
||||
const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-interop", "python-bridge"];
|
||||
const EXPECTED_CRATE_DIRS: &[&str] = &[
|
||||
"core",
|
||||
"config",
|
||||
"ai-gateway",
|
||||
"python-interop",
|
||||
"python-bridge",
|
||||
];
|
||||
|
||||
const MISMATCH: &str = "litellm-rust crate set changed — update this allowlist AND litellm-rust/AGENTS.md, and justify the crate per the rule (crate = layer needing independent compilation / its own deps / a separate artifact).";
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue