This commit is contained in:
yujonglee 2026-10-24 19:16:35 +00:00 • committed by GitHub
commit 56d89a4bae
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
138 changed files with 6212 additions and 1566 deletions

View file

@ -3647,12 +3647,26 @@ name = "litellm-config"
version = "0.1.0"
dependencies = [
"litellm-auth-types",
"litellm-python-compat",
"litellm-router-types",
"macro_rules_attribute",
"rstest",
"serde",
"serde_json",
"serde_yaml_ng",
"strum",
"thiserror 2.0.19",
]
[[package]]
name = "litellm-config-load"
version = "0.1.0"
dependencies = [
"litellm-config",
"litellm-core-utils",
"litellm-http-types",
"rstest",
"serde_yaml_ng",
"tempfile",
"thiserror 2.0.19",
]
@ -3737,6 +3751,7 @@ dependencies = [
"http-body-util",
"litellm-auth-types",
"litellm-config",
"litellm-config-load",
"litellm-gateway-auth",
"litellm-gateway-inference",
"litellm-gateway-mcp",
@ -3837,6 +3852,7 @@ dependencies = [
"http 1.4.2",
"litellm-auth-types",
"litellm-config",
"litellm-gateway-auth",
"litellm-secrets",
"moka",
"rmcp",
@ -3944,25 +3960,35 @@ dependencies = [
"http 1.4.2",
"hyper-util",
"litellm-core-utils",
"litellm-http-types",
"rcgen",
"reqwest 0.12.28",
"reqwest 0.13.5",
"rmcp",
"rstest",
"rustls 0.23.42",
"rustls-native-certs",
"serde",
"serde_json",
"strum",
"tempfile",
"thiserror 2.0.19",
"tokio",
"tokio-rustls 0.26.4",
"tokio-tungstenite",
"tracing",
"veil",
"webpki-roots",
]
[[package]]
name = "litellm-http-types"
version = "0.1.0"
dependencies = [
"litellm-core-utils",
"rstest",
"veil",
]
[[package]]
name = "litellm-inference"
version = "0.1.0"
@ -4291,6 +4317,7 @@ name = "litellm-router-types"
version = "0.1.0"
dependencies = [
"litellm-auth-types",
"litellm-python-compat",
"rstest",
"serde",
"serde_json",
@ -7403,11 +7430,7 @@ checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"
dependencies = [
"futures-util",
"log",
"rustls 0.23.42",
"rustls-native-certs",
"rustls-pki-types",
"tokio",
"tokio-rustls 0.26.4",
"tungstenite",
]
@ -7752,8 +7775,6 @@ dependencies = [
"httparse",
"log",
"rand 0.8.7",
"rustls 0.23.42",
"rustls-pki-types",
"sha1 0.10.7",
"thiserror 1.0.69",
"utf-8",

View file

@ -10,6 +10,7 @@ repository = "https://github.com/BerriAI/litellm"
[workspace.dependencies]
litellm-config = { path = "crates/config" }
litellm-config-load = { path = "crates/config-load" }
litellm-router = { path = "crates/router" }
litellm-router-types = { path = "crates/router-types" }
litellm-tracing = { path = "crates/tracing" }
@ -47,6 +48,7 @@ litellm-secrets-hashicorp = { path = "crates/secrets-hashicorp" }
litellm-secrets-azure = { path = "crates/secrets-azure" }
litellm-secrets-cyberark = { path = "crates/secrets-cyberark" }
litellm-http = { path = "crates/http" }
litellm-http-types = { path = "crates/http-types" }
litellm-llms = { path = "crates/llms" }
litellm-llms-types = { path = "crates/llms-types" }
litellm-core-utils = { path = "crates/core-utils" }
@ -97,7 +99,6 @@ rustls = { version = "0.23", default-features = false, features = ["ring", "std"
rustify = "=0.7.0"
rustify_derive = "=0.5.5"
vaultrs = { version = "=0.8.0", default-features = false, features = ["rustls"] }
rustls-native-certs = "0.8"
serde = { version = "1.0", features = ["derive"] }
serde_json = { version = "1.0", features = ["float_roundtrip"] }
serde_with = { version = "=3.21.0", default-features = false, features = ["std", "macros"] }
@ -111,7 +112,7 @@ thiserror = "2.0"
tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
tiktoken-rs = "0.12.0"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] }
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
tokio-tungstenite = { version = "0.24", default-features = false, features = ["handshake"] }
futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
base64 = "0.22"
flate2 = "1"

View file

@ -0,0 +1,35 @@
## Scope
- This crate is the vocabulary for authenticating LiteLLM to a provider: how a credential is named before it is known (`CredentialRef`, `CredentialPlan`, `ParamSpec`), how it is obtained (`CredentialResolver`, `TokenProvider`, `ResolvedCredential`), how it is carried (`SecretValue`, `Sourced`), how it lands on a request (`CredentialPlacement`, `ProviderAuthPolicy`) and how that fails (`Error`)
- Outbound only. Who is calling the gateway is `gateway-auth`. The one exception is `UiCredentials`, which is here because `gateway-auth` and `gateway-ui` both read it and neither may depend on the other
- No I/O and no clock. Resolving a reference, fetching a token or reading a file is a `CredentialResolver` or `TokenProvider` implementation in `auth-aws`, `auth-azure`, `auth-gcp` or a host. This crate only defines the traits and runs the plan over an injected handle
- Depends on `serde`, `serde_json`, `subtle`, `thiserror` and `veil` only. Nothing here may pull in an HTTP client, a cloud SDK or a provider crate
## Layering
- `auth-types` (contracts) is below `auth-aws`, `auth-azure` and `auth-gcp` (one cloud's acquisition each), which are below `auth` (the `AuthServices` facade that bundles them behind features). `llms`, `inference-*`, `secrets-*` and `cache-*` take the types from here and the services from `auth`
- Import every type from `litellm_auth_types`. `auth` still glob re-exports this crate and `secrets-types` re-exports `SecretValue`, both against the workspace rule in `../inference/AGENTS.md`. Do not add a third path, and move consumers off those two when touching them
- A provider's own credential shape (`AwsParams`, `VertexParams`) lives in `src/params/` only because routing, config and the provider all read it. A field or env name used by one provider stays in that provider's crate
## Contracts
- `SecretValue` is the only way a secret moves between crates. `Debug` is redacted through `veil`, equality is constant time, and `expose()` is the single read. Never add `Display`, `Serialize` or an `AsRef<str>` to it
- `Sourced<T>` records whether a value came from the request, the deployment or the environment. Precedence between those is decided by whoever builds the `Sourced`, never by this crate
- `CredentialPlan::resolve` has three outcomes and they are not interchangeable. `Resolved` carries a credential, `Unavailable` means the caller may fall back to the next plan, and `Err` is terminal. A resolver that cannot answer returns `Declined` or `Missing`, it never errors to mean "not mine"
- A `CredentialRef::Explicit` resolves without consulting the resolver. Everything else, including `Host`, is resolved at acquisition time so rotated secrets are read fresh
- `apply_credential` prepends the credential header and refuses an empty credential or a header already present. Whether an existing header is preserved or rejected is `ProviderAuthPolicy::existing_header_behavior`, chosen per provider, never a default here
- `ParamSpec` lists every place Python reads one connection param, in order: wire names, the `litellm.<name>` module global, then environment names. `resolve` walks that order and treats blank as absent. The static tables in `src/params/` are the one copy of those names
## Errors
- `Error` is the single credential error every provider nests. `auth-aws` maps its SDK failures into it with `From`, `auth-azure` and `auth-gcp` build the variants directly. Either way a consumer never sees a cloud SDK error type
- `MissingApiKey`, `MissingApiBase` and `MissingParam` are what the end user reads when a setting is absent. The message names what to pass or set, and `ParamSpec::guidance` derives that from the spec so the two never drift
- `ErrorDetail::Failed` wraps a source error by `Arc`, so equality is pointer identity. Tests compare the `Message`, `Empty`, `InvalidType` and `DuplicateHeader` variants and only match on `Failed`
## Known gaps
- `ProviderAuthPolicy` and `CredentialPlan` have no production consumer yet, `auth/tests/facade.rs` is the only user. Wire them through `llms/src/base_llm/auth.rs` or delete them, do not grow them unused
## Tests
- Test what crosses the boundary: redaction in `Debug` output, which header a placement writes, which plan outcome a resolver answer produces. Never test that a type derives something

View file

@ -7,6 +7,7 @@ mod params;
mod policy;
mod secret;
mod token;
mod ui;
use serde::{Deserialize, Serialize};
@ -57,3 +58,4 @@ pub use params::{AwsParams, ParamSpec, VertexParams};
pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
pub use secret::SecretValue;
pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};
pub use ui::UiCredentials;

View file

@ -0,0 +1,9 @@
use serde::Deserialize;
use crate::SecretValue;
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
pub struct UiCredentials {
pub username: String,
pub password: SecretValue,
}

View file

@ -0,0 +1,12 @@
## Scope
- This crate is the gateway's config loader. It takes the types `litellm-config` parses and turns them into the values the gateway runs on, so env lookups, the DB overlay, reference substitution, defaults and the domain types live here, never in `litellm-config`
- It never reads process state. The caller reads the process environment once and passes it in as an `Environment`, so tests pass their own variables
- Python never uses this crate. It shares only the `litellm-config` types, so behavior here may differ from Python where Python is wrong, and each difference is listed in the config compat doc
- Fixture and corpus rules live in `tests/AGENTS.md`
## Precedence
- Each lowered value takes the env, the file and then the default in the order Python uses for that key today, until that order is normalized
- HTTP client settings are lowered into `litellm-http-types` layers (`http_settings`), one layer per source, so this crate never defines its own transport types

View file

@ -0,0 +1,17 @@
[package]
name = "litellm-config-load"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-config.workspace = true
litellm-core-utils.workspace = true
litellm-http-types.workspace = true
serde_yaml_ng = "0.10.0"
thiserror.workspace = true
[dev-dependencies]
rstest.workspace = true
tempfile.workspace = true

View file

@ -0,0 +1,3 @@
fn main() {
println!("cargo::rerun-if-changed=tests/fixtures");
}

View file

@ -0,0 +1,26 @@
use std::fmt;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("could not read config")]
Read(#[from] std::io::Error),
#[error("invalid YAML config")]
Parse(#[from] serde_yaml_ng::Error),
#[error("invalid config")]
Config(#[from] litellm_config::Error),
#[error("could not load config {key}")]
Document {
key: String,
#[source]
source: Box<Error>,
},
}
impl Error {
pub(crate) fn document(key: &impl fmt::Debug, source: Error) -> Self {
Self::Document {
key: format!("{key:?}"),
source: Box::new(source),
}
}
}

View file

@ -0,0 +1,156 @@
use litellm_config::{LiteLlmSettings, Spelled};
use litellm_http_types::{HttpSettings, HttpSettingsLayer, SslVerify};
use crate::Environment;
pub fn http_settings(settings: &LiteLlmSettings, environment: &Environment) -> HttpSettings {
HttpSettings::from_layers([
HttpSettingsLayer::from_environment(environment),
written(settings),
])
}
fn written(settings: &LiteLlmSettings) -> HttpSettingsLayer {
HttpSettingsLayer {
ssl_verify: settings.tls.ssl_verify.as_ref().map(ssl_verify),
ssl_certificate: settings
.client_certificate
.ssl_certificate
.clone()
.map(Into::into),
ssl_security_level: settings.tls.ssl_security_level.clone(),
ssl_ecdh_curve: settings.tls.ssl_ecdh_curve.clone(),
force_ipv4: settings.socket.force_ipv4,
http2: settings.http2.http2,
aiohttp_trust_env: settings.aiohttp.aiohttp_trust_env,
disable_aiohttp_trust_env: settings.aiohttp.disable_aiohttp_trust_env,
disable_aiohttp_transport: settings.aiohttp.disable_aiohttp_transport,
..HttpSettingsLayer::default()
}
}
fn ssl_verify(value: &Spelled<bool>) -> SslVerify {
match value {
Spelled::Value(true) => SslVerify::Enabled,
Spelled::Value(false) => SslVerify::Disabled,
Spelled::Text(text) => SslVerify::parse(text),
}
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use rstest::rstest;
use litellm_config::Config;
use litellm_http_types::{HttpSettings, SslVerify};
use super::http_settings;
use crate::Environment;
fn lowered(yaml: &str, variables: &[(&str, &str)]) -> HttpSettings {
let config = Config::from_yaml(yaml).unwrap();
let environment = variables
.iter()
.map(|(name, value)| ((*name).to_owned(), (*value).to_owned()))
.collect();
http_settings(&config.litellm_settings, &environment)
}
#[test]
fn absent_everywhere_is_the_default() {
assert_eq!(lowered("{}", &[]), HttpSettings::default());
}
#[rstest]
#[case::default("{}", &[], None)]
#[case::written("litellm_settings: {ssl_verify: false}", &[], Some(SslVerify::Disabled))]
#[case::written_text("litellm_settings: {ssl_verify: 'False'}", &[], Some(SslVerify::Disabled))]
#[case::written_bundle("litellm_settings: {ssl_verify: /etc/roots.pem}", &[], Some(SslVerify::CaBundle(PathBuf::from("/etc/roots.pem"))))]
#[case::environment("{}", &[("SSL_VERIFY", "False")], Some(SslVerify::Disabled))]
#[case::environment_over_file("litellm_settings: {ssl_verify: false}", &[("SSL_VERIFY", "true")], Some(SslVerify::Enabled))]
fn ssl_verify_takes_environment_then_file(
#[case] yaml: &str,
#[case] variables: &[(&str, &str)],
#[case] expected: Option<SslVerify>,
) {
assert_eq!(lowered(yaml, variables).ssl_verify, expected);
}
#[rstest]
#[case::file("litellm_settings: {ssl_certificate: /etc/client.pem}", &[], "/etc/client.pem")]
#[case::environment_over_file("litellm_settings: {ssl_certificate: /etc/client.pem}", &[("SSL_CERTIFICATE", "/env/client.pem")], "/env/client.pem")]
fn client_certificate_takes_environment_then_file(
#[case] yaml: &str,
#[case] variables: &[(&str, &str)],
#[case] expected: &str,
) {
assert_eq!(
lowered(yaml, variables).ssl_certificate,
Some(PathBuf::from(expected))
);
}
#[rstest]
#[case::file("litellm_settings: {http2: true}", &[], true)]
#[case::environment("{}", &[("LITELLM_HTTP2", "TRUE")], true)]
#[case::environment_false_does_not_override_file("litellm_settings: {http2: true}", &[("LITELLM_HTTP2", "false")], true)]
fn flags_are_only_turned_on_by_the_environment(
#[case] yaml: &str,
#[case] variables: &[(&str, &str)],
#[case] expected: bool,
) {
assert_eq!(lowered(yaml, variables).http2, expected);
}
#[test]
fn force_ipv4_comes_from_the_file_only() {
assert!(lowered("litellm_settings: {force_ipv4: true}", &[]).force_ipv4);
assert!(!lowered("{}", &[("FORCE_IPV4", "true")]).force_ipv4);
}
#[rstest]
#[case::file("litellm_settings: {ssl_security_level: '1'}", &[], Some("1"))]
#[case::environment_over_file("litellm_settings: {ssl_security_level: '1'}", &[("SSL_SECURITY_LEVEL", "2")], Some("2"))]
#[case::empty_is_unset("{}", &[("SSL_SECURITY_LEVEL", "")], None)]
fn text_settings_skip_empty_values(
#[case] yaml: &str,
#[case] variables: &[(&str, &str)],
#[case] expected: Option<&str>,
) {
assert_eq!(
lowered(yaml, variables).ssl_security_level.as_deref(),
expected
);
}
#[test]
fn aiohttp_flags_from_the_file_decide_proxy_trust() {
assert!(
!lowered("litellm_settings: {disable_aiohttp_trust_env: true}", &[]).trust_proxy_env
);
assert!(
lowered(
"litellm_settings: {disable_aiohttp_trust_env: true, aiohttp_trust_env: true}",
&[]
)
.trust_proxy_env
);
}
#[test]
fn later_overlay_wins_and_unrelated_variables_are_ignored() {
let environment = Environment::from_iter([("SSL_VERIFY".to_owned(), "true".to_owned())])
.overlaid([
("SSL_VERIFY".to_owned(), "false".to_owned()),
("HTTP2".to_owned(), "true".to_owned()),
]);
let settings = http_settings(
&Config::from_yaml("{}").unwrap().litellm_settings,
&environment,
);
assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled));
assert!(!settings.http2);
}
}

View file

@ -0,0 +1,39 @@
use std::collections::{BTreeSet, VecDeque};
use serde_yaml_ng::{Mapping, Value};
use super::Store;
use crate::Error;
#[derive(Debug)]
pub struct Document<K> {
pub key: K,
pub body: Mapping,
}
pub fn documents<S: Store>(store: &S, root: S::Key) -> Result<Vec<Document<S::Key>>, Error> {
let mut pending = VecDeque::from([root.clone()]);
let mut loaded = BTreeSet::new();
let mut documents = Vec::new();
while let Some(key) = pending.pop_front() {
if !loaded.insert(key.clone()) {
continue;
}
let (body, includes) = read(store, &key).map_err(|error| Error::document(&key, error))?;
for entry in includes {
pending.push_back(store.locate(&entry, &key, &root)?);
}
documents.push(Document { key, body });
}
Ok(documents)
}
fn read<S: Store>(store: &S, key: &S::Key) -> Result<(Mapping, Vec<String>), Error> {
let mut body = litellm_config::yaml::mapping(&store.read(key)?)?;
let includes = body
.remove(Value::from("include"))
.map(serde_yaml_ng::from_value)
.transpose()?
.unwrap_or_default();
Ok((body, includes))
}

View file

@ -0,0 +1,90 @@
use std::collections::BTreeMap;
use serde_yaml_ng::{Mapping, Value};
use super::Document;
#[derive(Debug)]
pub struct Merged<K> {
pub document: Mapping,
origins: BTreeMap<String, Vec<(K, usize)>>,
}
impl<K> Merged<K> {
/// Maps item `index` of the merged top-level `field` back to its document and its index there.
/// A value that is not a list is item 0.
pub fn locate(&self, field: &str, index: usize) -> Option<(&K, usize)> {
self.origins
.get(field)?
.iter()
.scan(0, |start, (key, len)| {
let run = (key, *start, *len);
*start += len;
Some(run)
})
.find(|(_, start, len)| (*start..start + len).contains(&index))
.map(|(key, start, _)| (key, index - start))
}
}
pub fn merge<K: Clone>(documents: &[Document<K>]) -> Merged<K> {
let fields = documents
.iter()
.flat_map(|document| document.body.keys())
.fold(Vec::new(), |mut fields, field| {
if !fields.contains(&field) {
fields.push(field);
}
fields
});
let (document, origins) = fields
.into_iter()
.map(|field| {
let layers: Vec<_> = documents
.iter()
.filter_map(|document| Some((&document.key, document.body.get(field)?)))
.collect();
(field, combine(&layers))
})
.map(|(field, (value, runs))| {
(
(field.clone(), value),
(field.as_str().map(str::to_owned), runs),
)
})
.unzip::<_, _, Mapping, Vec<_>>();
Merged {
document,
origins: origins
.into_iter()
.filter_map(|(field, runs)| Some((field?, runs)))
.collect(),
}
}
/// Python's rule (config L16): a list extends a list, anything else replaces. So the merged value is the
/// lists after the last non-list concatenated, or that non-list alone when nothing follows it.
fn combine<K: Clone>(layers: &[(&K, &Value)]) -> (Value, Vec<(K, usize)>) {
let tail = layers
.iter()
.rposition(|(_, value)| !value.is_sequence())
.map_or(0, |last| last + 1);
match &layers[tail..] {
[] => {
let (key, value) = layers[layers.len() - 1];
(value.clone(), vec![(key.clone(), 1)])
}
lists => (
Value::Sequence(
lists
.iter()
.flat_map(|(_, value)| value.as_sequence().into_iter().flatten().cloned())
.collect(),
),
lists
.iter()
.map(|(key, value)| ((*key).clone(), value.as_sequence().map_or(0, Vec::len)))
.collect(),
),
}
}

View file

@ -0,0 +1,169 @@
mod documents;
mod merge;
mod store;
pub use documents::{Document, documents};
pub use merge::{Merged, merge};
pub use store::{Files, Store};
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use rstest::rstest;
use serde_yaml_ng::Value;
use super::*;
use crate::Error;
struct Memory(BTreeMap<&'static str, &'static str>);
impl<const N: usize> From<[(&'static str, &'static str); N]> for Memory {
fn from(files: [(&'static str, &'static str); N]) -> Self {
Self(files.into())
}
}
impl Store for Memory {
type Key = &'static str;
fn locate(
&self,
entry: &str,
_: &&'static str,
_: &&'static str,
) -> Result<&'static str, Error> {
self.0
.get_key_value(entry)
.map(|(key, _)| *key)
.ok_or_else(|| {
Error::document(
&entry,
std::io::Error::from(std::io::ErrorKind::NotFound).into(),
)
})
}
fn read(&self, key: &&'static str) -> Result<String, Error> {
Ok(self.0[key].to_owned())
}
}
fn load(files: impl Into<Memory>) -> Result<Merged<&'static str>, Error> {
documents(&files.into(), "root").map(|documents| merge(&documents))
}
fn yaml(text: &str) -> Value {
serde_yaml_ng::from_str(text).unwrap()
}
#[rstest]
fn reads_each_document_once_in_breadth_first_order() {
let order = documents(
&Memory::from([
("root", "include: [a, b]"),
("a", "include: [c, root]"),
("b", "include: [a]"),
("c", "{}"),
]),
"root",
)
.unwrap()
.into_iter()
.map(|document| document.key)
.collect::<Vec<_>>();
assert_eq!(order, ["root", "a", "b", "c"]);
}
#[rstest]
#[case::lists_concatenate("x: [1]", "x: [2]", "[1, 2]")]
#[case::mappings_replace_whole("x: {a: 1, b: 2}", "x: {c: 3}", "{c: 3}")]
#[case::scalar_replaces_list("x: [1]", "x: 2", "2")]
#[case::list_replaces_scalar("x: 1", "x: [2]", "[2]")]
fn merges_top_level_keys_like_python(
#[case] root: &'static str,
#[case] other: &'static str,
#[case] expected: &str,
) {
let root = Box::leak(format!("include: [other]\n{root}").into_boxed_str());
let merged = load([("root", &*root), ("other", other)]).unwrap();
assert_eq!(merged.document.get("x"), Some(&yaml(expected)));
assert!(!merged.document.contains_key("include"));
}
#[rstest]
fn keeps_concatenating_after_a_replace() {
let merged = load([
("root", "include: [a, b, c]\nx: [1]"),
("a", "x: 2"),
("b", "x: [3]"),
("c", "x: [4, 5]"),
])
.unwrap();
assert_eq!(merged.document.get("x"), Some(&yaml("[3, 4, 5]")));
assert_eq!(merged.locate("x", 0), Some((&"b", 0)));
assert_eq!(merged.locate("x", 2), Some((&"c", 1)));
assert_eq!(merged.locate("x", 3), None);
}
#[rstest]
fn locates_list_items_and_replaced_values_in_their_documents() {
let merged = load([
(
"root",
"include: [a]\nmodel_list: [r0, r1]\ngeneral_settings: {k: 1}\nonly_root: 1",
),
("a", "model_list: [a0]\ngeneral_settings: {k: 2}"),
])
.unwrap();
assert_eq!(merged.locate("model_list", 1), Some((&"root", 1)));
assert_eq!(merged.locate("model_list", 2), Some((&"a", 0)));
assert_eq!(merged.locate("general_settings", 0), Some((&"a", 0)));
assert_eq!(merged.locate("only_root", 0), Some((&"root", 0)));
assert_eq!(merged.locate("missing", 0), None);
}
#[rstest]
fn names_the_missing_include() {
let error = load([("root", "include: [gone.yaml]")]).unwrap_err();
assert!(
matches!(error, Error::Document { ref key, .. } if key.contains("gone.yaml")),
"{error:?}"
);
}
#[rstest]
#[case::bare_string("include: a.yaml")]
#[case::number_item("include: [1]")]
fn rejects_include_that_is_not_a_list_of_paths(#[case] root: &'static str) {
assert!(load([("root", root)]).is_err());
}
#[rstest]
#[case::declaring_directory_wins(&["nested/x.yaml", "x.yaml"], "nested/x.yaml")]
#[case::falls_back_to_root_directory(&["x.yaml"], "x.yaml")]
#[case::reports_declaring_path_when_neither_exists(&[], "nested/x.yaml")]
fn files_resolve_against_declaring_then_root_directory(
#[case] existing: &[&str],
#[case] expected: &str,
) {
let directory = tempfile::TempDir::new().unwrap();
let root = directory.path().canonicalize().unwrap();
std::fs::create_dir(root.join("nested")).unwrap();
for file in ["config.yaml", "nested/first.yaml"].iter().chain(existing) {
std::fs::write(root.join(file), "{}").unwrap();
}
let located = Files.locate(
"x.yaml",
&root.join("nested/first.yaml"),
&root.join("config.yaml"),
);
match located {
Ok(path) => assert_eq!(path, root.join(expected)),
Err(Error::Document { key, .. }) if !existing.contains(&expected) => {
assert!(key.contains(&*root.join(expected).to_string_lossy()))
}
Err(error) => panic!("{error:?}"),
}
}
}

View file

@ -0,0 +1,46 @@
use std::{
fmt,
path::{Path, PathBuf},
};
use crate::Error;
pub trait Store {
type Key: Ord + Clone + fmt::Debug;
fn locate(
&self,
entry: &str,
declaring: &Self::Key,
root: &Self::Key,
) -> Result<Self::Key, Error>;
fn read(&self, key: &Self::Key) -> Result<String, Error>;
}
pub struct Files;
impl Store for Files {
type Key = PathBuf;
fn locate(&self, entry: &str, declaring: &PathBuf, root: &PathBuf) -> Result<PathBuf, Error> {
let declared = sibling(declaring, entry);
let fallback = sibling(root, entry);
let location = if declared.exists() || !fallback.exists() {
declared
} else {
fallback
};
location
.canonicalize()
.map_err(|error| Error::document(&location, error.into()))
}
fn read(&self, key: &PathBuf) -> Result<String, Error> {
Ok(std::fs::read_to_string(key)?)
}
}
fn sibling(file: &Path, entry: &str) -> PathBuf {
file.parent().unwrap_or(Path::new(".")).join(entry)
}

View file

@ -0,0 +1,62 @@
mod error;
mod http;
pub mod include;
use std::{collections::BTreeMap, ffi::OsString, path::Path};
use litellm_config::Config;
use litellm_core_utils::settings::Lookup;
pub use self::{error::Error, http::http_settings};
pub fn load(path: impl AsRef<Path>) -> Result<Config, Error> {
let path = path.as_ref();
let root = path
.canonicalize()
.map_err(|error| Error::document(&path, error.into()))?;
load_from(&include::Files, root)
}
pub fn load_from<S: include::Store>(store: &S, root: S::Key) -> Result<Config, Error> {
Ok(Config::from_mapping(
include::merge(&include::documents(store, root)?).document,
)?)
}
#[derive(Clone, Default)]
pub struct Environment(BTreeMap<String, String>);
impl Environment {
pub fn from_os(variables: impl IntoIterator<Item = (OsString, OsString)>) -> Self {
variables
.into_iter()
.filter_map(|(name, value)| Some((name.into_string().ok()?, value.into_string().ok()?)))
.collect()
}
pub fn overlaid(self, higher: impl IntoIterator<Item = (String, String)>) -> Self {
Self(self.0.into_iter().chain(higher).collect())
}
pub fn var(&self, name: &str) -> Option<&str> {
self.0.get(name).map(String::as_str)
}
pub fn variables(&self) -> impl Iterator<Item = (String, String)> + '_ {
self.0
.iter()
.map(|(name, value)| (name.clone(), value.clone()))
}
}
impl FromIterator<(String, String)> for Environment {
fn from_iter<I: IntoIterator<Item = (String, String)>>(variables: I) -> Self {
Self(variables.into_iter().collect())
}
}
impl Lookup for Environment {
fn get(&self, name: &str) -> Option<String> {
self.0.get(name).cloned()
}
}

View file

@ -0,0 +1,5 @@
## Fixtures
- `corpus.rs` requires every config in `fixtures/valid/` to load and every config in `fixtures/invalid/` to fail. There is no known-gap list, so a valid config that fails to load is fixed in `src/`
- A config goes in `invalid/` only when Python rejects it too, either by raising on read or by crashing when the proxy loads it
- `#[files]` globs at compile time, so `build.rs` reruns the build when `fixtures/` changes

View file

@ -0,0 +1,16 @@
use std::path::PathBuf;
use litellm_config_load::load;
use rstest::rstest;
#[rstest]
fn loads_every_valid_fixture(#[files("tests/fixtures/valid/*.yaml")] path: PathBuf) {
if let Err(error) = load(&path) {
panic!("{}: {error:?}", path.display());
}
}
#[rstest]
fn rejects_every_invalid_fixture(#[files("tests/fixtures/invalid/*.yaml")] path: PathBuf) {
assert!(load(&path).is_err(), "{} loads", path.display());
}

View file

@ -0,0 +1,3 @@
These configs are copied as-is from `tests/unit/proxy/example_config_yaml` and `tests/unit/proxy/test_configs` for now. Keep each of `valid/` and `invalid/` flat so relative `include` paths still resolve
`invalid/config_with_missing_include.yaml` is in Python's `INVALID_FILES`. `invalid/langfuse_config.yaml` passes Python's read-only `get_config` test, but its model has no `litellm_params`, so Python's `load_config` raises `KeyError` on it

View file

@ -0,0 +1,5 @@
include:
- non-existent-file.yaml
litellm_settings:
callbacks: ["prometheus"]

View file

@ -0,0 +1,7 @@
model_list:
- model_name: gpt-3.5-turbo
litellm_settings:
drop_params: True
success_callback: ["langfuse"] # https://docs.litellm.ai/docs/observability/langfuse_integration

View file

@ -0,0 +1,30 @@
model_list:
- model_name: gpt-3.5-turbo-instruct
litellm_params:
model: ollama/zephyr
- model_name: gpt-4
litellm_params:
model: ollama/llama2
- model_name: gpt-3.5-turbo
litellm_params:
model: ollama/llama2
temperature: 0.1
max_tokens: 20
# request to gpt-4, response from ollama/llama2
# curl --location 'http://0.0.0.0:8000/chat/completions' \
# --header 'Content-Type: application/json' \
# --data ' {
# "model": "gpt-4",
# "messages": [
# {
# "role": "user",
# "content": "what llm are you"
# }
# ],
# }
# '
#
# {"id":"chatcmpl-27c85cf0-ab09-4bcf-8cb1-0ee950520743","choices":[{"finish_reason":"stop","index":0,"message":{"content":" Hello! I'm just an AI, I don't have personal experiences or emotions like humans do. However, I can help you with any questions or tasks you may have! Is there something specific you'd like to know or discuss?","role":"assistant","_logprobs":null}}],"created":1700094955.373751,"model":"ollama/llama2","object":"chat.completion","system_fingerprint":null,"usage":{"prompt_tokens":12,"completion_tokens":47,"total_tokens":59},"_response_ms":8028.017999999999}%

View file

@ -0,0 +1,15 @@
model_list:
- model_name: gpt-4-team1
litellm_params:
model: azure/gpt-4.1-mini
api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
api_version: "2023-05-15"
api_key: os.environ/AZURE_AI_API_KEY
tpm: 20_000
- model_name: gpt-4-team2
litellm_params:
model: azure/gpt-4
api_key: os.environ/AZURE_AI_API_KEY
api_base: https://openai-gpt-4-test-v-2.openai.azure.com/
tpm: 100_000

View file

@ -0,0 +1,7 @@
model_list:
- model_name: "openai-model"
litellm_params:
model: "gpt-3.5-turbo"
litellm_settings:
cache: True

View file

@ -0,0 +1,11 @@
model_list:
- model_name: "openai-model"
litellm_params:
model: "gpt-3.5-turbo"
litellm_settings:
cache: True
cache_params:
type: "redis"
supported_call_types: ["embedding", "aembedding"]
host: "os.environ/REDIS_HOST"

View file

@ -0,0 +1,48 @@
model_list:
################################################################################
# Azure
- model_name: gpt-4o-mini
litellm_params:
model: azure/gpt-4o-mini
api_base: https://amazin-prod.openai.azure.com
api_key: "os.environ/AZURE_GPT_4O"
deployment_id: gpt-4o-mini
- model_name: gpt-4o
litellm_params:
model: azure/gpt-4o
api_base: https://very-cool-prod.openai.azure.com
api_key: "os.environ/AZURE_GPT_4O"
deployment_id: gpt-4o
################################################################################
# Fireworks
- model_name: fireworks-llama-v3p1-405b-instruct
litellm_params:
model: fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct
api_key: "os.environ/FIREWORKS"
- model_name: fireworks-llama-v3p1-70b-instruct
litellm_params:
model: fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct
api_key: "os.environ/FIREWORKS"
general_settings:
alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+
litellm_settings: # module level litellm settings - https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py
success_callback: ["prometheus"]
service_callback: ["prometheus_system"]
drop_params: False # Raise an exception if the openai param being passed in isn't supported.
cache: false
default_internal_user_params:
user_role: os.environ/DEFAULT_USER_ROLE
success_callback: ["s3"]
s3_callback_params:
s3_bucket_name: logs-bucket-litellm # AWS Bucket Name for S3
s3_region_name: us-west-2 # AWS Region Name for S3
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets
router_settings:
routing_strategy: simple-shuffle # "simple-shuffle" shown to result in highest throughput. https://docs.litellm.ai/docs/proxy/configs#load-balancing

View file

@ -0,0 +1,5 @@
include:
- included_models.yaml
litellm_settings:
callbacks: ["prometheus"]

View file

@ -0,0 +1,6 @@
include:
- models_file_1.yaml
- models_file_2.yaml
litellm_settings:
callbacks: ["prometheus"]

View file

@ -0,0 +1,4 @@
model_list:
- model_name: included-model
litellm_params:
model: gpt-4

View file

@ -0,0 +1,28 @@
litellm_settings:
drop_params: True
# Model-specific settings
model_list: # use the same model_name for using the litellm router. LiteLLM will use the router between gpt-3.5-turbo
- model_name: gpt-3.5-turbo # litellm will
litellm_params:
model: gpt-3.5-turbo
api_key: sk-uj6F
tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm
rpm: 3 # [OPTIONAL] REPLACE with your openai rpm
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
api_key: sk-Imn
tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm
rpm: 3 # [OPTIONAL] REPLACE with your openai rpm
- model_name: gpt-3.5-turbo
litellm_params:
model: openrouter/gpt-3.5-turbo
- model_name: mistral-7b-instruct
litellm_params:
model: mistralai/mistral-7b-instruct
environment_variables:
REDIS_HOST: localhost
REDIS_PASSWORD:
REDIS_PORT:

View file

@ -0,0 +1,4 @@
model_list:
- model_name: included-model-1
litellm_params:
model: gpt-4

View file

@ -0,0 +1,4 @@
model_list:
- model_name: included-model-2
litellm_params:
model: gpt-3.5-turbo

View file

@ -0,0 +1,7 @@
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
general_settings:
otel: True # OpenTelemetry Logger this logs OTEL data to your collector

View file

@ -0,0 +1,4 @@
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo

View file

@ -0,0 +1,20 @@
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
api_key: bad-key
model: gpt-3.5-turbo
- model_name: working-azure-gpt-3.5-turbo
litellm_params:
model: azure/gpt-4.1-mini
api_base: https://azure-unit.example.com
api_key: bad-key
- model_name: azure-gpt-3.5-turbo
litellm_params:
model: azure/gpt-4.1-mini
api_base: https://azure-unit.example.com
api_key: bad-key
- model_name: azure-embedding
litellm_params:
model: azure/text-embedding-ada-002
api_base: https://azure-unit.example.com
api_key: bad-key

View file

@ -0,0 +1,17 @@
model_list:
- model_name: azure-cloudflare
litellm_params:
model: azure/gpt-4.1-mini
api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1
api_key: os.environ/AZURE_AI_API_KEY
api_version: 2023-07-01-preview
litellm_settings:
set_verbose: True
cache: True # set cache responses to True
cache_params: # set cache params for s3
type: s3
s3_bucket_name: litellm-my-test-bucket-2 # AWS Bucket Name for S3
s3_region_name: us-east-1 # AWS Region Name for S3
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # AWS Access Key ID for S3
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3

View file

@ -0,0 +1,28 @@
general_settings:
database_url: os.environ/DATABASE_URL
master_key: os.environ/PROXY_MASTER_KEY
litellm_settings:
drop_params: true
success_callback: ["langfuse"]
model_list:
- litellm_params:
api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
api_key: os.environ/AZURE_EUROPE_API_KEY
model: azure/gpt-35-turbo
model_name: azure-model
- litellm_params:
api_base: https://my-endpoint-canada-berri992.openai.azure.com
api_key: os.environ/AZURE_CANADA_API_KEY
model: azure/gpt-35-turbo
model_name: azure-model
- litellm_params:
api_base: https://openai-france-1234.openai.azure.com
api_key: os.environ/AZURE_FRANCE_API_KEY
model: azure/gpt-turbo
model_name: azure-model
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
model_name: test_openai_models

View file

@ -0,0 +1,11 @@
model_list:
- model_name: "openai-model"
litellm_params:
model: "gpt-3.5-turbo"
litellm_settings:
drop_params: True
set_verbose: True
general_settings:
custom_auth: custom_auth.user_api_key_auth

View file

@ -0,0 +1,6 @@
model_list:
- litellm_params:
model: hosted_vllm/embed_model
model_info:
description: this is a test embedding hosted_vllm model
model_name: vllm_embed_model

View file

@ -0,0 +1,127 @@
model_list:
- litellm_params:
api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
api_key: os.environ/AZURE_EUROPE_API_KEY
model: azure/gpt-35-turbo
model_name: azure-model
- litellm_params:
api_base: https://my-endpoint-canada-berri992.openai.azure.com
api_key: os.environ/AZURE_CANADA_API_KEY
model: azure/gpt-35-turbo
model_name: azure-model
- litellm_params:
api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1
api_key: os.environ/AZURE_AI_API_KEY
model: azure/gpt-4.1-mini
model_name: azure-cloudflare-model
- litellm_params:
api_base: https://openai-france-1234.openai.azure.com
api_key: os.environ/AZURE_FRANCE_API_KEY
model: azure/gpt-turbo
model_name: azure-model
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
model_name: test_openai_models
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
id: 56f1bd94-3b54-4b67-9ea2-7c70e9a3a709
model_name: test_openai_models
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
id: 4d1ee26c-abca-450c-8744-8e87fd6755e9
model_name: test_openai_models
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
id: 00e19c0f-b63d-42bb-88e9-016fb0c60764
model_name: test_openai_models
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
id: 79fc75bf-8e1b-47d5-8d24-9365a854af03
model_name: test_openai_models
- litellm_params:
api_base: os.environ/AZURE_AI_API_BASE
api_key: os.environ/AZURE_AI_API_KEY
api_version: 2023-07-01-preview
model: azure/text-embedding-ada-002
model_info:
mode: embedding
model_name: azure-embedding-model
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
id: 55848c55-4162-40f9-a6e2-9a722b9ef404
model_name: test_openai_models
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
id: 34339b1e-e030-4bcc-a531-c48559f10ce4
model_name: test_openai_models
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
id: f6f74e14-ac64-4403-9365-319e584dcdc5
model_name: test_openai_models
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
id: 9b1ef341-322c-410a-8992-903987fef439
model_name: test_openai_models
- litellm_params:
model: dall-e-3
model_info:
mode: image_generation
model_name: dall-e-3
- litellm_params:
api_base: os.environ/AZURE_SWEDEN_API_BASE
api_key: os.environ/AZURE_SWEDEN_API_KEY
api_version: 2023-12-01-preview
model: azure/dall-e-3-test
model_info:
mode: image_generation
model_name: dall-e-3
- litellm_params:
api_base: os.environ/AZURE_AI_API_BASE
api_key: os.environ/AZURE_AI_API_KEY
api_version: 2023-06-01-preview
model: azure/
model_info:
mode: image_generation
model_name: dall-e-2
- litellm_params:
api_base: os.environ/AZURE_AI_API_BASE
api_key: os.environ/AZURE_AI_API_KEY
api_version: 2023-07-01-preview
model: azure/text-embedding-ada-002
model_info:
base_model: text-embedding-ada-002
mode: embedding
model_name: text-embedding-ada-002
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
id: 34cb2419-7c63-44ae-a189-53f1d1ce5953
model_name: test_openai_models
- litellm_params:
model: amazon.titan-embed-text-v1
model_name: amazon-embeddings
- litellm_params:
model: gpt-3.5-turbo
model_info:
description: this is a test openai model
id: 753dca9a-898d-4ff7-9961-5acf7cdf38cf
model_name: test_openai_models

View file

@ -0,0 +1,21 @@
model_list:
- model_name: Azure OpenAI GPT-4 Canada
litellm_params:
model: gpt-4.1-nano
api_key: sk-unit-fake
model_info:
mode: chat
input_cost_per_token: 0.0002
id: gm
- model_name: azure-embedding-model
litellm_params:
model: text-embedding-ada-002
api_key: sk-unit-fake
model_info:
mode: embedding
input_cost_per_token: 0.002
id: hello
litellm_settings:
drop_params: True
callbacks: custom_callbacks.my_custom_logger

View file

@ -0,0 +1,32 @@
model_list:
- litellm_params:
api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
api_key: os.environ/AZURE_EUROPE_API_KEY
model: azure/gpt-35-turbo
model_name: azure-model
- litellm_params:
api_base: https://my-endpoint-canada-berri992.openai.azure.com
api_key: os.environ/AZURE_CANADA_API_KEY
model: azure/gpt-35-turbo
model_name: azure-model
- litellm_params:
api_base: https://openai-france-1234.openai.azure.com
api_key: os.environ/AZURE_FRANCE_API_KEY
model: azure/gpt-turbo
model_name: azure-model
litellm_settings:
guardrails:
- prompt_injection:
callbacks: [lakera_prompt_injection, detect_prompt_injection]
default_on: true
- hide_secrets:
callbacks: [hide_secrets]
default_on: true
- moderations:
callbacks: [openai_moderations]
default_on: false

View file

@ -0,0 +1,111 @@
use std::fs;
use litellm_config_load::{Error, load};
use rstest::{fixture, rstest};
use tempfile::TempDir;
#[fixture]
fn directory() -> TempDir {
tempfile::tempdir().unwrap()
}
#[rstest]
fn loads_model_list_from_file(directory: TempDir) {
let path = directory.path().join("config.yaml");
fs::write(
&path,
r#"
model_list:
- model_name: assistant
litellm_params:
model: anthropic/test-model
- model_name: local
litellm_params:
model: test-model
"#,
)
.unwrap();
let config = load(path).unwrap();
assert_eq!(
config
.model_list
.iter()
.map(|model| model.model_name.as_str())
.collect::<Vec<_>>(),
["assistant", "local"]
);
}
#[rstest]
#[case::missing_file(None, |error: &Error| matches!(error, Error::Read(error) if error.kind() == std::io::ErrorKind::NotFound))]
#[case::malformed_yaml(Some("model_list: ["), |error: &Error| matches!(error, Error::Parse(_)))]
fn distinguishes_read_errors_from_parse_errors(
directory: TempDir,
#[case] contents: Option<&str>,
#[case] expected: fn(&Error) -> bool,
) {
let path = directory.path().join("config.yaml");
if let Some(contents) = contents {
fs::write(&path, contents).unwrap();
}
let error = load(&path).unwrap_err();
let Error::Document { source, .. } = &error else {
panic!("{error:?}");
};
assert!(expected(source), "{error:?}");
}
#[rstest]
fn resolves_nested_includes_once_in_breadth_first_order(directory: TempDir) {
let root = directory.path();
fs::create_dir(root.join("nested")).unwrap();
fs::write(
root.join("config.yaml"),
r#"
include: [nested/first.yaml, second.yaml]
model_list: [{model_name: root, litellm_params: {model: root}}]
"#,
)
.unwrap();
fs::write(
root.join("nested/first.yaml"),
r#"
include: [third.yaml]
model_list: [{model_name: first, litellm_params: {model: first}}]
"#,
)
.unwrap();
fs::write(
root.join("second.yaml"),
r#"
general_settings: {master_key: second}
model_list: [{model_name: second, litellm_params: {model: second}}]
"#,
)
.unwrap();
fs::write(
root.join("nested/third.yaml"),
r#"
include: [../config.yaml]
general_settings: {master_key: third}
model_list: [{model_name: third, litellm_params: {model: third}}]
"#,
)
.unwrap();
let config = load(root.join("config.yaml")).unwrap();
assert_eq!(
config
.model_list
.iter()
.map(|model| model.model_name.as_str())
.collect::<Vec<_>>(),
["root", "first", "second", "third"]
);
assert_eq!(
config.general_settings.master_key.unwrap().expose(),
"third"
);
}

View file

@ -0,0 +1,61 @@
## Scope
- The crate is the type definition of `config.yaml`, what a user writes and nothing more. Parsing one YAML text into those types (`Config::from_yaml`, `Config::from_mapping`, the PyYAML reading rules in `yaml.rs`) is all it does
- Everything that turns those types into what the gateway runs on lives in `crates/config-load`: resolving `include`, overlaying DB rows, reading the environment, substituting references, applying defaults and building the domain types. So this crate never reads files, the environment or a secret manager
- It knows nothing about which settings a consumer implements or rolls out, so checks like "refuse setting X because it is not supported yet" live in the consuming crate (e.g. `gateway`), never here
## Defaults
- The value the existing proxy uses when a key is absent (`health_check_interval` is 300, `database_connection_pool_limit` is 10) is part of what `config.yaml` means, but it is applied in `crates/config-load`, never here. Parsing never fills a default in, so every key here keeps whether the file set it. The proxy treats a key set in the file as owned by the file, so the DB value and Admin UI edits for it are ignored
- Other sections have no DB overlay, so a key there whose default is empty (an empty list or map) can be the plain collection, filled in by `#[serde(default)]`
## Module layout
Each top-level `config.yaml` key belongs to exactly one module under `src/`. A module is a single `<name>.rs` file until it has flattened sub-groups, then it becomes a `<name>/` folder with one file per sub-group. This only applies one level down, so groups flattened inside a sub-group (like the cache backends in `litellm/caching.rs`) stay in that sub-group's file. Keys typed as a plain `Object` have no module and stay on `Config` in `lib.rs`. Keep this list in sync when adding a key
- `model.rs`: `model_list`
- `general/`: `general_settings`
- `router.rs`: `router_settings`
- `litellm/`: `litellm_settings`
- `callback.rs`: `callback_settings`
- `provider.rs`: `assistant_settings`, `default_vertex_config`, `files_settings`, `finetune_settings`, `worker_registry`
- `mcp.rs`: `mcp_servers`
- `credential.rs`: `credential_list`
- `guardrail.rs`: `guardrails`
- `prompt.rs`: `prompts`
- `tool.rs`: `sandbox_tools`, `search_tools`
- `vector_store.rs`: `vector_store_registry`
- `agent.rs`: `agents`, `agent_list`
- `policy.rs`: `policies`, `policy_attachments`
- `lib.rs`: `environment_variables`, `mcp_tools`, `include`
`error.rs`, `value.rs` and `yaml.rs` are crate infrastructure, not config keys
## Serde flatten
Sections split one YAML mapping into several structs with nested `#[serde(flatten)]` (`LitellmSettings` flattens `CachingSettings`, `GeneralSettings` flattens `AuthSettings`, and so on). Serde hands keys to flattened fields like this
- A flattened struct without a flatten field of its own takes only the keys it declares, and later siblings and the catch-all never see them
- A flattened struct or map that has its own flatten field is read as a map and claims nothing. It sees every key still unclaimed, including keys a later sibling declares, and those keys also reach the outer catch-all
- Fields claim keys in declaration order, so when two flattened structs declare the same key only the first one gets it
Which gives these rules
- Each mapping has exactly one `#[serde(flatten)] additional_fields: Object`, declared last on the outermost struct of that mapping. Flattened sub-structs stay plain, with no flatten field inside
- The exception is a mapping where Python gives every unnamed key one meaning, like the callback vars in a `default_team_settings` entry. Its catch-all is a typed map named for that meaning (`inline_callback_vars`), so a value of the wrong shape fails to parse
- A key belongs to exactly one struct. Which one is decided by who reads it, see `src/AGENTS.md`
- Never combine `deny_unknown_fields` with flatten, on either the outer or the flattened struct. It is fine on a struct that is never flattened and has no flatten fields
## Inline YAML
- Write inline YAML as a multi-line raw string (`r#"..."#`), not as a one-line string with `\n` escapes like `"key:\n value"`
- This includes `#[case(...)]` arguments. Open the raw string on the attribute line and start the YAML at column 0, e.g. `#[case::name(r#"` then the YAML lines then `"#)]`
- YAML that fits on one line, like flow-style `key: {a: 1}`, stays a plain string with no trailing `\n`
## Debug and redaction
Every struct derives `Debug`, and redaction lives on the field types, so no struct has a hand-written `Debug`
- Anything that is or can carry a secret (keys, tokens, passwords, URLs that may embed credentials, MCP `command` and `args`) is `SecretValue`, which prints `[REDACTED]`
- YAML we don't model is `Object` (prints its keys) or `Raw` (prints its kind). They are the only hand-written `Debug` impls. Never put a bare `Value` in a field
- Redaction tests put a unique `SENTINEL` under the slot and assert it never shows up in `format!("{config:?}")`. Words like `password` also match field names such as `redis_password`

View file

@ -7,7 +7,9 @@ repository.workspace = true
[dependencies]
litellm-auth-types.workspace = true
litellm-python-compat.workspace = true
litellm-router-types.workspace = true
macro_rules_attribute.workspace = true
serde.workspace = true
serde_yaml_ng = "0.10.0"
strum.workspace = true
@ -16,4 +18,3 @@ thiserror.workspace = true
[dev-dependencies]
rstest.workspace = true
serde_json.workspace = true
tempfile.workspace = true

View file

@ -0,0 +1,32 @@
## Grouping lives in the types
The YAML shape is Python's `config.yaml` and we never change it, so a mapping stays flat even when its keys serve different consumers. The grouping is encoded in the types instead, with flattened structs (see the "Serde flatten" rules in `../AGENTS.md`)
### Rules
- Every distinct set of readers gets its own flattened struct, even when only one key has that set. `namespace` in `cache_params` is read only by `redis`, so it gets its own struct and does not sit with `host` and `port`, which `redis-semantic` and `valkey-semantic` also read
- Only keys that every reader of the mapping uses stay on the parent struct. A key that some readers ignore never stays on the parent just because no group fits yet. Make the group
- Once a mapping has keys with different readers, all of them move into groups, never only some. `router_settings` typed as one flat struct, or a parent that keeps keys an existing group already covers, breaks this
### Finding the readers
- A reader is the code that receives or branches on the value, found in the Python that consumes the key. The key's name does not tell you. `default_in_redis_ttl` looks Redis-only but Valkey reads it too, and `default_redis_ttl` sits under `cache_params` yet configures the proxy's internal caches, not the response cache
- Follow the value to where it is used, not just to where it is passed. A constructor argument read in one `if type == ...` arm has that arm as its reader. A value applied behind an `isinstance` guard is read by that class and its subclasses only (`namespace` reaches `RedisCache` and `RedisClusterCache`, not `RedisSemanticCache`)
- A key read by the shared code path for every variant (e.g. `ttl` and `mode`, read by `Cache` itself) is read by all of them and stays on the parent
### Naming
- Name each struct for its reader set (`RedisServerCacheParams`, `SemanticCacheParams`), and name the parent's flattened field for the same set (`redis_server`, `semantic`)
- When the name alone does not say who reads the keys, add a one-line doc comment on the struct that lists the readers, like the one on `RedisServerCacheParams`
### Tests
- Each mapping has one rstest with a case per group, putting one key of that group in the YAML and asserting it lands in that group's field and not in `additional_fields`
- When a key's readers were easy to get wrong (the examples above), add a case for that key too, so moving it back into a broader group fails a test
## Lists are boxed slices
Config is parsed once and only read after that, so every list field on a config type is `Box<[T]>` (or `Option<Box<[T]>>`), never `Vec<T>`. The type then says the list can't grow, and serde deserializes into it directly
- `Vec<T>` is only for values that get built up and thrown away, like the layers and origins in `include/`. Those never become config fields
- Code that builds a field by hand collects into a `Vec` and converts at the end (`Box::from`, `.into_boxed_slice()`), and never keeps the `Vec` around

View file

@ -0,0 +1,75 @@
use crate::{Object, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct Agent {
pub agent_name: Option<String>,
pub agent_card_params: Option<Object>,
pub litellm_params: Option<Object>,
pub identity: Option<Object>,
pub enabled: Option<Spelled<bool>>,
pub execution_mode: Option<String>,
pub object_permission: Option<Object>,
pub tpm_limit: Option<Spelled<i64>>,
pub rpm_limit: Option<Spelled<i64>>,
pub session_tpm_limit: Option<Spelled<i64>>,
pub session_rpm_limit: Option<Spelled<i64>>,
pub static_headers: Option<Object>,
pub extra_headers: Option<Box<[String]>>,
pub access_group_ids: Option<Box<[String]>>,
pub kill_switch: Option<Object>,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::{Config, Spelled};
fn parse(yaml: &str) -> Config {
Config::from_yaml(yaml).unwrap()
}
#[rstest]
#[case::agents("agents")]
#[case::agent_list("agent_list")]
fn agent_entries_keep_typed_limits_and_flags(#[case] section: &str) {
let config = parse(&format!(
"{section}:\n - {{agent_name: reviewer, agent_card_params: {{name: x}}, tpm_limit: 100, enabled: false}}\n"
));
let agent = config.agents.first().or(config.agent_list.first()).unwrap();
assert_eq!(agent.agent_name.as_deref(), Some("reviewer"));
assert!(
agent
.agent_card_params
.as_ref()
.unwrap()
.contains_key("name")
);
assert_eq!(agent.tpm_limit, Some(Spelled::Value(100)));
assert_eq!(agent.enabled, Some(Spelled::Value(false)));
}
#[rstest]
#[case::agent_headers(
r#"
agents:
- static_headers: {Authorization: SENTINEL}
"#
)]
#[case::agent_params(
r#"
agents:
- litellm_params: {api_key: SENTINEL}
"#
)]
#[case::agent_identity(
r#"
agent_list:
- identity: {token: SENTINEL}
"#
)]
fn debug_never_prints_agent_values(#[case] yaml: &str) {
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
}
}

View file

@ -0,0 +1,77 @@
use litellm_auth_types::SecretValue;
use crate::{Object, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct CallbackSettings {
pub callback_type: Option<String>,
pub endpoint: Option<SecretValue>,
pub headers: Option<Object>,
pub event_types: Option<Box<[String]>>,
pub log_format: Option<String>,
pub max_retries: Option<Spelled<i64>>,
pub retry_delay: Option<Spelled<f64>>,
pub timeout: Option<Spelled<f64>>,
#[serde(flatten)]
pub additional_fields: Object,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::{Config, Spelled};
fn parse(yaml: &str) -> Config {
Config::from_yaml(yaml).unwrap()
}
#[rstest]
fn callback_settings_keep_typed_fields_and_unknown_keys() {
let config = parse(
"callback_settings:\n hook:\n callback_type: generic_api\n retry_delay: 0.5\n batch_size: 10\n",
);
let hook = &config.callback_settings["hook"];
assert_eq!(hook.callback_type.as_deref(), Some("generic_api"));
assert_eq!(hook.retry_delay, Some(Spelled::Value(0.5)));
assert!(hook.additional_fields.contains_key("batch_size"));
}
#[rstest]
#[case::env_reference("os.environ/RETRIES", Spelled::Text("os.environ/RETRIES".to_owned()))]
#[case::number("4", Spelled::Value(4))]
fn spelled_int_fields_keep_references_as_text(
#[case] spelled: &str,
#[case] expected: Spelled<i64>,
) {
let config = parse(&format!(
"callback_settings:\n hook:\n max_retries: {spelled}\n"
));
assert_eq!(config.callback_settings["hook"].max_retries, Some(expected));
}
#[rstest]
#[case::callback_endpoint(
r#"
callback_settings:
hook: {endpoint: SENTINEL}
"#
)]
#[case::callback_headers(
r#"
callback_settings:
hook: {headers: {Authorization: SENTINEL}}
"#
)]
#[case::callback_extra(
r#"
callback_settings:
hook: {vendor_secret: SENTINEL}
"#
)]
fn debug_never_prints_callback_values(#[case] yaml: &str) {
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
}
}

View file

@ -0,0 +1,65 @@
use crate::Object;
#[macro_rules_attribute::apply(crate::config_section)]
pub struct Credential {
pub credential_name: Option<String>,
pub credential_info: Option<Object>,
pub credential_values: Option<Object>,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::Config;
fn parse(yaml: &str) -> Config {
Config::from_yaml(yaml).unwrap()
}
#[rstest]
fn credential_entries_keep_name_info_and_values() {
let config = parse(
r#"
credential_list:
- credential_name: azure-prod
credential_info: {description: prod}
credential_values: {api_key: os.environ/AZURE_API_KEY}
"#,
);
let credential = &config.credential_list[0];
assert_eq!(credential.credential_name.as_deref(), Some("azure-prod"));
assert!(
credential
.credential_info
.as_ref()
.unwrap()
.contains_key("description")
);
assert!(
credential
.credential_values
.as_ref()
.unwrap()
.contains_key("api_key")
);
}
#[rstest]
#[case::credential_values(
r#"
credential_list:
- credential_values: {api_key: SENTINEL}
"#
)]
#[case::credential_info(
r#"
credential_list:
- credential_info: {owner: SENTINEL}
"#
)]
fn debug_never_prints_credential_values(#[case] yaml: &str) {
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
}
}

View file

@ -1,7 +1,5 @@
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("could not read config")]
Read(#[from] std::io::Error),
#[error("invalid YAML config")]
Parse(#[from] serde_yaml_ng::Error),
}

View file

@ -0,0 +1,14 @@
## Docs
- https://github.com/BerriAI/litellm-docs/blob/main/docs/proxy/config_settings.md
## Python
- `ConfigGeneralSettings` in `litellm/proxy/_types.py` is the Python model but is incomplete. What the proxy reads is `general_settings.get(...)`, mostly in `proxy_server.py`
- `general_settings` is one of the sections the proxy persists to the DB (`_CONFIG_PERSISTED_SECTIONS`) and the Admin UI edits, so at runtime the YAML is only one source of the effective values. Merging the DB overlay is the consumer's job, not this crate's
## Fields that need care
- `master_key`, `database_url` and `drain_endpoint_token` fall back to `LITELLM_MASTER_KEY`, `DATABASE_URL` and `DRAIN_ENDPOINT_TOKEN`. That fallback belongs to the consumer, so the fields stay optional here
- `custom_auth`, `custom_key_generate`, `custom_key_update`, `custom_key_policy`, `custom_team_metadata_validate`, `custom_sso`, `custom_ui_sso_sign_in_handler`, `litellm_jwtauth.custom_validate` and `pass_through_endpoints[].target` are Python module paths that the proxy imports. Type them as plain strings and never resolve or validate them here
- `key_management_system` and `key_management_settings` configure the secret manager itself, so they can only use `os.environ/` references, not secret manager ones

View file

@ -0,0 +1,39 @@
use serde::Deserialize;
use crate::{Object, Spelled};
#[derive(Clone, Debug, PartialEq, Deserialize)]
#[serde(untagged)]
pub enum Alerting {
List(Box<[String]>),
One(String),
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct AlertingSettings {
pub alerting: Option<Alerting>,
pub alerting_threshold: Option<Spelled<f64>>,
pub alert_types: Option<Box<[String]>>,
pub alert_to_webhook_url: Option<Object>,
pub alerting_args: Option<AlertingArgs>,
pub alert_type_config: Option<Object>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct AlertingArgs {
pub daily_report_frequency: Option<Spelled<i64>>,
pub report_check_interval: Option<Spelled<i64>>,
pub budget_alert_ttl: Option<Spelled<i64>>,
pub outage_alert_ttl: Option<Spelled<i64>>,
pub region_outage_alert_ttl: Option<Spelled<i64>>,
pub minor_outage_alert_threshold: Option<Spelled<i64>>,
pub major_outage_alert_threshold: Option<Spelled<i64>>,
pub max_outage_alert_list_size: Option<Spelled<i64>>,
pub log_to_console: Option<Spelled<bool>>,
pub daily_spend_per_user_threshold: Option<Spelled<f64>>,
pub monthly_spend_per_user_threshold: Option<Spelled<f64>>,
pub spend_anomaly_multiplier: Option<Spelled<f64>>,
pub spend_anomaly_baseline_days: Option<Spelled<i64>>,
pub spend_anomaly_min_spend: Option<Spelled<f64>>,
pub user_spend_check_interval: Option<Spelled<i64>>,
}

View file

@ -0,0 +1,77 @@
use litellm_auth_types::SecretValue;
use crate::{Object, Raw, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct AuthSettings {
pub litellm_key_header_name: Option<String>,
pub enable_jwt_auth: Option<Spelled<bool>>,
pub litellm_jwtauth: Option<Object>,
pub enable_oauth2_auth: Option<Spelled<bool>>,
pub enable_oauth2_proxy_auth: Option<Spelled<bool>>,
pub oauth2_config_mappings: Option<Object>,
pub custom_auth: Option<String>,
pub custom_auth_run_common_checks: Option<Spelled<bool>>,
pub allow_user_auth: Option<Spelled<bool>>,
pub allowed_routes: Option<Box<[String]>>,
pub admin_only_routes: Option<Box<[String]>>,
pub public_routes: Option<Box<[String]>>,
pub allowed_ips: Option<Box<[String]>>,
pub use_x_forwarded_for: Option<Spelled<bool>>,
pub trusted_proxy_ranges: Option<Box<[String]>>,
pub user_header_name: Option<String>,
pub user_header_mappings: Option<Box<[UserHeaderMapping]>>,
pub role_permissions: Option<Box<[RoleBasedPermissions]>>,
pub enforce_rbac: Option<Spelled<bool>>,
pub allow_client_side_credentials: Option<Spelled<bool>>,
pub enforce_user_param: Option<Spelled<bool>>,
pub reject_clientside_metadata_tags: Option<Spelled<bool>>,
pub enforced_params: Option<Box<[String]>>,
pub service_account_settings: Option<ServiceAccountSettings>,
pub litellm_license: Option<SecretValue>,
pub control_plane_url: Option<String>,
pub key_management_system: Option<String>,
pub key_management_settings: Option<KeyManagementSettings>,
pub use_google_kms: Option<Spelled<bool>>,
pub use_azure_key_vault: Option<Spelled<bool>>,
pub encryption_algorithm: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct UserHeaderMapping {
pub header_name: Option<String>,
pub litellm_user_role: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct RoleBasedPermissions {
pub role: Option<String>,
pub models: Option<Box<[String]>>,
pub routes: Option<Box<[String]>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct ServiceAccountSettings {
pub enforced_params: Option<Box<[String]>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct KeyManagementSettings {
pub hosted_keys: Option<Box<[Raw]>>,
pub store_virtual_keys: Option<Spelled<bool>>,
pub prefix_for_stored_virtual_keys: Option<String>,
pub access_mode: Option<String>,
pub primary_secret_name: Option<String>,
pub description: Option<String>,
pub tags: Option<Object>,
pub kms_key_id: Option<String>,
pub custom_secret_manager: Option<String>,
pub aws_region_name: Option<String>,
pub aws_role_name: Option<String>,
pub aws_session_name: Option<String>,
pub aws_external_id: Option<SecretValue>,
pub aws_profile_name: Option<String>,
pub aws_web_identity_token: Option<SecretValue>,
pub aws_sts_endpoint: Option<String>,
pub replica_regions: Option<Box<[String]>>,
}

View file

@ -0,0 +1,15 @@
use crate::{Object, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct DatabaseSettings {
pub database_connection_pool_timeout: Option<Spelled<f64>>,
pub database_extra_connection_params: Option<Object>,
pub database_disable_prepared_statements: Option<Spelled<bool>>,
pub database_statement_timeout: Option<Spelled<f64>>,
pub database_lock_timeout: Option<Spelled<f64>>,
pub database_idle_in_transaction_session_timeout: Option<Spelled<f64>>,
pub disable_prisma_schema_update: Option<Spelled<bool>>,
pub allow_requests_on_db_unavailable: Option<Spelled<bool>>,
pub supported_db_objects: Option<Box<[String]>>,
pub use_redis_transaction_buffer: Option<Spelled<bool>>,
}

View file

@ -0,0 +1,15 @@
use crate::Spelled;
#[macro_rules_attribute::apply(crate::config_section)]
pub struct FilesSettings {
pub max_file_size_mb: Option<Spelled<i64>>,
pub max_batch_file_size_mb: Option<Spelled<i64>>,
pub max_batch_file_records: Option<Spelled<i64>>,
pub max_batch_file_uploads_per_day: Option<Spelled<i64>>,
pub max_file_downloads_per_minute: Option<Spelled<i64>>,
pub allowed_file_extensions: Option<Box<[String]>>,
pub blocked_file_extensions: Option<Box<[String]>>,
pub disable_batch_input_file_rate_limiting: Option<Spelled<bool>>,
pub skip_batch_input_file_rate_limiting_for_providers: Option<Box<[String]>>,
pub skip_batch_input_file_rate_limiting_for_models: Option<Box<[String]>>,
}

View file

@ -0,0 +1,22 @@
use litellm_auth_types::SecretValue;
use crate::Spelled;
#[macro_rules_attribute::apply(crate::config_section)]
pub struct HealthSettings {
pub use_shared_health_check: Option<Spelled<bool>>,
pub health_check_details: Option<Spelled<bool>>,
pub enable_health_check_routing: Option<Spelled<bool>>,
pub health_check_staleness_threshold: Option<Spelled<f64>>,
pub health_check_ignore_transient_errors: Option<Spelled<bool>>,
pub health_check_skip_disabled_background_models: Option<Spelled<bool>>,
pub background_health_check_model_groups: Option<Box<[String]>>,
pub model_list_healthy_only: Option<Spelled<bool>>,
pub allow_public_health_readiness_details: Option<Spelled<bool>>,
pub enable_drain_endpoint: Option<Spelled<bool>>,
pub drain_endpoint_token: Option<SecretValue>,
pub disable_model_info_refresh: Option<Spelled<bool>>,
pub use_team_public_model_name: Option<Spelled<bool>>,
pub image_generation_model: Option<String>,
pub moderation_model: Option<String>,
}

View file

@ -0,0 +1,11 @@
use crate::Spelled;
#[macro_rules_attribute::apply(crate::config_section)]
pub struct LimitsSettings {
pub fail_closed_rate_limit_enforcement: Option<Spelled<bool>>,
pub token_rate_limit_type: Option<String>,
pub disable_retry_on_max_parallel_request_limit_error: Option<Spelled<bool>>,
pub disable_fallbacks_on_per_model_rate_limits: Option<Spelled<bool>>,
pub user_api_key_cache_ttl: Option<Spelled<f64>>,
pub user_api_key_cache_max_size: Option<Spelled<i64>>,
}

View file

@ -0,0 +1,20 @@
use crate::{Raw, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct McpSettings {
pub mcp_allowed_hosts: Option<Box<[String]>>,
pub mcp_allowed_origins: Box<[String]>,
pub mcp_advertised_versions: Option<Raw>,
pub mcp_disabled_capabilities: Option<Raw>,
pub mcp_allowed_clients: Option<Raw>,
pub mcp_require_explicit_server_scope: Option<Spelled<bool>>,
pub mcp_client_id_header: Option<String>,
pub mcp_client_side_auth_header_name: Option<String>,
pub mcp_allow_all_keys_respects_mcp_scope: Option<Spelled<bool>>,
pub mcp_prefer_client_id_metadata_document: Option<Spelled<bool>>,
pub mcp_internal_ip_ranges: Option<Raw>,
pub mcp_trusted_proxy_ranges: Option<Raw>,
pub mcp_xff_num_trusted_hops: Option<Spelled<i64>>,
pub mcp_required_fields: Option<Raw>,
pub mcp_session_token_signing: Option<Raw>,
}

View file

@ -0,0 +1,535 @@
mod alerting;
mod auth;
mod database;
mod files;
mod health;
mod limits;
mod mcp;
mod objects;
mod requests;
mod spend;
mod tracing;
mod ui;
use litellm_auth_types::SecretValue;
use serde::Deserialize;
pub use self::{
alerting::{Alerting, AlertingArgs, AlertingSettings},
auth::{
AuthSettings, KeyManagementSettings, RoleBasedPermissions, ServiceAccountSettings,
UserHeaderMapping,
},
database::DatabaseSettings,
files::FilesSettings,
health::HealthSettings,
limits::LimitsSettings,
mcp::McpSettings,
objects::{ObjectsSettings, TeamMetadataField},
requests::{PassThroughEndpoint, RequestTagHeader, RequestTagMapping, RequestsSettings},
spend::{
ScheduledJobStaggerSettings, SpendCaptureRateCheckSettings, SpendLogsMetadataFields,
SpendSettings,
},
tracing::{ClickHouseStoreSettings, TracingSettings, TracingStoreSettings},
ui::{UiAccessMode, UiAccessSsoGroup, UiSettings},
};
use crate::Object;
#[derive(Clone, Debug, Deserialize)]
#[serde(default)]
pub struct GeneralSettings {
pub completion_model: Option<String>,
pub max_in_flight_requests_per_worker: Option<u64>,
pub max_queued_requests_per_worker: Option<u64>,
pub admission_queue_timeout_seconds: f64,
pub master_key: Option<SecretValue>,
pub database_url: Option<SecretValue>,
pub tracing: Option<TracingSettings>,
pub database_connection_pool_limit: Option<u64>,
pub database_connection_timeout: Option<f64>,
pub database_connect_timeout: Option<f64>,
pub database_socket_timeout: Option<f64>,
pub database_max_idle_connection_lifetime: Option<f64>,
pub max_parallel_requests: Option<u64>,
pub global_max_parallel_requests: Option<u64>,
pub max_request_size_mb: Option<u64>,
pub max_response_size_mb: Option<u64>,
pub proxy_config_reload_interval_seconds: u64,
pub background_health_checks: Option<bool>,
pub health_check_interval: u64,
pub health_check_concurrency: Option<u64>,
pub store_model_in_db: Option<bool>,
pub forward_client_headers_to_llm_api: Option<bool>,
pub cancel_on_disconnect: Option<bool>,
pub infer_model_from_keys: Option<bool>,
pub enable_public_model_hub: bool,
pub dangerously_permit_weak_or_unset_master_key: Option<bool>,
pub plugins: Option<Box<[Object]>>,
pub coordination_redis: Option<Object>,
#[serde(flatten)]
pub auth: AuthSettings,
#[serde(flatten)]
pub database: DatabaseSettings,
#[serde(flatten)]
pub spend: SpendSettings,
#[serde(flatten)]
pub limits: LimitsSettings,
#[serde(flatten)]
pub health: HealthSettings,
#[serde(flatten)]
pub alerting: AlertingSettings,
#[serde(flatten)]
pub ui: UiSettings,
#[serde(flatten)]
pub objects: ObjectsSettings,
#[serde(flatten)]
pub requests: RequestsSettings,
#[serde(flatten)]
pub files: FilesSettings,
#[serde(flatten)]
pub mcp: McpSettings,
#[serde(flatten)]
pub additional_fields: Object,
}
impl Default for GeneralSettings {
fn default() -> Self {
Self {
completion_model: None,
max_in_flight_requests_per_worker: None,
max_queued_requests_per_worker: None,
admission_queue_timeout_seconds: 1.0,
master_key: None,
database_url: None,
tracing: None,
database_connection_pool_limit: Some(10),
database_connection_timeout: Some(60.0),
database_connect_timeout: None,
database_socket_timeout: None,
database_max_idle_connection_lifetime: Some(60.0),
max_parallel_requests: None,
global_max_parallel_requests: None,
max_request_size_mb: None,
max_response_size_mb: None,
proxy_config_reload_interval_seconds: 30,
background_health_checks: None,
health_check_interval: 300,
health_check_concurrency: None,
store_model_in_db: None,
forward_client_headers_to_llm_api: None,
cancel_on_disconnect: None,
infer_model_from_keys: None,
enable_public_model_hub: false,
dangerously_permit_weak_or_unset_master_key: None,
plugins: None,
coordination_redis: None,
auth: AuthSettings::default(),
database: DatabaseSettings::default(),
spend: SpendSettings::default(),
limits: LimitsSettings::default(),
health: HealthSettings::default(),
alerting: AlertingSettings::default(),
ui: UiSettings::default(),
objects: ObjectsSettings::default(),
requests: RequestsSettings::default(),
files: FilesSettings::default(),
mcp: McpSettings::default(),
additional_fields: Object::default(),
}
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::{Alerting, Config, GeneralSettings, Spelled, UiAccessMode};
fn general(yaml: &str) -> GeneralSettings {
Config::from_yaml(&format!("general_settings:\n{yaml}"))
.unwrap()
.general_settings
}
fn text<T>(value: &str) -> Option<Spelled<T>> {
Some(Spelled::Text(value.to_string()))
}
#[rstest]
#[case::yaml_bool("true", Some(Spelled::Value(true)))]
#[case::yaml_1_1_word("yes", text("yes"))]
#[case::quoted("'false'", text("false"))]
#[case::environment_reference("os.environ/ENABLE_JWT", text("os.environ/ENABLE_JWT"))]
fn bool_settings_keep_every_python_spelling(
#[case] spelling: &str,
#[case] expected: Option<Spelled<bool>>,
) {
let settings = general(&format!(
" enable_jwt_auth: {spelling}\n disable_spend_logs: {spelling}\n use_redis_transaction_buffer: {spelling}\n model_list_healthy_only: {spelling}\n"
));
assert_eq!(settings.auth.enable_jwt_auth, expected);
assert_eq!(settings.spend.disable_spend_logs, expected);
assert_eq!(settings.database.use_redis_transaction_buffer, expected);
assert_eq!(settings.health.model_list_healthy_only, expected);
}
#[rstest]
#[case::bare_days("7", Some(Spelled::Value(7)))]
#[case::duration("'60d'", text("60d"))]
#[case::environment_reference("os.environ/RETENTION", text("os.environ/RETENTION"))]
fn retention_periods_accept_days_or_a_duration(
#[case] spelling: &str,
#[case] expected: Option<Spelled<i64>>,
) {
let settings = general(&format!(
" maximum_spend_logs_retention_period: {spelling}\n"
));
assert_eq!(settings.spend.maximum_spend_logs_retention_period, expected);
}
#[rstest]
#[case::integer("5", Some(Spelled::Value(5.0)))]
#[case::fraction("0.5", Some(Spelled::Value(0.5)))]
#[case::environment_reference("os.environ/TTL", text("os.environ/TTL"))]
fn number_settings_accept_integers_fractions_and_references(
#[case] spelling: &str,
#[case] expected: Option<Spelled<f64>>,
) {
let settings = general(&format!(
" user_api_key_cache_ttl: {spelling}\n database_statement_timeout: {spelling}\n health_check_staleness_threshold: {spelling}\n"
));
assert_eq!(settings.limits.user_api_key_cache_ttl, expected);
assert_eq!(settings.database.database_statement_timeout, expected);
assert_eq!(settings.health.health_check_staleness_threshold, expected);
}
#[test]
fn nested_settings_are_typed_and_keep_unknown_keys() {
let settings = general(
r#"
allowed_routes: [/chat/completions, /models]
custom_auth: custom_auth.user_api_key_auth
user_header_mappings:
- header_name: X-OpenWebUI-User-Id
litellm_user_role: internal_user
role_permissions:
- role: team
models: [gpt]
routes: [/chat/completions]
service_account_settings:
enforced_params: [user]
key_management_settings:
access_mode: read_and_write
store_virtual_keys: true
future_option: 1
spend_logs_metadata_fields:
include: [user_api_key_alias]
spend_capture_rate_check:
threshold: 0.8
lookback_days: 3
scheduled_job_stagger:
window_seconds: 120
offsets: {reset_budget_job: 0}
"#,
);
assert_eq!(
settings.auth.allowed_routes.as_deref(),
Some(["/chat/completions".to_string(), "/models".to_string()].as_slice())
);
assert_eq!(
settings.auth.custom_auth.as_deref(),
Some("custom_auth.user_api_key_auth")
);
let mapping = &settings.auth.user_header_mappings.as_deref().unwrap()[0];
assert_eq!(mapping.header_name.as_deref(), Some("X-OpenWebUI-User-Id"));
assert_eq!(mapping.litellm_user_role.as_deref(), Some("internal_user"));
let permission = &settings.auth.role_permissions.as_deref().unwrap()[0];
assert_eq!(permission.role.as_deref(), Some("team"));
assert_eq!(
permission.routes.as_deref(),
Some(["/chat/completions".to_string()].as_slice())
);
assert_eq!(
settings
.auth
.service_account_settings
.as_ref()
.unwrap()
.enforced_params
.as_deref(),
Some(["user".to_string()].as_slice())
);
let key_management = settings.auth.key_management_settings.as_ref().unwrap();
assert_eq!(
key_management.access_mode.as_deref(),
Some("read_and_write")
);
assert_eq!(
key_management.store_virtual_keys,
Some(Spelled::Value(true))
);
assert_eq!(
settings
.spend
.spend_logs_metadata_fields
.as_ref()
.unwrap()
.include
.as_deref(),
Some(["user_api_key_alias".to_string()].as_slice())
);
let capture = settings.spend.spend_capture_rate_check.as_ref().unwrap();
assert_eq!(capture.threshold, Some(Spelled::Value(0.8)));
assert_eq!(capture.lookback_days, Some(Spelled::Value(3)));
let stagger = settings.spend.scheduled_job_stagger.as_ref().unwrap();
assert_eq!(stagger.window_seconds, Some(Spelled::Value(120)));
assert!(
stagger
.offsets
.as_ref()
.unwrap()
.contains_key("reset_budget_job")
);
assert!(settings.additional_fields.is_empty());
}
#[rstest]
#[case::license("litellm_license: SENTINEL")]
#[case::drain_token("drain_endpoint_token: SENTINEL")]
#[case::jwt_auth_object("litellm_jwtauth: {team_id_jwt_field: SENTINEL}")]
#[case::oauth2_mappings("oauth2_config_mappings: {X-Header: SENTINEL}")]
#[case::database_params("database_extra_connection_params: {sslpassword: SENTINEL}")]
#[case::key_management_token("key_management_settings: {aws_web_identity_token: SENTINEL}")]
#[case::key_management_external_id("key_management_settings: {aws_external_id: SENTINEL}")]
#[case::key_management_hosted_keys("key_management_settings: {hosted_keys: [SENTINEL]}")]
#[case::key_management_tags("key_management_settings: {tags: {owner: SENTINEL}}")]
#[case::mcp_session_signing("mcp_session_token_signing: {private_key: SENTINEL}")]
#[case::master_key("master_key: SENTINEL")]
#[case::database_url("database_url: postgres://user:SENTINEL@db/litellm")]
#[case::coordination_redis("coordination_redis: {password: SENTINEL}")]
#[case::plugins("plugins: [{token: SENTINEL}]")]
#[case::unknown_setting("future_setting: SENTINEL")]
fn debug_output_does_not_expose_setting_values(#[case] setting: &str) {
let config = Config::from_yaml(&format!("general_settings:\n {setting}\n")).unwrap();
let printed = format!("{config:?}");
assert!(!printed.contains("SENTINEL"), "{printed}");
}
#[test]
fn unknown_keys_still_land_in_additional_fields() {
let settings = general(" future_setting: 1\n custom_auth: auth.py\n");
assert!(settings.additional_fields.contains_key("future_setting"));
assert!(!settings.additional_fields.contains_key("custom_auth"));
}
#[rstest]
#[case("true", Spelled::Value(true))]
#[case("\"false\"", Spelled::Text("false".to_string()))]
#[case("yes", Spelled::Text("yes".to_string()))]
#[case("os.environ/GATEWAY_FLAG", Spelled::Text("os.environ/GATEWAY_FLAG".to_string()))]
fn bool_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled<bool>) {
let settings = general(&format!(" enable_claude_code_gateway: {value}\n"));
assert_eq!(settings.requests.enable_claude_code_gateway, Some(expected));
}
#[rstest]
#[case("100", Spelled::Value(100))]
#[case("os.environ/MAX_FILE_MB", Spelled::Text("os.environ/MAX_FILE_MB".to_string()))]
fn int_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled<i64>) {
let settings = general(&format!(" max_file_size_mb: {value}\n"));
assert_eq!(settings.files.max_file_size_mb, Some(expected));
}
#[rstest]
#[case("600", Spelled::Value(600.0))]
#[case("2.5", Spelled::Value(2.5))]
#[case("os.environ/ALERT_THRESHOLD", Spelled::Text("os.environ/ALERT_THRESHOLD".to_string()))]
fn float_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled<f64>) {
let settings = general(&format!(" alerting_threshold: {value}\n"));
assert_eq!(settings.alerting.alerting_threshold, Some(expected));
}
#[rstest]
fn nested_mappings_land_in_typed_structs() {
let settings = general(
r#"
alerting: [slack, email]
alerting_args:
daily_report_frequency: 3600
log_to_console: true
spend_anomaly_multiplier: 2.5
future_arg: 1
team_metadata_schema:
- key: cost_center
label: Cost center
request_tag_mappings:
- header: baggage
key: tenant
tag_prefix: tenant-
required: true
pass_through_endpoints:
- path: /bria
target: https://engine.prod.bria-api.com
include_subpath: true
cost_per_request: 0.5
methods: [GET, POST]
custom_handler: handler.fn
allowed_file_extensions: [jsonl, csv]
"#,
);
assert_eq!(
settings.alerting.alerting,
Some(Alerting::List(
["slack".to_string(), "email".to_string()].into()
))
);
let args = settings.alerting.alerting_args.unwrap();
assert_eq!(args.daily_report_frequency, Some(Spelled::Value(3600)));
assert_eq!(args.log_to_console, Some(Spelled::Value(true)));
assert_eq!(args.spend_anomaly_multiplier, Some(Spelled::Value(2.5)));
let schema = settings.objects.team_metadata_schema.unwrap();
assert_eq!(schema[0].key.as_deref(), Some("cost_center"));
assert_eq!(schema[0].label.as_deref(), Some("Cost center"));
let mapping = &settings.requests.request_tag_mappings.unwrap()[0];
let header: &'static str = mapping.header.unwrap().into();
assert_eq!(header, "baggage");
assert_eq!(mapping.key.as_deref(), Some("tenant"));
assert_eq!(mapping.tag_prefix.as_deref(), Some("tenant-"));
assert_eq!(mapping.required, Some(Spelled::Value(true)));
let endpoint = &settings.requests.pass_through_endpoints.unwrap()[0];
assert_eq!(endpoint.path.as_deref(), Some("/bria"));
assert_eq!(
endpoint.target.as_ref().unwrap().expose(),
"https://engine.prod.bria-api.com"
);
assert_eq!(endpoint.include_subpath, Some(Spelled::Value(true)));
assert_eq!(endpoint.cost_per_request, Some(Spelled::Value(0.5)));
assert_eq!(
endpoint.methods.as_deref(),
Some(&["GET".to_string(), "POST".to_string()][..])
);
assert_eq!(
settings.files.allowed_file_extensions.as_deref(),
Some(&["jsonl".to_string(), "csv".to_string()][..])
);
}
#[rstest]
fn ui_access_mode_accepts_a_mode_string() {
let settings = general(" ui_access_mode: admin_only\n");
assert!(
matches!(settings.ui.ui_access_mode, Some(UiAccessMode::Mode(mode)) if mode == "admin_only")
);
}
#[rstest]
fn ui_access_mode_accepts_an_sso_group_mapping() {
let settings = general(
" ui_access_mode: {type: restricted_sso_group, restricted_sso_group: ai-admins, sso_group_jwt_field: groups}\n",
);
let Some(UiAccessMode::SsoGroup(group)) = settings.ui.ui_access_mode else {
panic!("expected an SSO group mapping")
};
assert_eq!(group.kind.as_deref(), Some("restricted_sso_group"));
assert_eq!(group.restricted_sso_group.as_deref(), Some("ai-admins"));
assert_eq!(group.sso_group_jwt_field.as_deref(), Some("groups"));
}
#[rstest]
fn alerting_accepts_a_single_name() {
let settings = general(" alerting: slack\n");
assert_eq!(
settings.alerting.alerting,
Some(Alerting::One("slack".to_string()))
);
}
#[rstest]
fn rejects_request_tag_header_python_rejects_at_boot() {
assert!(
Config::from_yaml(
"general_settings:\n request_tag_mappings: [{header: x-tenant, key: tenant, tag: t}]\n"
)
.is_err()
);
}
#[rstest]
#[case("alert_to_webhook_url: {budget_alerts: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))]
#[case("alert_type_config: {budget_alerts: {webhook: SENTINEL}}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))]
#[case("alerting_args: {webhook: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))]
#[case("pass_through_endpoints: [{path: /x, target: SENTINEL}]", |settings: &GeneralSettings| format!("{:?}", settings.requests))]
#[case("pass_through_endpoints: [{path: /x, target: t, headers: {Authorization: SENTINEL}}]", |settings: &GeneralSettings| format!("{:?}", settings.requests))]
#[case("claude_code_gateway_managed_settings: {token: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.requests))]
#[case("search_tools: [{litellm_params: {api_key: SENTINEL}}]", |settings: &GeneralSettings| format!("{:?}", settings.objects))]
#[case("max_failed_login_attempts_per_source_overrides: {10.0.0.1: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.ui))]
fn debug_hides_values_that_can_carry_secrets(
#[case] entry: &str,
#[case] render: fn(&GeneralSettings) -> String,
) {
let settings = general(&format!(" {entry}\n"));
assert!(!render(&settings).contains("SENTINEL"));
}
#[rstest]
fn unknown_key_still_lands_in_additional_fields() {
let settings = general(" future_setting: 1\n forward_openai_org_id: true\n");
assert!(settings.additional_fields.contains_key("future_setting"));
assert!(
!settings
.additional_fields
.contains_key("forward_openai_org_id")
);
assert_eq!(
settings.requests.forward_openai_org_id,
Some(Spelled::Value(true))
);
}
#[rstest]
fn mcp_allowed_hosts_and_origins_land_typed() {
let settings = general(
" mcp_allowed_hosts: [localhost]\n mcp_allowed_origins: ['https://app.test']\n",
);
assert_eq!(
settings.mcp.mcp_allowed_hosts.as_deref(),
Some(&["localhost".to_string()][..])
);
assert_eq!(
settings.mcp.mcp_allowed_origins.as_ref(),
["https://app.test".to_string()]
);
}
#[rstest]
#[case::literal("secret-master-key")]
#[case::reference("os.environ/LITELLM_MASTER_KEY")]
fn loads_and_redacts_the_master_key(#[case] key: &str) {
let config = Config::from_yaml(&format!(
"model_list: []\ngeneral_settings:\n master_key: {key}\n"
))
.unwrap();
assert_eq!(
config
.general_settings
.master_key
.as_ref()
.unwrap()
.expose(),
key
);
assert!(!format!("{config:?}").contains(key));
}
#[rstest]
fn missing_general_settings_has_no_master_key() {
let config = Config::from_yaml("model_list: []").unwrap();
assert!(config.general_settings.master_key.is_none());
}
}

View file

@ -0,0 +1,26 @@
use crate::{Object, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct ObjectsSettings {
pub custom_key_generate: Option<String>,
pub custom_key_update: Option<String>,
pub custom_key_policy: Option<String>,
pub custom_team_metadata_validate: Option<String>,
pub team_metadata_schema: Option<Box<[TeamMetadataField]>>,
pub team_metadata_validation_timeout: Option<Spelled<f64>>,
pub team_metadata_validation_error_message: Option<String>,
pub disable_auto_add_proxy_admin_to_teams: Option<Spelled<bool>>,
pub self_serve_budget_policy: Option<String>,
pub enforce_rpm_tpm_on_model_add: Option<Spelled<bool>>,
pub agent_access_default_deny: Option<Spelled<bool>>,
pub vector_store_deny_by_default: Option<Spelled<bool>>,
pub search_tool_deny_by_default: Option<Spelled<bool>>,
pub search_tools: Option<Box<[Object]>>,
pub enforce_fallback_model_access: Option<Spelled<bool>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct TeamMetadataField {
pub key: Option<String>,
pub label: Option<String>,
}

View file

@ -0,0 +1,66 @@
use litellm_auth_types::SecretValue;
use serde::Deserialize;
use crate::{Object, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct RequestsSettings {
pub forward_llm_provider_auth_headers: Option<Spelled<bool>>,
pub forward_openai_org_id: Option<Spelled<bool>>,
pub always_include_stream_usage: Option<Spelled<bool>>,
pub expose_fallback_errors_to_caller: Option<Spelled<bool>>,
pub include_call_id_in_error_body: Option<Spelled<bool>>,
pub request_tag_mappings: Option<Box<[RequestTagMapping]>>,
pub pass_through_endpoints: Option<Box<[PassThroughEndpoint]>>,
pub pass_through_request_timeout: Option<Spelled<f64>>,
pub passthrough_managed_object_ids: Option<Spelled<bool>>,
pub disable_bedrock_agent_runtime_passthrough: Option<Spelled<bool>>,
pub enable_openai_websocket_passthrough: Option<Spelled<bool>>,
pub vertex_ai_pass_through_drop_anthropic_betas: Option<Box<[String]>>,
pub transcribe_media_buckets: Option<Box<[String]>>,
pub responses_websocket_session_limit_seconds: Option<Spelled<f64>>,
pub disable_responses_id_security: Option<Spelled<bool>>,
pub allow_unmanaged_response_ids: Option<Spelled<bool>>,
pub dangerously_allow_mock_testing_request_params: Option<Spelled<bool>>,
pub enable_claude_code_gateway: Option<Spelled<bool>>,
pub claude_code_gateway_managed_settings: Option<Object>,
pub context_management_summary_model: Option<String>,
pub context_management_summary_max_tokens: Option<Spelled<i64>>,
pub safeguards_classifier_model: Option<String>,
pub user_url_validation: Option<Spelled<bool>>,
pub user_url_allowed_hosts: Option<Box<[String]>>,
pub provider_url_destination_allowed_hosts: Option<Box<[String]>>,
}
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, strum::IntoStaticStr)]
#[serde(rename_all = "lowercase")]
pub enum RequestTagHeader {
#[strum(serialize = "baggage")]
Baggage,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct RequestTagMapping {
pub header: Option<RequestTagHeader>,
pub key: Option<String>,
pub value: Option<String>,
pub tag: Option<String>,
pub tag_prefix: Option<String>,
pub required: Option<Spelled<bool>>,
pub require_budgeted_tag: Option<Spelled<bool>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct PassThroughEndpoint {
pub id: Option<String>,
pub path: Option<String>,
pub target: Option<SecretValue>,
pub headers: Option<Object>,
pub default_query_params: Option<Object>,
pub include_subpath: Option<Spelled<bool>>,
pub cost_per_request: Option<Spelled<f64>>,
pub timeout: Option<Spelled<f64>>,
pub auth: Option<Spelled<bool>>,
pub guardrails: Option<Object>,
pub methods: Option<Box<[String]>>,
}

View file

@ -0,0 +1,61 @@
use crate::{Object, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct SpendSettings {
pub disable_spend_logs: Option<Spelled<bool>>,
pub disable_spend_updates: Option<Spelled<bool>>,
pub disable_reset_budget: Option<Spelled<bool>>,
pub disable_error_logs: Option<Spelled<bool>>,
pub store_prompts_in_spend_logs: Option<Spelled<bool>>,
pub disable_requester_ip_address_logging: Option<Spelled<bool>>,
pub spend_logs_metadata_fields: Option<SpendLogsMetadataFields>,
pub proxy_batch_write_at: Option<Spelled<f64>>,
pub proxy_batch_polling_interval: Option<Spelled<f64>>,
pub proxy_budget_rescheduler_min_time: Option<Spelled<f64>>,
pub proxy_budget_rescheduler_max_time: Option<Spelled<f64>>,
pub disable_budget_reservation: Option<Spelled<bool>>,
pub fail_closed_budget_enforcement: Option<Spelled<bool>>,
pub apply_user_budget_to_team_keys: Option<Spelled<bool>>,
pub enforce_fallback_budget: Option<Spelled<bool>>,
pub track_unmanaged_batch_cost: Option<Spelled<bool>>,
pub spend_report_frequency: Option<String>,
pub spend_capture_rate_check: Option<SpendCaptureRateCheckSettings>,
pub maximum_spend_logs_retention_period: Option<Spelled<i64>>,
pub maximum_autorouter_session_retention_period: Option<Spelled<i64>>,
pub maximum_health_check_retention_period: Option<Spelled<i64>>,
pub maximum_daily_tag_spend_retention_period: Option<Spelled<i64>>,
pub maximum_spend_logs_size: Option<Spelled<i64>>,
pub maximum_spend_logs_cleanup_target_size: Option<Spelled<i64>>,
pub maximum_spend_logs_size_check_interval: Option<String>,
pub maximum_spend_logs_retention_interval: Option<String>,
pub maximum_spend_logs_cleanup_cron: Option<String>,
pub maximum_spend_logs_cleanup_batch_size: Option<Spelled<i64>>,
pub maximum_spend_logs_cleanup_max_batches: Option<Spelled<i64>>,
pub maximum_spend_logs_cleanup_run_budget: Option<Spelled<f64>>,
pub maximum_spend_logs_cleanup_batch_timeout: Option<Spelled<f64>>,
pub use_spend_logs_partitioning: Option<Spelled<bool>>,
pub scheduled_job_stagger: Option<ScheduledJobStaggerSettings>,
pub missing_session_id: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct SpendLogsMetadataFields {
pub include: Option<Box<[String]>>,
pub exclude: Option<Box<[String]>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct SpendCaptureRateCheckSettings {
pub providers: Option<Box<[String]>>,
pub threshold: Option<Spelled<f64>>,
pub lookback_days: Option<Spelled<i64>>,
pub openai_project_ids: Option<Box<[String]>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct ScheduledJobStaggerSettings {
pub enabled: Option<Spelled<bool>>,
pub window_seconds: Option<Spelled<i64>>,
pub identity: Option<String>,
pub offsets: Option<Object>,
}

View file

@ -0,0 +1,84 @@
use litellm_auth_types::SecretValue;
use serde::Deserialize;
use crate::Spelled;
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum TracingStoreKind {
Clickhouse,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ClickHouseStoreSettings {
#[serde(rename = "type")]
pub kind: TracingStoreKind,
pub url: Option<SecretValue>,
pub database: Option<String>,
pub retention_days: Option<Spelled<f64>>,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(untagged)]
pub enum TracingStoreSettings {
ClickHouse(ClickHouseStoreSettings),
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct TracingSettings {
pub store: Option<TracingStoreSettings>,
}
#[cfg(test)]
mod tests {
use crate::{Config, Spelled, TracingStoreSettings};
#[test]
fn tracing_settings_are_typed_and_redact_the_url() {
let config = Config::from_yaml(
"general_settings:\n tracing:\n store:\n type: clickhouse\n url: https://writer:SENTINEL@example.com\n database: analytics\n retention_days: 7\n",
)
.unwrap();
let tracing = config.general_settings.tracing.as_ref().unwrap();
let Some(TracingStoreSettings::ClickHouse(store)) = tracing.store.as_ref() else {
panic!("expected ClickHouse tracing store")
};
assert_eq!(
store.url.as_ref().unwrap().expose(),
"https://writer:SENTINEL@example.com"
);
assert_eq!(store.database.as_deref(), Some("analytics"));
assert_eq!(store.retention_days, Some(Spelled::Value(7.0)));
assert!(!format!("{config:?}").contains("SENTINEL"));
}
#[test]
fn tracing_settings_accept_environment_references() {
let config = Config::from_yaml(
"general_settings:\n tracing:\n store:\n type: clickhouse\n url: os.environ/CLICKHOUSE_URL\n retention_days: os.environ/RETENTION_DAYS\n",
)
.unwrap();
let Some(TracingStoreSettings::ClickHouse(store)) =
config.general_settings.tracing.unwrap().store
else {
panic!("expected ClickHouse tracing store")
};
assert_eq!(
store.retention_days,
Some(Spelled::Text("os.environ/RETENTION_DAYS".to_owned()))
);
}
#[test]
fn tracing_settings_reject_string_store() {
assert!(
Config::from_yaml("general_settings:\n tracing:\n store: clickhouse\n").is_err()
);
}
#[test]
fn tracing_settings_reject_removed_reader_configuration() {
assert!(Config::from_yaml("general_settings:\n tracing:\n store:\n type: clickhouse\n reader_url: http://localhost:8123\n").is_err());
}
}

View file

@ -0,0 +1,50 @@
use serde::Deserialize;
use crate::{Object, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct UiSettings {
pub ui_access_mode: Option<UiAccessMode>,
pub custom_sso: Option<String>,
pub custom_ui_sso_sign_in_handler: Option<String>,
pub auto_redirect_ui_login_to_sso: Option<Spelled<bool>>,
pub default_team_disabled: Option<Spelled<bool>>,
pub allow_cli_sso_verification_uri_complete: Option<Spelled<bool>>,
pub disable_env_credential_login: Option<Spelled<bool>>,
pub disable_password_login_when_sso_enabled: Option<Spelled<bool>>,
pub password_policy_min_length: Option<Spelled<f64>>,
pub password_policy_require_uppercase: Option<Spelled<bool>>,
pub password_policy_require_lowercase: Option<Spelled<bool>>,
pub password_policy_require_numbers: Option<Spelled<bool>>,
pub password_policy_require_special_characters: Option<Spelled<bool>>,
pub password_policy_check_breached_passwords: Option<Spelled<bool>>,
pub max_failed_login_attempts_per_source: Option<Spelled<i64>>,
pub failed_login_window_seconds: Option<Spelled<i64>>,
pub failed_login_block_seconds: Option<Spelled<i64>>,
pub max_failed_login_attempts_per_source_overrides: Option<Object>,
pub hide_default_credentials_hint: Option<Spelled<bool>>,
pub disable_master_key_return: Option<Spelled<bool>>,
pub disable_custom_api_keys: Option<Spelled<bool>>,
pub disable_key_generate_for_org_admin: Option<Spelled<bool>>,
pub team_admin_editable_team_fields: Option<Box<[String]>>,
pub disable_model_add_for_internal_users: Option<Spelled<bool>>,
pub disable_agents_for_internal_users: Option<Spelled<bool>>,
pub allow_agents_for_team_admins: Option<Spelled<bool>>,
pub disable_vector_stores_for_internal_users: Option<Spelled<bool>>,
pub allow_vector_stores_for_team_admins: Option<Spelled<bool>>,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(untagged)]
pub enum UiAccessMode {
Mode(String),
SsoGroup(UiAccessSsoGroup),
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct UiAccessSsoGroup {
#[serde(rename = "type")]
pub kind: Option<String>,
pub restricted_sso_group: Option<String>,
pub sso_group_jwt_field: Option<String>,
}

View file

@ -0,0 +1,102 @@
use litellm_auth_types::SecretValue;
use crate::{Object, Raw, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct Guardrail {
pub guardrail_id: Option<String>,
pub guardrail_name: Option<String>,
pub litellm_params: Option<GuardrailParams>,
pub guardrail_info: Option<Object>,
pub policy_template: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct GuardrailParams {
pub guardrail: Option<String>,
pub mode: Option<Raw>,
pub default_on: Option<Spelled<bool>>,
pub api_key: Option<SecretValue>,
pub api_base: Option<String>,
#[serde(flatten)]
pub additional_fields: Object,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::{Config, Spelled, Value};
fn parse(yaml: &str) -> Config {
Config::from_yaml(yaml).unwrap()
}
#[rstest]
fn guardrail_entries_keep_typed_params_and_unknown_keys() {
let config = parse(
r#"
guardrails:
- guardrail_name: pii
litellm_params:
guardrail: presidio
mode: [pre_call, post_call]
default_on: "true"
api_key: os.environ/PRESIDIO_KEY
presidio_language: en
guardrail_info:
owner: security
"#,
);
let guardrail = &config.guardrails[0];
let params = guardrail.litellm_params.as_ref().unwrap();
assert_eq!(guardrail.guardrail_name.as_deref(), Some("pii"));
assert_eq!(params.guardrail.as_deref(), Some("presidio"));
assert!(
matches!(params.mode.as_ref().unwrap().get(), Value::Sequence(modes) if modes.len() == 2)
);
assert_eq!(params.default_on, Some(Spelled::Text("true".to_owned())));
assert_eq!(
params.api_key.as_ref().unwrap().expose(),
"os.environ/PRESIDIO_KEY"
);
assert!(params.additional_fields.contains_key("presidio_language"));
assert!(
guardrail
.guardrail_info
.as_ref()
.unwrap()
.contains_key("owner")
);
}
#[rstest]
#[case::guardrail_key(
r#"
guardrails:
- litellm_params: {api_key: SENTINEL}
"#
)]
#[case::guardrail_extra(
r#"
guardrails:
- litellm_params: {vendor_secret: SENTINEL}
"#
)]
#[case::guardrail_info(
r#"
guardrails:
- guardrail_info: {owner: SENTINEL}
"#
)]
#[case::guardrail_mode(
r#"
guardrails:
- litellm_params: {mode: SENTINEL}
"#
)]
fn debug_never_prints_guardrail_values(#[case] yaml: &str) {
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
}
}

View file

@ -1,61 +0,0 @@
use std::{
collections::{BTreeSet, VecDeque},
path::{Path, PathBuf},
};
use serde::Deserialize;
use serde_yaml_ng::{Mapping, Value};
use crate::Error;
#[derive(Deserialize)]
struct Includes {
#[serde(default)]
include: Vec<String>,
}
fn read(path: &Path) -> Result<Mapping, Error> {
Ok(serde_yaml_ng::from_str(&std::fs::read_to_string(path)?)?)
}
fn entries(config: &Mapping, path: &Path) -> Result<Vec<(String, PathBuf)>, Error> {
let includes: Includes = serde_yaml_ng::from_value(Value::Mapping(config.clone()))?;
Ok(includes
.include
.into_iter()
.map(|entry| (entry, path.to_owned()))
.collect())
}
pub(super) fn load(path: &Path) -> Result<Value, Error> {
let root = path.canonicalize()?;
let mut merged = read(&root)?;
let mut pending: VecDeque<_> = entries(&merged, &root)?.into();
let mut loaded = BTreeSet::from([root.clone()]);
merged.remove(Value::String("include".into()));
while let Some((entry, declaring)) = pending.pop_front() {
let declared = declaring.parent().unwrap_or(Path::new(".")).join(&entry);
let fallback = root.parent().unwrap_or(Path::new(".")).join(&entry);
let location = if declared.exists() {
declared
} else {
fallback
}
.canonicalize()?;
if !loaded.insert(location.clone()) {
continue;
}
let mut included = read(&location)?;
pending.extend(entries(&included, &location)?);
included.remove(Value::String("include".into()));
for (key, value) in included {
match (merged.get_mut(&key), value) {
(Some(Value::Sequence(base)), Value::Sequence(extra)) => base.extend(extra),
(_, value) => {
merged.insert(key, value);
}
}
}
}
Ok(Value::Mapping(merged))
}

View file

@ -1,94 +1,349 @@
macro_rules_attribute::attribute_alias! {
#[apply(config_section)] =
#[derive(Clone, Debug, Default, ::serde::Deserialize)]
#[serde(default)];
}
mod agent;
mod callback;
mod credential;
mod error;
mod includes;
mod general;
mod guardrail;
mod litellm;
mod mcp;
mod model;
mod settings;
mod policy;
mod prompt;
mod provider;
mod router;
mod tool;
mod value;
mod vector_store;
pub mod yaml;
use std::{fmt, path::Path};
use serde::Deserialize;
use std::collections::BTreeMap;
pub use agent::Agent;
pub use callback::CallbackSettings;
pub use credential::Credential;
pub use error::Error;
pub use litellm_router_types::{LitellmParams, Spelled};
pub use mcp::{McpAuth, McpServer, McpTransport};
pub use model::Model;
pub use settings::{
ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings,
TracingStoreSettings,
pub use general::{
Alerting, AlertingArgs, AlertingSettings, AuthSettings, ClickHouseStoreSettings,
DatabaseSettings, FilesSettings, GeneralSettings, HealthSettings, KeyManagementSettings,
LimitsSettings, McpSettings, ObjectsSettings, PassThroughEndpoint, RequestTagHeader,
RequestTagMapping, RequestsSettings, RoleBasedPermissions, ScheduledJobStaggerSettings,
ServiceAccountSettings, SpendCaptureRateCheckSettings, SpendLogsMetadataFields, SpendSettings,
TeamMetadataField, TracingSettings, TracingStoreSettings, UiAccessMode, UiAccessSsoGroup,
UiSettings, UserHeaderMapping,
};
pub use value::{AdditionalFields, Object, Value};
pub use guardrail::{Guardrail, GuardrailParams};
pub use litellm::{
AiohttpSettings, AzureBlobCacheParams, BackgroundMode, BudgetSettings, CacheParams,
CachingSettings, CallbackVar, CallerIdentity, ClientCertificateSettings, CustomProvider,
DiskCacheParams, FallbackSettings, GcsCacheParams, Http2Settings, LiteLlmSettings,
LoggingSettings, McpToolSettings, ModelGroupSettings, PriorityReservationSettings,
ProxyCacheTtlParams, QdrantCacheParams, RedisCacheParams, RedisServerCacheParams,
ResponsesSettings, S3CacheParams, SemanticCacheParams, SocketSettings, TeamDefaultSettings,
TlsSettings, UpperboundKeyGenerateParams, ValkeyCacheParams,
};
pub use litellm_python_compat::read::Spelled;
pub use litellm_router_types::LitellmParams;
pub use mcp::{McpAuth, McpServer, McpTransport};
pub use model::{AccessWindow, Model, ModelId, ModelInfo};
pub use policy::{
Policy, PolicyAttachment, PolicyCondition, PolicyGuardrailChanges, PolicyGuardrails,
PolicyPipeline, PolicyPipelineStep,
};
pub use prompt::{Prompt, PromptInfo, PromptParams};
pub use provider::{AssistantSettings, ProviderSettings, VertexConfig, Worker};
pub use router::{AlertingConfig, AllowedFailsPolicy, ProviderBudget, RouterSettings};
pub use tool::{SandboxTool, SandboxToolParams, SearchTool, SearchToolInfo, SearchToolParams};
pub use value::{Object, Raw, Value};
pub use vector_store::{VectorStore, VectorStoreParams};
#[derive(Clone, Default, Deserialize)]
#[serde(default)]
#[macro_rules_attribute::apply(config_section)]
pub struct Config {
pub model_list: Box<[Model]>,
pub general_settings: GeneralSettings,
pub router_settings: RouterSettings,
pub litellm_settings: LiteLlmSettings,
pub environment_variables: Object,
pub callback_settings: Object,
pub assistant_settings: Object,
pub default_vertex_config: Object,
pub mcp_servers: std::collections::BTreeMap<String, McpServer>,
pub credential_list: Box<[Object]>,
pub guardrails: Box<[Object]>,
pub prompts: Box<[Object]>,
pub sandbox_tools: Box<[Object]>,
pub search_tools: Box<[Object]>,
pub files_settings: Box<[Object]>,
pub finetune_settings: Box<[Object]>,
pub callback_settings: BTreeMap<String, CallbackSettings>,
pub assistant_settings: Option<AssistantSettings>,
pub default_vertex_config: Option<VertexConfig>,
pub mcp_servers: BTreeMap<String, McpServer>,
pub credential_list: Box<[Credential]>,
pub guardrails: Box<[Guardrail]>,
pub prompts: Box<[Prompt]>,
pub sandbox_tools: Box<[SandboxTool]>,
pub search_tools: Box<[SearchTool]>,
pub files_settings: Box<[ProviderSettings]>,
pub finetune_settings: Box<[ProviderSettings]>,
pub mcp_tools: Box<[Object]>,
pub vector_store_registry: Box<[Object]>,
pub worker_registry: Box<[Object]>,
pub agents: Box<[Object]>,
pub agent_list: Box<[Object]>,
pub policies: Object,
pub policy_attachments: Box<[Object]>,
pub vector_store_registry: Box<[VectorStore]>,
pub worker_registry: Box<[Worker]>,
pub agents: Box<[Agent]>,
pub agent_list: Box<[Agent]>,
pub policies: BTreeMap<String, Policy>,
pub policy_attachments: Box<[PolicyAttachment]>,
pub include: Box<[String]>,
#[serde(flatten)]
pub additional_fields: AdditionalFields,
}
impl fmt::Debug for Config {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Config")
.field("model_list", &self.model_list)
.field("general_settings", &self.general_settings)
.field("router_settings", &self.router_settings)
.field("litellm_settings", &self.litellm_settings)
.field("environment_variables", &self.environment_variables)
.field("callback_settings", &self.callback_settings)
.field("assistant_settings", &self.assistant_settings)
.field("default_vertex_config", &self.default_vertex_config)
.field("mcp_servers", &self.mcp_servers)
.field("credential_list", &self.credential_list)
.field("guardrails", &self.guardrails)
.field("prompts", &self.prompts)
.field("sandbox_tools", &self.sandbox_tools)
.field("search_tools", &self.search_tools)
.field("files_settings", &self.files_settings)
.field("finetune_settings", &self.finetune_settings)
.field("mcp_tools", &self.mcp_tools)
.field("vector_store_registry", &self.vector_store_registry)
.field("worker_registry", &self.worker_registry)
.field("agents", &self.agents)
.field("agent_list", &self.agent_list)
.field("policies", &self.policies)
.field("policy_attachments", &self.policy_attachments)
.field("include", &self.include)
.field("additional_fields", &self.additional_fields.keys())
.finish()
}
pub additional_fields: Object,
}
impl Config {
pub fn from_yaml(yaml: &str) -> Result<Self, Error> {
Ok(serde_yaml_ng::from_str(yaml)?)
Self::from_mapping(yaml::mapping(yaml)?)
}
pub fn load(path: impl AsRef<Path>) -> Result<Self, Error> {
Ok(serde_yaml_ng::from_value(includes::load(path.as_ref())?)?)
pub fn from_mapping(document: serde_yaml_ng::Mapping) -> Result<Self, Error> {
Ok(serde_yaml_ng::from_value(serde_yaml_ng::Value::Mapping(
document,
))?)
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::{Config, Error, Spelled};
#[rstest]
fn config_debug_redacts_api_keys() {
let config = Config::from_yaml(
"model_list: [{model_name: assistant, litellm_params: {model: anthropic/test-model, api_key: secret-value}}]",
)
.unwrap();
assert_eq!(
config.model_list[0]
.litellm_params
.api_key
.as_ref()
.unwrap()
.expose(),
"secret-value"
);
assert!(!format!("{config:?}").contains("secret-value"));
}
#[rstest]
#[case::malformed_yaml("model_list: [")]
#[case::missing_params("model_list: [{model_name: assistant}]")]
#[case::missing_model("model_list: [{model_name: assistant, litellm_params: {api_key: key}}]")]
#[case::provider_param_of_the_wrong_type(
"model_list: [{model_name: assistant, litellm_params: {model: test, aws_region_name: [eu-central-1]}}]"
)]
#[case::negative_model_rpm(
"model_list: [{model_name: assistant, rpm: -1, litellm_params: {model: test}}]"
)]
fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) {
let error = Config::from_yaml(yaml).unwrap_err();
assert!(matches!(error, Error::Parse(_)), "{error:?}");
}
#[rstest]
fn empty_config_matches_python_defaults() {
let config = Config::from_yaml("{}").unwrap();
assert!(config.model_list.is_empty());
assert_eq!(config.general_settings.admission_queue_timeout_seconds, 1.0);
assert_eq!(
config.general_settings.database_connection_pool_limit,
Some(10)
);
assert_eq!(
config.general_settings.proxy_config_reload_interval_seconds,
30
);
assert_eq!(config.general_settings.health_check_interval, 300);
}
#[rstest]
fn parses_typed_settings_and_preserves_extension_fields() {
let config = Config::from_yaml(
r#"
model_list:
- model_name: assistant
litellm_params:
model: vertex_ai/test-model
timeout: os.environ/REQUEST_TIMEOUT
tpm: os.environ/TPM_LIMIT
rpm: 5
drop_params: "true"
vertex_project: test-project
vertex_credentials:
type: service_account
future_param: 1
model_info:
mode: chat
access_groups: [internal]
general_settings:
master_key: secret-master-key
store_model_in_db: true
custom_auth: auth.py
router_settings:
routing_strategy: simple-shuffle
allowed_fails: 2
future_router_setting: cache.internal
litellm_settings:
drop_params: true
cache: true
custom_callback_name: audit
future_section:
enabled: true
"#,
)
.unwrap();
let model = &config.model_list[0];
assert_eq!(
model.litellm_params.timeout,
Some(Spelled::Text("os.environ/REQUEST_TIMEOUT".to_string()))
);
assert_eq!(
model.litellm_params.tpm,
Some(Spelled::Text("os.environ/TPM_LIMIT".to_string()))
);
assert_eq!(model.litellm_params.rpm, Some(Spelled::Value(5)));
assert_eq!(
model.litellm_params.drop_params,
Some(Spelled::Text("true".to_string()))
);
assert_eq!(
model.litellm_params.vertex.vertex_project.as_deref(),
Some("test-project")
);
assert_eq!(
model.litellm_params.vertex.vertex_credentials.as_deref(),
Some(r#"{"type":"service_account"}"#)
);
assert!(model.litellm_params.extra.contains_key("future_param"));
assert!(model.additional_fields.contains_key("access_groups"));
assert_eq!(config.router_settings.allowed_fails, Some(2));
assert!(
config
.router_settings
.additional_fields
.contains_key("future_router_setting")
);
assert!(
config
.litellm_settings
.additional_fields
.contains_key("custom_callback_name")
);
assert!(config.additional_fields.contains_key("future_section"));
}
#[rstest]
fn parses_python_config_sections() {
let config = Config::from_yaml(
r#"
environment_variables:
REDIS_PORT: 6379
callback_settings:
otel:
message_logging: false
assistant_settings:
custom_llm_provider: openai
credential_list:
- credential_name: bedrock
credential_values:
aws_region_name: us-east-1
guardrails:
- guardrail_name: pii
litellm_params:
guardrail: presidio
prompts:
- prompt_id: support
sandbox_tools:
- sandbox_tool_name: e2b
search_tools:
- search_tool_name: web
files_settings:
- custom_llm_provider: openai
finetune_settings:
- custom_llm_provider: openai
mcp_tools:
- name: lookup
mcp_servers:
docs:
url: https://example.test/mcp
vector_store_registry:
- vector_store_name: docs
worker_registry:
- worker_id: regional
agents:
- agent_name: reviewer
agent_list:
- agent_name: legacy-reviewer
policies:
safe:
guardrails:
add: [pii]
policy_attachments:
- policy_id: safe
"#,
)
.unwrap();
assert_eq!(config.environment_variables.len(), 1);
assert_eq!(config.credential_list.len(), 1);
assert_eq!(config.guardrails.len(), 1);
assert_eq!(config.prompts.len(), 1);
assert_eq!(config.sandbox_tools.len(), 1);
assert_eq!(config.search_tools.len(), 1);
assert_eq!(config.files_settings.len(), 1);
assert_eq!(config.finetune_settings.len(), 1);
assert_eq!(config.mcp_tools.len(), 1);
assert_eq!(config.mcp_servers.len(), 1);
assert_eq!(config.vector_store_registry.len(), 1);
assert_eq!(config.worker_registry.len(), 1);
assert_eq!(config.agents.len(), 1);
assert_eq!(config.agent_list.len(), 1);
assert!(config.policies.contains_key("safe"));
assert_eq!(config.policy_attachments.len(), 1);
}
#[rstest]
#[case::api_key("api_key", "provider-secret")]
#[case::aws_secret("aws_secret_access_key", "aws-secret")]
#[case::vertex_credentials("vertex_credentials", "vertex-secret")]
#[case::provider_extension("azure_ad_token", "azure-secret")]
#[case::general_extension("custom_auth_secret", "auth-secret")]
#[case::router_extension("future_router_token", "router-secret")]
#[case::litellm_extension("callback_token", "callback-secret")]
#[case::root_extension("private_token", "root-secret")]
fn debug_output_does_not_expose_config_values(#[case] field: &str, #[case] secret: &str) {
let yaml = match field {
"api_key" => format!(
"model_list: [{{model_name: assistant, litellm_params: {{model: test, api_key: {secret}}}}}]"
),
"aws_secret_access_key" => format!(
"model_list: [{{model_name: assistant, litellm_params: {{model: test, aws_secret_access_key: {secret}}}}}]"
),
"vertex_credentials" => format!(
"model_list: [{{model_name: assistant, litellm_params: {{model: test, vertex_credentials: {secret}}}}}]"
),
"azure_ad_token" => format!(
"model_list: [{{model_name: assistant, litellm_params: {{model: test, azure_ad_token: {secret}}}}}]"
),
"custom_auth_secret" => format!("general_settings: {{{field}: {secret}}}"),
"future_router_token" => format!("router_settings: {{{field}: {secret}}}"),
"callback_token" => format!("litellm_settings: {{{field}: {secret}}}"),
"private_token" => format!("{field}: {secret}"),
_ => unreachable!(),
};
let config = Config::from_yaml(&yaml).unwrap();
assert!(!format!("{config:?}").contains(secret));
}
#[rstest]
fn unknown_top_level_key_lands_in_additional_fields() {
let config = Config::from_yaml("future_section:\n a: 1\nenable_jwt_auth: true\n").unwrap();
assert!(config.additional_fields.contains_key("future_section"));
assert!(config.additional_fields.contains_key("enable_jwt_auth"));
}
}

View file

@ -0,0 +1,9 @@
## Docs
- https://github.com/BerriAI/litellm-docs/blob/main/docs/proxy/config_settings.md
## Python
- There is no Python model. `ConfigYAML.litellm_settings` in `litellm/proxy/_types.py` is a plain `dict`
- `ProxyConfig.load_config` in `proxy_server.py` handles a few keys specially (`cache`, `cache_params`, `callbacks`, `success_callback`, `guardrails`, `drop_params` and others) and does `setattr(litellm, key, value)` for everything else. So any module-level attribute in `litellm/__init__.py` is a valid key, and that is the real key set
- `litellm_settings` is one of the sections the proxy persists to the DB (`_CONFIG_PERSISTED_SECTIONS`) and the Admin UI edits, so at runtime the YAML is only one source of the effective values. Merging the DB overlay is the consumer's job, not this crate's

View file

@ -0,0 +1,128 @@
use serde::Deserialize;
use std::collections::BTreeMap;
use litellm_auth_types::SecretValue;
use crate::{Raw, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct BudgetSettings {
pub max_budget: Option<Spelled<f64>>,
pub max_internal_user_budget: Option<Spelled<f64>>,
pub default_max_internal_user_budget: Option<Spelled<f64>>,
pub default_internal_user_params: Option<Raw>,
pub max_ui_session_budget: Option<Spelled<f64>>,
pub default_team_settings: Option<Box<[TeamDefaultSettings]>>,
pub upperbound_key_generate_params: Option<UpperboundKeyGenerateParams>,
pub key_alias_pattern: Option<String>,
pub budget_reset_time: Option<String>,
pub disable_end_user_cost_tracking: Option<Spelled<bool>>,
}
#[derive(Clone, Debug, Deserialize)]
pub struct TeamDefaultSettings {
pub team_id: String,
#[serde(default)]
pub success_callback: Option<Box<[String]>>,
#[serde(default)]
pub failure_callback: Option<Box<[String]>>,
#[serde(default)]
pub callback_vars: Option<BTreeMap<String, CallbackVar>>,
#[serde(flatten)]
pub inline_callback_vars: BTreeMap<String, CallbackVar>,
}
#[derive(Clone, Debug, Deserialize, PartialEq)]
#[serde(untagged)]
pub enum CallbackVar {
Bool(bool),
Number(serde_yaml_ng::Number),
Text(SecretValue),
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct UpperboundKeyGenerateParams {
pub max_budget: Option<Spelled<f64>>,
pub budget_duration: Option<String>,
pub duration: Option<String>,
pub max_parallel_requests: Option<Spelled<i64>>,
pub tpm_limit: Option<Spelled<i64>>,
pub rpm_limit: Option<Spelled<i64>>,
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use litellm_auth_types::SecretValue;
use rstest::rstest;
use super::{CallbackVar, TeamDefaultSettings};
use crate::Config;
fn teams(yaml: &str) -> Result<Box<[TeamDefaultSettings]>, crate::Error> {
Config::from_yaml(&format!(
"litellm_settings:\n default_team_settings: {yaml}"
))
.map(|config| {
config
.litellm_settings
.budget
.default_team_settings
.unwrap()
})
}
fn team(yaml: &str) -> TeamDefaultSettings {
teams(yaml).unwrap()[0].clone()
}
#[rstest]
fn named_keys_stay_out_of_inline_callback_vars() {
let team = team(
"[{team_id: t, success_callback: [langfuse], failure_callback: [otel], callback_vars: {k: v}, pk: x}]",
);
assert_eq!(team.team_id, "t");
assert_eq!(
team.success_callback.as_deref(),
Some(&["langfuse".to_owned()][..])
);
assert_eq!(
team.failure_callback.as_deref(),
Some(&["otel".to_owned()][..])
);
assert_eq!(
team.callback_vars,
Some(BTreeMap::from([(
"k".to_owned(),
CallbackVar::Text(SecretValue::new("v"))
)]))
);
assert_eq!(team.inline_callback_vars.keys().collect::<Vec<_>>(), ["pk"]);
}
#[rstest]
#[case::text("pk", CallbackVar::Text(SecretValue::new("pk")))]
#[case::reference("os.environ/PK", CallbackVar::Text(SecretValue::new("os.environ/PK")))]
#[case::quoted_number("'5'", CallbackVar::Text(SecretValue::new("5")))]
#[case::integer("5", CallbackVar::Number(5.into()))]
#[case::float("0.5", CallbackVar::Number(0.5.into()))]
#[case::bool("true", CallbackVar::Bool(true))]
fn callback_vars_accept_scalars(#[case] yaml: &str, #[case] expected: CallbackVar) {
let inline = team(&format!("[{{team_id: t, var: {yaml}}}]"));
let nested = team(&format!("[{{team_id: t, callback_vars: {{var: {yaml}}}}}]"));
assert_eq!(inline.inline_callback_vars.get("var"), Some(&expected));
assert_eq!(nested.callback_vars.unwrap().get("var"), Some(&expected));
}
#[rstest]
#[case::missing_team_id("[{success_callback: [langfuse]}]")]
#[case::inline_list("[{team_id: t, var: [a]}]")]
#[case::inline_mapping("[{team_id: t, var: {a: 1}}]")]
#[case::inline_null("[{team_id: t, var: null}]")]
#[case::nested_mapping("[{team_id: t, callback_vars: {var: {a: 1}}}]")]
#[case::single_callback("[{team_id: t, success_callback: langfuse}]")]
fn invalid_team_settings_fail_to_parse(#[case] yaml: &str) {
assert!(teams(yaml).is_err());
}
}

View file

@ -0,0 +1,137 @@
use serde::{Deserialize, Deserializer};
use litellm_auth_types::SecretValue;
use crate::{Object, Raw, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct CachingSettings {
pub cache: Option<bool>,
pub cache_params: Option<CacheParams>,
pub enable_redis_auth_cache: Option<Spelled<bool>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct CacheParams {
#[serde(rename = "type")]
pub cache_type: Option<String>,
pub mode: Option<String>,
pub namespace: Option<String>,
pub ttl: Option<Spelled<f64>>,
pub supported_call_types: Option<Box<[String]>>,
#[serde(deserialize_with = "present")]
pub max_messages: Option<Option<Spelled<i64>>>,
#[serde(flatten)]
pub redis_server: RedisServerCacheParams,
#[serde(flatten)]
pub semantic: SemanticCacheParams,
#[serde(flatten)]
pub proxy_ttl: ProxyCacheTtlParams,
#[serde(flatten)]
pub redis: RedisCacheParams,
#[serde(flatten)]
pub valkey: ValkeyCacheParams,
#[serde(flatten)]
pub s3: S3CacheParams,
#[serde(flatten)]
pub gcs: GcsCacheParams,
#[serde(flatten)]
pub azure_blob: AzureBlobCacheParams,
#[serde(flatten)]
pub qdrant: QdrantCacheParams,
#[serde(flatten)]
pub disk: DiskCacheParams,
#[serde(flatten)]
pub additional_fields: Object,
}
/// Read by every backend that talks to a Redis-protocol server: `redis`, `redis-semantic` and `valkey-semantic`.
#[macro_rules_attribute::apply(crate::config_section)]
pub struct RedisServerCacheParams {
pub host: Option<String>,
pub port: Option<Spelled<i64>>,
pub password: Option<SecretValue>,
pub redis_startup_nodes: Option<Raw>,
pub default_in_redis_ttl: Option<Spelled<f64>>,
}
/// Read by the semantic backends: `redis-semantic`, `valkey-semantic` and `qdrant-semantic`.
#[macro_rules_attribute::apply(crate::config_section)]
pub struct SemanticCacheParams {
pub similarity_threshold: Option<Spelled<f64>>,
pub semantic_cache_embedding_max_input_tokens: Option<Spelled<i64>>,
pub semantic_cache_embedding_timeout: Option<Spelled<f64>>,
pub semantic_cache_scope: Option<String>,
}
/// Not response-cache settings. The proxy copies them onto `litellm.default_*_ttl` for its own internal caches.
/// `default_in_memory_ttl` also replaces `ttl` when `type` is `local`.
#[macro_rules_attribute::apply(crate::config_section)]
pub struct ProxyCacheTtlParams {
pub default_in_memory_ttl: Option<Spelled<f64>>,
pub default_redis_ttl: Option<Spelled<f64>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct RedisCacheParams {
pub redis_semantic_cache_embedding_model: Option<String>,
pub redis_semantic_cache_index_name: Option<String>,
pub redis_flush_size: Option<Spelled<i64>>,
pub gcp_service_account: Option<String>,
pub gcp_ssl_ca_certs: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct ValkeyCacheParams {
pub valkey_semantic_cache_embedding_model: Option<String>,
pub valkey_semantic_cache_index_name: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct S3CacheParams {
pub s3_bucket_name: Option<String>,
pub s3_region_name: Option<String>,
pub s3_api_version: Option<String>,
pub s3_use_ssl: Option<Spelled<bool>>,
pub s3_verify: Option<Spelled<bool>>,
pub s3_endpoint_url: Option<SecretValue>,
pub s3_aws_access_key_id: Option<SecretValue>,
pub s3_aws_secret_access_key: Option<SecretValue>,
pub s3_aws_session_token: Option<SecretValue>,
pub s3_config: Option<Raw>,
pub s3_path: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct GcsCacheParams {
pub gcs_bucket_name: Option<String>,
pub gcs_path_service_account: Option<String>,
pub gcs_path: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct AzureBlobCacheParams {
pub azure_account_url: Option<SecretValue>,
pub azure_blob_container: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct QdrantCacheParams {
pub qdrant_api_base: Option<SecretValue>,
pub qdrant_api_key: Option<SecretValue>,
pub qdrant_collection_name: Option<String>,
pub qdrant_quantization_config: Option<String>,
pub qdrant_semantic_cache_embedding_model: Option<String>,
pub qdrant_semantic_cache_vector_size: Option<Spelled<i64>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct DiskCacheParams {
pub disk_cache_dir: Option<String>,
}
fn present<'de, D: Deserializer<'de>, T: Deserialize<'de>>(
deserializer: D,
) -> Result<Option<Option<T>>, D::Error> {
Ok(Some(Option::deserialize(deserializer)?))
}

View file

@ -0,0 +1,7 @@
use crate::Raw;
#[macro_rules_attribute::apply(crate::config_section)]
pub struct FallbackSettings {
pub default_fallbacks: Option<Raw>,
pub context_window_fallbacks: Option<Raw>,
}

View file

@ -0,0 +1,24 @@
use litellm_python_compat::read::one_or_many;
use crate::{Object, Raw, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct LoggingSettings {
#[serde(deserialize_with = "one_or_many")]
pub callbacks: Option<Box<[Raw]>>,
#[serde(deserialize_with = "one_or_many")]
pub success_callback: Option<Box<[Raw]>>,
#[serde(deserialize_with = "one_or_many")]
pub failure_callback: Option<Box<[Raw]>>,
pub service_callback: Option<Box<[String]>>,
pub audit_log_callbacks: Option<Box<[String]>>,
pub s3_callback_params: Option<Object>,
pub s3_audit_callback_params: Option<Object>,
pub store_audit_logs: Option<Spelled<bool>>,
pub json_logs: Option<bool>,
pub set_verbose: Option<bool>,
pub turn_off_message_logging: Option<Spelled<bool>>,
pub redact_messages_in_exceptions: Option<Spelled<bool>>,
pub log_raw_request_response: Option<Spelled<bool>>,
pub presidio_logging_only: Option<Spelled<bool>>,
}

View file

@ -0,0 +1,8 @@
use crate::Raw;
#[macro_rules_attribute::apply(crate::config_section)]
pub struct McpToolSettings {
pub mcp_aliases: Option<Raw>,
pub mcp_semantic_tool_filter: Option<Raw>,
pub mcp_tool_search: Option<Raw>,
}

View file

@ -0,0 +1,406 @@
mod budget;
mod caching;
mod fallbacks;
mod logging;
mod mcp;
mod transport;
use serde::Deserialize;
pub use self::{
budget::{BudgetSettings, CallbackVar, TeamDefaultSettings, UpperboundKeyGenerateParams},
caching::{
AzureBlobCacheParams, CacheParams, CachingSettings, DiskCacheParams, GcsCacheParams,
ProxyCacheTtlParams, QdrantCacheParams, RedisCacheParams, RedisServerCacheParams,
S3CacheParams, SemanticCacheParams, ValkeyCacheParams,
},
fallbacks::FallbackSettings,
logging::LoggingSettings,
mcp::McpToolSettings,
transport::{
AiohttpSettings, ClientCertificateSettings, Http2Settings, SocketSettings, TlsSettings,
},
};
use crate::{Object, Raw, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct LiteLlmSettings {
pub drop_params: Option<Spelled<bool>>,
pub request_timeout: Option<Spelled<f64>>,
pub num_retries: Option<u64>,
pub prometheus_deployment_and_latency_caller_identity: Option<CallerIdentity>,
pub guardrails: Option<Box<[Object]>>,
pub prompt_injection_params: Option<Object>,
pub azure_content_safety_params: Option<Object>,
pub global_prompt_directory: Option<String>,
pub global_bitbucket_config: Option<Object>,
pub global_gitlab_config: Option<Object>,
pub priority_reservation_settings: Option<PriorityReservationSettings>,
pub model_group_settings: Option<ModelGroupSettings>,
pub post_call_rules: Option<String>,
pub custom_provider_map: Option<Box<[CustomProvider]>>,
pub responses: Option<ResponsesSettings>,
pub enable_preview_features: Option<Spelled<bool>>,
pub public_model_groups: Option<Box<[String]>>,
pub timezone: Option<String>,
#[serde(flatten)]
pub tls: TlsSettings,
#[serde(flatten)]
pub client_certificate: ClientCertificateSettings,
#[serde(flatten)]
pub socket: SocketSettings,
#[serde(flatten)]
pub http2: Http2Settings,
#[serde(flatten)]
pub aiohttp: AiohttpSettings,
#[serde(flatten)]
pub caching: CachingSettings,
#[serde(flatten)]
pub budget: BudgetSettings,
#[serde(flatten)]
pub logging: LoggingSettings,
#[serde(flatten)]
pub fallbacks: FallbackSettings,
#[serde(flatten)]
pub mcp: McpToolSettings,
#[serde(flatten)]
pub additional_fields: Object,
}
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, strum::IntoStaticStr)]
#[serde(rename_all = "snake_case")]
pub enum CallerIdentity {
#[strum(serialize = "api_key_alias")]
ApiKeyAlias,
#[strum(serialize = "user_email")]
UserEmail,
#[strum(serialize = "both")]
Both,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct PriorityReservationSettings {
pub default_priority: Option<Spelled<f64>>,
pub saturation_threshold: Option<Spelled<f64>>,
pub saturation_check_cache_ttl: Option<Spelled<i64>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct ModelGroupSettings {
pub forward_client_headers_to_llm_api: Option<Box<[String]>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct CustomProvider {
pub provider: Option<String>,
pub custom_handler: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct ResponsesSettings {
pub background_mode: Option<BackgroundMode>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct BackgroundMode {
pub polling_via_cache: Option<Raw>,
pub native_background_mode: Option<Box<[String]>>,
pub ttl: Option<Spelled<i64>>,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::{CallerIdentity, Config, LiteLlmSettings, Spelled};
fn settings(yaml: &str) -> LiteLlmSettings {
Config::from_yaml(&format!("litellm_settings:\n{yaml}"))
.unwrap()
.litellm_settings
}
#[rstest]
#[case::bool_word(" turn_off_message_logging: 'false'", Spelled::Text("false".into()))]
#[case::bool_value(" turn_off_message_logging: true", Spelled::Value(true))]
#[case::env_reference(" turn_off_message_logging: os.environ/REDACT", Spelled::Text("os.environ/REDACT".into()))]
fn bool_settings_keep_python_spellings(#[case] yaml: &str, #[case] expected: Spelled<bool>) {
assert_eq!(
settings(yaml).logging.turn_off_message_logging,
Some(expected)
);
}
#[rstest]
#[case::number(" max_budget: 10", Spelled::Value(10.0))]
#[case::string(" max_budget: '10'", Spelled::Text("10".into()))]
#[case::env_reference(" max_budget: os.environ/MAX_BUDGET", Spelled::Text("os.environ/MAX_BUDGET".into()))]
fn budget_accepts_number_or_string(#[case] yaml: &str, #[case] expected: Spelled<f64>) {
assert_eq!(settings(yaml).budget.max_budget, Some(expected));
}
#[rstest]
#[case::api_key_alias("api_key_alias", CallerIdentity::ApiKeyAlias)]
#[case::user_email("user_email", CallerIdentity::UserEmail)]
#[case::both("both", CallerIdentity::Both)]
fn caller_identity_accepts_python_values(
#[case] value: &str,
#[case] expected: CallerIdentity,
) {
assert_eq!(
settings(&format!(
" prometheus_deployment_and_latency_caller_identity: {value}"
))
.prometheus_deployment_and_latency_caller_identity,
Some(expected)
);
}
#[rstest]
fn caller_identity_rejects_values_python_refuses() {
assert!(
Config::from_yaml(
"litellm_settings:\n prometheus_deployment_and_latency_caller_identity: team"
)
.is_err()
);
}
#[rstest]
fn cache_params_reads_cache_keywords() {
let cache = settings(
" cache_params:\n type: redis\n port: 6379\n password: os.environ/REDIS_PASSWORD\n ttl: 600\n supported_call_types: [acompletion]\n namespace_extra: x",
)
.caching
.cache_params
.unwrap();
assert_eq!(cache.cache_type.as_deref(), Some("redis"));
assert_eq!(cache.redis_server.port, Some(Spelled::Value(6379)));
assert_eq!(
cache.redis_server.password.unwrap().expose(),
"os.environ/REDIS_PASSWORD"
);
assert_eq!(cache.ttl, Some(Spelled::Value(600.0)));
assert_eq!(
cache.supported_call_types.as_deref(),
Some(&["acompletion".to_owned()][..])
);
assert!(cache.additional_fields.contains_key("namespace_extra"));
}
#[rstest]
fn cache_params_route_each_key_to_its_backend_group() {
let cache = settings(
r#" cache_params:
type: s3
namespace: shared
port: 6379
similarity_threshold: 0.8
default_redis_ttl: 30
redis_flush_size: 10
valkey_semantic_cache_index_name: valkey-index
s3_bucket_name: bucket
gcs_path: gcs/path
azure_blob_container: container
qdrant_collection_name: collection
disk_cache_dir: /tmp/cache
namespace_extra: x"#,
)
.caching
.cache_params
.unwrap();
assert_eq!(cache.namespace.as_deref(), Some("shared"));
assert_eq!(cache.redis_server.port, Some(Spelled::Value(6379)));
assert_eq!(
cache.semantic.similarity_threshold,
Some(Spelled::Value(0.8))
);
assert_eq!(
cache.proxy_ttl.default_redis_ttl,
Some(Spelled::Value(30.0))
);
assert_eq!(cache.redis.redis_flush_size, Some(Spelled::Value(10)));
assert_eq!(
cache.valkey.valkey_semantic_cache_index_name.as_deref(),
Some("valkey-index")
);
assert_eq!(cache.s3.s3_bucket_name.as_deref(), Some("bucket"));
assert_eq!(cache.gcs.gcs_path.as_deref(), Some("gcs/path"));
assert_eq!(
cache.azure_blob.azure_blob_container.as_deref(),
Some("container")
);
assert_eq!(
cache.qdrant.qdrant_collection_name.as_deref(),
Some("collection")
);
assert_eq!(cache.disk.disk_cache_dir.as_deref(), Some("/tmp/cache"));
assert_eq!(
cache.additional_fields.keys().collect::<Vec<_>>(),
["namespace_extra"]
);
}
#[rstest]
#[case::absent(" ttl: 1", None)]
#[case::explicit_null(" max_messages: null", Some(None))]
#[case::limit(" max_messages: 8", Some(Some(Spelled::Value(8))))]
fn cache_max_messages_tells_null_from_absent(
#[case] yaml: &str,
#[case] expected: Option<Option<Spelled<i64>>>,
) {
assert_eq!(
settings(&format!(" cache_params:\n{yaml}"))
.caching
.cache_params
.unwrap()
.max_messages,
expected
);
}
#[rstest]
fn nested_settings_land_in_typed_fields() {
let settings = settings(
" priority_reservation_settings: {default_priority: 0.5, saturation_check_cache_ttl: 30}
model_group_settings: {forward_client_headers_to_llm_api: [gpt]}
custom_provider_map: [{provider: my-llm, custom_handler: handler.instance}]
responses: {background_mode: {polling_via_cache: all, native_background_mode: [o3], ttl: 60}}
default_team_settings: [{team_id: team-1, success_callback: [langfuse]}]
upperbound_key_generate_params: {max_budget: 100, duration: 30d, rpm_limit: os.environ/RPM}",
);
let priority = settings.priority_reservation_settings.unwrap();
assert_eq!(priority.default_priority, Some(Spelled::Value(0.5)));
assert_eq!(
priority.saturation_check_cache_ttl,
Some(Spelled::Value(30))
);
assert_eq!(
settings
.model_group_settings
.unwrap()
.forward_client_headers_to_llm_api
.as_deref(),
Some(&["gpt".to_owned()][..])
);
let provider = &settings.custom_provider_map.unwrap()[0];
assert_eq!(provider.provider.as_deref(), Some("my-llm"));
assert_eq!(provider.custom_handler.as_deref(), Some("handler.instance"));
let background = settings.responses.unwrap().background_mode.unwrap();
assert_eq!(
background.polling_via_cache.unwrap().get().as_str(),
Some("all")
);
assert_eq!(
background.native_background_mode.as_deref(),
Some(&["o3".to_owned()][..])
);
assert_eq!(background.ttl, Some(Spelled::Value(60)));
let team = &settings.budget.default_team_settings.unwrap()[0];
assert_eq!(team.team_id, "team-1");
assert_eq!(
team.success_callback.as_deref(),
Some(&["langfuse".to_owned()][..])
);
let upperbound = settings.budget.upperbound_key_generate_params.unwrap();
assert_eq!(upperbound.max_budget, Some(Spelled::Value(100.0)));
assert_eq!(upperbound.duration.as_deref(), Some("30d"));
assert_eq!(
upperbound.rpm_limit,
Some(Spelled::Text("os.environ/RPM".into()))
);
}
#[rstest]
#[case::cache_password(
r#"
cache: true
cache_params: {password: SENTINEL}
"#
)]
#[case::cache_endpoint(" cache_params: {s3_endpoint_url: 'https://user:SENTINEL@s3.local'}")]
#[case::cache_raw(" cache_params: {s3_config: {secret: SENTINEL}}")]
#[case::gitlab_object(" global_gitlab_config: {access_token: SENTINEL}")]
#[case::s3_callback_object(" s3_callback_params: {s3_aws_secret_access_key: SENTINEL}")]
#[case::team_inline_var(" default_team_settings: [{team_id: t, langfuse_secret: SENTINEL}]")]
#[case::team_callback_vars(
" default_team_settings: [{team_id: t, callback_vars: {langfuse_secret: SENTINEL}}]"
)]
#[case::fallbacks_raw(" default_fallbacks: [SENTINEL]")]
#[case::user_params_raw(" default_internal_user_params: {models: [SENTINEL]}")]
#[case::callback_entry(" callbacks: [SENTINEL]")]
#[case::success_callback_map(" success_callback: {name: SENTINEL}")]
#[case::failure_callback_one(" failure_callback: SENTINEL")]
#[case::audit_object(" s3_audit_callback_params: {secret: SENTINEL}")]
#[case::context_window_raw(" context_window_fallbacks: [{gpt: [SENTINEL]}]")]
#[case::mcp_raw(" mcp_aliases: {docs: SENTINEL}")]
#[case::injection_object(" prompt_injection_params: {key: SENTINEL}")]
#[case::unknown_setting(" future_setting: SENTINEL")]
fn debug_output_does_not_expose_settings_values(#[case] yaml: &str) {
let config = Config::from_yaml(&format!("litellm_settings:\n{yaml}")).unwrap();
assert!(!format!("{config:?}").contains("SENTINEL"));
}
#[rstest]
fn unknown_setting_stays_in_additional_fields() {
assert!(
settings(" future_setting: 1")
.additional_fields
.contains_key("future_setting")
);
}
#[rstest]
fn transport_keys_land_in_their_reader_groups() {
let settings = settings(
r#" ssl_verify: false
ssl_security_level: '1'
ssl_certificate: /etc/client.pem
force_ipv4: true
http2: true
aiohttp_trust_env: true"#,
);
assert_eq!(settings.tls.ssl_verify, Some(Spelled::Value(false)));
assert_eq!(settings.tls.ssl_security_level.as_deref(), Some("1"));
assert_eq!(
settings.client_certificate.ssl_certificate.as_deref(),
Some("/etc/client.pem")
);
assert_eq!(settings.socket.force_ipv4, Some(true));
assert_eq!(settings.http2.http2, Some(true));
assert_eq!(settings.aiohttp.aiohttp_trust_env, Some(true));
assert!(settings.additional_fields.is_empty());
}
#[rstest]
#[case::tls(" ssl_verify: false", "ssl_verify")]
#[case::client_certificate(" ssl_certificate: /etc/client.pem", "ssl_certificate")]
#[case::socket(" force_ipv4: true", "force_ipv4")]
#[case::http2(" http2: true", "http2")]
#[case::aiohttp(" disable_aiohttp_transport: true", "disable_aiohttp_transport")]
#[case::caching(" cache: true", "cache")]
#[case::budget(" max_budget: 10", "max_budget")]
#[case::logging(" json_logs: true", "json_logs")]
#[case::fallbacks(" default_fallbacks: [gpt]", "default_fallbacks")]
#[case::mcp(" mcp_aliases: {docs: docs_server}", "mcp_aliases")]
fn grouped_settings_stay_out_of_additional_fields(#[case] yaml: &str, #[case] key: &str) {
assert!(!settings(yaml).additional_fields.contains_key(key));
}
#[rstest]
#[case::one("callbacks: custom_callbacks.logger", 1)]
#[case::many("callbacks: [prometheus, otel]", 2)]
fn accepts_python_callback_shorthand(#[case] setting: &str, #[case] expected_len: usize) {
let config = Config::from_yaml(&format!("litellm_settings:\n {setting}")).unwrap();
assert_eq!(
config
.litellm_settings
.logging
.callbacks
.as_ref()
.unwrap()
.len(),
expected_len
);
}
}

View file

@ -0,0 +1,34 @@
use crate::Spelled;
/// Read by the SSL context builder, `get_ssl_configuration`.
#[macro_rules_attribute::apply(crate::config_section)]
pub struct TlsSettings {
pub ssl_verify: Option<Spelled<bool>>,
pub ssl_security_level: Option<String>,
pub ssl_ecdh_curve: Option<String>,
}
/// Read by the sync and async HTTP clients, which send it as the client certificate.
#[macro_rules_attribute::apply(crate::config_section)]
pub struct ClientCertificateSettings {
pub ssl_certificate: Option<String>,
}
/// Read by the httpx and aiohttp transports when they bind the local socket.
#[macro_rules_attribute::apply(crate::config_section)]
pub struct SocketSettings {
pub force_ipv4: Option<bool>,
}
/// Read by the httpx client.
#[macro_rules_attribute::apply(crate::config_section)]
pub struct Http2Settings {
pub http2: Option<bool>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct AiohttpSettings {
pub aiohttp_trust_env: Option<bool>,
pub disable_aiohttp_trust_env: Option<bool>,
pub disable_aiohttp_transport: Option<bool>,
}

View file

@ -1,11 +1,11 @@
use std::{collections::BTreeMap, fmt};
use std::collections::BTreeMap;
use litellm_auth_types::SecretValue;
use serde::Deserialize;
use crate::Object;
#[derive(Clone, Default, Deserialize)]
#[derive(Clone, Debug, Default, Deserialize)]
#[serde(default)]
pub struct McpServer {
pub server_id: Option<String>,
@ -14,8 +14,8 @@ pub struct McpServer {
pub mcp_info: Object,
pub transport: McpTransport,
pub url: Option<SecretValue>,
pub command: Option<String>,
pub args: Box<[String]>,
pub command: Option<SecretValue>,
pub args: Box<[SecretValue]>,
pub env: BTreeMap<String, SecretValue>,
pub auth_type: Option<McpAuth>,
#[serde(alias = "auth_value")]
@ -29,18 +29,6 @@ pub struct McpServer {
pub unsupported: Object,
}
impl fmt::Debug for McpServer {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("McpServer")
.field("transport", &self.transport)
.field("auth_type", &self.auth_type)
.field("timeout", &self.timeout)
.field("max_concurrent_requests", &self.max_concurrent_requests)
.finish_non_exhaustive()
}
}
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, strum::IntoStaticStr)]
#[serde(rename_all = "snake_case")]
pub enum McpTransport {
@ -81,3 +69,75 @@ pub enum McpAuth {
#[strum(serialize = "oauth_delegate")]
OauthDelegate,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use crate::{Config, McpAuth, McpTransport};
#[rstest]
#[case::http(McpTransport::Http, "http")]
#[case::sse(McpTransport::Sse, "sse")]
#[case::stdio(McpTransport::Stdio, "stdio")]
fn mcp_transport_as_str_matches_the_serde_spelling(
#[case] transport: McpTransport,
#[case] name: &str,
) {
assert_eq!(<&'static str>::from(transport), name);
assert_eq!(
serde_json::from_value::<McpTransport>(json!(name)).unwrap(),
transport
);
}
#[rstest]
#[case::none(McpAuth::None, "none")]
#[case::api_key(McpAuth::ApiKey, "api_key")]
#[case::bearer_token(McpAuth::BearerToken, "bearer_token")]
#[case::basic(McpAuth::Basic, "basic")]
#[case::authorization(McpAuth::Authorization, "authorization")]
#[case::token(McpAuth::Token, "token")]
#[case::oauth2(McpAuth::Oauth2, "oauth2")]
#[case::aws_sigv4(McpAuth::AwsSigv4, "aws_sigv4")]
#[case::oauth2_token_exchange(McpAuth::Oauth2TokenExchange, "oauth2_token_exchange")]
#[case::oauth2_id_jag(McpAuth::Oauth2IdJag, "oauth2_id_jag")]
#[case::true_passthrough(McpAuth::TruePassthrough, "true_passthrough")]
#[case::oauth_delegate(McpAuth::OauthDelegate, "oauth_delegate")]
fn mcp_auth_as_str_matches_the_serde_spelling(#[case] auth: McpAuth, #[case] name: &str) {
assert_eq!(<&'static str>::from(auth), name);
assert_eq!(
serde_json::from_value::<McpAuth>(json!(name)).unwrap(),
auth
);
}
#[rstest]
fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() {
let config = Config::from_yaml("mcp_servers:\n docs:\n url: https://example.test/private-secret/mcp\n authentication_token: upstream-secret\n static_headers: {x-token: header-secret}\n env: {TOKEN: env-secret}\n args: [argument-secret]\n client_secret: oauth-secret\n allowed_tools: [search]\n").unwrap();
let server = &config.mcp_servers["docs"];
assert_eq!(server.allowed_tools.as_deref().unwrap(), ["search"]);
assert!(server.unsupported.contains_key("client_secret"));
let debug = format!("{config:?}");
for secret in [
"private-secret",
"upstream-secret",
"header-secret",
"env-secret",
"oauth-secret",
"argument-secret",
] {
assert!(!debug.contains(secret));
}
}
#[rstest]
#[case::transport("transport: invalid")]
#[case::auth("auth_type: invalid")]
#[case::concurrency("max_concurrent_requests: -1")]
#[case::headers("static_headers: {x-token: [not, a, string]}")]
fn rejects_invalid_typed_mcp_settings(#[case] setting: &str) {
assert!(Config::from_yaml(&format!("mcp_servers:\n docs:\n {setting}\n")).is_err());
}
}

View file

@ -1,30 +1,187 @@
use std::fmt;
use litellm_router_types::LitellmParams;
use serde::Deserialize;
use crate::{AdditionalFields, Object};
use crate::{Object, Raw, Spelled};
#[derive(Clone, Deserialize)]
#[derive(Clone, Debug, Deserialize)]
pub struct Model {
pub model_name: String,
pub litellm_params: LitellmParams,
#[serde(default)]
pub model_info: Object,
pub model_info: ModelInfo,
pub blocked: Option<bool>,
pub rpm: Option<Spelled<u64>>,
pub tpm: Option<Spelled<u64>>,
#[serde(flatten)]
pub additional_fields: AdditionalFields,
pub additional_fields: Object,
}
impl fmt::Debug for Model {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Model")
.field("model_name", &self.model_name)
.field("litellm_params", &self.litellm_params)
.field("model_info", &self.model_info)
.field("blocked", &self.blocked)
.field("additional_fields", &self.additional_fields.keys())
.finish()
#[macro_rules_attribute::apply(crate::config_section)]
pub struct ModelInfo {
pub id: Option<ModelId>,
pub db_model: Option<Spelled<bool>>,
pub updated_at: Option<Raw>,
pub created_at: Option<Raw>,
pub updated_by: Option<String>,
pub created_by: Option<String>,
pub base_model: Option<String>,
pub pricing_provider: Option<String>,
pub tier: Option<String>,
pub team_id: Option<String>,
pub team_public_model_name: Option<String>,
pub member_auto_router: Option<Spelled<bool>>,
pub blocked: Option<Spelled<bool>>,
pub discoverable: Option<Spelled<bool>>,
pub access_windows: Option<Box<[AccessWindow]>>,
pub ptu_count: Option<Spelled<i64>>,
pub cost_per_ptu_per_hour: Option<Spelled<f64>>,
pub ptu_effective_from: Option<Raw>,
pub ptu_effective_to: Option<Raw>,
pub ptu_shares: Option<Object>,
pub allow_fail_open: Option<Spelled<bool>>,
pub enable_tag_filtering: Option<Spelled<bool>>,
pub internal_router_model: Option<Spelled<bool>>,
pub max_embedding_batch_size: Option<Spelled<i64>>,
pub input_cost_per_token: Option<Spelled<f64>>,
pub output_cost_per_token: Option<Spelled<f64>>,
pub input_cost_per_character: Option<Spelled<f64>>,
pub output_cost_per_character: Option<Spelled<f64>>,
pub cache_read_input_token_cost: Option<Spelled<f64>>,
pub explicit_cache_read_input_token_cost: Option<Spelled<f64>>,
pub cache_creation_input_token_cost: Option<Spelled<f64>>,
pub cache_creation_input_token_cost_above_1hr: Option<Spelled<f64>>,
pub tiered_pricing: Option<Box<[Object]>>,
pub mode: Option<String>,
pub access_groups: Option<Box<[String]>>,
pub supported_environments: Option<Box<[String]>>,
#[serde(flatten)]
pub additional_fields: Object,
}
#[derive(Clone, Debug, PartialEq, Deserialize)]
#[serde(untagged)]
pub enum ModelId {
Text(String),
Number(i64),
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct AccessWindow {
pub start: Option<Raw>,
pub end: Option<Raw>,
pub timezone: Option<String>,
pub team_ids: Option<Box<[String]>>,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::{Config, Model, ModelId, Spelled};
fn model(entry: &str) -> Model {
let config = Config::from_yaml(&format!(
"model_list:\n - model_name: assistant\n litellm_params: {{model: openai/test}}\n{entry}"
))
.unwrap();
config.model_list[0].clone()
}
#[rstest]
#[case::text(" model_info: {id: primary}", Some(ModelId::Text("primary".to_string())))]
#[case::number(" model_info: {id: 7}", Some(ModelId::Number(7)))]
#[case::null(" model_info: {id: null}", None)]
#[case::absent(" model_info: {mode: chat}", None)]
fn model_info_id_accepts_what_python_accepts(#[case] entry: &str, #[case] id: Option<ModelId>) {
assert_eq!(model(entry).model_info.id, id);
}
#[rstest]
#[case::missing("")]
#[case::null(" model_info:")]
fn model_info_may_be_missing_or_null(#[case] entry: &str) {
let model = model(entry);
assert!(model.model_info.id.is_none());
assert!(model.model_info.additional_fields.is_empty());
}
#[rstest]
fn model_info_fields_land_typed() {
let model = model(
r#" model_info:
mode: embedding
base_model: azure/test-base
team_id: team-1
access_groups: [internal, beta]
db_model: "1"
blocked: true
ptu_count: 4
cost_per_ptu_per_hour: 2
input_cost_per_token: os.environ/INPUT_COST
tiered_pricing: [{range: [0, 1000]}]
access_windows:
- {start: "09:00", end: "17:00", timezone: UTC, team_ids: [team-1]}
supports_vision: true
"#,
);
let info = &model.model_info;
assert_eq!(info.mode.as_deref(), Some("embedding"));
assert_eq!(info.base_model.as_deref(), Some("azure/test-base"));
assert_eq!(info.team_id.as_deref(), Some("team-1"));
assert_eq!(
info.access_groups.as_deref(),
Some(&["internal".to_string(), "beta".to_string()][..])
);
assert_eq!(info.db_model, Some(Spelled::Text("1".to_string())));
assert_eq!(info.blocked, Some(Spelled::Value(true)));
assert_eq!(info.ptu_count, Some(Spelled::Value(4)));
assert_eq!(info.cost_per_ptu_per_hour, Some(Spelled::Value(2.0)));
assert_eq!(
info.input_cost_per_token,
Some(Spelled::Text("os.environ/INPUT_COST".to_string()))
);
assert_eq!(
info.tiered_pricing.as_ref().map(|tiers| tiers.len()),
Some(1)
);
let window = &info.access_windows.as_ref().unwrap()[0];
assert_eq!(window.timezone.as_deref(), Some("UTC"));
assert_eq!(window.start.as_ref().unwrap().get().as_str(), Some("09:00"));
assert_eq!(
window.team_ids.as_deref(),
Some(&["team-1".to_string()][..])
);
assert!(info.additional_fields.contains_key("supports_vision"));
}
#[rstest]
fn top_level_limits_and_unknown_keys_land_on_the_model() {
let model = model(" rpm: 100\n tpm: os.environ/TPM\n team: research\n");
assert_eq!(model.rpm, Some(Spelled::Value(100)));
assert_eq!(model.tpm, Some(Spelled::Text("os.environ/TPM".to_string())));
assert!(model.additional_fields.contains_key("team"));
assert!(!model.additional_fields.contains_key("rpm"));
}
#[rstest]
#[case::ptu_shares(
" model_info: {ptu_shares: {team-1: sentinel-share-value}}",
"sentinel-share-value"
)]
#[case::created_at(
" model_info: {created_at: sentinel-created-at}",
"sentinel-created-at"
)]
#[case::tiered_pricing(
" model_info: {tiered_pricing: [{label: sentinel-tier-value}]}",
"sentinel-tier-value"
)]
#[case::window_start(
" model_info: {access_windows: [{start: sentinel-window-start, end: '17:00', timezone: UTC, team_ids: [t]}]}",
"sentinel-window-start"
)]
#[case::model_extension(" team_token: sentinel-model-extension", "sentinel-model-extension")]
fn opaque_model_info_values_stay_out_of_debug(#[case] entry: &str, #[case] sentinel: &str) {
assert!(!format!("{:?}", model(entry)).contains(sentinel));
}
}

View file

@ -0,0 +1,144 @@
use serde::Deserialize;
use crate::{Raw, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct Policy {
pub guardrails: Option<PolicyGuardrails>,
pub condition: Option<PolicyCondition>,
pub pipeline: Option<PolicyPipeline>,
pub inherit: Option<String>,
pub description: Option<String>,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(untagged)]
pub enum PolicyGuardrails {
Added(Box<[String]>),
Changes(PolicyGuardrailChanges),
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct PolicyGuardrailChanges {
pub add: Option<Box<[String]>>,
pub remove: Option<Box<[String]>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct PolicyCondition {
pub model: Option<Raw>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct PolicyPipeline {
pub mode: Option<String>,
pub steps: Option<Box<[PolicyPipelineStep]>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct PolicyPipelineStep {
pub guardrail: Option<String>,
pub on_fail: Option<String>,
pub on_pass: Option<String>,
pub on_error: Option<String>,
pub pass_data: Option<Spelled<bool>>,
pub modify_response_message: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct PolicyAttachment {
pub policy: Option<String>,
pub scope: Option<String>,
pub teams: Option<Box<[String]>>,
pub keys: Option<Box<[String]>>,
pub models: Option<Box<[String]>>,
pub tags: Option<Box<[String]>>,
pub priority: Option<Spelled<i64>>,
pub default: Option<Spelled<bool>>,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::{Config, PolicyGuardrails, Spelled, Value};
fn parse(yaml: &str) -> Config {
Config::from_yaml(yaml).unwrap()
}
#[rstest]
#[case::legacy_list("guardrails: [pii, toxicity]", &["pii", "toxicity"], &[])]
#[case::add_remove("guardrails: {add: [pii], remove: [toxicity]}", &["pii"], &["toxicity"])]
fn policy_guardrails_accept_both_python_spellings(
#[case] guardrails: &str,
#[case] added: &[&str],
#[case] removed: &[&str],
) {
let config = parse(&format!("policies:\n safe:\n {guardrails}\n"));
let (add, remove) = match config.policies["safe"].guardrails.as_ref().unwrap() {
PolicyGuardrails::Added(add) => (add.to_vec(), Vec::new()),
PolicyGuardrails::Changes(changes) => (
changes.add.as_deref().unwrap_or_default().to_vec(),
changes.remove.as_deref().unwrap_or_default().to_vec(),
),
};
assert_eq!(add, added);
assert_eq!(remove, removed);
}
#[rstest]
fn policies_and_attachments_parse_pipeline_condition_and_scope() {
let config = parse(
r#"
policies:
safe:
inherit: base
condition:
model: [gpt-a, gpt-b]
pipeline:
mode: pre_call
steps:
- guardrail: pii
on_fail: next
pass_data: true
policy_attachments:
- policy: safe
teams: [research]
priority: 10
default: true
"#,
);
let policy = &config.policies["safe"];
let step = &policy.pipeline.as_ref().unwrap().steps.as_ref().unwrap()[0];
let attachment = &config.policy_attachments[0];
assert_eq!(policy.inherit.as_deref(), Some("base"));
assert!(matches!(
policy.condition.as_ref().unwrap().model.as_ref().unwrap().get(),
Value::Sequence(models) if models.len() == 2
));
assert_eq!(step.guardrail.as_deref(), Some("pii"));
assert_eq!(step.on_fail.as_deref(), Some("next"));
assert_eq!(step.pass_data, Some(Spelled::Value(true)));
assert_eq!(attachment.policy.as_deref(), Some("safe"));
assert_eq!(
attachment.teams.as_deref(),
Some(&["research".to_owned()][..])
);
assert_eq!(attachment.priority, Some(Spelled::Value(10)));
assert_eq!(attachment.default, Some(Spelled::Value(true)));
}
#[rstest]
#[case::policy_model(
r#"
policies:
safe: {condition: {model: SENTINEL}}
"#
)]
fn debug_never_prints_policy_values(#[case] yaml: &str) {
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
}
}

View file

@ -0,0 +1,99 @@
use litellm_auth_types::SecretValue;
use crate::{Object, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct Prompt {
pub prompt_id: Option<String>,
pub litellm_params: Option<PromptParams>,
pub prompt_info: Option<PromptInfo>,
pub version: Option<Spelled<i64>>,
pub environment: Option<String>,
pub created_by: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct PromptParams {
pub prompt_id: Option<String>,
pub prompt_integration: Option<String>,
pub api_base: Option<String>,
pub api_key: Option<SecretValue>,
pub provider_specific_query_params: Option<Object>,
pub ignore_prompt_manager_model: Option<Spelled<bool>>,
pub ignore_prompt_manager_optional_params: Option<Spelled<bool>>,
pub dotprompt_content: Option<String>,
#[serde(flatten)]
pub additional_fields: Object,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct PromptInfo {
pub prompt_type: Option<String>,
pub environment: Option<String>,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::{Config, Spelled};
fn parse(yaml: &str) -> Config {
Config::from_yaml(yaml).unwrap()
}
#[rstest]
fn prompt_entries_keep_typed_params_and_info() {
let config = parse(
r#"
prompts:
- prompt_id: greeting
version: 2
litellm_params:
prompt_integration: dotprompt
ignore_prompt_manager_model: true
prompt_extension: x
prompt_info:
prompt_type: config
"#,
);
let prompt = &config.prompts[0];
let params = prompt.litellm_params.as_ref().unwrap();
assert_eq!(prompt.prompt_id.as_deref(), Some("greeting"));
assert_eq!(prompt.version, Some(Spelled::Value(2)));
assert_eq!(params.prompt_integration.as_deref(), Some("dotprompt"));
assert_eq!(
params.ignore_prompt_manager_model,
Some(Spelled::Value(true))
);
assert!(params.additional_fields.contains_key("prompt_extension"));
assert_eq!(
prompt.prompt_info.as_ref().unwrap().prompt_type.as_deref(),
Some("config")
);
}
#[rstest]
#[case::prompt_key(
r#"
prompts:
- litellm_params: {api_key: SENTINEL}
"#
)]
#[case::prompt_query_params(
r#"
prompts:
- litellm_params: {provider_specific_query_params: {token: SENTINEL}}
"#
)]
#[case::prompt_extra(
r#"
prompts:
- litellm_params: {vendor_secret: SENTINEL}
"#
)]
fn debug_never_prints_prompt_values(#[case] yaml: &str) {
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
}
}

View file

@ -0,0 +1,118 @@
use litellm_auth_types::SecretValue;
use crate::{Object, Raw};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct AssistantSettings {
pub custom_llm_provider: Option<String>,
pub litellm_params: Option<Object>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct ProviderSettings {
pub custom_llm_provider: Option<String>,
pub api_key: Option<SecretValue>,
pub api_base: Option<String>,
#[serde(flatten)]
pub additional_fields: Object,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct VertexConfig {
pub vertex_project: Option<String>,
pub vertex_location: Option<String>,
pub vertex_credentials: Option<Raw>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct Worker {
pub worker_id: Option<String>,
pub name: Option<String>,
pub url: Option<String>,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::Config;
fn parse(yaml: &str) -> Config {
Config::from_yaml(yaml).unwrap()
}
#[rstest]
#[case::files("files_settings")]
#[case::finetune("finetune_settings")]
fn provider_settings_keep_the_provider_and_unknown_keys(#[case] section: &str) {
let config = parse(&format!(
"{section}:\n - custom_llm_provider: azure\n api_version: '2024-01-01'\n"
));
let settings = config
.files_settings
.first()
.or(config.finetune_settings.first())
.unwrap();
assert_eq!(settings.custom_llm_provider.as_deref(), Some("azure"));
assert!(settings.additional_fields.contains_key("api_version"));
}
#[rstest]
fn single_provider_sections_land_in_typed_fields() {
let config = parse(
r#"
assistant_settings:
custom_llm_provider: openai
litellm_params:
api_key: os.environ/OPENAI_API_KEY
default_vertex_config:
vertex_project: proj
vertex_location: us-central1
worker_registry:
- worker_id: eu
name: Europe
url: https://eu.example.test
"#,
);
let assistant = config.assistant_settings.as_ref().unwrap();
let vertex = config.default_vertex_config.as_ref().unwrap();
let worker = &config.worker_registry[0];
assert_eq!(assistant.custom_llm_provider.as_deref(), Some("openai"));
assert!(
assistant
.litellm_params
.as_ref()
.unwrap()
.contains_key("api_key")
);
assert_eq!(vertex.vertex_project.as_deref(), Some("proj"));
assert_eq!(vertex.vertex_location.as_deref(), Some("us-central1"));
assert_eq!(worker.worker_id.as_deref(), Some("eu"));
assert_eq!(worker.name.as_deref(), Some("Europe"));
assert_eq!(worker.url.as_deref(), Some("https://eu.example.test"));
}
#[rstest]
#[case::vertex_credentials("default_vertex_config: {vertex_credentials: SENTINEL}")]
#[case::vertex_credentials_map(
"default_vertex_config: {vertex_credentials: {private_key: SENTINEL}}"
)]
#[case::assistant_params("assistant_settings: {litellm_params: {api_key: SENTINEL}}")]
#[case::files_key(
r#"
files_settings:
- api_key: SENTINEL
"#
)]
#[case::finetune_extra(
r#"
finetune_settings:
- vendor_secret: SENTINEL
"#
)]
fn debug_never_prints_provider_values(#[case] yaml: &str) {
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
}
}

View file

@ -0,0 +1,191 @@
use std::collections::BTreeMap;
use litellm_auth_types::SecretValue;
use crate::{Object, Raw, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct RouterSettings {
pub redis_url: Option<SecretValue>,
pub redis_host: Option<String>,
pub redis_port: Option<Spelled<i64>>,
pub redis_password: Option<SecretValue>,
pub redis_db: Option<Spelled<i64>>,
pub cache_responses: Option<Spelled<bool>>,
pub cache_kwargs: Option<Object>,
pub caching_groups: Option<Raw>,
pub client_ttl: Option<Spelled<i64>>,
pub polling_interval: Option<Spelled<f64>>,
pub default_priority: Option<Raw>,
pub num_retries: Option<u64>,
pub max_fallbacks: Option<Spelled<i64>>,
pub timeout: Option<f64>,
pub stream_timeout: Option<Spelled<f64>>,
pub default_litellm_params: Option<Object>,
pub default_max_parallel_requests: Option<Spelled<i64>>,
pub set_verbose: Option<Spelled<bool>>,
pub debug_level: Option<String>,
pub default_fallbacks: Option<Box<[String]>>,
pub fallbacks: Option<Box<[Object]>>,
pub context_window_fallbacks: Option<Box<[Object]>>,
pub content_policy_fallbacks: Option<Box<[Object]>>,
pub model_group_alias: Option<Object>,
pub enable_pre_call_checks: Option<Spelled<bool>>,
pub enable_tag_filtering: Option<bool>,
pub tag_filtering_match_any: Option<Spelled<bool>>,
pub tag_routing_prefix: Option<String>,
pub plugins: Option<Box<[String]>>,
pub retry_after: Option<f64>,
pub retry_policy: Option<Object>,
pub model_group_retry_policy: Option<Object>,
pub allowed_fails: Option<u64>,
pub allowed_fails_policy: Option<AllowedFailsPolicy>,
pub cooldown_time: Option<f64>,
pub disable_cooldowns: Option<Spelled<bool>>,
pub fallback_on_unsupported_params: Option<Spelled<bool>>,
pub routing_strategy: Option<String>,
pub optional_pre_call_checks: Option<Box<[String]>>,
pub routing_strategy_args: Option<Object>,
pub routing_groups: Option<Box<[Object]>>,
pub provider_budget_config: Option<BTreeMap<String, ProviderBudget>>,
pub alerting_config: Option<AlertingConfig>,
pub deployment_affinity_ttl_seconds: Option<Spelled<i64>>,
pub model_group_affinity_config: Option<Object>,
pub enable_health_check_routing: Option<Spelled<bool>>,
pub health_check_staleness_threshold: Option<Spelled<f64>>,
pub health_check_ignore_transient_errors: Option<Spelled<bool>>,
pub background_health_check_model_groups: Option<Box<[String]>>,
pub enable_weighted_failover: Option<Spelled<bool>>,
pub enable_mid_stream_fallback_continuation: Option<Spelled<bool>>,
pub guardrail_list: Option<Box<[Object]>>,
pub max_retries: Option<u64>,
pub weights: Option<Object>,
#[serde(flatten)]
pub additional_fields: Object,
}
#[macro_rules_attribute::apply(crate::config_section)]
#[serde(rename_all = "PascalCase")]
pub struct AllowedFailsPolicy {
pub bad_request_error_allowed_fails: Option<Spelled<i64>>,
pub authentication_error_allowed_fails: Option<Spelled<i64>>,
pub timeout_error_allowed_fails: Option<Spelled<i64>>,
pub rate_limit_error_allowed_fails: Option<Spelled<i64>>,
pub content_policy_violation_error_allowed_fails: Option<Spelled<i64>>,
pub internal_server_error_allowed_fails: Option<Spelled<i64>>,
pub service_unavailable_error_allowed_fails: Option<Spelled<i64>>,
pub bad_gateway_error_allowed_fails: Option<Spelled<i64>>,
pub not_found_error_allowed_fails: Option<Spelled<i64>>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct ProviderBudget {
pub budget_limit: Option<Spelled<f64>>,
pub time_period: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct AlertingConfig {
pub webhook_url: Option<SecretValue>,
pub alerting_threshold: Option<Spelled<f64>>,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::{Config, RouterSettings, Spelled};
fn router(yaml: &str) -> RouterSettings {
Config::from_yaml(&format!("router_settings:\n{yaml}"))
.unwrap()
.router_settings
}
#[rstest]
#[case::int(" redis_port: 6379", |s: &RouterSettings| s.redis_port == Some(Spelled::Value(6379)))]
#[case::int_env(
" client_ttl: os.environ/CLIENT_TTL",
|s: &RouterSettings| s.client_ttl == Some(Spelled::Text("os.environ/CLIENT_TTL".to_string()))
)]
#[case::float_from_int(" stream_timeout: 30", |s: &RouterSettings| s.stream_timeout == Some(Spelled::Value(30.0)))]
#[case::float(
" health_check_staleness_threshold: 2.5",
|s: &RouterSettings| s.health_check_staleness_threshold == Some(Spelled::Value(2.5))
)]
#[case::bool(" enable_weighted_failover: true", |s: &RouterSettings| s.enable_weighted_failover == Some(Spelled::Value(true)))]
#[case::bool_word(
" enable_mid_stream_fallback_continuation: \"true\"",
|s: &RouterSettings| s.enable_mid_stream_fallback_continuation == Some(Spelled::Text("true".to_string()))
)]
#[case::string(" redis_host: cache.internal", |s: &RouterSettings| s.redis_host.as_deref() == Some("cache.internal"))]
#[case::string_list(
" default_fallbacks: [gpt-small, gpt-large]",
|s: &RouterSettings| s.default_fallbacks.as_deref() == Some(&["gpt-small".to_string(), "gpt-large".to_string()][..])
)]
#[case::mapping_list(
" content_policy_fallbacks: [{gpt: [claude]}]",
|s: &RouterSettings| s.content_policy_fallbacks.as_ref().is_some_and(|items| items[0].contains_key("gpt"))
)]
#[case::mapping(
" default_litellm_params: {temperature: 0}",
|s: &RouterSettings| s.default_litellm_params.as_ref().is_some_and(|params| params.contains_key("temperature"))
)]
#[case::non_integer_priority(" default_priority: 1.5", |s: &RouterSettings| s.default_priority.is_some())]
#[case::bool_priority(" default_priority: true", |s: &RouterSettings| s.default_priority.is_some())]
#[case::caching_groups(
" caching_groups: [[gpt, [gpt-a, gpt-b]]]",
|s: &RouterSettings| s.caching_groups.as_ref().is_some_and(|groups| groups.get().is_sequence())
)]
#[case::allowed_fails_policy(
" allowed_fails_policy: {RateLimitErrorAllowedFails: \"3\", BadGatewayErrorAllowedFails: 2, FutureAllowedFails: 1}",
|s: &RouterSettings| s.allowed_fails_policy.as_ref().is_some_and(|policy| {
policy.rate_limit_error_allowed_fails == Some(Spelled::Text("3".to_string()))
&& policy.bad_gateway_error_allowed_fails == Some(Spelled::Value(2))
})
)]
#[case::provider_budget_config(
" provider_budget_config: {openai: {budget_limit: 100, time_period: 1d, rpm_limit: 5}}",
|s: &RouterSettings| s.provider_budget_config.as_ref().is_some_and(|budgets| {
let openai = &budgets["openai"];
openai.budget_limit == Some(Spelled::Value(100.0))
&& openai.time_period.as_deref() == Some("1d")
})
)]
#[case::alerting_config(
" alerting_config: {webhook_url: https://hooks.example/a, alerting_threshold: 10, channel: ops}",
|s: &RouterSettings| s.alerting_config.as_ref().is_some_and(|alerting| {
alerting.webhook_url.as_ref().map(|url| url.expose()) == Some("https://hooks.example/a")
&& alerting.alerting_threshold == Some(Spelled::Value(10.0))
})
)]
fn parses_python_router_spellings(
#[case] yaml: &str,
#[case] check: fn(&RouterSettings) -> bool,
) {
assert!(check(&router(yaml)));
}
#[rstest]
#[case::unknown("future_router_setting")]
#[case::moved_to_general_settings("health_check_interval")]
#[case::managed_outside_config("model_list")]
fn keeps_keys_outside_the_router_signature(#[case] key: &str) {
let settings = router(&format!(" {key}: 60"));
assert!(settings.additional_fields.contains_key(key));
}
#[rstest]
#[case::redis_password(" redis_password: router-sentinel")]
#[case::redis_url(" redis_url: redis://user:router-sentinel@cache:6379")]
#[case::webhook_url(" alerting_config: {webhook_url: https://hooks.example/router-sentinel}")]
#[case::cache_kwargs(" cache_kwargs: {password: router-sentinel}")]
#[case::default_priority(" default_priority: router-sentinel")]
#[case::caching_groups(" caching_groups: [[router-sentinel, [a]]]")]
#[case::guardrail_list(" guardrail_list: [{api_key: router-sentinel}]")]
#[case::nested_extension(" allowed_fails_policy: {FutureAllowedFails: router-sentinel}")]
fn debug_output_does_not_expose_router_values(#[case] yaml: &str) {
let config = Config::from_yaml(&format!("router_settings:\n{yaml}")).unwrap();
assert!(!format!("{config:?}").contains("router-sentinel"));
}
}

View file

@ -1,268 +0,0 @@
use std::fmt;
use litellm_auth_types::SecretValue;
use serde::Deserialize;
use crate::{AdditionalFields, Object, Spelled, Value, value::one_or_many};
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum TracingStoreKind {
Clickhouse,
}
#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ClickHouseStoreSettings {
#[serde(rename = "type")]
pub kind: TracingStoreKind,
pub url: Option<SecretValue>,
pub database: Option<String>,
pub retention_days: Option<Spelled<f64>>,
}
impl fmt::Debug for ClickHouseStoreSettings {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ClickHouseStoreSettings")
.field("kind", &self.kind)
.field("database", &self.database)
.field("retention_days", &self.retention_days)
.finish()
}
}
#[derive(Clone, Debug, Deserialize)]
#[serde(untagged)]
pub enum TracingStoreSettings {
ClickHouse(ClickHouseStoreSettings),
}
#[derive(Clone, Default, Debug, Deserialize)]
#[serde(default)]
pub struct TracingSettings {
pub store: Option<TracingStoreSettings>,
#[serde(flatten)]
pub additional_fields: AdditionalFields,
}
#[derive(Clone, Deserialize)]
#[serde(default)]
pub struct GeneralSettings {
pub completion_model: Option<String>,
pub max_in_flight_requests_per_worker: Option<u64>,
pub max_queued_requests_per_worker: Option<u64>,
pub admission_queue_timeout_seconds: f64,
pub master_key: Option<SecretValue>,
pub database_url: Option<SecretValue>,
pub tracing: Option<TracingSettings>,
pub database_connection_pool_limit: Option<u64>,
pub database_connection_timeout: Option<f64>,
pub database_connect_timeout: Option<f64>,
pub database_socket_timeout: Option<f64>,
pub database_max_idle_connection_lifetime: Option<f64>,
pub max_parallel_requests: Option<u64>,
pub global_max_parallel_requests: Option<u64>,
pub max_request_size_mb: Option<u64>,
pub max_response_size_mb: Option<u64>,
pub proxy_config_reload_interval_seconds: u64,
pub background_health_checks: Option<bool>,
pub health_check_interval: u64,
pub health_check_concurrency: Option<u64>,
pub store_model_in_db: Option<bool>,
pub forward_client_headers_to_llm_api: Option<bool>,
pub cancel_on_disconnect: Option<bool>,
pub infer_model_from_keys: Option<bool>,
pub enable_public_model_hub: bool,
pub dangerously_permit_weak_or_unset_master_key: Option<bool>,
pub plugins: Option<Box<[Object]>>,
pub coordination_redis: Option<Object>,
pub mcp_allowed_hosts: Option<Box<[String]>>,
pub mcp_allowed_origins: Box<[String]>,
#[serde(flatten)]
pub additional_fields: AdditionalFields,
}
impl Default for GeneralSettings {
fn default() -> Self {
Self {
completion_model: None,
max_in_flight_requests_per_worker: None,
max_queued_requests_per_worker: None,
admission_queue_timeout_seconds: 1.0,
master_key: None,
database_url: None,
tracing: None,
database_connection_pool_limit: Some(10),
database_connection_timeout: Some(60.0),
database_connect_timeout: None,
database_socket_timeout: None,
database_max_idle_connection_lifetime: Some(60.0),
max_parallel_requests: None,
global_max_parallel_requests: None,
max_request_size_mb: None,
max_response_size_mb: None,
proxy_config_reload_interval_seconds: 30,
background_health_checks: None,
health_check_interval: 300,
health_check_concurrency: None,
store_model_in_db: None,
forward_client_headers_to_llm_api: None,
cancel_on_disconnect: None,
infer_model_from_keys: None,
enable_public_model_hub: false,
dangerously_permit_weak_or_unset_master_key: None,
plugins: None,
coordination_redis: None,
mcp_allowed_hosts: None,
mcp_allowed_origins: Box::default(),
additional_fields: AdditionalFields::new(),
}
}
}
impl fmt::Debug for GeneralSettings {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("GeneralSettings")
.field("completion_model", &self.completion_model)
.field(
"max_in_flight_requests_per_worker",
&self.max_in_flight_requests_per_worker,
)
.field(
"max_queued_requests_per_worker",
&self.max_queued_requests_per_worker,
)
.field(
"admission_queue_timeout_seconds",
&self.admission_queue_timeout_seconds,
)
.field("master_key", &self.master_key)
.field("database_url", &self.database_url)
.field("tracing", &self.tracing)
.field("store_model_in_db", &self.store_model_in_db)
.field("additional_fields", &self.additional_fields.keys())
.finish_non_exhaustive()
}
}
#[derive(Clone, Default, Deserialize)]
#[serde(default)]
pub struct RouterSettings {
pub routing_strategy: Option<String>,
pub routing_strategy_args: Option<Object>,
pub routing_groups: Option<Box<[Object]>>,
pub retry_policy: Option<Object>,
pub model_group_retry_policy: Option<Object>,
pub model_group_affinity_config: Option<Object>,
pub allowed_fails: Option<u64>,
pub cooldown_time: Option<f64>,
pub num_retries: Option<u64>,
pub timeout: Option<f64>,
pub max_retries: Option<u64>,
pub retry_after: Option<f64>,
pub fallbacks: Option<Box<[Object]>>,
pub context_window_fallbacks: Option<Box<[Object]>>,
pub model_group_alias: Option<Object>,
pub enable_tag_filtering: Option<bool>,
pub weights: Option<Object>,
pub tag_routing_prefix: Option<String>,
pub optional_pre_call_checks: Option<Box<[String]>>,
#[serde(flatten)]
pub additional_fields: AdditionalFields,
}
impl fmt::Debug for RouterSettings {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("RouterSettings")
.field("routing_strategy", &self.routing_strategy)
.field("routing_strategy_args", &self.routing_strategy_args)
.field("routing_groups", &self.routing_groups)
.field("retry_policy", &self.retry_policy)
.field("model_group_retry_policy", &self.model_group_retry_policy)
.field(
"model_group_affinity_config",
&self.model_group_affinity_config,
)
.field("allowed_fails", &self.allowed_fails)
.field("cooldown_time", &self.cooldown_time)
.field("num_retries", &self.num_retries)
.field("timeout", &self.timeout)
.field("max_retries", &self.max_retries)
.field("retry_after", &self.retry_after)
.field("fallbacks", &self.fallbacks)
.field("context_window_fallbacks", &self.context_window_fallbacks)
.field("model_group_alias", &self.model_group_alias)
.field("enable_tag_filtering", &self.enable_tag_filtering)
.field("weights", &self.weights)
.field("tag_routing_prefix", &self.tag_routing_prefix)
.field("optional_pre_call_checks", &self.optional_pre_call_checks)
.field("additional_fields", &self.additional_fields.keys())
.finish()
}
}
#[derive(Clone, Default, Deserialize)]
#[serde(default)]
pub struct LiteLlmSettings {
pub ssl_verify: Option<Spelled<bool>>,
pub ssl_certificate: Option<String>,
pub ssl_security_level: Option<String>,
pub ssl_ecdh_curve: Option<String>,
pub force_ipv4: Option<bool>,
pub http2: Option<bool>,
pub aiohttp_trust_env: Option<bool>,
pub disable_aiohttp_trust_env: Option<bool>,
pub disable_aiohttp_transport: Option<bool>,
pub drop_params: Option<Spelled<bool>>,
pub request_timeout: Option<Spelled<f64>>,
pub num_retries: Option<u64>,
pub cache: Option<bool>,
pub cache_params: Option<Object>,
#[serde(deserialize_with = "one_or_many")]
pub callbacks: Option<Vec<Value>>,
#[serde(deserialize_with = "one_or_many")]
pub success_callback: Option<Vec<Value>>,
#[serde(deserialize_with = "one_or_many")]
pub failure_callback: Option<Vec<Value>>,
pub json_logs: Option<bool>,
pub set_verbose: Option<bool>,
#[serde(flatten)]
pub additional_fields: AdditionalFields,
}
impl fmt::Debug for LiteLlmSettings {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("LiteLlmSettings")
.field("drop_params", &self.drop_params)
.field("request_timeout", &self.request_timeout)
.field("num_retries", &self.num_retries)
.field("cache", &self.cache)
.field("cache_params", &self.cache_params)
.field(
"callbacks",
&self.callbacks.as_ref().map(|callbacks| callbacks.len()),
)
.field(
"success_callback",
&self
.success_callback
.as_ref()
.map(|callbacks| callbacks.len()),
)
.field(
"failure_callback",
&self
.failure_callback
.as_ref()
.map(|callbacks| callbacks.len()),
)
.field("json_logs", &self.json_logs)
.field("set_verbose", &self.set_verbose)
.field("additional_fields", &self.additional_fields.keys())
.finish()
}
}

View file

@ -0,0 +1,128 @@
use litellm_auth_types::SecretValue;
use crate::{Object, Spelled};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct SearchTool {
pub search_tool_name: Option<String>,
pub litellm_params: Option<SearchToolParams>,
pub search_tool_info: Option<SearchToolInfo>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct SearchToolParams {
pub search_provider: Option<String>,
pub api_key: Option<SecretValue>,
pub api_base: Option<String>,
pub timeout: Option<Spelled<f64>>,
pub max_retries: Option<Spelled<i64>>,
pub input_cost_per_query: Option<Spelled<f64>>,
#[serde(flatten)]
pub additional_fields: Object,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct SearchToolInfo {
pub description: Option<String>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct SandboxTool {
pub sandbox_tool_name: Option<String>,
pub litellm_params: Option<SandboxToolParams>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct SandboxToolParams {
pub sandbox_provider: Option<String>,
pub api_key: Option<SecretValue>,
pub api_base: Option<String>,
#[serde(flatten)]
pub additional_fields: Object,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::{Config, Spelled};
fn parse(yaml: &str) -> Config {
Config::from_yaml(yaml).unwrap()
}
#[rstest]
fn search_tools_keep_typed_params_and_info() {
let config = parse(
r#"
search_tools:
- search_tool_name: web
litellm_params:
search_provider: perplexity
max_retries: 3
timeout: 2.5
region: eu
search_tool_info:
description: web search
"#,
);
let tool = &config.search_tools[0];
let params = tool.litellm_params.as_ref().unwrap();
assert_eq!(tool.search_tool_name.as_deref(), Some("web"));
assert_eq!(params.search_provider.as_deref(), Some("perplexity"));
assert_eq!(params.max_retries, Some(Spelled::Value(3)));
assert_eq!(params.timeout, Some(Spelled::Value(2.5)));
assert!(params.additional_fields.contains_key("region"));
assert_eq!(
tool.search_tool_info
.as_ref()
.unwrap()
.description
.as_deref(),
Some("web search")
);
}
#[rstest]
fn sandbox_tools_keep_typed_params() {
let config = parse(
"sandbox_tools:\n - sandbox_tool_name: e2b\n litellm_params: {sandbox_provider: e2b, template: base}\n",
);
let tool = &config.sandbox_tools[0];
let params = tool.litellm_params.as_ref().unwrap();
assert_eq!(tool.sandbox_tool_name.as_deref(), Some("e2b"));
assert_eq!(params.sandbox_provider.as_deref(), Some("e2b"));
assert!(params.additional_fields.contains_key("template"));
}
#[rstest]
#[case::search_key(
r#"
search_tools:
- litellm_params: {api_key: SENTINEL}
"#
)]
#[case::search_extra(
r#"
search_tools:
- litellm_params: {vendor_secret: SENTINEL}
"#
)]
#[case::sandbox_key(
r#"
sandbox_tools:
- litellm_params: {api_key: SENTINEL}
"#
)]
#[case::sandbox_extra(
r#"
sandbox_tools:
- litellm_params: {vendor_secret: SENTINEL}
"#
)]
fn debug_never_prints_tool_values(#[case] yaml: &str) {
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
}
}

View file

@ -1,28 +1,13 @@
use std::{collections::BTreeMap, fmt, ops::Deref};
use serde::{Deserialize, Deserializer};
use serde::Deserialize;
pub type Value = serde_yaml_ng::Value;
pub type AdditionalFields = BTreeMap<String, Value>;
#[derive(Clone, Default, Deserialize)]
#[serde(transparent)]
pub struct Object(BTreeMap<String, Value>);
impl Object {
pub fn get(&self, key: &str) -> Option<&Value> {
self.0.get(key)
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn len(&self) -> usize {
self.0.len()
}
}
impl Deref for Object {
type Target = BTreeMap<String, Value>;
@ -40,14 +25,27 @@ impl fmt::Debug for Object {
}
}
/// Python's callback shorthand: one entry or a list of them, kept as the values written.
pub(crate) fn one_or_many<'de, D: Deserializer<'de>>(
deserializer: D,
) -> Result<Option<Vec<Value>>, D::Error> {
Ok(
Option::<Value>::deserialize(deserializer)?.map(|value| match value {
Value::Sequence(values) => values,
one => vec![one],
}),
)
#[derive(Clone, Deserialize)]
#[serde(transparent)]
pub struct Raw(Value);
impl Raw {
pub fn get(&self) -> &Value {
&self.0
}
}
impl fmt::Debug for Raw {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
let kind = match &self.0 {
Value::Null => "null",
Value::Bool(_) => "bool",
Value::Number(_) => "number",
Value::String(_) => "string",
Value::Sequence(_) => "sequence",
Value::Mapping(_) => "mapping",
Value::Tagged(_) => "tagged",
};
formatter.debug_tuple("Raw").field(&kind).finish()
}
}

View file

@ -0,0 +1,63 @@
use crate::{Object, Raw};
#[macro_rules_attribute::apply(crate::config_section)]
pub struct VectorStore {
pub vector_store_name: Option<String>,
pub litellm_params: Option<VectorStoreParams>,
}
#[macro_rules_attribute::apply(crate::config_section)]
pub struct VectorStoreParams {
pub vector_store_id: Option<String>,
pub custom_llm_provider: Option<String>,
pub vector_store_description: Option<String>,
pub vector_store_metadata: Option<Raw>,
#[serde(flatten)]
pub additional_fields: Object,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use crate::Config;
fn parse(yaml: &str) -> Config {
Config::from_yaml(yaml).unwrap()
}
#[rstest]
fn vector_store_entries_keep_typed_params_and_unknown_keys() {
let config = parse(
r#"
vector_store_registry:
- vector_store_name: kb
litellm_params: {vector_store_id: vs_123, custom_llm_provider: openai, region: eu}
"#,
);
let store = &config.vector_store_registry[0];
let params = store.litellm_params.as_ref().unwrap();
assert_eq!(store.vector_store_name.as_deref(), Some("kb"));
assert_eq!(params.vector_store_id.as_deref(), Some("vs_123"));
assert_eq!(params.custom_llm_provider.as_deref(), Some("openai"));
assert!(params.additional_fields.contains_key("region"));
}
#[rstest]
#[case::vector_store_metadata(
r#"
vector_store_registry:
- litellm_params: {vector_store_metadata: SENTINEL}
"#
)]
#[case::vector_store_extra(
r#"
vector_store_registry:
- litellm_params: {api_key: SENTINEL}
"#
)]
fn debug_never_prints_vector_store_values(#[case] yaml: &str) {
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
}
}

View file

@ -0,0 +1,177 @@
use std::fmt;
use serde::{
Deserialize, Deserializer,
de::{MapAccess, SeqAccess, Visitor},
};
use serde_yaml_ng::{Mapping, Value};
pub fn mapping(text: &str) -> Result<Mapping, serde_yaml_ng::Error> {
serde_yaml_ng::from_value(serde_yaml_ng::from_str::<LastWins>(text)?.0)
}
struct LastWins(Value);
impl<'de> Deserialize<'de> for LastWins {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
deserializer.deserialize_any(LastWinsVisitor).map(Self)
}
}
// PyYAML safe_load keeps the later value of a duplicate key, serde_yaml_ng rejects it
struct LastWinsVisitor;
impl<'de> Visitor<'de> for LastWinsVisitor {
type Value = Value;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("any YAML value")
}
fn visit_bool<E>(self, value: bool) -> Result<Value, E> {
Ok(Value::Bool(value))
}
fn visit_i64<E>(self, value: i64) -> Result<Value, E> {
Ok(Value::Number(value.into()))
}
fn visit_u64<E>(self, value: u64) -> Result<Value, E> {
Ok(Value::Number(value.into()))
}
fn visit_f64<E>(self, value: f64) -> Result<Value, E> {
Ok(Value::Number(value.into()))
}
fn visit_str<E>(self, value: &str) -> Result<Value, E> {
Ok(Value::String(value.to_owned()))
}
fn visit_string<E>(self, value: String) -> Result<Value, E> {
Ok(Value::String(value))
}
fn visit_unit<E>(self) -> Result<Value, E> {
Ok(Value::Null)
}
fn visit_none<E>(self) -> Result<Value, E> {
Ok(Value::Null)
}
fn visit_some<D: Deserializer<'de>>(self, deserializer: D) -> Result<Value, D::Error> {
LastWins::deserialize(deserializer).map(|value| value.0)
}
fn visit_seq<A: SeqAccess<'de>>(self, mut items: A) -> Result<Value, A::Error> {
let mut sequence = Vec::new();
while let Some(LastWins(item)) = items.next_element()? {
sequence.push(item);
}
Ok(Value::Sequence(sequence))
}
fn visit_map<A: MapAccess<'de>>(self, mut entries: A) -> Result<Value, A::Error> {
let mut mapping = Mapping::new();
while let Some((LastWins(key), LastWins(value))) = entries.next_entry()? {
mapping.insert(key, value);
}
Ok(Value::Mapping(mapping))
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
fn yaml(text: &str) -> Mapping {
serde_yaml_ng::from_str(text).unwrap()
}
#[rstest]
#[case::top_level(
r#"
a: 1
b: 2
a: 3
"#,
"{a: 3, b: 2}"
)]
#[case::sequence_value(
r#"
cb: [prometheus]
cb: [s3]
"#,
"cb: [s3]"
)]
#[case::nested("outer: {x: [1], y: 2, x: [3]}", "outer: {x: [3], y: 2}")]
#[case::deeply_nested("a: {b: {c: 1, c: 2}}", "a: {b: {c: 2}}")]
#[case::inside_sequence("list: [{k: old, k: new}]", "list: [{k: new}]")]
#[case::mapping_replaced_by_scalar(
r#"
a: {b: 1}
a: 2
"#,
"a: 2"
)]
#[case::value_replaced_by_null(
r#"
a: 1
a:
"#,
"a: null"
)]
#[case::three_occurrences(
r#"
a: 1
a: 2
a: 3
"#,
"a: 3"
)]
#[case::non_string_keys(
r#"
1: a
true: b
1: c
"#,
"{1: c, true: b}"
)]
#[case::scalars_without_duplicates(
r#"
i: -1
u: 18446744073709551615
f: 1.5
b: false
n: ~
s: text
"#,
"{i: -1, u: 18446744073709551615, f: 1.5, b: false, n: ~, s: text}"
)]
#[case::empty_document("", "{}")]
fn later_duplicate_keys_override_earlier_ones(#[case] text: &str, #[case] expected: &str) {
assert_eq!(mapping(text).unwrap(), yaml(expected));
}
#[rstest]
fn duplicate_key_keeps_its_first_position() {
let keys: Vec<_> = mapping("a: 1\nb: 2\na: 3\n").unwrap().into_keys().collect();
assert_eq!(keys, [Value::from("a"), Value::from("b")]);
}
#[rstest]
#[case::sequence(
r#"
- a
- b
"#
)]
#[case::scalar("text")]
#[case::malformed("a: [")]
fn rejects_a_document_that_is_not_a_mapping(#[case] text: &str) {
assert!(mapping(text).is_err());
}
}

View file

@ -1,436 +0,0 @@
use litellm_config::{Config, Error, Spelled, TracingStoreSettings};
use rstest::{fixture, rstest};
use tempfile::TempDir;
#[fixture]
fn directory() -> TempDir {
tempfile::tempdir().unwrap()
}
#[fixture]
fn model_list_yaml() -> &'static str {
r#"
model_list:
- model_name: assistant
litellm_params:
model: anthropic/test-model
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: local
litellm_params:
model: test-model
api_base: http://localhost:8000/v1
custom_llm_provider: openai
"#
}
#[rstest]
fn loads_model_list_from_file(directory: TempDir, model_list_yaml: &str) {
let path = directory.path().join("config.yaml");
std::fs::write(&path, model_list_yaml).unwrap();
let config = Config::load(path).unwrap();
assert_eq!(config.model_list.len(), 2);
let anthropic = &config.model_list[0];
assert_eq!(anthropic.model_name, "assistant");
assert_eq!(anthropic.litellm_params.model, "anthropic/test-model");
assert_eq!(
anthropic.litellm_params.api_key.as_ref().unwrap().expose(),
"os.environ/ANTHROPIC_API_KEY"
);
assert!(anthropic.litellm_params.api_base.is_none());
assert!(anthropic.litellm_params.custom_llm_provider.is_none());
let local = &config.model_list[1];
assert_eq!(local.model_name, "local");
assert_eq!(local.litellm_params.model, "test-model");
assert!(local.litellm_params.api_key.is_none());
assert_eq!(
local.litellm_params.api_base.as_deref(),
Some("http://localhost:8000/v1")
);
assert_eq!(
local.litellm_params.custom_llm_provider.as_deref(),
Some("openai")
);
}
#[rstest]
fn config_debug_redacts_api_keys() {
let config = Config::from_yaml(
"model_list: [{model_name: assistant, litellm_params: {model: anthropic/test-model, api_key: secret-value}}]",
)
.unwrap();
assert_eq!(
config.model_list[0]
.litellm_params
.api_key
.as_ref()
.unwrap()
.expose(),
"secret-value"
);
assert!(!format!("{config:?}").contains("secret-value"));
}
#[rstest]
#[case::malformed_yaml("model_list: [")]
#[case::missing_params("model_list: [{model_name: assistant}]")]
#[case::missing_model("model_list: [{model_name: assistant, litellm_params: {api_key: key}}]")]
#[case::provider_param_of_the_wrong_type(
"model_list: [{model_name: assistant, litellm_params: {model: test, aws_region_name: [eu-central-1]}}]"
)]
fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) {
assert!(matches!(Config::from_yaml(yaml), Err(Error::Parse(_))));
}
#[rstest]
fn distinguishes_read_errors_from_parse_errors(directory: TempDir) {
assert!(matches!(
Config::load(directory.path().join("missing.yaml")),
Err(Error::Read(error)) if error.kind() == std::io::ErrorKind::NotFound
));
}
#[rstest]
#[case::literal("secret-master-key")]
#[case::reference("os.environ/LITELLM_MASTER_KEY")]
fn loads_and_redacts_the_master_key(#[case] key: &str) {
let config = Config::from_yaml(&format!(
"model_list: []\ngeneral_settings:\n master_key: {key}\n"
))
.unwrap();
assert_eq!(
config
.general_settings
.master_key
.as_ref()
.unwrap()
.expose(),
key
);
assert!(!format!("{config:?}").contains(key));
}
#[rstest]
fn missing_general_settings_has_no_master_key() {
let config = Config::from_yaml("model_list: []").unwrap();
assert!(config.general_settings.master_key.is_none());
}
#[test]
fn tracing_settings_are_typed_and_redact_the_url() {
let config = Config::from_yaml(
"general_settings:\n tracing:\n store:\n type: clickhouse\n url: https://writer:password@example.com\n database: analytics\n retention_days: 7\n",
)
.unwrap();
let tracing = config.general_settings.tracing.as_ref().unwrap();
let Some(TracingStoreSettings::ClickHouse(store)) = tracing.store.as_ref() else {
panic!("expected ClickHouse tracing store")
};
assert_eq!(
store.url.as_ref().unwrap().expose(),
"https://writer:password@example.com"
);
assert_eq!(store.database.as_deref(), Some("analytics"));
assert_eq!(store.retention_days, Some(Spelled::Value(7.0)));
assert!(!format!("{config:?}").contains("password"));
}
#[test]
fn tracing_settings_accept_environment_references() {
let config = Config::from_yaml(
"general_settings:\n tracing:\n store:\n type: clickhouse\n url: os.environ/CLICKHOUSE_URL\n retention_days: os.environ/RETENTION_DAYS\n",
)
.unwrap();
let Some(TracingStoreSettings::ClickHouse(store)) =
config.general_settings.tracing.unwrap().store
else {
panic!("expected ClickHouse tracing store")
};
assert_eq!(
store.retention_days,
Some(Spelled::Text("os.environ/RETENTION_DAYS".to_owned()))
);
}
#[test]
fn tracing_settings_reject_string_store() {
assert!(Config::from_yaml("general_settings:\n tracing:\n store: clickhouse\n").is_err());
}
#[test]
fn tracing_settings_reject_removed_reader_configuration() {
assert!(Config::from_yaml("general_settings:\n tracing:\n store:\n type: clickhouse\n reader_url: http://localhost:8123\n").is_err());
}
#[rstest]
fn empty_config_matches_python_defaults() {
let config = Config::from_yaml("{}").unwrap();
assert!(config.model_list.is_empty());
assert_eq!(config.general_settings.admission_queue_timeout_seconds, 1.0);
assert_eq!(
config.general_settings.database_connection_pool_limit,
Some(10)
);
assert_eq!(
config.general_settings.proxy_config_reload_interval_seconds,
30
);
assert_eq!(config.general_settings.health_check_interval, 300);
}
#[rstest]
fn parses_typed_settings_and_preserves_extension_fields() {
let config = Config::from_yaml(
r#"
model_list:
- model_name: assistant
litellm_params:
model: vertex_ai/test-model
timeout: os.environ/REQUEST_TIMEOUT
tpm: os.environ/TPM_LIMIT
rpm: 5
drop_params: "true"
vertex_project: test-project
vertex_credentials:
type: service_account
future_param: 1
model_info:
mode: chat
access_groups: [internal]
general_settings:
master_key: secret-master-key
store_model_in_db: true
custom_auth: auth.py
router_settings:
routing_strategy: simple-shuffle
allowed_fails: 2
redis_host: cache.internal
litellm_settings:
drop_params: true
cache: true
custom_callback_name: audit
future_section:
enabled: true
"#,
)
.unwrap();
let model = &config.model_list[0];
assert_eq!(
model.litellm_params.timeout,
Some(Spelled::Text("os.environ/REQUEST_TIMEOUT".to_string()))
);
assert_eq!(
model.litellm_params.tpm,
Some(Spelled::Text("os.environ/TPM_LIMIT".to_string()))
);
assert_eq!(model.litellm_params.rpm, Some(Spelled::Value(5.0)));
assert_eq!(
model.litellm_params.drop_params,
Some(Spelled::Text("true".to_string()))
);
assert_eq!(
model.litellm_params.vertex.vertex_project.as_deref(),
Some("test-project")
);
assert_eq!(
model.litellm_params.vertex.vertex_credentials.as_deref(),
Some(r#"{"type":"service_account"}"#)
);
assert!(model.litellm_params.extra.contains_key("future_param"));
assert!(model.additional_fields.contains_key("access_groups"));
assert_eq!(config.router_settings.allowed_fails, Some(2));
assert!(
config
.router_settings
.additional_fields
.contains_key("redis_host")
);
assert!(
config
.litellm_settings
.additional_fields
.contains_key("custom_callback_name")
);
assert!(config.additional_fields.contains_key("future_section"));
}
#[rstest]
fn parses_python_config_sections() {
let config = Config::from_yaml(
r#"
environment_variables:
REDIS_PORT: 6379
callback_settings:
otel:
message_logging: false
assistant_settings:
custom_llm_provider: openai
credential_list:
- credential_name: bedrock
credential_values:
aws_region_name: us-east-1
guardrails:
- guardrail_name: pii
litellm_params:
guardrail: presidio
prompts:
- prompt_id: support
sandbox_tools:
- sandbox_tool_name: e2b
search_tools:
- search_tool_name: web
files_settings:
- custom_llm_provider: openai
finetune_settings:
- custom_llm_provider: openai
mcp_tools:
- name: lookup
mcp_servers:
docs:
url: https://example.test/mcp
vector_store_registry:
- vector_store_name: docs
worker_registry:
- worker_id: regional
agents:
- agent_name: reviewer
agent_list:
- agent_name: legacy-reviewer
policies:
safe:
guardrails:
add: [pii]
policy_attachments:
- policy_id: safe
include:
- models.yaml
"#,
)
.unwrap();
assert_eq!(config.environment_variables.len(), 1);
assert_eq!(config.credential_list.len(), 1);
assert_eq!(config.guardrails.len(), 1);
assert_eq!(config.prompts.len(), 1);
assert_eq!(config.sandbox_tools.len(), 1);
assert_eq!(config.search_tools.len(), 1);
assert_eq!(config.files_settings.len(), 1);
assert_eq!(config.finetune_settings.len(), 1);
assert_eq!(config.mcp_tools.len(), 1);
assert_eq!(config.mcp_servers.len(), 1);
assert_eq!(config.vector_store_registry.len(), 1);
assert_eq!(config.worker_registry.len(), 1);
assert_eq!(config.agents.len(), 1);
assert_eq!(config.agent_list.len(), 1);
assert!(config.policies.contains_key("safe"));
assert_eq!(config.policy_attachments.len(), 1);
assert_eq!(&*config.include, &["models.yaml"]);
}
#[rstest]
#[case::policy_pipeline("../../../litellm/proxy/example_config_yaml/test_pipeline_config.yaml")]
#[case::gateway("../../../tests/e2e/gateway/stage_mirror_ci_config.yml")]
fn parses_representative_python_configs(#[case] relative_path: &str) {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(relative_path);
let config = Config::load(path).unwrap();
assert!(!config.model_list.is_empty());
}
#[rstest]
#[case::one("callbacks: custom_callbacks.logger", 1)]
#[case::many("callbacks: [prometheus, otel]", 2)]
fn accepts_python_callback_shorthand(#[case] setting: &str, #[case] expected_len: usize) {
let config = Config::from_yaml(&format!("litellm_settings:\n {setting}")).unwrap();
assert_eq!(
config.litellm_settings.callbacks.as_ref().unwrap().len(),
expected_len
);
}
#[rstest]
#[case::api_key("api_key", "provider-secret")]
#[case::aws_secret("aws_secret_access_key", "aws-secret")]
#[case::vertex_credentials("vertex_credentials", "vertex-secret")]
#[case::provider_extension("azure_ad_token", "azure-secret")]
#[case::general_extension("custom_auth_secret", "auth-secret")]
#[case::router_extension("redis_password", "redis-secret")]
#[case::litellm_extension("callback_token", "callback-secret")]
#[case::root_extension("private_token", "root-secret")]
fn debug_output_does_not_expose_config_values(#[case] field: &str, #[case] secret: &str) {
let yaml = match field {
"api_key" => format!(
"model_list: [{{model_name: assistant, litellm_params: {{model: test, api_key: {secret}}}}}]"
),
"aws_secret_access_key" => format!(
"model_list: [{{model_name: assistant, litellm_params: {{model: test, aws_secret_access_key: {secret}}}}}]"
),
"vertex_credentials" => format!(
"model_list: [{{model_name: assistant, litellm_params: {{model: test, vertex_credentials: {secret}}}}}]"
),
"azure_ad_token" => format!(
"model_list: [{{model_name: assistant, litellm_params: {{model: test, azure_ad_token: {secret}}}}}]"
),
"custom_auth_secret" => format!("general_settings: {{{field}: {secret}}}"),
"redis_password" => format!("router_settings: {{{field}: {secret}}}"),
"callback_token" => format!("litellm_settings: {{{field}: {secret}}}"),
"private_token" => format!("{field}: {secret}"),
_ => unreachable!(),
};
let config = Config::from_yaml(&yaml).unwrap();
assert!(!format!("{config:?}").contains(secret));
}
#[rstest]
fn resolves_nested_includes_once_in_breadth_first_order() {
let directory = TempDir::new().unwrap();
let root = directory.path();
std::fs::create_dir(root.join("nested")).unwrap();
std::fs::write(root.join("config.yaml"), "include: [nested/first.yaml, second.yaml]\nmodel_list: [{model_name: root, litellm_params: {model: root}}]\n").unwrap();
std::fs::write(root.join("nested/first.yaml"), "include: [third.yaml]\nmodel_list: [{model_name: first, litellm_params: {model: first}}]\n").unwrap();
std::fs::write(root.join("second.yaml"), "general_settings: {master_key: second}\nmodel_list: [{model_name: second, litellm_params: {model: second}}]\n").unwrap();
std::fs::write(root.join("nested/third.yaml"), "include: [../config.yaml]\ngeneral_settings: {master_key: third}\nmodel_list: [{model_name: third, litellm_params: {model: third}}]\n").unwrap();
let config = Config::load(root.join("config.yaml")).unwrap();
assert_eq!(
config
.model_list
.iter()
.map(|model| model.model_name.as_str())
.collect::<Vec<_>>(),
["root", "first", "second", "third"]
);
assert_eq!(
config.general_settings.master_key.unwrap().expose(),
"third"
);
assert!(config.include.is_empty());
}
#[rstest]
fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() {
let config = Config::from_yaml("mcp_servers:\n docs:\n url: https://example.test/private-secret/mcp\n authentication_token: upstream-secret\n static_headers: {x-token: header-secret}\n env: {TOKEN: env-secret}\n args: [argument-secret]\n client_secret: oauth-secret\n allowed_tools: [search]\n").unwrap();
let server = &config.mcp_servers["docs"];
assert_eq!(server.allowed_tools.as_deref().unwrap(), ["search"]);
assert!(server.unsupported.contains_key("client_secret"));
let debug = format!("{config:?}");
for secret in [
"private-secret",
"upstream-secret",
"header-secret",
"env-secret",
"oauth-secret",
"argument-secret",
] {
assert!(!debug.contains(secret));
}
}
#[rstest]
#[case::transport("transport: invalid")]
#[case::auth("auth_type: invalid")]
#[case::concurrency("max_concurrent_requests: -1")]
#[case::headers("static_headers: {x-token: [not, a, string]}")]
fn rejects_invalid_typed_mcp_settings(#[case] setting: &str) {
assert!(Config::from_yaml(&format!("mcp_servers:\n docs:\n {setting}\n")).is_err());
}

View file

@ -1,39 +0,0 @@
use litellm_config::{McpAuth, McpTransport};
use rstest::rstest;
use serde_json::json;
#[rstest]
#[case::http(McpTransport::Http, "http")]
#[case::sse(McpTransport::Sse, "sse")]
#[case::stdio(McpTransport::Stdio, "stdio")]
fn mcp_transport_as_str_matches_the_serde_spelling(
#[case] transport: McpTransport,
#[case] name: &str,
) {
assert_eq!(<&'static str>::from(transport), name);
assert_eq!(
serde_json::from_value::<McpTransport>(json!(name)).unwrap(),
transport
);
}
#[rstest]
#[case::none(McpAuth::None, "none")]
#[case::api_key(McpAuth::ApiKey, "api_key")]
#[case::bearer_token(McpAuth::BearerToken, "bearer_token")]
#[case::basic(McpAuth::Basic, "basic")]
#[case::authorization(McpAuth::Authorization, "authorization")]
#[case::token(McpAuth::Token, "token")]
#[case::oauth2(McpAuth::Oauth2, "oauth2")]
#[case::aws_sigv4(McpAuth::AwsSigv4, "aws_sigv4")]
#[case::oauth2_token_exchange(McpAuth::Oauth2TokenExchange, "oauth2_token_exchange")]
#[case::oauth2_id_jag(McpAuth::Oauth2IdJag, "oauth2_id_jag")]
#[case::true_passthrough(McpAuth::TruePassthrough, "true_passthrough")]
#[case::oauth_delegate(McpAuth::OauthDelegate, "oauth_delegate")]
fn mcp_auth_as_str_matches_the_serde_spelling(#[case] auth: McpAuth, #[case] name: &str) {
assert_eq!(<&'static str>::from(auth), name);
assert_eq!(
serde_json::from_value::<McpAuth>(json!(name)).unwrap(),
auth
);
}

View file

@ -22,7 +22,7 @@ pub use identity::{
AuthenticatedCaller, Authentication, AuthenticationMethod, Principal, PrincipalKind,
ResolvedIdentity, SharedCaller, VerifiedIdentity,
};
pub use ui::{UI_CSRF_KEY, UiAuthSession, UiBackend, UiCredentials, UiSession, UiUser};
pub use ui::{UI_CSRF_KEY, UiAuthSession, UiBackend, UiSession, UiUser};
pub fn hash_token(token: &str) -> String {
format!("{:x}", Sha256::digest(token.as_bytes()))

View file

@ -3,8 +3,7 @@ use std::{convert::Infallible, sync::Arc};
use axum::extract::FromRequestParts;
use axum::http::{header::AUTHORIZATION, request::Parts};
use axum_login::{AuthUser, AuthnBackend, UserId};
use litellm_auth_types::SecretValue;
use serde::Deserialize;
use litellm_auth_types::{SecretValue, UiCredentials};
use sha2::{Digest, Sha256};
use subtle::ConstantTimeEq;
use tower_sessions::Session;
@ -38,12 +37,6 @@ impl AuthUser for UiUser {
}
}
#[derive(Deserialize)]
pub struct UiCredentials {
pub username: String,
pub password: SecretValue,
}
#[derive(Clone)]
pub struct UiBackend {
user: UiUser,

View file

@ -1,6 +1,6 @@
use axum_login::{AuthUser, AuthnBackend};
use litellm_auth_types::SecretValue;
use litellm_gateway_auth::{UiBackend, UiCredentials};
use litellm_auth_types::{SecretValue, UiCredentials};
use litellm_gateway_auth::UiBackend;
use rstest::{fixture, rstest};
#[fixture]

View file

@ -7,11 +7,13 @@ license.workspace = true
repository.workspace = true
[dependencies]
axum = { workspace = true, features = ["json", "query"] }
futures-util.workspace = true
litellm-config.workspace = true
litellm-gateway-auth.workspace = true
litellm-secrets.workspace = true
litellm-auth-types.workspace = true
axum = { workspace = true, features = ["json", "query"] }
futures-util.workspace = true
base64.workspace = true
http.workspace = true
uuid.workspace = true

View file

@ -1,13 +1,13 @@
use std::sync::Arc;
use axum::{extract::Request, middleware::Next, response::Response};
use litellm_gateway_auth::{AccessRequest, AuthenticatedRequest, Error, McpAction};
use litellm_gateway_mcp::{
Authorization, GatewayFuture, OperationAuthorizer, ServerInfo, SessionOwner,
rmcp::model::ClientRequest,
};
use rmcp::model::ClientRequest;
pub(crate) async fn bind_session_owner(
use litellm_gateway_auth::{AccessRequest, AuthenticatedRequest, McpAction};
use crate::{Authorization, GatewayFuture, OperationAuthorizer, ServerInfo, SessionOwner};
pub async fn bind_caller(
identity: AuthenticatedRequest,
mut request: Request,
next: Next,
@ -47,7 +47,7 @@ impl OperationAuthorizer for McpPolicy {
Some(ClientRequest::ReadResourceRequest(request)) => {
McpAction::ReadResource(request.params.uri.clone())
}
Some(_) => return Err(litellm_gateway_mcp::Error::Forbidden),
Some(_) => return Err(crate::Error::Forbidden),
};
let access = AccessRequest::Mcp {
server: server.server_id.clone(),
@ -55,19 +55,9 @@ impl OperationAuthorizer for McpPolicy {
};
self.0
.authorize(access.clone())
.await
.map_err(mcp_error)?
.consume(self.0.caller(), &access)
.map_err(mcp_error)
.await?
.consume(self.0.caller(), &access)?;
Ok(())
})
}
}
fn mcp_error(error: Error) -> litellm_gateway_mcp::Error {
match error {
Error::InvalidToken | Error::Expired | Error::Forbidden => {
litellm_gateway_mcp::Error::Forbidden
}
_ => litellm_gateway_mcp::Error::Configuration("MCP authorization unavailable".into()),
}
}

View file

@ -136,7 +136,7 @@ fn validate(name: &str, config: &McpServer) -> Result<(), ConnectError> {
if config
.command
.as_ref()
.is_none_or(|command| command.trim().is_empty())
.is_none_or(|command| command.expose().trim().is_empty())
|| config.url.is_some()
{
return Err(invalid(
@ -377,11 +377,12 @@ async fn connect_service<C: StreamableHttpClient>(
.await?;
let program = config
.command
.as_deref()
.as_ref()
.map(SecretValue::expose)
.ok_or_else(|| invalid(name, "stdio requires command"))?;
let mut command = tokio::process::Command::new(program);
command
.args(&config.args)
.args(config.args.iter().map(SecretValue::expose))
.envs(environment)
.kill_on_drop(true);
().serve_with_ct(TokioChildProcess::new(command)?, shutdown)

View file

@ -3,6 +3,7 @@ use axum::{
http::StatusCode,
response::{IntoResponse, Response},
};
use litellm_gateway_auth::Error as AuthError;
use rmcp::{
model::{ErrorCode, ErrorData},
service::ServiceError,
@ -23,6 +24,8 @@ pub enum Error {
UnexpectedResult,
#[error("MCP request cancelled")]
Cancelled,
#[error(transparent)]
Auth(#[from] AuthError),
}
impl Error {
@ -30,7 +33,10 @@ impl Error {
match self {
Self::Upstream(ServiceError::McpError(error)) => error,
Self::InvalidRequest(message) => ErrorData::invalid_params(message, None),
Self::Forbidden => ErrorData::new(ErrorCode(-32003), self.to_string(), None),
Self::Forbidden
| Self::Auth(AuthError::Forbidden | AuthError::InvalidToken | AuthError::Expired) => {
ErrorData::new(ErrorCode(-32003), Self::Forbidden.to_string(), None)
}
_ => ErrorData::internal_error(self.to_string(), None),
}
}
@ -44,6 +50,7 @@ impl IntoResponse for Error {
Self::Upstream(_) => StatusCode::BAD_GATEWAY,
Self::Cancelled => StatusCode::REQUEST_TIMEOUT,
Self::Configuration(_) | Self::UnexpectedResult => StatusCode::INTERNAL_SERVER_ERROR,
Self::Auth(error) => error.status(),
};
(status, Json(json!({"detail": self.to_string()}))).into_response()
}

View file

@ -1,3 +1,4 @@
mod auth;
mod configured;
mod error;
mod legacy;
@ -12,6 +13,7 @@ use std::{collections::BTreeMap, future::Future, pin::Pin, sync::Arc};
use http::request::Parts;
use rmcp::{RoleServer, model::*, service::RequestContext};
pub use auth::bind_caller;
pub use configured::ConfiguredGateway;
pub use error::{ConnectError, Error};
pub use native::{Limits, NativeGateway, Registry, Server, ServerResolver};

View file

@ -11,6 +11,7 @@ axum-login.workspace = true
base64.workspace = true
governor = { version = "0.10.4", default-features = false, features = ["std"] }
jsonwebtoken = { workspace = true, features = ["rust_crypto"] }
litellm-auth-types.workspace = true
litellm-gateway-auth.workspace = true
rand.workspace = true
serde.workspace = true
@ -24,7 +25,6 @@ tower-sessions.workspace = true
[dev-dependencies]
serde_json.workspace = true
tower = { version = "0.5", features = ["util"] }
litellm-auth-types.workspace = true
rstest.workspace = true
tempfile.workspace = true
tokio.workspace = true

View file

@ -10,8 +10,9 @@ use tower_cookies::{Cookie, Cookies};
use tower_sessions::{Expiry, Session, cookie::SameSite};
use crate::Error;
use litellm_auth_types::UiCredentials;
use litellm_gateway_auth::{
AccessRequest, UI_CSRF_KEY, UiAction, UiAuthError, UiAuthSession, UiCredentials, UiSession,
AccessRequest, UI_CSRF_KEY, UiAction, UiAuthError, UiAuthSession, UiSession,
};
pub struct State {

View file

@ -19,6 +19,7 @@ litellm-gateway-auth.workspace = true
litellm-gateway-ui.workspace = true
litellm-auth-types.workspace = true
litellm-config.workspace = true
litellm-config-load.workspace = true
litellm-http = { workspace = true, features = ["mcp"] }
litellm-llms.workspace = true
litellm-secrets.workspace = true

View file

@ -32,7 +32,7 @@ MCP clients can connect to `/mcp` or the scoped `/math/mcp` endpoint with the sa
## Configuration behavior
`Config::load` resolves nested `include` files once in breadth-first order. Included lists append and other top-level values replace earlier values. Paths resolve relative to the declaring file, with the root config directory as a fallback
`litellm_config_load::load` resolves nested `include` files once in breadth-first order. Included lists append and other top-level values replace earlier values. Paths resolve relative to the declaring file, with the root config directory as a fallback
`environment_variables` provides scalar overrides to the injected secret source and HTTP settings without changing process globals. Credentials, static headers, URLs, and stdio environment values can reference `os.environ/NAME`. Model providers and MCP share the host's HTTP settings, including TLS certificates and environment proxies. Explicit environment HTTP settings take precedence over `litellm_settings`

View file

@ -4,8 +4,6 @@ pub enum Error {
Http(#[from] litellm_http::Error),
#[error("environment_variables values must be strings, numbers or booleans")]
Environment,
#[error("MCP host does not yet support configured setting {0}")]
McpSetting(String),
#[error(transparent)]
Mcp(#[from] litellm_gateway_mcp::ConnectError),
#[error("MCP requires a configured master key")]

Some files were not shown because too many files have changed in this diff Show more