diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index c868d3e4b8a..6632fd279fd 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3647,12 +3647,25 @@ name = "litellm-config" version = "0.1.0" dependencies = [ "litellm-auth-types", + "litellm-python-compat", "litellm-router-types", + "macro_rules_attribute", "rstest", "serde", "serde_json", "serde_yaml_ng", "strum", + "thiserror 2.0.19", +] + +[[package]] +name = "litellm-config-load" +version = "0.1.0" +dependencies = [ + "litellm-config", + "litellm-core-utils", + "rstest", + "serde_yaml_ng", "tempfile", "thiserror 2.0.19", ] @@ -3737,6 +3750,7 @@ dependencies = [ "http-body-util", "litellm-auth-types", "litellm-config", + "litellm-config-load", "litellm-gateway-auth", "litellm-gateway-inference", "litellm-gateway-mcp", @@ -4291,6 +4305,7 @@ name = "litellm-router-types" version = "0.1.0" dependencies = [ "litellm-auth-types", + "litellm-python-compat", "rstest", "serde", "serde_json", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 9e5e0a96b73..ce86003de28 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -10,6 +10,7 @@ repository = "https://github.com/BerriAI/litellm" [workspace.dependencies] litellm-config = { path = "crates/config" } +litellm-config-load = { path = "crates/config-load" } litellm-router = { path = "crates/router" } litellm-router-types = { path = "crates/router-types" } litellm-tracing = { path = "crates/tracing" } diff --git a/litellm-rust/crates/config-load/AGENTS.md b/litellm-rust/crates/config-load/AGENTS.md new file mode 100644 index 00000000000..bd4892d8d40 --- /dev/null +++ b/litellm-rust/crates/config-load/AGENTS.md @@ -0,0 +1,11 @@ +## Scope + +- This crate is the gateway's config loader. It takes the types `litellm-config` parses and turns them into the values the gateway runs on, so env lookups, the DB overlay, reference substitution, defaults and the domain types live here, never in `litellm-config` +- It never reads process state. The caller reads the process environment once and passes it in as an `Environment`, so tests pass their own variables +- Python never uses this crate. It shares only the `litellm-config` types, so behavior here may differ from Python where Python is wrong, and each difference is listed in the config compat doc + +- Fixture and corpus rules live in `tests/AGENTS.md` + +## Precedence + +- Each lowered value takes the env, the file and then the default in the order Python uses for that key today, until that order is normalized diff --git a/litellm-rust/crates/config-load/Cargo.toml b/litellm-rust/crates/config-load/Cargo.toml new file mode 100644 index 00000000000..5810cccb77b --- /dev/null +++ b/litellm-rust/crates/config-load/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "litellm-config-load" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-config.workspace = true +litellm-core-utils.workspace = true +serde_yaml_ng = "0.10.0" +thiserror.workspace = true + +[dev-dependencies] +rstest.workspace = true +tempfile.workspace = true diff --git a/litellm-rust/crates/config-load/build.rs b/litellm-rust/crates/config-load/build.rs new file mode 100644 index 00000000000..74f67a8473c --- /dev/null +++ b/litellm-rust/crates/config-load/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo::rerun-if-changed=tests/fixtures"); +} diff --git a/litellm-rust/crates/config-load/src/error.rs b/litellm-rust/crates/config-load/src/error.rs new file mode 100644 index 00000000000..b10aff338c8 --- /dev/null +++ b/litellm-rust/crates/config-load/src/error.rs @@ -0,0 +1,26 @@ +use std::fmt; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("could not read config")] + Read(#[from] std::io::Error), + #[error("invalid YAML config")] + Parse(#[from] serde_yaml_ng::Error), + #[error("invalid config")] + Config(#[from] litellm_config::Error), + #[error("could not load config {key}")] + Document { + key: String, + #[source] + source: Box, + }, +} + +impl Error { + pub(crate) fn document(key: &impl fmt::Debug, source: Error) -> Self { + Self::Document { + key: format!("{key:?}"), + source: Box::new(source), + } + } +} diff --git a/litellm-rust/crates/config-load/src/include/documents.rs b/litellm-rust/crates/config-load/src/include/documents.rs new file mode 100644 index 00000000000..6dad6b39e65 --- /dev/null +++ b/litellm-rust/crates/config-load/src/include/documents.rs @@ -0,0 +1,39 @@ +use std::collections::{BTreeSet, VecDeque}; + +use serde_yaml_ng::{Mapping, Value}; + +use super::Store; +use crate::Error; + +#[derive(Debug)] +pub struct Document { + pub key: K, + pub body: Mapping, +} + +pub fn documents(store: &S, root: S::Key) -> Result>, Error> { + let mut pending = VecDeque::from([root.clone()]); + let mut loaded = BTreeSet::new(); + let mut documents = Vec::new(); + while let Some(key) = pending.pop_front() { + if !loaded.insert(key.clone()) { + continue; + } + let (body, includes) = read(store, &key).map_err(|error| Error::document(&key, error))?; + for entry in includes { + pending.push_back(store.locate(&entry, &key, &root)?); + } + documents.push(Document { key, body }); + } + Ok(documents) +} + +fn read(store: &S, key: &S::Key) -> Result<(Mapping, Vec), Error> { + let mut body = litellm_config::yaml::mapping(&store.read(key)?)?; + let includes = body + .remove(Value::from("include")) + .map(serde_yaml_ng::from_value) + .transpose()? + .unwrap_or_default(); + Ok((body, includes)) +} diff --git a/litellm-rust/crates/config-load/src/include/merge.rs b/litellm-rust/crates/config-load/src/include/merge.rs new file mode 100644 index 00000000000..b319b6e7558 --- /dev/null +++ b/litellm-rust/crates/config-load/src/include/merge.rs @@ -0,0 +1,90 @@ +use std::collections::BTreeMap; + +use serde_yaml_ng::{Mapping, Value}; + +use super::Document; + +#[derive(Debug)] +pub struct Merged { + pub document: Mapping, + origins: BTreeMap>, +} + +impl Merged { + /// Maps item `index` of the merged top-level `field` back to its document and its index there. + /// A value that is not a list is item 0. + pub fn locate(&self, field: &str, index: usize) -> Option<(&K, usize)> { + self.origins + .get(field)? + .iter() + .scan(0, |start, (key, len)| { + let run = (key, *start, *len); + *start += len; + Some(run) + }) + .find(|(_, start, len)| (*start..start + len).contains(&index)) + .map(|(key, start, _)| (key, index - start)) + } +} + +pub fn merge(documents: &[Document]) -> Merged { + let fields = documents + .iter() + .flat_map(|document| document.body.keys()) + .fold(Vec::new(), |mut fields, field| { + if !fields.contains(&field) { + fields.push(field); + } + fields + }); + let (document, origins) = fields + .into_iter() + .map(|field| { + let layers: Vec<_> = documents + .iter() + .filter_map(|document| Some((&document.key, document.body.get(field)?))) + .collect(); + (field, combine(&layers)) + }) + .map(|(field, (value, runs))| { + ( + (field.clone(), value), + (field.as_str().map(str::to_owned), runs), + ) + }) + .unzip::<_, _, Mapping, Vec<_>>(); + Merged { + document, + origins: origins + .into_iter() + .filter_map(|(field, runs)| Some((field?, runs))) + .collect(), + } +} + +/// Python's rule (config L16): a list extends a list, anything else replaces. So the merged value is the +/// lists after the last non-list concatenated, or that non-list alone when nothing follows it. +fn combine(layers: &[(&K, &Value)]) -> (Value, Vec<(K, usize)>) { + let tail = layers + .iter() + .rposition(|(_, value)| !value.is_sequence()) + .map_or(0, |last| last + 1); + match &layers[tail..] { + [] => { + let (key, value) = layers[layers.len() - 1]; + (value.clone(), vec![(key.clone(), 1)]) + } + lists => ( + Value::Sequence( + lists + .iter() + .flat_map(|(_, value)| value.as_sequence().into_iter().flatten().cloned()) + .collect(), + ), + lists + .iter() + .map(|(key, value)| ((*key).clone(), value.as_sequence().map_or(0, Vec::len))) + .collect(), + ), + } +} diff --git a/litellm-rust/crates/config-load/src/include/mod.rs b/litellm-rust/crates/config-load/src/include/mod.rs new file mode 100644 index 00000000000..8eec7c42e86 --- /dev/null +++ b/litellm-rust/crates/config-load/src/include/mod.rs @@ -0,0 +1,169 @@ +mod documents; +mod merge; +mod store; + +pub use documents::{Document, documents}; +pub use merge::{Merged, merge}; +pub use store::{Files, Store}; + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use rstest::rstest; + use serde_yaml_ng::Value; + + use super::*; + use crate::Error; + + struct Memory(BTreeMap<&'static str, &'static str>); + + impl From<[(&'static str, &'static str); N]> for Memory { + fn from(files: [(&'static str, &'static str); N]) -> Self { + Self(files.into()) + } + } + + impl Store for Memory { + type Key = &'static str; + + fn locate( + &self, + entry: &str, + _: &&'static str, + _: &&'static str, + ) -> Result<&'static str, Error> { + self.0 + .get_key_value(entry) + .map(|(key, _)| *key) + .ok_or_else(|| { + Error::document( + &entry, + std::io::Error::from(std::io::ErrorKind::NotFound).into(), + ) + }) + } + + fn read(&self, key: &&'static str) -> Result { + Ok(self.0[key].to_owned()) + } + } + + fn load(files: impl Into) -> Result, Error> { + documents(&files.into(), "root").map(|documents| merge(&documents)) + } + + fn yaml(text: &str) -> Value { + serde_yaml_ng::from_str(text).unwrap() + } + + #[rstest] + fn reads_each_document_once_in_breadth_first_order() { + let order = documents( + &Memory::from([ + ("root", "include: [a, b]"), + ("a", "include: [c, root]"), + ("b", "include: [a]"), + ("c", "{}"), + ]), + "root", + ) + .unwrap() + .into_iter() + .map(|document| document.key) + .collect::>(); + assert_eq!(order, ["root", "a", "b", "c"]); + } + + #[rstest] + #[case::lists_concatenate("x: [1]", "x: [2]", "[1, 2]")] + #[case::mappings_replace_whole("x: {a: 1, b: 2}", "x: {c: 3}", "{c: 3}")] + #[case::scalar_replaces_list("x: [1]", "x: 2", "2")] + #[case::list_replaces_scalar("x: 1", "x: [2]", "[2]")] + fn merges_top_level_keys_like_python( + #[case] root: &'static str, + #[case] other: &'static str, + #[case] expected: &str, + ) { + let root = Box::leak(format!("include: [other]\n{root}").into_boxed_str()); + let merged = load([("root", &*root), ("other", other)]).unwrap(); + assert_eq!(merged.document.get("x"), Some(&yaml(expected))); + assert!(!merged.document.contains_key("include")); + } + + #[rstest] + fn keeps_concatenating_after_a_replace() { + let merged = load([ + ("root", "include: [a, b, c]\nx: [1]"), + ("a", "x: 2"), + ("b", "x: [3]"), + ("c", "x: [4, 5]"), + ]) + .unwrap(); + assert_eq!(merged.document.get("x"), Some(&yaml("[3, 4, 5]"))); + assert_eq!(merged.locate("x", 0), Some((&"b", 0))); + assert_eq!(merged.locate("x", 2), Some((&"c", 1))); + assert_eq!(merged.locate("x", 3), None); + } + + #[rstest] + fn locates_list_items_and_replaced_values_in_their_documents() { + let merged = load([ + ( + "root", + "include: [a]\nmodel_list: [r0, r1]\ngeneral_settings: {k: 1}\nonly_root: 1", + ), + ("a", "model_list: [a0]\ngeneral_settings: {k: 2}"), + ]) + .unwrap(); + assert_eq!(merged.locate("model_list", 1), Some((&"root", 1))); + assert_eq!(merged.locate("model_list", 2), Some((&"a", 0))); + assert_eq!(merged.locate("general_settings", 0), Some((&"a", 0))); + assert_eq!(merged.locate("only_root", 0), Some((&"root", 0))); + assert_eq!(merged.locate("missing", 0), None); + } + + #[rstest] + fn names_the_missing_include() { + let error = load([("root", "include: [gone.yaml]")]).unwrap_err(); + assert!( + matches!(error, Error::Document { ref key, .. } if key.contains("gone.yaml")), + "{error:?}" + ); + } + + #[rstest] + #[case::bare_string("include: a.yaml")] + #[case::number_item("include: [1]")] + fn rejects_include_that_is_not_a_list_of_paths(#[case] root: &'static str) { + assert!(load([("root", root)]).is_err()); + } + + #[rstest] + #[case::declaring_directory_wins(&["nested/x.yaml", "x.yaml"], "nested/x.yaml")] + #[case::falls_back_to_root_directory(&["x.yaml"], "x.yaml")] + #[case::reports_declaring_path_when_neither_exists(&[], "nested/x.yaml")] + fn files_resolve_against_declaring_then_root_directory( + #[case] existing: &[&str], + #[case] expected: &str, + ) { + let directory = tempfile::TempDir::new().unwrap(); + let root = directory.path().canonicalize().unwrap(); + std::fs::create_dir(root.join("nested")).unwrap(); + for file in ["config.yaml", "nested/first.yaml"].iter().chain(existing) { + std::fs::write(root.join(file), "{}").unwrap(); + } + let located = Files.locate( + "x.yaml", + &root.join("nested/first.yaml"), + &root.join("config.yaml"), + ); + match located { + Ok(path) => assert_eq!(path, root.join(expected)), + Err(Error::Document { key, .. }) if !existing.contains(&expected) => { + assert!(key.contains(&*root.join(expected).to_string_lossy())) + } + Err(error) => panic!("{error:?}"), + } + } +} diff --git a/litellm-rust/crates/config-load/src/include/store.rs b/litellm-rust/crates/config-load/src/include/store.rs new file mode 100644 index 00000000000..af4d5a01b51 --- /dev/null +++ b/litellm-rust/crates/config-load/src/include/store.rs @@ -0,0 +1,46 @@ +use std::{ + fmt, + path::{Path, PathBuf}, +}; + +use crate::Error; + +pub trait Store { + type Key: Ord + Clone + fmt::Debug; + + fn locate( + &self, + entry: &str, + declaring: &Self::Key, + root: &Self::Key, + ) -> Result; + + fn read(&self, key: &Self::Key) -> Result; +} + +pub struct Files; + +impl Store for Files { + type Key = PathBuf; + + fn locate(&self, entry: &str, declaring: &PathBuf, root: &PathBuf) -> Result { + let declared = sibling(declaring, entry); + let fallback = sibling(root, entry); + let location = if declared.exists() || !fallback.exists() { + declared + } else { + fallback + }; + location + .canonicalize() + .map_err(|error| Error::document(&location, error.into())) + } + + fn read(&self, key: &PathBuf) -> Result { + Ok(std::fs::read_to_string(key)?) + } +} + +fn sibling(file: &Path, entry: &str) -> PathBuf { + file.parent().unwrap_or(Path::new(".")).join(entry) +} diff --git a/litellm-rust/crates/config-load/src/lib.rs b/litellm-rust/crates/config-load/src/lib.rs new file mode 100644 index 00000000000..62f1ed876b5 --- /dev/null +++ b/litellm-rust/crates/config-load/src/lib.rs @@ -0,0 +1,65 @@ +mod error; +pub mod include; +mod transport; + +use std::{collections::BTreeMap, ffi::OsString, path::Path}; + +use litellm_config::Config; +use litellm_core_utils::settings::Lookup; + +pub use self::{ + error::Error, + transport::{SslVerify, Transport}, +}; + +pub fn load(path: impl AsRef) -> Result { + let path = path.as_ref(); + let root = path + .canonicalize() + .map_err(|error| Error::document(&path, error.into()))?; + load_from(&include::Files, root) +} + +pub fn load_from(store: &S, root: S::Key) -> Result { + Ok(Config::from_mapping( + include::merge(&include::documents(store, root)?).document, + )?) +} + +#[derive(Clone, Default)] +pub struct Environment(BTreeMap); + +impl Environment { + pub fn from_os(variables: impl IntoIterator) -> Self { + variables + .into_iter() + .filter_map(|(name, value)| Some((name.into_string().ok()?, value.into_string().ok()?))) + .collect() + } + + pub fn overlaid(self, higher: impl IntoIterator) -> Self { + Self(self.0.into_iter().chain(higher).collect()) + } + + pub fn var(&self, name: &str) -> Option<&str> { + self.0.get(name).map(String::as_str) + } + + pub fn variables(&self) -> impl Iterator + '_ { + self.0 + .iter() + .map(|(name, value)| (name.clone(), value.clone())) + } +} + +impl FromIterator<(String, String)> for Environment { + fn from_iter>(variables: I) -> Self { + Self(variables.into_iter().collect()) + } +} + +impl Lookup for Environment { + fn get(&self, name: &str) -> Option { + self.0.get(name).cloned() + } +} diff --git a/litellm-rust/crates/config-load/src/transport.rs b/litellm-rust/crates/config-load/src/transport.rs new file mode 100644 index 00000000000..ba1f95b8296 --- /dev/null +++ b/litellm-rust/crates/config-load/src/transport.rs @@ -0,0 +1,166 @@ +use std::path::PathBuf; + +use litellm_core_utils::{serde_compat::parse_str_bool, settings::Lookup}; + +use litellm_config::{LiteLlmSettings, Spelled}; + +use crate::Environment; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum SslVerify { + #[default] + Enabled, + Disabled, + CaBundle(PathBuf), +} + +impl SslVerify { + fn parse(value: &str) -> Self { + match parse_str_bool(value) { + Some(true) => Self::Enabled, + Some(false) => Self::Disabled, + None => Self::CaBundle(PathBuf::from(value)), + } + } + + fn written(value: &Spelled) -> Self { + match value { + Spelled::Value(true) => Self::Enabled, + Spelled::Value(false) => Self::Disabled, + Spelled::Text(text) => Self::parse(text), + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Transport { + pub ssl_verify: SslVerify, + pub ssl_certificate: Option, + pub ssl_security_level: Option, + pub ssl_ecdh_curve: Option, + pub force_ipv4: bool, + pub http2: bool, + pub aiohttp_trust_env: bool, + pub disable_aiohttp_trust_env: bool, + pub disable_aiohttp_transport: bool, +} + +impl Transport { + pub fn lower(settings: &LiteLlmSettings, environment: &Environment) -> Self { + let text = |name: &str, written: &Option| { + environment + .get(name) + .or_else(|| written.clone()) + .filter(|value| !value.is_empty()) + }; + let flag = |name: &str, written: Option| { + environment.enabled(name).or(written).unwrap_or(false) + }; + Self { + ssl_verify: environment + .get("SSL_VERIFY") + .map(|value| SslVerify::parse(&value)) + .or_else(|| settings.tls.ssl_verify.as_ref().map(SslVerify::written)) + .unwrap_or_default(), + ssl_certificate: environment + .get("SSL_CERTIFICATE") + .or_else(|| settings.client_certificate.ssl_certificate.clone()) + .map(PathBuf::from), + ssl_security_level: text("SSL_SECURITY_LEVEL", &settings.tls.ssl_security_level), + ssl_ecdh_curve: text("SSL_ECDH_CURVE", &settings.tls.ssl_ecdh_curve), + force_ipv4: settings.socket.force_ipv4.unwrap_or(false), + http2: flag("LITELLM_HTTP2", settings.http2.http2), + aiohttp_trust_env: flag("AIOHTTP_TRUST_ENV", settings.aiohttp.aiohttp_trust_env), + disable_aiohttp_trust_env: flag( + "DISABLE_AIOHTTP_TRUST_ENV", + settings.aiohttp.disable_aiohttp_trust_env, + ), + disable_aiohttp_transport: flag( + "DISABLE_AIOHTTP_TRANSPORT", + settings.aiohttp.disable_aiohttp_transport, + ), + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use rstest::rstest; + + use super::{SslVerify, Transport}; + use litellm_config::Config; + + use crate::Environment; + + fn lowered(yaml: &str, variables: &[(&str, &str)]) -> Transport { + let config = Config::from_yaml(yaml).unwrap(); + let environment = variables + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + Transport::lower(&config.litellm_settings, &environment) + } + + #[rstest] + fn absent_everywhere_is_the_default() { + assert_eq!(lowered("{}", &[]), Transport::default()); + } + + #[rstest] + #[case::default("{}", &[], SslVerify::Enabled)] + #[case::written("litellm_settings: {ssl_verify: false}", &[], SslVerify::Disabled)] + #[case::written_bundle("litellm_settings: {ssl_verify: /etc/roots.pem}", &[], SslVerify::CaBundle(PathBuf::from("/etc/roots.pem")))] + #[case::environment("{}", &[("SSL_VERIFY", "False")], SslVerify::Disabled)] + #[case::environment_over_file("litellm_settings: {ssl_verify: false}", &[("SSL_VERIFY", "true")], SslVerify::Enabled)] + fn ssl_verify_takes_environment_then_file_then_default( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: SslVerify, + ) { + assert_eq!(lowered(yaml, variables).ssl_verify, expected); + } + + #[rstest] + #[case::file("litellm_settings: {http2: true}", &[], true)] + #[case::environment("{}", &[("LITELLM_HTTP2", "TRUE")], true)] + #[case::environment_false_does_not_override_file("litellm_settings: {http2: true}", &[("LITELLM_HTTP2", "false")], true)] + fn flags_are_only_turned_on_by_the_environment( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: bool, + ) { + assert_eq!(lowered(yaml, variables).http2, expected); + } + + #[rstest] + #[case::file("litellm_settings: {ssl_security_level: '1'}", &[], Some("1"))] + #[case::environment_over_file("litellm_settings: {ssl_security_level: '1'}", &[("SSL_SECURITY_LEVEL", "2")], Some("2"))] + #[case::empty_is_unset("{}", &[("SSL_SECURITY_LEVEL", "")], None)] + fn text_settings_skip_empty_values( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: Option<&str>, + ) { + assert_eq!( + lowered(yaml, variables).ssl_security_level.as_deref(), + expected + ); + } + + #[rstest] + fn later_overlay_wins_and_unrelated_variables_are_ignored() { + let environment = Environment::from_iter([("SSL_VERIFY".to_owned(), "true".to_owned())]) + .overlaid([ + ("SSL_VERIFY".to_owned(), "false".to_owned()), + ("HTTP2".to_owned(), "true".to_owned()), + ]); + let transport = Transport::lower( + &Config::from_yaml("{}").unwrap().litellm_settings, + &environment, + ); + assert_eq!(transport.ssl_verify, SslVerify::Disabled); + assert!(!transport.http2); + } +} diff --git a/litellm-rust/crates/config-load/tests/AGENTS.md b/litellm-rust/crates/config-load/tests/AGENTS.md new file mode 100644 index 00000000000..e6484b746c9 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/AGENTS.md @@ -0,0 +1,5 @@ +## Fixtures + +- `corpus.rs` requires every config in `fixtures/valid/` to load and every config in `fixtures/invalid/` to fail. There is no known-gap list, so a valid config that fails to load is fixed in `src/` +- A config goes in `invalid/` only when Python rejects it too, either by raising on read or by crashing when the proxy loads it +- `#[files]` globs at compile time, so `build.rs` reruns the build when `fixtures/` changes diff --git a/litellm-rust/crates/config-load/tests/corpus.rs b/litellm-rust/crates/config-load/tests/corpus.rs new file mode 100644 index 00000000000..ffaf8864c05 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/corpus.rs @@ -0,0 +1,16 @@ +use std::path::PathBuf; + +use litellm_config_load::load; +use rstest::rstest; + +#[rstest] +fn loads_every_valid_fixture(#[files("tests/fixtures/valid/*.yaml")] path: PathBuf) { + if let Err(error) = load(&path) { + panic!("{}: {error:?}", path.display()); + } +} + +#[rstest] +fn rejects_every_invalid_fixture(#[files("tests/fixtures/invalid/*.yaml")] path: PathBuf) { + assert!(load(&path).is_err(), "{} loads", path.display()); +} diff --git a/litellm-rust/crates/config-load/tests/fixtures/AGENTS.md b/litellm-rust/crates/config-load/tests/fixtures/AGENTS.md new file mode 100644 index 00000000000..6ad1ed3978d --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/AGENTS.md @@ -0,0 +1,3 @@ +These configs are copied as-is from `tests/unit/proxy/example_config_yaml` and `tests/unit/proxy/test_configs` for now. Keep each of `valid/` and `invalid/` flat so relative `include` paths still resolve + +`invalid/config_with_missing_include.yaml` is in Python's `INVALID_FILES`. `invalid/langfuse_config.yaml` passes Python's read-only `get_config` test, but its model has no `litellm_params`, so Python's `load_config` raises `KeyError` on it diff --git a/litellm-rust/crates/config-load/tests/fixtures/invalid/config_with_missing_include.yaml b/litellm-rust/crates/config-load/tests/fixtures/invalid/config_with_missing_include.yaml new file mode 100644 index 00000000000..40d3e9e7f8d --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/invalid/config_with_missing_include.yaml @@ -0,0 +1,5 @@ +include: + - non-existent-file.yaml + +litellm_settings: + callbacks: ["prometheus"] \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/invalid/langfuse_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/invalid/langfuse_config.yaml new file mode 100644 index 00000000000..c2a77b5adc4 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/invalid/langfuse_config.yaml @@ -0,0 +1,7 @@ +model_list: + - model_name: gpt-3.5-turbo + +litellm_settings: + drop_params: True + success_callback: ["langfuse"] # https://docs.litellm.ai/docs/observability/langfuse_integration + diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/aliases_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/aliases_config.yaml new file mode 100644 index 00000000000..43681f64ba0 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/aliases_config.yaml @@ -0,0 +1,30 @@ +model_list: + - model_name: gpt-3.5-turbo-instruct + litellm_params: + model: ollama/zephyr + - model_name: gpt-4 + litellm_params: + model: ollama/llama2 + - model_name: gpt-3.5-turbo + litellm_params: + model: ollama/llama2 + temperature: 0.1 + max_tokens: 20 + + +# request to gpt-4, response from ollama/llama2 +# curl --location 'http://0.0.0.0:8000/chat/completions' \ +# --header 'Content-Type: application/json' \ +# --data ' { +# "model": "gpt-4", +# "messages": [ +# { +# "role": "user", +# "content": "what llm are you" +# } +# ], +# } +# ' +# + +# {"id":"chatcmpl-27c85cf0-ab09-4bcf-8cb1-0ee950520743","choices":[{"finish_reason":"stop","index":0,"message":{"content":" Hello! I'm just an AI, I don't have personal experiences or emotions like humans do. However, I can help you with any questions or tasks you may have! Is there something specific you'd like to know or discuss?","role":"assistant","_logprobs":null}}],"created":1700094955.373751,"model":"ollama/llama2","object":"chat.completion","system_fingerprint":null,"usage":{"prompt_tokens":12,"completion_tokens":47,"total_tokens":59},"_response_ms":8028.017999999999}% \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/azure_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/azure_config.yaml new file mode 100644 index 00000000000..05ba0c9bf54 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/azure_config.yaml @@ -0,0 +1,15 @@ +model_list: + - model_name: gpt-4-team1 + litellm_params: + model: azure/gpt-4.1-mini + api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ + api_version: "2023-05-15" + api_key: os.environ/AZURE_AI_API_KEY + tpm: 20_000 + - model_name: gpt-4-team2 + litellm_params: + model: azure/gpt-4 + api_key: os.environ/AZURE_AI_API_KEY + api_base: https://openai-gpt-4-test-v-2.openai.azure.com/ + tpm: 100_000 + diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/cache_no_params.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/cache_no_params.yaml new file mode 100644 index 00000000000..20ed919dd56 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/cache_no_params.yaml @@ -0,0 +1,7 @@ +model_list: + - model_name: "openai-model" + litellm_params: + model: "gpt-3.5-turbo" + +litellm_settings: + cache: True diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/cache_with_params.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/cache_with_params.yaml new file mode 100644 index 00000000000..068e2cc4a2b --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/cache_with_params.yaml @@ -0,0 +1,11 @@ +model_list: + - model_name: "openai-model" + litellm_params: + model: "gpt-3.5-turbo" + +litellm_settings: + cache: True + cache_params: + type: "redis" + supported_call_types: ["embedding", "aembedding"] + host: "os.environ/REDIS_HOST" \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_env_vars.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_env_vars.yaml new file mode 100644 index 00000000000..bae738c735d --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_env_vars.yaml @@ -0,0 +1,48 @@ +model_list: + ################################################################################ + # Azure + - model_name: gpt-4o-mini + litellm_params: + model: azure/gpt-4o-mini + api_base: https://amazin-prod.openai.azure.com + api_key: "os.environ/AZURE_GPT_4O" + deployment_id: gpt-4o-mini + - model_name: gpt-4o + litellm_params: + model: azure/gpt-4o + api_base: https://very-cool-prod.openai.azure.com + api_key: "os.environ/AZURE_GPT_4O" + deployment_id: gpt-4o + + ################################################################################ + # Fireworks + - model_name: fireworks-llama-v3p1-405b-instruct + litellm_params: + model: fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct + api_key: "os.environ/FIREWORKS" + - model_name: fireworks-llama-v3p1-70b-instruct + litellm_params: + model: fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct + api_key: "os.environ/FIREWORKS" + +general_settings: + alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+ +litellm_settings: # module level litellm settings - https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py + success_callback: ["prometheus"] + service_callback: ["prometheus_system"] + drop_params: False # Raise an exception if the openai param being passed in isn't supported. + cache: false + default_internal_user_params: + user_role: os.environ/DEFAULT_USER_ROLE + + success_callback: ["s3"] + s3_callback_params: + s3_bucket_name: logs-bucket-litellm # AWS Bucket Name for S3 + s3_region_name: us-west-2 # AWS Region Name for S3 + s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/ to pass environment variables. This is AWS Access Key ID for S3 + s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3 + s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to + s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets + +router_settings: + routing_strategy: simple-shuffle # "simple-shuffle" shown to result in highest throughput. https://docs.litellm.ai/docs/proxy/configs#load-balancing diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_include.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_include.yaml new file mode 100644 index 00000000000..0a0c9434b07 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_include.yaml @@ -0,0 +1,5 @@ +include: + - included_models.yaml + +litellm_settings: + callbacks: ["prometheus"] \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_multiple_includes.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_multiple_includes.yaml new file mode 100644 index 00000000000..c46adacd737 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_multiple_includes.yaml @@ -0,0 +1,6 @@ +include: + - models_file_1.yaml + - models_file_2.yaml + +litellm_settings: + callbacks: ["prometheus"] \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/included_models.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/included_models.yaml new file mode 100644 index 00000000000..c1526b203a6 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/included_models.yaml @@ -0,0 +1,4 @@ +model_list: + - model_name: included-model + litellm_params: + model: gpt-4 \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/load_balancer.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/load_balancer.yaml new file mode 100644 index 00000000000..502b90ff92a --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/load_balancer.yaml @@ -0,0 +1,28 @@ +litellm_settings: + drop_params: True + +# Model-specific settings +model_list: # use the same model_name for using the litellm router. LiteLLM will use the router between gpt-3.5-turbo + - model_name: gpt-3.5-turbo # litellm will + litellm_params: + model: gpt-3.5-turbo + api_key: sk-uj6F + tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm + rpm: 3 # [OPTIONAL] REPLACE with your openai rpm + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + api_key: sk-Imn + tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm + rpm: 3 # [OPTIONAL] REPLACE with your openai rpm + - model_name: gpt-3.5-turbo + litellm_params: + model: openrouter/gpt-3.5-turbo + - model_name: mistral-7b-instruct + litellm_params: + model: mistralai/mistral-7b-instruct + +environment_variables: + REDIS_HOST: localhost + REDIS_PASSWORD: + REDIS_PORT: \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_1.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_1.yaml new file mode 100644 index 00000000000..344f67128fb --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_1.yaml @@ -0,0 +1,4 @@ +model_list: + - model_name: included-model-1 + litellm_params: + model: gpt-4 \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_2.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_2.yaml new file mode 100644 index 00000000000..56bc3b1aa08 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_2.yaml @@ -0,0 +1,4 @@ +model_list: + - model_name: included-model-2 + litellm_params: + model: gpt-3.5-turbo \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/opentelemetry_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/opentelemetry_config.yaml new file mode 100644 index 00000000000..92d3454d7d5 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/opentelemetry_config.yaml @@ -0,0 +1,7 @@ +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + +general_settings: + otel: True # OpenTelemetry Logger this logs OTEL data to your collector diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/simple_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/simple_config.yaml new file mode 100644 index 00000000000..14b39a12518 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/simple_config.yaml @@ -0,0 +1,4 @@ +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_bad_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_bad_config.yaml new file mode 100644 index 00000000000..72ceefcda2d --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_bad_config.yaml @@ -0,0 +1,20 @@ +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + api_key: bad-key + model: gpt-3.5-turbo + - model_name: working-azure-gpt-3.5-turbo + litellm_params: + model: azure/gpt-4.1-mini + api_base: https://azure-unit.example.com + api_key: bad-key + - model_name: azure-gpt-3.5-turbo + litellm_params: + model: azure/gpt-4.1-mini + api_base: https://azure-unit.example.com + api_key: bad-key + - model_name: azure-embedding + litellm_params: + model: azure/text-embedding-ada-002 + api_base: https://azure-unit.example.com + api_key: bad-key diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_cloudflare_azure_with_cache_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_cloudflare_azure_with_cache_config.yaml new file mode 100644 index 00000000000..24240008fe2 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_cloudflare_azure_with_cache_config.yaml @@ -0,0 +1,17 @@ +model_list: + - model_name: azure-cloudflare + litellm_params: + model: azure/gpt-4.1-mini + api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1 + api_key: os.environ/AZURE_AI_API_KEY + api_version: 2023-07-01-preview + +litellm_settings: + set_verbose: True + cache: True # set cache responses to True + cache_params: # set cache params for s3 + type: s3 + s3_bucket_name: litellm-my-test-bucket-2 # AWS Bucket Name for S3 + s3_region_name: us-east-1 # AWS Region Name for S3 + s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # AWS Access Key ID for S3 + s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3 \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config.yaml new file mode 100644 index 00000000000..a711b65ea03 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config.yaml @@ -0,0 +1,28 @@ +general_settings: + database_url: os.environ/DATABASE_URL + master_key: os.environ/PROXY_MASTER_KEY +litellm_settings: + drop_params: true + success_callback: ["langfuse"] + +model_list: +- litellm_params: + api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ + api_key: os.environ/AZURE_EUROPE_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://my-endpoint-canada-berri992.openai.azure.com + api_key: os.environ/AZURE_CANADA_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://openai-france-1234.openai.azure.com + api_key: os.environ/AZURE_FRANCE_API_KEY + model: azure/gpt-turbo + model_name: azure-model +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + model_name: test_openai_models diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_custom_auth.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_custom_auth.yaml new file mode 100644 index 00000000000..33088bd1ce1 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_custom_auth.yaml @@ -0,0 +1,11 @@ +model_list: + - model_name: "openai-model" + litellm_params: + model: "gpt-3.5-turbo" + +litellm_settings: + drop_params: True + set_verbose: True + +general_settings: + custom_auth: custom_auth.user_api_key_auth \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_hosted_vllm_embedding.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_hosted_vllm_embedding.yaml new file mode 100644 index 00000000000..1b6b9ad198e --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_hosted_vllm_embedding.yaml @@ -0,0 +1,6 @@ +model_list: +- litellm_params: + model: hosted_vllm/embed_model + model_info: + description: this is a test embedding hosted_vllm model + model_name: vllm_embed_model diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_no_auth.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_no_auth.yaml new file mode 100644 index 00000000000..f4896217049 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_no_auth.yaml @@ -0,0 +1,127 @@ +model_list: +- litellm_params: + api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ + api_key: os.environ/AZURE_EUROPE_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://my-endpoint-canada-berri992.openai.azure.com + api_key: os.environ/AZURE_CANADA_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1 + api_key: os.environ/AZURE_AI_API_KEY + model: azure/gpt-4.1-mini + model_name: azure-cloudflare-model +- litellm_params: + api_base: https://openai-france-1234.openai.azure.com + api_key: os.environ/AZURE_FRANCE_API_KEY + model: azure/gpt-turbo + model_name: azure-model +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 56f1bd94-3b54-4b67-9ea2-7c70e9a3a709 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 4d1ee26c-abca-450c-8744-8e87fd6755e9 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 00e19c0f-b63d-42bb-88e9-016fb0c60764 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 79fc75bf-8e1b-47d5-8d24-9365a854af03 + model_name: test_openai_models +- litellm_params: + api_base: os.environ/AZURE_AI_API_BASE + api_key: os.environ/AZURE_AI_API_KEY + api_version: 2023-07-01-preview + model: azure/text-embedding-ada-002 + model_info: + mode: embedding + model_name: azure-embedding-model +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 55848c55-4162-40f9-a6e2-9a722b9ef404 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 34339b1e-e030-4bcc-a531-c48559f10ce4 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: f6f74e14-ac64-4403-9365-319e584dcdc5 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 9b1ef341-322c-410a-8992-903987fef439 + model_name: test_openai_models +- litellm_params: + model: dall-e-3 + model_info: + mode: image_generation + model_name: dall-e-3 +- litellm_params: + api_base: os.environ/AZURE_SWEDEN_API_BASE + api_key: os.environ/AZURE_SWEDEN_API_KEY + api_version: 2023-12-01-preview + model: azure/dall-e-3-test + model_info: + mode: image_generation + model_name: dall-e-3 +- litellm_params: + api_base: os.environ/AZURE_AI_API_BASE + api_key: os.environ/AZURE_AI_API_KEY + api_version: 2023-06-01-preview + model: azure/ + model_info: + mode: image_generation + model_name: dall-e-2 +- litellm_params: + api_base: os.environ/AZURE_AI_API_BASE + api_key: os.environ/AZURE_AI_API_KEY + api_version: 2023-07-01-preview + model: azure/text-embedding-ada-002 + model_info: + base_model: text-embedding-ada-002 + mode: embedding + model_name: text-embedding-ada-002 +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 34cb2419-7c63-44ae-a189-53f1d1ce5953 + model_name: test_openai_models +- litellm_params: + model: amazon.titan-embed-text-v1 + model_name: amazon-embeddings +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 753dca9a-898d-4ff7-9961-5acf7cdf38cf + model_name: test_openai_models diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_custom_logger.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_custom_logger.yaml new file mode 100644 index 00000000000..c58df01b602 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_custom_logger.yaml @@ -0,0 +1,21 @@ +model_list: + - model_name: Azure OpenAI GPT-4 Canada + litellm_params: + model: gpt-4.1-nano + api_key: sk-unit-fake + model_info: + mode: chat + input_cost_per_token: 0.0002 + id: gm + - model_name: azure-embedding-model + litellm_params: + model: text-embedding-ada-002 + api_key: sk-unit-fake + model_info: + mode: embedding + input_cost_per_token: 0.002 + id: hello + +litellm_settings: + drop_params: True + callbacks: custom_callbacks.my_custom_logger diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_guardrails_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_guardrails_config.yaml new file mode 100644 index 00000000000..f09ff9d1bc4 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_guardrails_config.yaml @@ -0,0 +1,32 @@ + + +model_list: +- litellm_params: + api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ + api_key: os.environ/AZURE_EUROPE_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://my-endpoint-canada-berri992.openai.azure.com + api_key: os.environ/AZURE_CANADA_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://openai-france-1234.openai.azure.com + api_key: os.environ/AZURE_FRANCE_API_KEY + model: azure/gpt-turbo + model_name: azure-model + + + +litellm_settings: + guardrails: + - prompt_injection: + callbacks: [lakera_prompt_injection, detect_prompt_injection] + default_on: true + - hide_secrets: + callbacks: [hide_secrets] + default_on: true + - moderations: + callbacks: [openai_moderations] + default_on: false \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/load.rs b/litellm-rust/crates/config-load/tests/load.rs new file mode 100644 index 00000000000..60772507861 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/load.rs @@ -0,0 +1,111 @@ +use std::fs; + +use litellm_config_load::{Error, load}; +use rstest::{fixture, rstest}; +use tempfile::TempDir; + +#[fixture] +fn directory() -> TempDir { + tempfile::tempdir().unwrap() +} + +#[rstest] +fn loads_model_list_from_file(directory: TempDir) { + let path = directory.path().join("config.yaml"); + fs::write( + &path, + r#" +model_list: + - model_name: assistant + litellm_params: + model: anthropic/test-model + - model_name: local + litellm_params: + model: test-model +"#, + ) + .unwrap(); + + let config = load(path).unwrap(); + assert_eq!( + config + .model_list + .iter() + .map(|model| model.model_name.as_str()) + .collect::>(), + ["assistant", "local"] + ); +} + +#[rstest] +#[case::missing_file(None, |error: &Error| matches!(error, Error::Read(error) if error.kind() == std::io::ErrorKind::NotFound))] +#[case::malformed_yaml(Some("model_list: ["), |error: &Error| matches!(error, Error::Parse(_)))] +fn distinguishes_read_errors_from_parse_errors( + directory: TempDir, + #[case] contents: Option<&str>, + #[case] expected: fn(&Error) -> bool, +) { + let path = directory.path().join("config.yaml"); + if let Some(contents) = contents { + fs::write(&path, contents).unwrap(); + } + + let error = load(&path).unwrap_err(); + let Error::Document { source, .. } = &error else { + panic!("{error:?}"); + }; + assert!(expected(source), "{error:?}"); +} + +#[rstest] +fn resolves_nested_includes_once_in_breadth_first_order(directory: TempDir) { + let root = directory.path(); + fs::create_dir(root.join("nested")).unwrap(); + fs::write( + root.join("config.yaml"), + r#" +include: [nested/first.yaml, second.yaml] +model_list: [{model_name: root, litellm_params: {model: root}}] +"#, + ) + .unwrap(); + fs::write( + root.join("nested/first.yaml"), + r#" +include: [third.yaml] +model_list: [{model_name: first, litellm_params: {model: first}}] +"#, + ) + .unwrap(); + fs::write( + root.join("second.yaml"), + r#" +general_settings: {master_key: second} +model_list: [{model_name: second, litellm_params: {model: second}}] +"#, + ) + .unwrap(); + fs::write( + root.join("nested/third.yaml"), + r#" +include: [../config.yaml] +general_settings: {master_key: third} +model_list: [{model_name: third, litellm_params: {model: third}}] +"#, + ) + .unwrap(); + + let config = load(root.join("config.yaml")).unwrap(); + assert_eq!( + config + .model_list + .iter() + .map(|model| model.model_name.as_str()) + .collect::>(), + ["root", "first", "second", "third"] + ); + assert_eq!( + config.general_settings.master_key.unwrap().expose(), + "third" + ); +} diff --git a/litellm-rust/crates/config/AGENTS.md b/litellm-rust/crates/config/AGENTS.md new file mode 100644 index 00000000000..310c97bb566 --- /dev/null +++ b/litellm-rust/crates/config/AGENTS.md @@ -0,0 +1,60 @@ +## Scope + +- The crate is the type definition of `config.yaml`, what a user writes and nothing more. Parsing one YAML text into those types (`Config::from_yaml`, `Config::from_mapping`, the PyYAML reading rules in `yaml.rs`) is all it does +- Everything that turns those types into what the gateway runs on lives in `crates/config-load`: resolving `include`, overlaying DB rows, reading the environment, substituting references, applying defaults and building the domain types. So this crate never reads files, the environment or a secret manager +- It knows nothing about which settings a consumer implements or rolls out, so checks like "refuse setting X because it is not supported yet" live in the consuming crate (e.g. `gateway`), never here + +## Defaults + +- The value the existing proxy uses when a key is absent (`health_check_interval` is 300, `database_connection_pool_limit` is 10) is part of what `config.yaml` means, but it is applied in `crates/config-load`, never here. Parsing never fills a default in, so every key here keeps whether the file set it. The proxy treats a key set in the file as owned by the file, so the DB value and Admin UI edits for it are ignored +- Other sections have no DB overlay, so a key there whose default is empty (an empty list or map) can be the plain collection, filled in by `#[serde(default)]` + +## Module layout + +Each top-level `config.yaml` key belongs to exactly one module under `src/`. A module is a single `.rs` file until it has flattened sub-groups, then it becomes a `/` folder with one file per sub-group. This only applies one level down, so groups flattened inside a sub-group (like the cache backends in `litellm/caching.rs`) stay in that sub-group's file. Keys typed as a plain `Object` have no module and stay on `Config` in `lib.rs`. Keep this list in sync when adding a key + +- `model.rs`: `model_list` +- `general/`: `general_settings` +- `router.rs`: `router_settings` +- `litellm/`: `litellm_settings` +- `callback.rs`: `callback_settings` +- `provider.rs`: `assistant_settings`, `default_vertex_config`, `files_settings`, `finetune_settings`, `worker_registry` +- `mcp.rs`: `mcp_servers` +- `credential.rs`: `credential_list` +- `guardrail.rs`: `guardrails` +- `prompt.rs`: `prompts` +- `tool.rs`: `sandbox_tools`, `search_tools` +- `vector_store.rs`: `vector_store_registry` +- `agent.rs`: `agents`, `agent_list` +- `policy.rs`: `policies`, `policy_attachments` +- `lib.rs`: `environment_variables`, `mcp_tools`, `include` + +`error.rs`, `value.rs` and `yaml.rs` are crate infrastructure, not config keys + +## Serde flatten + +Sections split one YAML mapping into several structs with nested `#[serde(flatten)]` (`LitellmSettings` flattens `CachingSettings`, `GeneralSettings` flattens `AuthSettings`, and so on). Serde hands keys to flattened fields like this + +- A flattened struct without a flatten field of its own takes only the keys it declares, and later siblings and the catch-all never see them +- A flattened struct or map that has its own flatten field is read as a map and claims nothing. It sees every key still unclaimed, including keys a later sibling declares, and those keys also reach the outer catch-all +- Fields claim keys in declaration order, so when two flattened structs declare the same key only the first one gets it + +Which gives these rules + +- Each mapping has exactly one `#[serde(flatten)] additional_fields: Object`, declared last on the outermost struct of that mapping. Flattened sub-structs stay plain, with no flatten field inside +- A key belongs to exactly one struct. Which one is decided by who reads it, see `src/AGENTS.md` +- Never combine `deny_unknown_fields` with flatten, on either the outer or the flattened struct. It is fine on a struct that is never flattened and has no flatten fields + +## Inline YAML + +- Write inline YAML as a multi-line raw string (`r#"..."#`), not as a one-line string with `\n` escapes like `"key:\n value"` +- This includes `#[case(...)]` arguments. Open the raw string on the attribute line and start the YAML at column 0, e.g. `#[case::name(r#"` then the YAML lines then `"#)]` +- YAML that fits on one line, like flow-style `key: {a: 1}`, stays a plain string with no trailing `\n` + +## Debug and redaction + +Every struct derives `Debug`, and redaction lives on the field types, so no struct has a hand-written `Debug` + +- Anything that is or can carry a secret (keys, tokens, passwords, URLs that may embed credentials, MCP `command` and `args`) is `SecretValue`, which prints `[REDACTED]` +- YAML we don't model is `Object` (prints its keys) or `Raw` (prints its kind). They are the only hand-written `Debug` impls. Never put a bare `Value` in a field +- Redaction tests put a unique `SENTINEL` under the slot and assert it never shows up in `format!("{config:?}")`. Words like `password` also match field names such as `redis_password` diff --git a/litellm-rust/crates/config/Cargo.toml b/litellm-rust/crates/config/Cargo.toml index 23d53923362..cda0bf53b45 100644 --- a/litellm-rust/crates/config/Cargo.toml +++ b/litellm-rust/crates/config/Cargo.toml @@ -7,7 +7,9 @@ repository.workspace = true [dependencies] litellm-auth-types.workspace = true +litellm-python-compat.workspace = true litellm-router-types.workspace = true +macro_rules_attribute.workspace = true serde.workspace = true serde_yaml_ng = "0.10.0" strum.workspace = true @@ -16,4 +18,3 @@ thiserror.workspace = true [dev-dependencies] rstest.workspace = true serde_json.workspace = true -tempfile.workspace = true diff --git a/litellm-rust/crates/config/src/AGENTS.md b/litellm-rust/crates/config/src/AGENTS.md new file mode 100644 index 00000000000..9c967d7fa1f --- /dev/null +++ b/litellm-rust/crates/config/src/AGENTS.md @@ -0,0 +1,32 @@ +## Grouping lives in the types + +The YAML shape is Python's `config.yaml` and we never change it, so a mapping stays flat even when its keys serve different consumers. The grouping is encoded in the types instead, with flattened structs (see the "Serde flatten" rules in `../AGENTS.md`) + +### Rules + +- Every distinct set of readers gets its own flattened struct, even when only one key has that set. `namespace` in `cache_params` is read only by `redis`, so it gets its own struct and does not sit with `host` and `port`, which `redis-semantic` and `valkey-semantic` also read +- Only keys that every reader of the mapping uses stay on the parent struct. A key that some readers ignore never stays on the parent just because no group fits yet. Make the group +- Once a mapping has keys with different readers, all of them move into groups, never only some. `router_settings` typed as one flat struct, or a parent that keeps keys an existing group already covers, breaks this + +### Finding the readers + +- A reader is the code that receives or branches on the value, found in the Python that consumes the key. The key's name does not tell you. `default_in_redis_ttl` looks Redis-only but Valkey reads it too, and `default_redis_ttl` sits under `cache_params` yet configures the proxy's internal caches, not the response cache +- Follow the value to where it is used, not just to where it is passed. A constructor argument read in one `if type == ...` arm has that arm as its reader. A value applied behind an `isinstance` guard is read by that class and its subclasses only (`namespace` reaches `RedisCache` and `RedisClusterCache`, not `RedisSemanticCache`) +- A key read by the shared code path for every variant (e.g. `ttl` and `mode`, read by `Cache` itself) is read by all of them and stays on the parent + +### Naming + +- Name each struct for its reader set (`RedisServerCacheParams`, `SemanticCacheParams`), and name the parent's flattened field for the same set (`redis_server`, `semantic`) +- When the name alone does not say who reads the keys, add a one-line doc comment on the struct that lists the readers, like the one on `RedisServerCacheParams` + +### Tests + +- Each mapping has one rstest with a case per group, putting one key of that group in the YAML and asserting it lands in that group's field and not in `additional_fields` +- When a key's readers were easy to get wrong (the examples above), add a case for that key too, so moving it back into a broader group fails a test + +## Lists are boxed slices + +Config is parsed once and only read after that, so every list field on a config type is `Box<[T]>` (or `Option>`), never `Vec`. The type then says the list can't grow, and serde deserializes into it directly + +- `Vec` is only for values that get built up and thrown away, like the layers and origins in `include/`. Those never become config fields +- Code that builds a field by hand collects into a `Vec` and converts at the end (`Box::from`, `.into_boxed_slice()`), and never keeps the `Vec` around diff --git a/litellm-rust/crates/config/src/agent.rs b/litellm-rust/crates/config/src/agent.rs new file mode 100644 index 00000000000..c2de9654557 --- /dev/null +++ b/litellm-rust/crates/config/src/agent.rs @@ -0,0 +1,75 @@ +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Agent { + pub agent_name: Option, + pub agent_card_params: Option, + pub litellm_params: Option, + pub identity: Option, + pub enabled: Option>, + pub execution_mode: Option, + pub object_permission: Option, + pub tpm_limit: Option>, + pub rpm_limit: Option>, + pub session_tpm_limit: Option>, + pub session_rpm_limit: Option>, + pub static_headers: Option, + pub extra_headers: Option>, + pub access_group_ids: Option>, + pub kill_switch: Option, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Spelled}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + #[case::agents("agents")] + #[case::agent_list("agent_list")] + fn agent_entries_keep_typed_limits_and_flags(#[case] section: &str) { + let config = parse(&format!( + "{section}:\n - {{agent_name: reviewer, agent_card_params: {{name: x}}, tpm_limit: 100, enabled: false}}\n" + )); + let agent = config.agents.first().or(config.agent_list.first()).unwrap(); + + assert_eq!(agent.agent_name.as_deref(), Some("reviewer")); + assert!( + agent + .agent_card_params + .as_ref() + .unwrap() + .contains_key("name") + ); + assert_eq!(agent.tpm_limit, Some(Spelled::Value(100))); + assert_eq!(agent.enabled, Some(Spelled::Value(false))); + } + + #[rstest] + #[case::agent_headers( + r#" +agents: + - static_headers: {Authorization: SENTINEL} +"# + )] + #[case::agent_params( + r#" +agents: + - litellm_params: {api_key: SENTINEL} +"# + )] + #[case::agent_identity( + r#" +agent_list: + - identity: {token: SENTINEL} +"# + )] + fn debug_never_prints_agent_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/callback.rs b/litellm-rust/crates/config/src/callback.rs new file mode 100644 index 00000000000..5a6c0ab57f4 --- /dev/null +++ b/litellm-rust/crates/config/src/callback.rs @@ -0,0 +1,77 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct CallbackSettings { + pub callback_type: Option, + pub endpoint: Option, + pub headers: Option, + pub event_types: Option>, + pub log_format: Option, + pub max_retries: Option>, + pub retry_delay: Option>, + pub timeout: Option>, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Spelled}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn callback_settings_keep_typed_fields_and_unknown_keys() { + let config = parse( + "callback_settings:\n hook:\n callback_type: generic_api\n retry_delay: 0.5\n batch_size: 10\n", + ); + let hook = &config.callback_settings["hook"]; + + assert_eq!(hook.callback_type.as_deref(), Some("generic_api")); + assert_eq!(hook.retry_delay, Some(Spelled::Value(0.5))); + assert!(hook.additional_fields.contains_key("batch_size")); + } + + #[rstest] + #[case::env_reference("os.environ/RETRIES", Spelled::Text("os.environ/RETRIES".to_owned()))] + #[case::number("4", Spelled::Value(4))] + fn spelled_int_fields_keep_references_as_text( + #[case] spelled: &str, + #[case] expected: Spelled, + ) { + let config = parse(&format!( + "callback_settings:\n hook:\n max_retries: {spelled}\n" + )); + + assert_eq!(config.callback_settings["hook"].max_retries, Some(expected)); + } + + #[rstest] + #[case::callback_endpoint( + r#" +callback_settings: + hook: {endpoint: SENTINEL} +"# + )] + #[case::callback_headers( + r#" +callback_settings: + hook: {headers: {Authorization: SENTINEL}} +"# + )] + #[case::callback_extra( + r#" +callback_settings: + hook: {vendor_secret: SENTINEL} +"# + )] + fn debug_never_prints_callback_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/credential.rs b/litellm-rust/crates/config/src/credential.rs new file mode 100644 index 00000000000..9db78717e88 --- /dev/null +++ b/litellm-rust/crates/config/src/credential.rs @@ -0,0 +1,65 @@ +use crate::Object; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Credential { + pub credential_name: Option, + pub credential_info: Option, + pub credential_values: Option, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::Config; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn credential_entries_keep_name_info_and_values() { + let config = parse( + r#" +credential_list: + - credential_name: azure-prod + credential_info: {description: prod} + credential_values: {api_key: os.environ/AZURE_API_KEY} +"#, + ); + let credential = &config.credential_list[0]; + + assert_eq!(credential.credential_name.as_deref(), Some("azure-prod")); + assert!( + credential + .credential_info + .as_ref() + .unwrap() + .contains_key("description") + ); + assert!( + credential + .credential_values + .as_ref() + .unwrap() + .contains_key("api_key") + ); + } + + #[rstest] + #[case::credential_values( + r#" +credential_list: + - credential_values: {api_key: SENTINEL} +"# + )] + #[case::credential_info( + r#" +credential_list: + - credential_info: {owner: SENTINEL} +"# + )] + fn debug_never_prints_credential_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/error.rs b/litellm-rust/crates/config/src/error.rs index 61d19491abc..e3db06bfdd2 100644 --- a/litellm-rust/crates/config/src/error.rs +++ b/litellm-rust/crates/config/src/error.rs @@ -1,7 +1,5 @@ #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("could not read config")] - Read(#[from] std::io::Error), #[error("invalid YAML config")] Parse(#[from] serde_yaml_ng::Error), } diff --git a/litellm-rust/crates/config/src/general/AGENTS.md b/litellm-rust/crates/config/src/general/AGENTS.md new file mode 100644 index 00000000000..42578db1ed4 --- /dev/null +++ b/litellm-rust/crates/config/src/general/AGENTS.md @@ -0,0 +1,14 @@ +## Docs + +- https://github.com/BerriAI/litellm-docs/blob/main/docs/proxy/config_settings.md + +## Python + +- `ConfigGeneralSettings` in `litellm/proxy/_types.py` is the Python model but is incomplete. What the proxy reads is `general_settings.get(...)`, mostly in `proxy_server.py` +- `general_settings` is one of the sections the proxy persists to the DB (`_CONFIG_PERSISTED_SECTIONS`) and the Admin UI edits, so at runtime the YAML is only one source of the effective values. Merging the DB overlay is the consumer's job, not this crate's + +## Fields that need care + +- `master_key`, `database_url` and `drain_endpoint_token` fall back to `LITELLM_MASTER_KEY`, `DATABASE_URL` and `DRAIN_ENDPOINT_TOKEN`. That fallback belongs to the consumer, so the fields stay optional here +- `custom_auth`, `custom_key_generate`, `custom_key_update`, `custom_key_policy`, `custom_team_metadata_validate`, `custom_sso`, `custom_ui_sso_sign_in_handler`, `litellm_jwtauth.custom_validate` and `pass_through_endpoints[].target` are Python module paths that the proxy imports. Type them as plain strings and never resolve or validate them here +- `key_management_system` and `key_management_settings` configure the secret manager itself, so they can only use `os.environ/` references, not secret manager ones diff --git a/litellm-rust/crates/config/src/general/alerting.rs b/litellm-rust/crates/config/src/general/alerting.rs new file mode 100644 index 00000000000..66aed7441e4 --- /dev/null +++ b/litellm-rust/crates/config/src/general/alerting.rs @@ -0,0 +1,39 @@ +use serde::Deserialize; + +use crate::{Object, Spelled}; + +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum Alerting { + List(Box<[String]>), + One(String), +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AlertingSettings { + pub alerting: Option, + pub alerting_threshold: Option>, + pub alert_types: Option>, + pub alert_to_webhook_url: Option, + pub alerting_args: Option, + pub alert_type_config: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AlertingArgs { + pub daily_report_frequency: Option>, + pub report_check_interval: Option>, + pub budget_alert_ttl: Option>, + pub outage_alert_ttl: Option>, + pub region_outage_alert_ttl: Option>, + pub minor_outage_alert_threshold: Option>, + pub major_outage_alert_threshold: Option>, + pub max_outage_alert_list_size: Option>, + pub log_to_console: Option>, + pub daily_spend_per_user_threshold: Option>, + pub monthly_spend_per_user_threshold: Option>, + pub spend_anomaly_multiplier: Option>, + pub spend_anomaly_baseline_days: Option>, + pub spend_anomaly_min_spend: Option>, + pub user_spend_check_interval: Option>, +} diff --git a/litellm-rust/crates/config/src/general/auth.rs b/litellm-rust/crates/config/src/general/auth.rs new file mode 100644 index 00000000000..8b62febc152 --- /dev/null +++ b/litellm-rust/crates/config/src/general/auth.rs @@ -0,0 +1,77 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AuthSettings { + pub litellm_key_header_name: Option, + pub enable_jwt_auth: Option>, + pub litellm_jwtauth: Option, + pub enable_oauth2_auth: Option>, + pub enable_oauth2_proxy_auth: Option>, + pub oauth2_config_mappings: Option, + pub custom_auth: Option, + pub custom_auth_run_common_checks: Option>, + pub allow_user_auth: Option>, + pub allowed_routes: Option>, + pub admin_only_routes: Option>, + pub public_routes: Option>, + pub allowed_ips: Option>, + pub use_x_forwarded_for: Option>, + pub trusted_proxy_ranges: Option>, + pub user_header_name: Option, + pub user_header_mappings: Option>, + pub role_permissions: Option>, + pub enforce_rbac: Option>, + pub allow_client_side_credentials: Option>, + pub enforce_user_param: Option>, + pub reject_clientside_metadata_tags: Option>, + pub enforced_params: Option>, + pub service_account_settings: Option, + pub litellm_license: Option, + pub control_plane_url: Option, + pub key_management_system: Option, + pub key_management_settings: Option, + pub use_google_kms: Option>, + pub use_azure_key_vault: Option>, + pub encryption_algorithm: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct UserHeaderMapping { + pub header_name: Option, + pub litellm_user_role: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RoleBasedPermissions { + pub role: Option, + pub models: Option>, + pub routes: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ServiceAccountSettings { + pub enforced_params: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct KeyManagementSettings { + pub hosted_keys: Option>, + pub store_virtual_keys: Option>, + pub prefix_for_stored_virtual_keys: Option, + pub access_mode: Option, + pub primary_secret_name: Option, + pub description: Option, + pub tags: Option, + pub kms_key_id: Option, + pub custom_secret_manager: Option, + pub aws_region_name: Option, + pub aws_role_name: Option, + pub aws_session_name: Option, + pub aws_external_id: Option, + pub aws_profile_name: Option, + pub aws_web_identity_token: Option, + pub aws_sts_endpoint: Option, + pub replica_regions: Option>, +} diff --git a/litellm-rust/crates/config/src/general/database.rs b/litellm-rust/crates/config/src/general/database.rs new file mode 100644 index 00000000000..e3f823d93d5 --- /dev/null +++ b/litellm-rust/crates/config/src/general/database.rs @@ -0,0 +1,15 @@ +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct DatabaseSettings { + pub database_connection_pool_timeout: Option>, + pub database_extra_connection_params: Option, + pub database_disable_prepared_statements: Option>, + pub database_statement_timeout: Option>, + pub database_lock_timeout: Option>, + pub database_idle_in_transaction_session_timeout: Option>, + pub disable_prisma_schema_update: Option>, + pub allow_requests_on_db_unavailable: Option>, + pub supported_db_objects: Option>, + pub use_redis_transaction_buffer: Option>, +} diff --git a/litellm-rust/crates/config/src/general/files.rs b/litellm-rust/crates/config/src/general/files.rs new file mode 100644 index 00000000000..e54aec83e90 --- /dev/null +++ b/litellm-rust/crates/config/src/general/files.rs @@ -0,0 +1,15 @@ +use crate::Spelled; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct FilesSettings { + pub max_file_size_mb: Option>, + pub max_batch_file_size_mb: Option>, + pub max_batch_file_records: Option>, + pub max_batch_file_uploads_per_day: Option>, + pub max_file_downloads_per_minute: Option>, + pub allowed_file_extensions: Option>, + pub blocked_file_extensions: Option>, + pub disable_batch_input_file_rate_limiting: Option>, + pub skip_batch_input_file_rate_limiting_for_providers: Option>, + pub skip_batch_input_file_rate_limiting_for_models: Option>, +} diff --git a/litellm-rust/crates/config/src/general/health.rs b/litellm-rust/crates/config/src/general/health.rs new file mode 100644 index 00000000000..169be606700 --- /dev/null +++ b/litellm-rust/crates/config/src/general/health.rs @@ -0,0 +1,22 @@ +use litellm_auth_types::SecretValue; + +use crate::Spelled; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct HealthSettings { + pub use_shared_health_check: Option>, + pub health_check_details: Option>, + pub enable_health_check_routing: Option>, + pub health_check_staleness_threshold: Option>, + pub health_check_ignore_transient_errors: Option>, + pub health_check_skip_disabled_background_models: Option>, + pub background_health_check_model_groups: Option>, + pub model_list_healthy_only: Option>, + pub allow_public_health_readiness_details: Option>, + pub enable_drain_endpoint: Option>, + pub drain_endpoint_token: Option, + pub disable_model_info_refresh: Option>, + pub use_team_public_model_name: Option>, + pub image_generation_model: Option, + pub moderation_model: Option, +} diff --git a/litellm-rust/crates/config/src/general/limits.rs b/litellm-rust/crates/config/src/general/limits.rs new file mode 100644 index 00000000000..d1522903e2c --- /dev/null +++ b/litellm-rust/crates/config/src/general/limits.rs @@ -0,0 +1,11 @@ +use crate::Spelled; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct LimitsSettings { + pub fail_closed_rate_limit_enforcement: Option>, + pub token_rate_limit_type: Option, + pub disable_retry_on_max_parallel_request_limit_error: Option>, + pub disable_fallbacks_on_per_model_rate_limits: Option>, + pub user_api_key_cache_ttl: Option>, + pub user_api_key_cache_max_size: Option>, +} diff --git a/litellm-rust/crates/config/src/general/mcp.rs b/litellm-rust/crates/config/src/general/mcp.rs new file mode 100644 index 00000000000..c60e7665c3c --- /dev/null +++ b/litellm-rust/crates/config/src/general/mcp.rs @@ -0,0 +1,20 @@ +use crate::{Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct McpSettings { + pub mcp_allowed_hosts: Option>, + pub mcp_allowed_origins: Box<[String]>, + pub mcp_advertised_versions: Option, + pub mcp_disabled_capabilities: Option, + pub mcp_allowed_clients: Option, + pub mcp_require_explicit_server_scope: Option>, + pub mcp_client_id_header: Option, + pub mcp_client_side_auth_header_name: Option, + pub mcp_allow_all_keys_respects_mcp_scope: Option>, + pub mcp_prefer_client_id_metadata_document: Option>, + pub mcp_internal_ip_ranges: Option, + pub mcp_trusted_proxy_ranges: Option, + pub mcp_xff_num_trusted_hops: Option>, + pub mcp_required_fields: Option, + pub mcp_session_token_signing: Option, +} diff --git a/litellm-rust/crates/config/src/general/mod.rs b/litellm-rust/crates/config/src/general/mod.rs new file mode 100644 index 00000000000..72871437f00 --- /dev/null +++ b/litellm-rust/crates/config/src/general/mod.rs @@ -0,0 +1,535 @@ +mod alerting; +mod auth; +mod database; +mod files; +mod health; +mod limits; +mod mcp; +mod objects; +mod requests; +mod spend; +mod tracing; +mod ui; + +use litellm_auth_types::SecretValue; +use serde::Deserialize; + +pub use self::{ + alerting::{Alerting, AlertingArgs, AlertingSettings}, + auth::{ + AuthSettings, KeyManagementSettings, RoleBasedPermissions, ServiceAccountSettings, + UserHeaderMapping, + }, + database::DatabaseSettings, + files::FilesSettings, + health::HealthSettings, + limits::LimitsSettings, + mcp::McpSettings, + objects::{ObjectsSettings, TeamMetadataField}, + requests::{PassThroughEndpoint, RequestTagHeader, RequestTagMapping, RequestsSettings}, + spend::{ + ScheduledJobStaggerSettings, SpendCaptureRateCheckSettings, SpendLogsMetadataFields, + SpendSettings, + }, + tracing::{ClickHouseStoreSettings, TracingSettings, TracingStoreSettings}, + ui::{UiAccessMode, UiAccessSsoGroup, UiSettings}, +}; +use crate::Object; + +#[derive(Clone, Debug, Deserialize)] +#[serde(default)] +pub struct GeneralSettings { + pub completion_model: Option, + pub max_in_flight_requests_per_worker: Option, + pub max_queued_requests_per_worker: Option, + pub admission_queue_timeout_seconds: f64, + pub master_key: Option, + pub database_url: Option, + pub tracing: Option, + pub database_connection_pool_limit: Option, + pub database_connection_timeout: Option, + pub database_connect_timeout: Option, + pub database_socket_timeout: Option, + pub database_max_idle_connection_lifetime: Option, + pub max_parallel_requests: Option, + pub global_max_parallel_requests: Option, + pub max_request_size_mb: Option, + pub max_response_size_mb: Option, + pub proxy_config_reload_interval_seconds: u64, + pub background_health_checks: Option, + pub health_check_interval: u64, + pub health_check_concurrency: Option, + pub store_model_in_db: Option, + pub forward_client_headers_to_llm_api: Option, + pub cancel_on_disconnect: Option, + pub infer_model_from_keys: Option, + pub enable_public_model_hub: bool, + pub dangerously_permit_weak_or_unset_master_key: Option, + pub plugins: Option>, + pub coordination_redis: Option, + #[serde(flatten)] + pub auth: AuthSettings, + #[serde(flatten)] + pub database: DatabaseSettings, + #[serde(flatten)] + pub spend: SpendSettings, + #[serde(flatten)] + pub limits: LimitsSettings, + #[serde(flatten)] + pub health: HealthSettings, + #[serde(flatten)] + pub alerting: AlertingSettings, + #[serde(flatten)] + pub ui: UiSettings, + #[serde(flatten)] + pub objects: ObjectsSettings, + #[serde(flatten)] + pub requests: RequestsSettings, + #[serde(flatten)] + pub files: FilesSettings, + #[serde(flatten)] + pub mcp: McpSettings, + #[serde(flatten)] + pub additional_fields: Object, +} + +impl Default for GeneralSettings { + fn default() -> Self { + Self { + completion_model: None, + max_in_flight_requests_per_worker: None, + max_queued_requests_per_worker: None, + admission_queue_timeout_seconds: 1.0, + master_key: None, + database_url: None, + tracing: None, + database_connection_pool_limit: Some(10), + database_connection_timeout: Some(60.0), + database_connect_timeout: None, + database_socket_timeout: None, + database_max_idle_connection_lifetime: Some(60.0), + max_parallel_requests: None, + global_max_parallel_requests: None, + max_request_size_mb: None, + max_response_size_mb: None, + proxy_config_reload_interval_seconds: 30, + background_health_checks: None, + health_check_interval: 300, + health_check_concurrency: None, + store_model_in_db: None, + forward_client_headers_to_llm_api: None, + cancel_on_disconnect: None, + infer_model_from_keys: None, + enable_public_model_hub: false, + dangerously_permit_weak_or_unset_master_key: None, + plugins: None, + coordination_redis: None, + auth: AuthSettings::default(), + database: DatabaseSettings::default(), + spend: SpendSettings::default(), + limits: LimitsSettings::default(), + health: HealthSettings::default(), + alerting: AlertingSettings::default(), + ui: UiSettings::default(), + objects: ObjectsSettings::default(), + requests: RequestsSettings::default(), + files: FilesSettings::default(), + mcp: McpSettings::default(), + additional_fields: Object::default(), + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Alerting, Config, GeneralSettings, Spelled, UiAccessMode}; + + fn general(yaml: &str) -> GeneralSettings { + Config::from_yaml(&format!("general_settings:\n{yaml}")) + .unwrap() + .general_settings + } + + fn text(value: &str) -> Option> { + Some(Spelled::Text(value.to_string())) + } + + #[rstest] + #[case::yaml_bool("true", Some(Spelled::Value(true)))] + #[case::yaml_1_1_word("yes", text("yes"))] + #[case::quoted("'false'", text("false"))] + #[case::environment_reference("os.environ/ENABLE_JWT", text("os.environ/ENABLE_JWT"))] + fn bool_settings_keep_every_python_spelling( + #[case] spelling: &str, + #[case] expected: Option>, + ) { + let settings = general(&format!( + " enable_jwt_auth: {spelling}\n disable_spend_logs: {spelling}\n use_redis_transaction_buffer: {spelling}\n model_list_healthy_only: {spelling}\n" + )); + assert_eq!(settings.auth.enable_jwt_auth, expected); + assert_eq!(settings.spend.disable_spend_logs, expected); + assert_eq!(settings.database.use_redis_transaction_buffer, expected); + assert_eq!(settings.health.model_list_healthy_only, expected); + } + + #[rstest] + #[case::bare_days("7", Some(Spelled::Value(7)))] + #[case::duration("'60d'", text("60d"))] + #[case::environment_reference("os.environ/RETENTION", text("os.environ/RETENTION"))] + fn retention_periods_accept_days_or_a_duration( + #[case] spelling: &str, + #[case] expected: Option>, + ) { + let settings = general(&format!( + " maximum_spend_logs_retention_period: {spelling}\n" + )); + assert_eq!(settings.spend.maximum_spend_logs_retention_period, expected); + } + + #[rstest] + #[case::integer("5", Some(Spelled::Value(5.0)))] + #[case::fraction("0.5", Some(Spelled::Value(0.5)))] + #[case::environment_reference("os.environ/TTL", text("os.environ/TTL"))] + fn number_settings_accept_integers_fractions_and_references( + #[case] spelling: &str, + #[case] expected: Option>, + ) { + let settings = general(&format!( + " user_api_key_cache_ttl: {spelling}\n database_statement_timeout: {spelling}\n health_check_staleness_threshold: {spelling}\n" + )); + assert_eq!(settings.limits.user_api_key_cache_ttl, expected); + assert_eq!(settings.database.database_statement_timeout, expected); + assert_eq!(settings.health.health_check_staleness_threshold, expected); + } + + #[test] + fn nested_settings_are_typed_and_keep_unknown_keys() { + let settings = general( + r#" + allowed_routes: [/chat/completions, /models] + custom_auth: custom_auth.user_api_key_auth + user_header_mappings: + - header_name: X-OpenWebUI-User-Id + litellm_user_role: internal_user + role_permissions: + - role: team + models: [gpt] + routes: [/chat/completions] + service_account_settings: + enforced_params: [user] + key_management_settings: + access_mode: read_and_write + store_virtual_keys: true + future_option: 1 + spend_logs_metadata_fields: + include: [user_api_key_alias] + spend_capture_rate_check: + threshold: 0.8 + lookback_days: 3 + scheduled_job_stagger: + window_seconds: 120 + offsets: {reset_budget_job: 0} +"#, + ); + assert_eq!( + settings.auth.allowed_routes.as_deref(), + Some(["/chat/completions".to_string(), "/models".to_string()].as_slice()) + ); + assert_eq!( + settings.auth.custom_auth.as_deref(), + Some("custom_auth.user_api_key_auth") + ); + let mapping = &settings.auth.user_header_mappings.as_deref().unwrap()[0]; + assert_eq!(mapping.header_name.as_deref(), Some("X-OpenWebUI-User-Id")); + assert_eq!(mapping.litellm_user_role.as_deref(), Some("internal_user")); + let permission = &settings.auth.role_permissions.as_deref().unwrap()[0]; + assert_eq!(permission.role.as_deref(), Some("team")); + assert_eq!( + permission.routes.as_deref(), + Some(["/chat/completions".to_string()].as_slice()) + ); + assert_eq!( + settings + .auth + .service_account_settings + .as_ref() + .unwrap() + .enforced_params + .as_deref(), + Some(["user".to_string()].as_slice()) + ); + let key_management = settings.auth.key_management_settings.as_ref().unwrap(); + assert_eq!( + key_management.access_mode.as_deref(), + Some("read_and_write") + ); + assert_eq!( + key_management.store_virtual_keys, + Some(Spelled::Value(true)) + ); + assert_eq!( + settings + .spend + .spend_logs_metadata_fields + .as_ref() + .unwrap() + .include + .as_deref(), + Some(["user_api_key_alias".to_string()].as_slice()) + ); + let capture = settings.spend.spend_capture_rate_check.as_ref().unwrap(); + assert_eq!(capture.threshold, Some(Spelled::Value(0.8))); + assert_eq!(capture.lookback_days, Some(Spelled::Value(3))); + let stagger = settings.spend.scheduled_job_stagger.as_ref().unwrap(); + assert_eq!(stagger.window_seconds, Some(Spelled::Value(120))); + assert!( + stagger + .offsets + .as_ref() + .unwrap() + .contains_key("reset_budget_job") + ); + assert!(settings.additional_fields.is_empty()); + } + + #[rstest] + #[case::license("litellm_license: SENTINEL")] + #[case::drain_token("drain_endpoint_token: SENTINEL")] + #[case::jwt_auth_object("litellm_jwtauth: {team_id_jwt_field: SENTINEL}")] + #[case::oauth2_mappings("oauth2_config_mappings: {X-Header: SENTINEL}")] + #[case::database_params("database_extra_connection_params: {sslpassword: SENTINEL}")] + #[case::key_management_token("key_management_settings: {aws_web_identity_token: SENTINEL}")] + #[case::key_management_external_id("key_management_settings: {aws_external_id: SENTINEL}")] + #[case::key_management_hosted_keys("key_management_settings: {hosted_keys: [SENTINEL]}")] + #[case::key_management_tags("key_management_settings: {tags: {owner: SENTINEL}}")] + #[case::mcp_session_signing("mcp_session_token_signing: {private_key: SENTINEL}")] + #[case::master_key("master_key: SENTINEL")] + #[case::database_url("database_url: postgres://user:SENTINEL@db/litellm")] + #[case::coordination_redis("coordination_redis: {password: SENTINEL}")] + #[case::plugins("plugins: [{token: SENTINEL}]")] + #[case::unknown_setting("future_setting: SENTINEL")] + fn debug_output_does_not_expose_setting_values(#[case] setting: &str) { + let config = Config::from_yaml(&format!("general_settings:\n {setting}\n")).unwrap(); + let printed = format!("{config:?}"); + assert!(!printed.contains("SENTINEL"), "{printed}"); + } + + #[test] + fn unknown_keys_still_land_in_additional_fields() { + let settings = general(" future_setting: 1\n custom_auth: auth.py\n"); + assert!(settings.additional_fields.contains_key("future_setting")); + assert!(!settings.additional_fields.contains_key("custom_auth")); + } + + #[rstest] + #[case("true", Spelled::Value(true))] + #[case("\"false\"", Spelled::Text("false".to_string()))] + #[case("yes", Spelled::Text("yes".to_string()))] + #[case("os.environ/GATEWAY_FLAG", Spelled::Text("os.environ/GATEWAY_FLAG".to_string()))] + fn bool_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled) { + let settings = general(&format!(" enable_claude_code_gateway: {value}\n")); + assert_eq!(settings.requests.enable_claude_code_gateway, Some(expected)); + } + + #[rstest] + #[case("100", Spelled::Value(100))] + #[case("os.environ/MAX_FILE_MB", Spelled::Text("os.environ/MAX_FILE_MB".to_string()))] + fn int_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled) { + let settings = general(&format!(" max_file_size_mb: {value}\n")); + assert_eq!(settings.files.max_file_size_mb, Some(expected)); + } + + #[rstest] + #[case("600", Spelled::Value(600.0))] + #[case("2.5", Spelled::Value(2.5))] + #[case("os.environ/ALERT_THRESHOLD", Spelled::Text("os.environ/ALERT_THRESHOLD".to_string()))] + fn float_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled) { + let settings = general(&format!(" alerting_threshold: {value}\n")); + assert_eq!(settings.alerting.alerting_threshold, Some(expected)); + } + + #[rstest] + fn nested_mappings_land_in_typed_structs() { + let settings = general( + r#" + alerting: [slack, email] + alerting_args: + daily_report_frequency: 3600 + log_to_console: true + spend_anomaly_multiplier: 2.5 + future_arg: 1 + team_metadata_schema: + - key: cost_center + label: Cost center + request_tag_mappings: + - header: baggage + key: tenant + tag_prefix: tenant- + required: true + pass_through_endpoints: + - path: /bria + target: https://engine.prod.bria-api.com + include_subpath: true + cost_per_request: 0.5 + methods: [GET, POST] + custom_handler: handler.fn + allowed_file_extensions: [jsonl, csv] +"#, + ); + + assert_eq!( + settings.alerting.alerting, + Some(Alerting::List( + ["slack".to_string(), "email".to_string()].into() + )) + ); + let args = settings.alerting.alerting_args.unwrap(); + assert_eq!(args.daily_report_frequency, Some(Spelled::Value(3600))); + assert_eq!(args.log_to_console, Some(Spelled::Value(true))); + assert_eq!(args.spend_anomaly_multiplier, Some(Spelled::Value(2.5))); + + let schema = settings.objects.team_metadata_schema.unwrap(); + assert_eq!(schema[0].key.as_deref(), Some("cost_center")); + assert_eq!(schema[0].label.as_deref(), Some("Cost center")); + + let mapping = &settings.requests.request_tag_mappings.unwrap()[0]; + let header: &'static str = mapping.header.unwrap().into(); + assert_eq!(header, "baggage"); + assert_eq!(mapping.key.as_deref(), Some("tenant")); + assert_eq!(mapping.tag_prefix.as_deref(), Some("tenant-")); + assert_eq!(mapping.required, Some(Spelled::Value(true))); + + let endpoint = &settings.requests.pass_through_endpoints.unwrap()[0]; + assert_eq!(endpoint.path.as_deref(), Some("/bria")); + assert_eq!( + endpoint.target.as_ref().unwrap().expose(), + "https://engine.prod.bria-api.com" + ); + assert_eq!(endpoint.include_subpath, Some(Spelled::Value(true))); + assert_eq!(endpoint.cost_per_request, Some(Spelled::Value(0.5))); + assert_eq!( + endpoint.methods.as_deref(), + Some(&["GET".to_string(), "POST".to_string()][..]) + ); + + assert_eq!( + settings.files.allowed_file_extensions.as_deref(), + Some(&["jsonl".to_string(), "csv".to_string()][..]) + ); + } + + #[rstest] + fn ui_access_mode_accepts_a_mode_string() { + let settings = general(" ui_access_mode: admin_only\n"); + assert!( + matches!(settings.ui.ui_access_mode, Some(UiAccessMode::Mode(mode)) if mode == "admin_only") + ); + } + + #[rstest] + fn ui_access_mode_accepts_an_sso_group_mapping() { + let settings = general( + " ui_access_mode: {type: restricted_sso_group, restricted_sso_group: ai-admins, sso_group_jwt_field: groups}\n", + ); + let Some(UiAccessMode::SsoGroup(group)) = settings.ui.ui_access_mode else { + panic!("expected an SSO group mapping") + }; + assert_eq!(group.kind.as_deref(), Some("restricted_sso_group")); + assert_eq!(group.restricted_sso_group.as_deref(), Some("ai-admins")); + assert_eq!(group.sso_group_jwt_field.as_deref(), Some("groups")); + } + + #[rstest] + fn alerting_accepts_a_single_name() { + let settings = general(" alerting: slack\n"); + assert_eq!( + settings.alerting.alerting, + Some(Alerting::One("slack".to_string())) + ); + } + + #[rstest] + fn rejects_request_tag_header_python_rejects_at_boot() { + assert!( + Config::from_yaml( + "general_settings:\n request_tag_mappings: [{header: x-tenant, key: tenant, tag: t}]\n" + ) + .is_err() + ); + } + + #[rstest] + #[case("alert_to_webhook_url: {budget_alerts: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))] + #[case("alert_type_config: {budget_alerts: {webhook: SENTINEL}}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))] + #[case("alerting_args: {webhook: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))] + #[case("pass_through_endpoints: [{path: /x, target: SENTINEL}]", |settings: &GeneralSettings| format!("{:?}", settings.requests))] + #[case("pass_through_endpoints: [{path: /x, target: t, headers: {Authorization: SENTINEL}}]", |settings: &GeneralSettings| format!("{:?}", settings.requests))] + #[case("claude_code_gateway_managed_settings: {token: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.requests))] + #[case("search_tools: [{litellm_params: {api_key: SENTINEL}}]", |settings: &GeneralSettings| format!("{:?}", settings.objects))] + #[case("max_failed_login_attempts_per_source_overrides: {10.0.0.1: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.ui))] + fn debug_hides_values_that_can_carry_secrets( + #[case] entry: &str, + #[case] render: fn(&GeneralSettings) -> String, + ) { + let settings = general(&format!(" {entry}\n")); + assert!(!render(&settings).contains("SENTINEL")); + } + + #[rstest] + fn unknown_key_still_lands_in_additional_fields() { + let settings = general(" future_setting: 1\n forward_openai_org_id: true\n"); + assert!(settings.additional_fields.contains_key("future_setting")); + assert!( + !settings + .additional_fields + .contains_key("forward_openai_org_id") + ); + assert_eq!( + settings.requests.forward_openai_org_id, + Some(Spelled::Value(true)) + ); + } + + #[rstest] + fn mcp_allowed_hosts_and_origins_land_typed() { + let settings = general( + " mcp_allowed_hosts: [localhost]\n mcp_allowed_origins: ['https://app.test']\n", + ); + assert_eq!( + settings.mcp.mcp_allowed_hosts.as_deref(), + Some(&["localhost".to_string()][..]) + ); + assert_eq!( + settings.mcp.mcp_allowed_origins.as_ref(), + ["https://app.test".to_string()] + ); + } + + #[rstest] + #[case::literal("secret-master-key")] + #[case::reference("os.environ/LITELLM_MASTER_KEY")] + fn loads_and_redacts_the_master_key(#[case] key: &str) { + let config = Config::from_yaml(&format!( + "model_list: []\ngeneral_settings:\n master_key: {key}\n" + )) + .unwrap(); + assert_eq!( + config + .general_settings + .master_key + .as_ref() + .unwrap() + .expose(), + key + ); + assert!(!format!("{config:?}").contains(key)); + } + + #[rstest] + fn missing_general_settings_has_no_master_key() { + let config = Config::from_yaml("model_list: []").unwrap(); + assert!(config.general_settings.master_key.is_none()); + } +} diff --git a/litellm-rust/crates/config/src/general/objects.rs b/litellm-rust/crates/config/src/general/objects.rs new file mode 100644 index 00000000000..d3cb7bed753 --- /dev/null +++ b/litellm-rust/crates/config/src/general/objects.rs @@ -0,0 +1,26 @@ +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ObjectsSettings { + pub custom_key_generate: Option, + pub custom_key_update: Option, + pub custom_key_policy: Option, + pub custom_team_metadata_validate: Option, + pub team_metadata_schema: Option>, + pub team_metadata_validation_timeout: Option>, + pub team_metadata_validation_error_message: Option, + pub disable_auto_add_proxy_admin_to_teams: Option>, + pub self_serve_budget_policy: Option, + pub enforce_rpm_tpm_on_model_add: Option>, + pub agent_access_default_deny: Option>, + pub vector_store_deny_by_default: Option>, + pub search_tool_deny_by_default: Option>, + pub search_tools: Option>, + pub enforce_fallback_model_access: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct TeamMetadataField { + pub key: Option, + pub label: Option, +} diff --git a/litellm-rust/crates/config/src/general/requests.rs b/litellm-rust/crates/config/src/general/requests.rs new file mode 100644 index 00000000000..11b5782ae2f --- /dev/null +++ b/litellm-rust/crates/config/src/general/requests.rs @@ -0,0 +1,66 @@ +use litellm_auth_types::SecretValue; +use serde::Deserialize; + +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RequestsSettings { + pub forward_llm_provider_auth_headers: Option>, + pub forward_openai_org_id: Option>, + pub always_include_stream_usage: Option>, + pub expose_fallback_errors_to_caller: Option>, + pub include_call_id_in_error_body: Option>, + pub request_tag_mappings: Option>, + pub pass_through_endpoints: Option>, + pub pass_through_request_timeout: Option>, + pub passthrough_managed_object_ids: Option>, + pub disable_bedrock_agent_runtime_passthrough: Option>, + pub enable_openai_websocket_passthrough: Option>, + pub vertex_ai_pass_through_drop_anthropic_betas: Option>, + pub transcribe_media_buckets: Option>, + pub responses_websocket_session_limit_seconds: Option>, + pub disable_responses_id_security: Option>, + pub allow_unmanaged_response_ids: Option>, + pub dangerously_allow_mock_testing_request_params: Option>, + pub enable_claude_code_gateway: Option>, + pub claude_code_gateway_managed_settings: Option, + pub context_management_summary_model: Option, + pub context_management_summary_max_tokens: Option>, + pub safeguards_classifier_model: Option, + pub user_url_validation: Option>, + pub user_url_allowed_hosts: Option>, + pub provider_url_destination_allowed_hosts: Option>, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, strum::IntoStaticStr)] +#[serde(rename_all = "lowercase")] +pub enum RequestTagHeader { + #[strum(serialize = "baggage")] + Baggage, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RequestTagMapping { + pub header: Option, + pub key: Option, + pub value: Option, + pub tag: Option, + pub tag_prefix: Option, + pub required: Option>, + pub require_budgeted_tag: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PassThroughEndpoint { + pub id: Option, + pub path: Option, + pub target: Option, + pub headers: Option, + pub default_query_params: Option, + pub include_subpath: Option>, + pub cost_per_request: Option>, + pub timeout: Option>, + pub auth: Option>, + pub guardrails: Option, + pub methods: Option>, +} diff --git a/litellm-rust/crates/config/src/general/spend.rs b/litellm-rust/crates/config/src/general/spend.rs new file mode 100644 index 00000000000..a86e3ebe756 --- /dev/null +++ b/litellm-rust/crates/config/src/general/spend.rs @@ -0,0 +1,61 @@ +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SpendSettings { + pub disable_spend_logs: Option>, + pub disable_spend_updates: Option>, + pub disable_reset_budget: Option>, + pub disable_error_logs: Option>, + pub store_prompts_in_spend_logs: Option>, + pub disable_requester_ip_address_logging: Option>, + pub spend_logs_metadata_fields: Option, + pub proxy_batch_write_at: Option>, + pub proxy_batch_polling_interval: Option>, + pub proxy_budget_rescheduler_min_time: Option>, + pub proxy_budget_rescheduler_max_time: Option>, + pub disable_budget_reservation: Option>, + pub fail_closed_budget_enforcement: Option>, + pub apply_user_budget_to_team_keys: Option>, + pub enforce_fallback_budget: Option>, + pub track_unmanaged_batch_cost: Option>, + pub spend_report_frequency: Option, + pub spend_capture_rate_check: Option, + pub maximum_spend_logs_retention_period: Option>, + pub maximum_autorouter_session_retention_period: Option>, + pub maximum_health_check_retention_period: Option>, + pub maximum_daily_tag_spend_retention_period: Option>, + pub maximum_spend_logs_size: Option>, + pub maximum_spend_logs_cleanup_target_size: Option>, + pub maximum_spend_logs_size_check_interval: Option, + pub maximum_spend_logs_retention_interval: Option, + pub maximum_spend_logs_cleanup_cron: Option, + pub maximum_spend_logs_cleanup_batch_size: Option>, + pub maximum_spend_logs_cleanup_max_batches: Option>, + pub maximum_spend_logs_cleanup_run_budget: Option>, + pub maximum_spend_logs_cleanup_batch_timeout: Option>, + pub use_spend_logs_partitioning: Option>, + pub scheduled_job_stagger: Option, + pub missing_session_id: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SpendLogsMetadataFields { + pub include: Option>, + pub exclude: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SpendCaptureRateCheckSettings { + pub providers: Option>, + pub threshold: Option>, + pub lookback_days: Option>, + pub openai_project_ids: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ScheduledJobStaggerSettings { + pub enabled: Option>, + pub window_seconds: Option>, + pub identity: Option, + pub offsets: Option, +} diff --git a/litellm-rust/crates/config/src/general/tracing.rs b/litellm-rust/crates/config/src/general/tracing.rs new file mode 100644 index 00000000000..b6bf1a4de2b --- /dev/null +++ b/litellm-rust/crates/config/src/general/tracing.rs @@ -0,0 +1,84 @@ +use litellm_auth_types::SecretValue; +use serde::Deserialize; + +use crate::Spelled; + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum TracingStoreKind { + Clickhouse, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClickHouseStoreSettings { + #[serde(rename = "type")] + pub kind: TracingStoreKind, + pub url: Option, + pub database: Option, + pub retention_days: Option>, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum TracingStoreSettings { + ClickHouse(ClickHouseStoreSettings), +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct TracingSettings { + pub store: Option, +} + +#[cfg(test)] +mod tests { + use crate::{Config, Spelled, TracingStoreSettings}; + + #[test] + fn tracing_settings_are_typed_and_redact_the_url() { + let config = Config::from_yaml( + "general_settings:\n tracing:\n store:\n type: clickhouse\n url: https://writer:SENTINEL@example.com\n database: analytics\n retention_days: 7\n", + ) + .unwrap(); + let tracing = config.general_settings.tracing.as_ref().unwrap(); + let Some(TracingStoreSettings::ClickHouse(store)) = tracing.store.as_ref() else { + panic!("expected ClickHouse tracing store") + }; + assert_eq!( + store.url.as_ref().unwrap().expose(), + "https://writer:SENTINEL@example.com" + ); + assert_eq!(store.database.as_deref(), Some("analytics")); + assert_eq!(store.retention_days, Some(Spelled::Value(7.0))); + assert!(!format!("{config:?}").contains("SENTINEL")); + } + + #[test] + fn tracing_settings_accept_environment_references() { + let config = Config::from_yaml( + "general_settings:\n tracing:\n store:\n type: clickhouse\n url: os.environ/CLICKHOUSE_URL\n retention_days: os.environ/RETENTION_DAYS\n", + ) + .unwrap(); + let Some(TracingStoreSettings::ClickHouse(store)) = + config.general_settings.tracing.unwrap().store + else { + panic!("expected ClickHouse tracing store") + }; + assert_eq!( + store.retention_days, + Some(Spelled::Text("os.environ/RETENTION_DAYS".to_owned())) + ); + } + + #[test] + fn tracing_settings_reject_string_store() { + assert!( + Config::from_yaml("general_settings:\n tracing:\n store: clickhouse\n").is_err() + ); + } + + #[test] + fn tracing_settings_reject_removed_reader_configuration() { + assert!(Config::from_yaml("general_settings:\n tracing:\n store:\n type: clickhouse\n reader_url: http://localhost:8123\n").is_err()); + } +} diff --git a/litellm-rust/crates/config/src/general/ui.rs b/litellm-rust/crates/config/src/general/ui.rs new file mode 100644 index 00000000000..5129d8f829c --- /dev/null +++ b/litellm-rust/crates/config/src/general/ui.rs @@ -0,0 +1,50 @@ +use serde::Deserialize; + +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct UiSettings { + pub ui_access_mode: Option, + pub custom_sso: Option, + pub custom_ui_sso_sign_in_handler: Option, + pub auto_redirect_ui_login_to_sso: Option>, + pub default_team_disabled: Option>, + pub allow_cli_sso_verification_uri_complete: Option>, + pub disable_env_credential_login: Option>, + pub disable_password_login_when_sso_enabled: Option>, + pub password_policy_min_length: Option>, + pub password_policy_require_uppercase: Option>, + pub password_policy_require_lowercase: Option>, + pub password_policy_require_numbers: Option>, + pub password_policy_require_special_characters: Option>, + pub password_policy_check_breached_passwords: Option>, + pub max_failed_login_attempts_per_source: Option>, + pub failed_login_window_seconds: Option>, + pub failed_login_block_seconds: Option>, + pub max_failed_login_attempts_per_source_overrides: Option, + pub hide_default_credentials_hint: Option>, + pub disable_master_key_return: Option>, + pub disable_custom_api_keys: Option>, + pub disable_key_generate_for_org_admin: Option>, + pub team_admin_editable_team_fields: Option>, + pub disable_model_add_for_internal_users: Option>, + pub disable_agents_for_internal_users: Option>, + pub allow_agents_for_team_admins: Option>, + pub disable_vector_stores_for_internal_users: Option>, + pub allow_vector_stores_for_team_admins: Option>, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum UiAccessMode { + Mode(String), + SsoGroup(UiAccessSsoGroup), +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct UiAccessSsoGroup { + #[serde(rename = "type")] + pub kind: Option, + pub restricted_sso_group: Option, + pub sso_group_jwt_field: Option, +} diff --git a/litellm-rust/crates/config/src/guardrail.rs b/litellm-rust/crates/config/src/guardrail.rs new file mode 100644 index 00000000000..387418cd039 --- /dev/null +++ b/litellm-rust/crates/config/src/guardrail.rs @@ -0,0 +1,102 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Guardrail { + pub guardrail_id: Option, + pub guardrail_name: Option, + pub litellm_params: Option, + pub guardrail_info: Option, + pub policy_template: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct GuardrailParams { + pub guardrail: Option, + pub mode: Option, + pub default_on: Option>, + pub api_key: Option, + pub api_base: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Spelled, Value}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn guardrail_entries_keep_typed_params_and_unknown_keys() { + let config = parse( + r#" +guardrails: + - guardrail_name: pii + litellm_params: + guardrail: presidio + mode: [pre_call, post_call] + default_on: "true" + api_key: os.environ/PRESIDIO_KEY + presidio_language: en + guardrail_info: + owner: security +"#, + ); + let guardrail = &config.guardrails[0]; + let params = guardrail.litellm_params.as_ref().unwrap(); + + assert_eq!(guardrail.guardrail_name.as_deref(), Some("pii")); + assert_eq!(params.guardrail.as_deref(), Some("presidio")); + assert!( + matches!(params.mode.as_ref().unwrap().get(), Value::Sequence(modes) if modes.len() == 2) + ); + assert_eq!(params.default_on, Some(Spelled::Text("true".to_owned()))); + assert_eq!( + params.api_key.as_ref().unwrap().expose(), + "os.environ/PRESIDIO_KEY" + ); + assert!(params.additional_fields.contains_key("presidio_language")); + assert!( + guardrail + .guardrail_info + .as_ref() + .unwrap() + .contains_key("owner") + ); + } + + #[rstest] + #[case::guardrail_key( + r#" +guardrails: + - litellm_params: {api_key: SENTINEL} +"# + )] + #[case::guardrail_extra( + r#" +guardrails: + - litellm_params: {vendor_secret: SENTINEL} +"# + )] + #[case::guardrail_info( + r#" +guardrails: + - guardrail_info: {owner: SENTINEL} +"# + )] + #[case::guardrail_mode( + r#" +guardrails: + - litellm_params: {mode: SENTINEL} +"# + )] + fn debug_never_prints_guardrail_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/includes.rs b/litellm-rust/crates/config/src/includes.rs deleted file mode 100644 index 78df8e9a33a..00000000000 --- a/litellm-rust/crates/config/src/includes.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::{ - collections::{BTreeSet, VecDeque}, - path::{Path, PathBuf}, -}; - -use serde::Deserialize; -use serde_yaml_ng::{Mapping, Value}; - -use crate::Error; - -#[derive(Deserialize)] -struct Includes { - #[serde(default)] - include: Vec, -} - -fn read(path: &Path) -> Result { - Ok(serde_yaml_ng::from_str(&std::fs::read_to_string(path)?)?) -} - -fn entries(config: &Mapping, path: &Path) -> Result, Error> { - let includes: Includes = serde_yaml_ng::from_value(Value::Mapping(config.clone()))?; - Ok(includes - .include - .into_iter() - .map(|entry| (entry, path.to_owned())) - .collect()) -} - -pub(super) fn load(path: &Path) -> Result { - let root = path.canonicalize()?; - let mut merged = read(&root)?; - let mut pending: VecDeque<_> = entries(&merged, &root)?.into(); - let mut loaded = BTreeSet::from([root.clone()]); - merged.remove(Value::String("include".into())); - while let Some((entry, declaring)) = pending.pop_front() { - let declared = declaring.parent().unwrap_or(Path::new(".")).join(&entry); - let fallback = root.parent().unwrap_or(Path::new(".")).join(&entry); - let location = if declared.exists() { - declared - } else { - fallback - } - .canonicalize()?; - if !loaded.insert(location.clone()) { - continue; - } - let mut included = read(&location)?; - pending.extend(entries(&included, &location)?); - included.remove(Value::String("include".into())); - for (key, value) in included { - match (merged.get_mut(&key), value) { - (Some(Value::Sequence(base)), Value::Sequence(extra)) => base.extend(extra), - (_, value) => { - merged.insert(key, value); - } - } - } - } - Ok(Value::Mapping(merged)) -} diff --git a/litellm-rust/crates/config/src/lib.rs b/litellm-rust/crates/config/src/lib.rs index 8603ef3279c..f831dbd710a 100644 --- a/litellm-rust/crates/config/src/lib.rs +++ b/litellm-rust/crates/config/src/lib.rs @@ -1,94 +1,349 @@ +macro_rules_attribute::attribute_alias! { + #[apply(config_section)] = + #[derive(Clone, Debug, Default, ::serde::Deserialize)] + #[serde(default)]; +} + +mod agent; +mod callback; +mod credential; mod error; -mod includes; +mod general; +mod guardrail; +mod litellm; mod mcp; mod model; -mod settings; +mod policy; +mod prompt; +mod provider; +mod router; +mod tool; mod value; +mod vector_store; +pub mod yaml; -use std::{fmt, path::Path}; - -use serde::Deserialize; +use std::collections::BTreeMap; +pub use agent::Agent; +pub use callback::CallbackSettings; +pub use credential::Credential; pub use error::Error; -pub use litellm_router_types::{LitellmParams, Spelled}; -pub use mcp::{McpAuth, McpServer, McpTransport}; -pub use model::Model; -pub use settings::{ - ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings, - TracingStoreSettings, +pub use general::{ + Alerting, AlertingArgs, AlertingSettings, AuthSettings, ClickHouseStoreSettings, + DatabaseSettings, FilesSettings, GeneralSettings, HealthSettings, KeyManagementSettings, + LimitsSettings, McpSettings, ObjectsSettings, PassThroughEndpoint, RequestTagHeader, + RequestTagMapping, RequestsSettings, RoleBasedPermissions, ScheduledJobStaggerSettings, + ServiceAccountSettings, SpendCaptureRateCheckSettings, SpendLogsMetadataFields, SpendSettings, + TeamMetadataField, TracingSettings, TracingStoreSettings, UiAccessMode, UiAccessSsoGroup, + UiSettings, UserHeaderMapping, }; -pub use value::{AdditionalFields, Object, Value}; +pub use guardrail::{Guardrail, GuardrailParams}; +pub use litellm::{ + AiohttpSettings, AzureBlobCacheParams, BackgroundMode, BudgetSettings, CacheParams, + CachingSettings, CallerIdentity, ClientCertificateSettings, CustomProvider, DiskCacheParams, + FallbackSettings, GcsCacheParams, Http2Settings, LiteLlmSettings, LoggingSettings, + McpToolSettings, ModelGroupSettings, PriorityReservationSettings, ProxyCacheTtlParams, + QdrantCacheParams, RedisCacheParams, RedisServerCacheParams, ResponsesSettings, S3CacheParams, + SemanticCacheParams, SocketSettings, TeamDefaultSettings, TlsSettings, + UpperboundKeyGenerateParams, ValkeyCacheParams, +}; +pub use litellm_python_compat::read::Spelled; +pub use litellm_router_types::LitellmParams; +pub use mcp::{McpAuth, McpServer, McpTransport}; +pub use model::{AccessWindow, Model, ModelId, ModelInfo}; +pub use policy::{ + Policy, PolicyAttachment, PolicyCondition, PolicyGuardrailChanges, PolicyGuardrails, + PolicyPipeline, PolicyPipelineStep, +}; +pub use prompt::{Prompt, PromptInfo, PromptParams}; +pub use provider::{AssistantSettings, ProviderSettings, VertexConfig, Worker}; +pub use router::{AlertingConfig, AllowedFailsPolicy, ProviderBudget, RouterSettings}; +pub use tool::{SandboxTool, SandboxToolParams, SearchTool, SearchToolInfo, SearchToolParams}; +pub use value::{Object, Raw, Value}; +pub use vector_store::{VectorStore, VectorStoreParams}; -#[derive(Clone, Default, Deserialize)] -#[serde(default)] +#[macro_rules_attribute::apply(config_section)] pub struct Config { pub model_list: Box<[Model]>, pub general_settings: GeneralSettings, pub router_settings: RouterSettings, pub litellm_settings: LiteLlmSettings, pub environment_variables: Object, - pub callback_settings: Object, - pub assistant_settings: Object, - pub default_vertex_config: Object, - pub mcp_servers: std::collections::BTreeMap, - pub credential_list: Box<[Object]>, - pub guardrails: Box<[Object]>, - pub prompts: Box<[Object]>, - pub sandbox_tools: Box<[Object]>, - pub search_tools: Box<[Object]>, - pub files_settings: Box<[Object]>, - pub finetune_settings: Box<[Object]>, + pub callback_settings: BTreeMap, + pub assistant_settings: Option, + pub default_vertex_config: Option, + pub mcp_servers: BTreeMap, + pub credential_list: Box<[Credential]>, + pub guardrails: Box<[Guardrail]>, + pub prompts: Box<[Prompt]>, + pub sandbox_tools: Box<[SandboxTool]>, + pub search_tools: Box<[SearchTool]>, + pub files_settings: Box<[ProviderSettings]>, + pub finetune_settings: Box<[ProviderSettings]>, pub mcp_tools: Box<[Object]>, - pub vector_store_registry: Box<[Object]>, - pub worker_registry: Box<[Object]>, - pub agents: Box<[Object]>, - pub agent_list: Box<[Object]>, - pub policies: Object, - pub policy_attachments: Box<[Object]>, + pub vector_store_registry: Box<[VectorStore]>, + pub worker_registry: Box<[Worker]>, + pub agents: Box<[Agent]>, + pub agent_list: Box<[Agent]>, + pub policies: BTreeMap, + pub policy_attachments: Box<[PolicyAttachment]>, pub include: Box<[String]>, #[serde(flatten)] - pub additional_fields: AdditionalFields, -} - -impl fmt::Debug for Config { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("Config") - .field("model_list", &self.model_list) - .field("general_settings", &self.general_settings) - .field("router_settings", &self.router_settings) - .field("litellm_settings", &self.litellm_settings) - .field("environment_variables", &self.environment_variables) - .field("callback_settings", &self.callback_settings) - .field("assistant_settings", &self.assistant_settings) - .field("default_vertex_config", &self.default_vertex_config) - .field("mcp_servers", &self.mcp_servers) - .field("credential_list", &self.credential_list) - .field("guardrails", &self.guardrails) - .field("prompts", &self.prompts) - .field("sandbox_tools", &self.sandbox_tools) - .field("search_tools", &self.search_tools) - .field("files_settings", &self.files_settings) - .field("finetune_settings", &self.finetune_settings) - .field("mcp_tools", &self.mcp_tools) - .field("vector_store_registry", &self.vector_store_registry) - .field("worker_registry", &self.worker_registry) - .field("agents", &self.agents) - .field("agent_list", &self.agent_list) - .field("policies", &self.policies) - .field("policy_attachments", &self.policy_attachments) - .field("include", &self.include) - .field("additional_fields", &self.additional_fields.keys()) - .finish() - } + pub additional_fields: Object, } impl Config { pub fn from_yaml(yaml: &str) -> Result { - Ok(serde_yaml_ng::from_str(yaml)?) + Self::from_mapping(yaml::mapping(yaml)?) } - pub fn load(path: impl AsRef) -> Result { - Ok(serde_yaml_ng::from_value(includes::load(path.as_ref())?)?) + pub fn from_mapping(document: serde_yaml_ng::Mapping) -> Result { + Ok(serde_yaml_ng::from_value(serde_yaml_ng::Value::Mapping( + document, + ))?) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Error, Spelled}; + + #[rstest] + fn config_debug_redacts_api_keys() { + let config = Config::from_yaml( + "model_list: [{model_name: assistant, litellm_params: {model: anthropic/test-model, api_key: secret-value}}]", + ) + .unwrap(); + assert_eq!( + config.model_list[0] + .litellm_params + .api_key + .as_ref() + .unwrap() + .expose(), + "secret-value" + ); + assert!(!format!("{config:?}").contains("secret-value")); + } + + #[rstest] + #[case::malformed_yaml("model_list: [")] + #[case::missing_params("model_list: [{model_name: assistant}]")] + #[case::missing_model("model_list: [{model_name: assistant, litellm_params: {api_key: key}}]")] + #[case::provider_param_of_the_wrong_type( + "model_list: [{model_name: assistant, litellm_params: {model: test, aws_region_name: [eu-central-1]}}]" + )] + #[case::negative_model_rpm( + "model_list: [{model_name: assistant, rpm: -1, litellm_params: {model: test}}]" + )] + fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) { + let error = Config::from_yaml(yaml).unwrap_err(); + assert!(matches!(error, Error::Parse(_)), "{error:?}"); + } + + #[rstest] + fn empty_config_matches_python_defaults() { + let config = Config::from_yaml("{}").unwrap(); + assert!(config.model_list.is_empty()); + assert_eq!(config.general_settings.admission_queue_timeout_seconds, 1.0); + assert_eq!( + config.general_settings.database_connection_pool_limit, + Some(10) + ); + assert_eq!( + config.general_settings.proxy_config_reload_interval_seconds, + 30 + ); + assert_eq!(config.general_settings.health_check_interval, 300); + } + + #[rstest] + fn parses_typed_settings_and_preserves_extension_fields() { + let config = Config::from_yaml( + r#" +model_list: + - model_name: assistant + litellm_params: + model: vertex_ai/test-model + timeout: os.environ/REQUEST_TIMEOUT + tpm: os.environ/TPM_LIMIT + rpm: 5 + drop_params: "true" + vertex_project: test-project + vertex_credentials: + type: service_account + future_param: 1 + model_info: + mode: chat + access_groups: [internal] +general_settings: + master_key: secret-master-key + store_model_in_db: true + custom_auth: auth.py +router_settings: + routing_strategy: simple-shuffle + allowed_fails: 2 + future_router_setting: cache.internal +litellm_settings: + drop_params: true + cache: true + custom_callback_name: audit +future_section: + enabled: true +"#, + ) + .unwrap(); + + let model = &config.model_list[0]; + assert_eq!( + model.litellm_params.timeout, + Some(Spelled::Text("os.environ/REQUEST_TIMEOUT".to_string())) + ); + assert_eq!( + model.litellm_params.tpm, + Some(Spelled::Text("os.environ/TPM_LIMIT".to_string())) + ); + assert_eq!(model.litellm_params.rpm, Some(Spelled::Value(5))); + assert_eq!( + model.litellm_params.drop_params, + Some(Spelled::Text("true".to_string())) + ); + assert_eq!( + model.litellm_params.vertex.vertex_project.as_deref(), + Some("test-project") + ); + assert_eq!( + model.litellm_params.vertex.vertex_credentials.as_deref(), + Some(r#"{"type":"service_account"}"#) + ); + assert!(model.litellm_params.extra.contains_key("future_param")); + assert!(model.additional_fields.contains_key("access_groups")); + assert_eq!(config.router_settings.allowed_fails, Some(2)); + assert!( + config + .router_settings + .additional_fields + .contains_key("future_router_setting") + ); + assert!( + config + .litellm_settings + .additional_fields + .contains_key("custom_callback_name") + ); + assert!(config.additional_fields.contains_key("future_section")); + } + + #[rstest] + fn parses_python_config_sections() { + let config = Config::from_yaml( + r#" +environment_variables: + REDIS_PORT: 6379 +callback_settings: + otel: + message_logging: false +assistant_settings: + custom_llm_provider: openai +credential_list: + - credential_name: bedrock + credential_values: + aws_region_name: us-east-1 +guardrails: + - guardrail_name: pii + litellm_params: + guardrail: presidio +prompts: + - prompt_id: support +sandbox_tools: + - sandbox_tool_name: e2b +search_tools: + - search_tool_name: web +files_settings: + - custom_llm_provider: openai +finetune_settings: + - custom_llm_provider: openai +mcp_tools: + - name: lookup +mcp_servers: + docs: + url: https://example.test/mcp +vector_store_registry: + - vector_store_name: docs +worker_registry: + - worker_id: regional +agents: + - agent_name: reviewer +agent_list: + - agent_name: legacy-reviewer +policies: + safe: + guardrails: + add: [pii] +policy_attachments: + - policy_id: safe +"#, + ) + .unwrap(); + + assert_eq!(config.environment_variables.len(), 1); + assert_eq!(config.credential_list.len(), 1); + assert_eq!(config.guardrails.len(), 1); + assert_eq!(config.prompts.len(), 1); + assert_eq!(config.sandbox_tools.len(), 1); + assert_eq!(config.search_tools.len(), 1); + assert_eq!(config.files_settings.len(), 1); + assert_eq!(config.finetune_settings.len(), 1); + assert_eq!(config.mcp_tools.len(), 1); + assert_eq!(config.mcp_servers.len(), 1); + assert_eq!(config.vector_store_registry.len(), 1); + assert_eq!(config.worker_registry.len(), 1); + assert_eq!(config.agents.len(), 1); + assert_eq!(config.agent_list.len(), 1); + assert!(config.policies.contains_key("safe")); + assert_eq!(config.policy_attachments.len(), 1); + } + + #[rstest] + #[case::api_key("api_key", "provider-secret")] + #[case::aws_secret("aws_secret_access_key", "aws-secret")] + #[case::vertex_credentials("vertex_credentials", "vertex-secret")] + #[case::provider_extension("azure_ad_token", "azure-secret")] + #[case::general_extension("custom_auth_secret", "auth-secret")] + #[case::router_extension("future_router_token", "router-secret")] + #[case::litellm_extension("callback_token", "callback-secret")] + #[case::root_extension("private_token", "root-secret")] + fn debug_output_does_not_expose_config_values(#[case] field: &str, #[case] secret: &str) { + let yaml = match field { + "api_key" => format!( + "model_list: [{{model_name: assistant, litellm_params: {{model: test, api_key: {secret}}}}}]" + ), + "aws_secret_access_key" => format!( + "model_list: [{{model_name: assistant, litellm_params: {{model: test, aws_secret_access_key: {secret}}}}}]" + ), + "vertex_credentials" => format!( + "model_list: [{{model_name: assistant, litellm_params: {{model: test, vertex_credentials: {secret}}}}}]" + ), + "azure_ad_token" => format!( + "model_list: [{{model_name: assistant, litellm_params: {{model: test, azure_ad_token: {secret}}}}}]" + ), + "custom_auth_secret" => format!("general_settings: {{{field}: {secret}}}"), + "future_router_token" => format!("router_settings: {{{field}: {secret}}}"), + "callback_token" => format!("litellm_settings: {{{field}: {secret}}}"), + "private_token" => format!("{field}: {secret}"), + _ => unreachable!(), + }; + let config = Config::from_yaml(&yaml).unwrap(); + assert!(!format!("{config:?}").contains(secret)); + } + + #[rstest] + fn unknown_top_level_key_lands_in_additional_fields() { + let config = Config::from_yaml("future_section:\n a: 1\nenable_jwt_auth: true\n").unwrap(); + + assert!(config.additional_fields.contains_key("future_section")); + assert!(config.additional_fields.contains_key("enable_jwt_auth")); } } diff --git a/litellm-rust/crates/config/src/litellm/AGENTS.md b/litellm-rust/crates/config/src/litellm/AGENTS.md new file mode 100644 index 00000000000..e19b6c87bbc --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/AGENTS.md @@ -0,0 +1,9 @@ +## Docs + +- https://github.com/BerriAI/litellm-docs/blob/main/docs/proxy/config_settings.md + +## Python + +- There is no Python model. `ConfigYAML.litellm_settings` in `litellm/proxy/_types.py` is a plain `dict` +- `ProxyConfig.load_config` in `proxy_server.py` handles a few keys specially (`cache`, `cache_params`, `callbacks`, `success_callback`, `guardrails`, `drop_params` and others) and does `setattr(litellm, key, value)` for everything else. So any module-level attribute in `litellm/__init__.py` is a valid key, and that is the real key set +- `litellm_settings` is one of the sections the proxy persists to the DB (`_CONFIG_PERSISTED_SECTIONS`) and the Admin UI edits, so at runtime the YAML is only one source of the effective values. Merging the DB overlay is the consumer's job, not this crate's diff --git a/litellm-rust/crates/config/src/litellm/budget.rs b/litellm-rust/crates/config/src/litellm/budget.rs new file mode 100644 index 00000000000..12acac2c1c2 --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/budget.rs @@ -0,0 +1,32 @@ +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct BudgetSettings { + pub max_budget: Option>, + pub max_internal_user_budget: Option>, + pub default_max_internal_user_budget: Option>, + pub default_internal_user_params: Option, + pub max_ui_session_budget: Option>, + pub default_team_settings: Option>, + pub upperbound_key_generate_params: Option, + pub key_alias_pattern: Option, + pub budget_reset_time: Option, + pub disable_end_user_cost_tracking: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct TeamDefaultSettings { + pub team_id: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct UpperboundKeyGenerateParams { + pub max_budget: Option>, + pub budget_duration: Option, + pub duration: Option, + pub max_parallel_requests: Option>, + pub tpm_limit: Option>, + pub rpm_limit: Option>, +} diff --git a/litellm-rust/crates/config/src/litellm/caching.rs b/litellm-rust/crates/config/src/litellm/caching.rs new file mode 100644 index 00000000000..d7ce2e3028f --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/caching.rs @@ -0,0 +1,137 @@ +use serde::{Deserialize, Deserializer}; + +use litellm_auth_types::SecretValue; + +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct CachingSettings { + pub cache: Option, + pub cache_params: Option, + pub enable_redis_auth_cache: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct CacheParams { + #[serde(rename = "type")] + pub cache_type: Option, + pub mode: Option, + pub namespace: Option, + pub ttl: Option>, + pub supported_call_types: Option>, + #[serde(deserialize_with = "present")] + pub max_messages: Option>>, + #[serde(flatten)] + pub redis_server: RedisServerCacheParams, + #[serde(flatten)] + pub semantic: SemanticCacheParams, + #[serde(flatten)] + pub proxy_ttl: ProxyCacheTtlParams, + #[serde(flatten)] + pub redis: RedisCacheParams, + #[serde(flatten)] + pub valkey: ValkeyCacheParams, + #[serde(flatten)] + pub s3: S3CacheParams, + #[serde(flatten)] + pub gcs: GcsCacheParams, + #[serde(flatten)] + pub azure_blob: AzureBlobCacheParams, + #[serde(flatten)] + pub qdrant: QdrantCacheParams, + #[serde(flatten)] + pub disk: DiskCacheParams, + #[serde(flatten)] + pub additional_fields: Object, +} + +/// Read by every backend that talks to a Redis-protocol server: `redis`, `redis-semantic` and `valkey-semantic`. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RedisServerCacheParams { + pub host: Option, + pub port: Option>, + pub password: Option, + pub redis_startup_nodes: Option, + pub default_in_redis_ttl: Option>, +} + +/// Read by the semantic backends: `redis-semantic`, `valkey-semantic` and `qdrant-semantic`. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SemanticCacheParams { + pub similarity_threshold: Option>, + pub semantic_cache_embedding_max_input_tokens: Option>, + pub semantic_cache_embedding_timeout: Option>, + pub semantic_cache_scope: Option, +} + +/// Not response-cache settings. The proxy copies them onto `litellm.default_*_ttl` for its own internal caches. +/// `default_in_memory_ttl` also replaces `ttl` when `type` is `local`. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ProxyCacheTtlParams { + pub default_in_memory_ttl: Option>, + pub default_redis_ttl: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RedisCacheParams { + pub redis_semantic_cache_embedding_model: Option, + pub redis_semantic_cache_index_name: Option, + pub redis_flush_size: Option>, + pub gcp_service_account: Option, + pub gcp_ssl_ca_certs: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ValkeyCacheParams { + pub valkey_semantic_cache_embedding_model: Option, + pub valkey_semantic_cache_index_name: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct S3CacheParams { + pub s3_bucket_name: Option, + pub s3_region_name: Option, + pub s3_api_version: Option, + pub s3_use_ssl: Option>, + pub s3_verify: Option>, + pub s3_endpoint_url: Option, + pub s3_aws_access_key_id: Option, + pub s3_aws_secret_access_key: Option, + pub s3_aws_session_token: Option, + pub s3_config: Option, + pub s3_path: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct GcsCacheParams { + pub gcs_bucket_name: Option, + pub gcs_path_service_account: Option, + pub gcs_path: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AzureBlobCacheParams { + pub azure_account_url: Option, + pub azure_blob_container: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct QdrantCacheParams { + pub qdrant_api_base: Option, + pub qdrant_api_key: Option, + pub qdrant_collection_name: Option, + pub qdrant_quantization_config: Option, + pub qdrant_semantic_cache_embedding_model: Option, + pub qdrant_semantic_cache_vector_size: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct DiskCacheParams { + pub disk_cache_dir: Option, +} + +fn present<'de, D: Deserializer<'de>, T: Deserialize<'de>>( + deserializer: D, +) -> Result>, D::Error> { + Ok(Some(Option::deserialize(deserializer)?)) +} diff --git a/litellm-rust/crates/config/src/litellm/fallbacks.rs b/litellm-rust/crates/config/src/litellm/fallbacks.rs new file mode 100644 index 00000000000..f930ffaa208 --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/fallbacks.rs @@ -0,0 +1,7 @@ +use crate::Raw; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct FallbackSettings { + pub default_fallbacks: Option, + pub context_window_fallbacks: Option, +} diff --git a/litellm-rust/crates/config/src/litellm/logging.rs b/litellm-rust/crates/config/src/litellm/logging.rs new file mode 100644 index 00000000000..46f74c17ab0 --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/logging.rs @@ -0,0 +1,24 @@ +use litellm_python_compat::read::one_or_many; + +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct LoggingSettings { + #[serde(deserialize_with = "one_or_many")] + pub callbacks: Option>, + #[serde(deserialize_with = "one_or_many")] + pub success_callback: Option>, + #[serde(deserialize_with = "one_or_many")] + pub failure_callback: Option>, + pub service_callback: Option>, + pub audit_log_callbacks: Option>, + pub s3_callback_params: Option, + pub s3_audit_callback_params: Option, + pub store_audit_logs: Option>, + pub json_logs: Option, + pub set_verbose: Option, + pub turn_off_message_logging: Option>, + pub redact_messages_in_exceptions: Option>, + pub log_raw_request_response: Option>, + pub presidio_logging_only: Option>, +} diff --git a/litellm-rust/crates/config/src/litellm/mcp.rs b/litellm-rust/crates/config/src/litellm/mcp.rs new file mode 100644 index 00000000000..43245638835 --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/mcp.rs @@ -0,0 +1,8 @@ +use crate::Raw; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct McpToolSettings { + pub mcp_aliases: Option, + pub mcp_semantic_tool_filter: Option, + pub mcp_tool_search: Option, +} diff --git a/litellm-rust/crates/config/src/litellm/mod.rs b/litellm-rust/crates/config/src/litellm/mod.rs new file mode 100644 index 00000000000..a5d3ee3ea1a --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/mod.rs @@ -0,0 +1,400 @@ +mod budget; +mod caching; +mod fallbacks; +mod logging; +mod mcp; +mod transport; + +use serde::Deserialize; + +pub use self::{ + budget::{BudgetSettings, TeamDefaultSettings, UpperboundKeyGenerateParams}, + caching::{ + AzureBlobCacheParams, CacheParams, CachingSettings, DiskCacheParams, GcsCacheParams, + ProxyCacheTtlParams, QdrantCacheParams, RedisCacheParams, RedisServerCacheParams, + S3CacheParams, SemanticCacheParams, ValkeyCacheParams, + }, + fallbacks::FallbackSettings, + logging::LoggingSettings, + mcp::McpToolSettings, + transport::{ + AiohttpSettings, ClientCertificateSettings, Http2Settings, SocketSettings, TlsSettings, + }, +}; +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct LiteLlmSettings { + pub drop_params: Option>, + pub request_timeout: Option>, + pub num_retries: Option, + pub prometheus_deployment_and_latency_caller_identity: Option, + pub guardrails: Option>, + pub prompt_injection_params: Option, + pub azure_content_safety_params: Option, + pub global_prompt_directory: Option, + pub global_bitbucket_config: Option, + pub global_gitlab_config: Option, + pub priority_reservation_settings: Option, + pub model_group_settings: Option, + pub post_call_rules: Option, + pub custom_provider_map: Option>, + pub responses: Option, + pub enable_preview_features: Option>, + pub public_model_groups: Option>, + pub timezone: Option, + #[serde(flatten)] + pub tls: TlsSettings, + #[serde(flatten)] + pub client_certificate: ClientCertificateSettings, + #[serde(flatten)] + pub socket: SocketSettings, + #[serde(flatten)] + pub http2: Http2Settings, + #[serde(flatten)] + pub aiohttp: AiohttpSettings, + #[serde(flatten)] + pub caching: CachingSettings, + #[serde(flatten)] + pub budget: BudgetSettings, + #[serde(flatten)] + pub logging: LoggingSettings, + #[serde(flatten)] + pub fallbacks: FallbackSettings, + #[serde(flatten)] + pub mcp: McpToolSettings, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, strum::IntoStaticStr)] +#[serde(rename_all = "snake_case")] +pub enum CallerIdentity { + #[strum(serialize = "api_key_alias")] + ApiKeyAlias, + #[strum(serialize = "user_email")] + UserEmail, + #[strum(serialize = "both")] + Both, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PriorityReservationSettings { + pub default_priority: Option>, + pub saturation_threshold: Option>, + pub saturation_check_cache_ttl: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ModelGroupSettings { + pub forward_client_headers_to_llm_api: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct CustomProvider { + pub provider: Option, + pub custom_handler: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ResponsesSettings { + pub background_mode: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct BackgroundMode { + pub polling_via_cache: Option, + pub native_background_mode: Option>, + pub ttl: Option>, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{CallerIdentity, Config, LiteLlmSettings, Spelled}; + + fn settings(yaml: &str) -> LiteLlmSettings { + Config::from_yaml(&format!("litellm_settings:\n{yaml}")) + .unwrap() + .litellm_settings + } + + #[rstest] + #[case::bool_word(" turn_off_message_logging: 'false'", Spelled::Text("false".into()))] + #[case::bool_value(" turn_off_message_logging: true", Spelled::Value(true))] + #[case::env_reference(" turn_off_message_logging: os.environ/REDACT", Spelled::Text("os.environ/REDACT".into()))] + fn bool_settings_keep_python_spellings(#[case] yaml: &str, #[case] expected: Spelled) { + assert_eq!( + settings(yaml).logging.turn_off_message_logging, + Some(expected) + ); + } + + #[rstest] + #[case::number(" max_budget: 10", Spelled::Value(10.0))] + #[case::string(" max_budget: '10'", Spelled::Text("10".into()))] + #[case::env_reference(" max_budget: os.environ/MAX_BUDGET", Spelled::Text("os.environ/MAX_BUDGET".into()))] + fn budget_accepts_number_or_string(#[case] yaml: &str, #[case] expected: Spelled) { + assert_eq!(settings(yaml).budget.max_budget, Some(expected)); + } + + #[rstest] + #[case::api_key_alias("api_key_alias", CallerIdentity::ApiKeyAlias)] + #[case::user_email("user_email", CallerIdentity::UserEmail)] + #[case::both("both", CallerIdentity::Both)] + fn caller_identity_accepts_python_values( + #[case] value: &str, + #[case] expected: CallerIdentity, + ) { + assert_eq!( + settings(&format!( + " prometheus_deployment_and_latency_caller_identity: {value}" + )) + .prometheus_deployment_and_latency_caller_identity, + Some(expected) + ); + } + + #[rstest] + fn caller_identity_rejects_values_python_refuses() { + assert!( + Config::from_yaml( + "litellm_settings:\n prometheus_deployment_and_latency_caller_identity: team" + ) + .is_err() + ); + } + + #[rstest] + fn cache_params_reads_cache_keywords() { + let cache = settings( + " cache_params:\n type: redis\n port: 6379\n password: os.environ/REDIS_PASSWORD\n ttl: 600\n supported_call_types: [acompletion]\n namespace_extra: x", + ) + .caching + .cache_params + .unwrap(); + assert_eq!(cache.cache_type.as_deref(), Some("redis")); + assert_eq!(cache.redis_server.port, Some(Spelled::Value(6379))); + assert_eq!( + cache.redis_server.password.unwrap().expose(), + "os.environ/REDIS_PASSWORD" + ); + assert_eq!(cache.ttl, Some(Spelled::Value(600.0))); + assert_eq!( + cache.supported_call_types.as_deref(), + Some(&["acompletion".to_owned()][..]) + ); + assert!(cache.additional_fields.contains_key("namespace_extra")); + } + + #[rstest] + fn cache_params_route_each_key_to_its_backend_group() { + let cache = settings( + r#" cache_params: + type: s3 + namespace: shared + port: 6379 + similarity_threshold: 0.8 + default_redis_ttl: 30 + redis_flush_size: 10 + valkey_semantic_cache_index_name: valkey-index + s3_bucket_name: bucket + gcs_path: gcs/path + azure_blob_container: container + qdrant_collection_name: collection + disk_cache_dir: /tmp/cache + namespace_extra: x"#, + ) + .caching + .cache_params + .unwrap(); + assert_eq!(cache.namespace.as_deref(), Some("shared")); + assert_eq!(cache.redis_server.port, Some(Spelled::Value(6379))); + assert_eq!( + cache.semantic.similarity_threshold, + Some(Spelled::Value(0.8)) + ); + assert_eq!( + cache.proxy_ttl.default_redis_ttl, + Some(Spelled::Value(30.0)) + ); + assert_eq!(cache.redis.redis_flush_size, Some(Spelled::Value(10))); + assert_eq!( + cache.valkey.valkey_semantic_cache_index_name.as_deref(), + Some("valkey-index") + ); + assert_eq!(cache.s3.s3_bucket_name.as_deref(), Some("bucket")); + assert_eq!(cache.gcs.gcs_path.as_deref(), Some("gcs/path")); + assert_eq!( + cache.azure_blob.azure_blob_container.as_deref(), + Some("container") + ); + assert_eq!( + cache.qdrant.qdrant_collection_name.as_deref(), + Some("collection") + ); + assert_eq!(cache.disk.disk_cache_dir.as_deref(), Some("/tmp/cache")); + assert_eq!( + cache.additional_fields.keys().collect::>(), + ["namespace_extra"] + ); + } + + #[rstest] + #[case::absent(" ttl: 1", None)] + #[case::explicit_null(" max_messages: null", Some(None))] + #[case::limit(" max_messages: 8", Some(Some(Spelled::Value(8))))] + fn cache_max_messages_tells_null_from_absent( + #[case] yaml: &str, + #[case] expected: Option>>, + ) { + assert_eq!( + settings(&format!(" cache_params:\n{yaml}")) + .caching + .cache_params + .unwrap() + .max_messages, + expected + ); + } + + #[rstest] + fn nested_settings_land_in_typed_fields() { + let settings = settings( + " priority_reservation_settings: {default_priority: 0.5, saturation_check_cache_ttl: 30} + model_group_settings: {forward_client_headers_to_llm_api: [gpt]} + custom_provider_map: [{provider: my-llm, custom_handler: handler.instance}] + responses: {background_mode: {polling_via_cache: all, native_background_mode: [o3], ttl: 60}} + default_team_settings: [{team_id: team-1, success_callback: [langfuse]}] + upperbound_key_generate_params: {max_budget: 100, duration: 30d, rpm_limit: os.environ/RPM}", + ); + let priority = settings.priority_reservation_settings.unwrap(); + assert_eq!(priority.default_priority, Some(Spelled::Value(0.5))); + assert_eq!( + priority.saturation_check_cache_ttl, + Some(Spelled::Value(30)) + ); + assert_eq!( + settings + .model_group_settings + .unwrap() + .forward_client_headers_to_llm_api + .as_deref(), + Some(&["gpt".to_owned()][..]) + ); + let provider = &settings.custom_provider_map.unwrap()[0]; + assert_eq!(provider.provider.as_deref(), Some("my-llm")); + assert_eq!(provider.custom_handler.as_deref(), Some("handler.instance")); + let background = settings.responses.unwrap().background_mode.unwrap(); + assert_eq!( + background.polling_via_cache.unwrap().get().as_str(), + Some("all") + ); + assert_eq!( + background.native_background_mode.as_deref(), + Some(&["o3".to_owned()][..]) + ); + assert_eq!(background.ttl, Some(Spelled::Value(60))); + let team = &settings.budget.default_team_settings.unwrap()[0]; + assert_eq!(team.team_id.as_deref(), Some("team-1")); + assert!(team.additional_fields.contains_key("success_callback")); + let upperbound = settings.budget.upperbound_key_generate_params.unwrap(); + assert_eq!(upperbound.max_budget, Some(Spelled::Value(100.0))); + assert_eq!(upperbound.duration.as_deref(), Some("30d")); + assert_eq!( + upperbound.rpm_limit, + Some(Spelled::Text("os.environ/RPM".into())) + ); + } + + #[rstest] + #[case::cache_password( + r#" + cache: true + cache_params: {password: SENTINEL} +"# + )] + #[case::cache_endpoint(" cache_params: {s3_endpoint_url: 'https://user:SENTINEL@s3.local'}")] + #[case::cache_raw(" cache_params: {s3_config: {secret: SENTINEL}}")] + #[case::gitlab_object(" global_gitlab_config: {access_token: SENTINEL}")] + #[case::s3_callback_object(" s3_callback_params: {s3_aws_secret_access_key: SENTINEL}")] + #[case::team_object(" default_team_settings: [{team_id: t, langfuse_secret: SENTINEL}]")] + #[case::fallbacks_raw(" default_fallbacks: [SENTINEL]")] + #[case::user_params_raw(" default_internal_user_params: {models: [SENTINEL]}")] + #[case::callback_entry(" callbacks: [SENTINEL]")] + #[case::success_callback_map(" success_callback: {name: SENTINEL}")] + #[case::failure_callback_one(" failure_callback: SENTINEL")] + #[case::audit_object(" s3_audit_callback_params: {secret: SENTINEL}")] + #[case::context_window_raw(" context_window_fallbacks: [{gpt: [SENTINEL]}]")] + #[case::mcp_raw(" mcp_aliases: {docs: SENTINEL}")] + #[case::injection_object(" prompt_injection_params: {key: SENTINEL}")] + #[case::unknown_setting(" future_setting: SENTINEL")] + fn debug_output_does_not_expose_settings_values(#[case] yaml: &str) { + let config = Config::from_yaml(&format!("litellm_settings:\n{yaml}")).unwrap(); + assert!(!format!("{config:?}").contains("SENTINEL")); + } + + #[rstest] + fn unknown_setting_stays_in_additional_fields() { + assert!( + settings(" future_setting: 1") + .additional_fields + .contains_key("future_setting") + ); + } + + #[rstest] + fn transport_keys_land_in_their_reader_groups() { + let settings = settings( + r#" ssl_verify: false + ssl_security_level: '1' + ssl_certificate: /etc/client.pem + force_ipv4: true + http2: true + aiohttp_trust_env: true"#, + ); + assert_eq!(settings.tls.ssl_verify, Some(Spelled::Value(false))); + assert_eq!(settings.tls.ssl_security_level.as_deref(), Some("1")); + assert_eq!( + settings.client_certificate.ssl_certificate.as_deref(), + Some("/etc/client.pem") + ); + assert_eq!(settings.socket.force_ipv4, Some(true)); + assert_eq!(settings.http2.http2, Some(true)); + assert_eq!(settings.aiohttp.aiohttp_trust_env, Some(true)); + assert!(settings.additional_fields.is_empty()); + } + + #[rstest] + #[case::tls(" ssl_verify: false", "ssl_verify")] + #[case::client_certificate(" ssl_certificate: /etc/client.pem", "ssl_certificate")] + #[case::socket(" force_ipv4: true", "force_ipv4")] + #[case::http2(" http2: true", "http2")] + #[case::aiohttp(" disable_aiohttp_transport: true", "disable_aiohttp_transport")] + #[case::caching(" cache: true", "cache")] + #[case::budget(" max_budget: 10", "max_budget")] + #[case::logging(" json_logs: true", "json_logs")] + #[case::fallbacks(" default_fallbacks: [gpt]", "default_fallbacks")] + #[case::mcp(" mcp_aliases: {docs: docs_server}", "mcp_aliases")] + fn grouped_settings_stay_out_of_additional_fields(#[case] yaml: &str, #[case] key: &str) { + assert!(!settings(yaml).additional_fields.contains_key(key)); + } + + #[rstest] + #[case::one("callbacks: custom_callbacks.logger", 1)] + #[case::many("callbacks: [prometheus, otel]", 2)] + fn accepts_python_callback_shorthand(#[case] setting: &str, #[case] expected_len: usize) { + let config = Config::from_yaml(&format!("litellm_settings:\n {setting}")).unwrap(); + assert_eq!( + config + .litellm_settings + .logging + .callbacks + .as_ref() + .unwrap() + .len(), + expected_len + ); + } +} diff --git a/litellm-rust/crates/config/src/litellm/transport.rs b/litellm-rust/crates/config/src/litellm/transport.rs new file mode 100644 index 00000000000..bd1a055b97f --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/transport.rs @@ -0,0 +1,34 @@ +use crate::Spelled; + +/// Read by the SSL context builder, `get_ssl_configuration`. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct TlsSettings { + pub ssl_verify: Option>, + pub ssl_security_level: Option, + pub ssl_ecdh_curve: Option, +} + +/// Read by the sync and async HTTP clients, which send it as the client certificate. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ClientCertificateSettings { + pub ssl_certificate: Option, +} + +/// Read by the httpx and aiohttp transports when they bind the local socket. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SocketSettings { + pub force_ipv4: Option, +} + +/// Read by the httpx client. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Http2Settings { + pub http2: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AiohttpSettings { + pub aiohttp_trust_env: Option, + pub disable_aiohttp_trust_env: Option, + pub disable_aiohttp_transport: Option, +} diff --git a/litellm-rust/crates/config/src/mcp.rs b/litellm-rust/crates/config/src/mcp.rs index cbe992f9956..c2ecb7824fc 100644 --- a/litellm-rust/crates/config/src/mcp.rs +++ b/litellm-rust/crates/config/src/mcp.rs @@ -1,11 +1,11 @@ -use std::{collections::BTreeMap, fmt}; +use std::collections::BTreeMap; use litellm_auth_types::SecretValue; use serde::Deserialize; use crate::Object; -#[derive(Clone, Default, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize)] #[serde(default)] pub struct McpServer { pub server_id: Option, @@ -14,8 +14,8 @@ pub struct McpServer { pub mcp_info: Object, pub transport: McpTransport, pub url: Option, - pub command: Option, - pub args: Box<[String]>, + pub command: Option, + pub args: Box<[SecretValue]>, pub env: BTreeMap, pub auth_type: Option, #[serde(alias = "auth_value")] @@ -29,18 +29,6 @@ pub struct McpServer { pub unsupported: Object, } -impl fmt::Debug for McpServer { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("McpServer") - .field("transport", &self.transport) - .field("auth_type", &self.auth_type) - .field("timeout", &self.timeout) - .field("max_concurrent_requests", &self.max_concurrent_requests) - .finish_non_exhaustive() - } -} - #[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, strum::IntoStaticStr)] #[serde(rename_all = "snake_case")] pub enum McpTransport { @@ -81,3 +69,75 @@ pub enum McpAuth { #[strum(serialize = "oauth_delegate")] OauthDelegate, } + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use crate::{Config, McpAuth, McpTransport}; + + #[rstest] + #[case::http(McpTransport::Http, "http")] + #[case::sse(McpTransport::Sse, "sse")] + #[case::stdio(McpTransport::Stdio, "stdio")] + fn mcp_transport_as_str_matches_the_serde_spelling( + #[case] transport: McpTransport, + #[case] name: &str, + ) { + assert_eq!(<&'static str>::from(transport), name); + assert_eq!( + serde_json::from_value::(json!(name)).unwrap(), + transport + ); + } + + #[rstest] + #[case::none(McpAuth::None, "none")] + #[case::api_key(McpAuth::ApiKey, "api_key")] + #[case::bearer_token(McpAuth::BearerToken, "bearer_token")] + #[case::basic(McpAuth::Basic, "basic")] + #[case::authorization(McpAuth::Authorization, "authorization")] + #[case::token(McpAuth::Token, "token")] + #[case::oauth2(McpAuth::Oauth2, "oauth2")] + #[case::aws_sigv4(McpAuth::AwsSigv4, "aws_sigv4")] + #[case::oauth2_token_exchange(McpAuth::Oauth2TokenExchange, "oauth2_token_exchange")] + #[case::oauth2_id_jag(McpAuth::Oauth2IdJag, "oauth2_id_jag")] + #[case::true_passthrough(McpAuth::TruePassthrough, "true_passthrough")] + #[case::oauth_delegate(McpAuth::OauthDelegate, "oauth_delegate")] + fn mcp_auth_as_str_matches_the_serde_spelling(#[case] auth: McpAuth, #[case] name: &str) { + assert_eq!(<&'static str>::from(auth), name); + assert_eq!( + serde_json::from_value::(json!(name)).unwrap(), + auth + ); + } + + #[rstest] + fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() { + let config = Config::from_yaml("mcp_servers:\n docs:\n url: https://example.test/private-secret/mcp\n authentication_token: upstream-secret\n static_headers: {x-token: header-secret}\n env: {TOKEN: env-secret}\n args: [argument-secret]\n client_secret: oauth-secret\n allowed_tools: [search]\n").unwrap(); + let server = &config.mcp_servers["docs"]; + assert_eq!(server.allowed_tools.as_deref().unwrap(), ["search"]); + assert!(server.unsupported.contains_key("client_secret")); + let debug = format!("{config:?}"); + for secret in [ + "private-secret", + "upstream-secret", + "header-secret", + "env-secret", + "oauth-secret", + "argument-secret", + ] { + assert!(!debug.contains(secret)); + } + } + + #[rstest] + #[case::transport("transport: invalid")] + #[case::auth("auth_type: invalid")] + #[case::concurrency("max_concurrent_requests: -1")] + #[case::headers("static_headers: {x-token: [not, a, string]}")] + fn rejects_invalid_typed_mcp_settings(#[case] setting: &str) { + assert!(Config::from_yaml(&format!("mcp_servers:\n docs:\n {setting}\n")).is_err()); + } +} diff --git a/litellm-rust/crates/config/src/model.rs b/litellm-rust/crates/config/src/model.rs index c0f8516103d..8ec24f354f0 100644 --- a/litellm-rust/crates/config/src/model.rs +++ b/litellm-rust/crates/config/src/model.rs @@ -1,30 +1,187 @@ -use std::fmt; - use litellm_router_types::LitellmParams; use serde::Deserialize; -use crate::{AdditionalFields, Object}; +use crate::{Object, Raw, Spelled}; -#[derive(Clone, Deserialize)] +#[derive(Clone, Debug, Deserialize)] pub struct Model { pub model_name: String, pub litellm_params: LitellmParams, #[serde(default)] - pub model_info: Object, + pub model_info: ModelInfo, pub blocked: Option, + pub rpm: Option>, + pub tpm: Option>, #[serde(flatten)] - pub additional_fields: AdditionalFields, + pub additional_fields: Object, } -impl fmt::Debug for Model { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("Model") - .field("model_name", &self.model_name) - .field("litellm_params", &self.litellm_params) - .field("model_info", &self.model_info) - .field("blocked", &self.blocked) - .field("additional_fields", &self.additional_fields.keys()) - .finish() +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ModelInfo { + pub id: Option, + pub db_model: Option>, + pub updated_at: Option, + pub created_at: Option, + pub updated_by: Option, + pub created_by: Option, + pub base_model: Option, + pub pricing_provider: Option, + pub tier: Option, + pub team_id: Option, + pub team_public_model_name: Option, + pub member_auto_router: Option>, + pub blocked: Option>, + pub discoverable: Option>, + pub access_windows: Option>, + pub ptu_count: Option>, + pub cost_per_ptu_per_hour: Option>, + pub ptu_effective_from: Option, + pub ptu_effective_to: Option, + pub ptu_shares: Option, + pub allow_fail_open: Option>, + pub enable_tag_filtering: Option>, + pub internal_router_model: Option>, + pub max_embedding_batch_size: Option>, + pub input_cost_per_token: Option>, + pub output_cost_per_token: Option>, + pub input_cost_per_character: Option>, + pub output_cost_per_character: Option>, + pub cache_read_input_token_cost: Option>, + pub explicit_cache_read_input_token_cost: Option>, + pub cache_creation_input_token_cost: Option>, + pub cache_creation_input_token_cost_above_1hr: Option>, + pub tiered_pricing: Option>, + pub mode: Option, + pub access_groups: Option>, + pub supported_environments: Option>, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum ModelId { + Text(String), + Number(i64), +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AccessWindow { + pub start: Option, + pub end: Option, + pub timezone: Option, + pub team_ids: Option>, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Model, ModelId, Spelled}; + + fn model(entry: &str) -> Model { + let config = Config::from_yaml(&format!( + "model_list:\n - model_name: assistant\n litellm_params: {{model: openai/test}}\n{entry}" + )) + .unwrap(); + config.model_list[0].clone() + } + + #[rstest] + #[case::text(" model_info: {id: primary}", Some(ModelId::Text("primary".to_string())))] + #[case::number(" model_info: {id: 7}", Some(ModelId::Number(7)))] + #[case::null(" model_info: {id: null}", None)] + #[case::absent(" model_info: {mode: chat}", None)] + fn model_info_id_accepts_what_python_accepts(#[case] entry: &str, #[case] id: Option) { + assert_eq!(model(entry).model_info.id, id); + } + + #[rstest] + #[case::missing("")] + #[case::null(" model_info:")] + fn model_info_may_be_missing_or_null(#[case] entry: &str) { + let model = model(entry); + assert!(model.model_info.id.is_none()); + assert!(model.model_info.additional_fields.is_empty()); + } + + #[rstest] + fn model_info_fields_land_typed() { + let model = model( + r#" model_info: + mode: embedding + base_model: azure/test-base + team_id: team-1 + access_groups: [internal, beta] + db_model: "1" + blocked: true + ptu_count: 4 + cost_per_ptu_per_hour: 2 + input_cost_per_token: os.environ/INPUT_COST + tiered_pricing: [{range: [0, 1000]}] + access_windows: + - {start: "09:00", end: "17:00", timezone: UTC, team_ids: [team-1]} + supports_vision: true +"#, + ); + let info = &model.model_info; + assert_eq!(info.mode.as_deref(), Some("embedding")); + assert_eq!(info.base_model.as_deref(), Some("azure/test-base")); + assert_eq!(info.team_id.as_deref(), Some("team-1")); + assert_eq!( + info.access_groups.as_deref(), + Some(&["internal".to_string(), "beta".to_string()][..]) + ); + assert_eq!(info.db_model, Some(Spelled::Text("1".to_string()))); + assert_eq!(info.blocked, Some(Spelled::Value(true))); + assert_eq!(info.ptu_count, Some(Spelled::Value(4))); + assert_eq!(info.cost_per_ptu_per_hour, Some(Spelled::Value(2.0))); + assert_eq!( + info.input_cost_per_token, + Some(Spelled::Text("os.environ/INPUT_COST".to_string())) + ); + assert_eq!( + info.tiered_pricing.as_ref().map(|tiers| tiers.len()), + Some(1) + ); + let window = &info.access_windows.as_ref().unwrap()[0]; + assert_eq!(window.timezone.as_deref(), Some("UTC")); + assert_eq!(window.start.as_ref().unwrap().get().as_str(), Some("09:00")); + assert_eq!( + window.team_ids.as_deref(), + Some(&["team-1".to_string()][..]) + ); + assert!(info.additional_fields.contains_key("supports_vision")); + } + + #[rstest] + fn top_level_limits_and_unknown_keys_land_on_the_model() { + let model = model(" rpm: 100\n tpm: os.environ/TPM\n team: research\n"); + assert_eq!(model.rpm, Some(Spelled::Value(100))); + assert_eq!(model.tpm, Some(Spelled::Text("os.environ/TPM".to_string()))); + assert!(model.additional_fields.contains_key("team")); + assert!(!model.additional_fields.contains_key("rpm")); + } + + #[rstest] + #[case::ptu_shares( + " model_info: {ptu_shares: {team-1: sentinel-share-value}}", + "sentinel-share-value" + )] + #[case::created_at( + " model_info: {created_at: sentinel-created-at}", + "sentinel-created-at" + )] + #[case::tiered_pricing( + " model_info: {tiered_pricing: [{label: sentinel-tier-value}]}", + "sentinel-tier-value" + )] + #[case::window_start( + " model_info: {access_windows: [{start: sentinel-window-start, end: '17:00', timezone: UTC, team_ids: [t]}]}", + "sentinel-window-start" + )] + #[case::model_extension(" team_token: sentinel-model-extension", "sentinel-model-extension")] + fn opaque_model_info_values_stay_out_of_debug(#[case] entry: &str, #[case] sentinel: &str) { + assert!(!format!("{:?}", model(entry)).contains(sentinel)); } } diff --git a/litellm-rust/crates/config/src/policy.rs b/litellm-rust/crates/config/src/policy.rs new file mode 100644 index 00000000000..4c68e8d1dce --- /dev/null +++ b/litellm-rust/crates/config/src/policy.rs @@ -0,0 +1,144 @@ +use serde::Deserialize; + +use crate::{Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Policy { + pub guardrails: Option, + pub condition: Option, + pub pipeline: Option, + pub inherit: Option, + pub description: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum PolicyGuardrails { + Added(Box<[String]>), + Changes(PolicyGuardrailChanges), +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PolicyGuardrailChanges { + pub add: Option>, + pub remove: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PolicyCondition { + pub model: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PolicyPipeline { + pub mode: Option, + pub steps: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PolicyPipelineStep { + pub guardrail: Option, + pub on_fail: Option, + pub on_pass: Option, + pub on_error: Option, + pub pass_data: Option>, + pub modify_response_message: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PolicyAttachment { + pub policy: Option, + pub scope: Option, + pub teams: Option>, + pub keys: Option>, + pub models: Option>, + pub tags: Option>, + pub priority: Option>, + pub default: Option>, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, PolicyGuardrails, Spelled, Value}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + #[case::legacy_list("guardrails: [pii, toxicity]", &["pii", "toxicity"], &[])] + #[case::add_remove("guardrails: {add: [pii], remove: [toxicity]}", &["pii"], &["toxicity"])] + fn policy_guardrails_accept_both_python_spellings( + #[case] guardrails: &str, + #[case] added: &[&str], + #[case] removed: &[&str], + ) { + let config = parse(&format!("policies:\n safe:\n {guardrails}\n")); + let (add, remove) = match config.policies["safe"].guardrails.as_ref().unwrap() { + PolicyGuardrails::Added(add) => (add.to_vec(), Vec::new()), + PolicyGuardrails::Changes(changes) => ( + changes.add.as_deref().unwrap_or_default().to_vec(), + changes.remove.as_deref().unwrap_or_default().to_vec(), + ), + }; + + assert_eq!(add, added); + assert_eq!(remove, removed); + } + + #[rstest] + fn policies_and_attachments_parse_pipeline_condition_and_scope() { + let config = parse( + r#" +policies: + safe: + inherit: base + condition: + model: [gpt-a, gpt-b] + pipeline: + mode: pre_call + steps: + - guardrail: pii + on_fail: next + pass_data: true +policy_attachments: + - policy: safe + teams: [research] + priority: 10 + default: true +"#, + ); + let policy = &config.policies["safe"]; + let step = &policy.pipeline.as_ref().unwrap().steps.as_ref().unwrap()[0]; + let attachment = &config.policy_attachments[0]; + + assert_eq!(policy.inherit.as_deref(), Some("base")); + assert!(matches!( + policy.condition.as_ref().unwrap().model.as_ref().unwrap().get(), + Value::Sequence(models) if models.len() == 2 + )); + assert_eq!(step.guardrail.as_deref(), Some("pii")); + assert_eq!(step.on_fail.as_deref(), Some("next")); + assert_eq!(step.pass_data, Some(Spelled::Value(true))); + assert_eq!(attachment.policy.as_deref(), Some("safe")); + assert_eq!( + attachment.teams.as_deref(), + Some(&["research".to_owned()][..]) + ); + assert_eq!(attachment.priority, Some(Spelled::Value(10))); + assert_eq!(attachment.default, Some(Spelled::Value(true))); + } + + #[rstest] + #[case::policy_model( + r#" +policies: + safe: {condition: {model: SENTINEL}} +"# + )] + fn debug_never_prints_policy_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/prompt.rs b/litellm-rust/crates/config/src/prompt.rs new file mode 100644 index 00000000000..5a298a9efda --- /dev/null +++ b/litellm-rust/crates/config/src/prompt.rs @@ -0,0 +1,99 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Prompt { + pub prompt_id: Option, + pub litellm_params: Option, + pub prompt_info: Option, + pub version: Option>, + pub environment: Option, + pub created_by: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PromptParams { + pub prompt_id: Option, + pub prompt_integration: Option, + pub api_base: Option, + pub api_key: Option, + pub provider_specific_query_params: Option, + pub ignore_prompt_manager_model: Option>, + pub ignore_prompt_manager_optional_params: Option>, + pub dotprompt_content: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PromptInfo { + pub prompt_type: Option, + pub environment: Option, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Spelled}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn prompt_entries_keep_typed_params_and_info() { + let config = parse( + r#" +prompts: + - prompt_id: greeting + version: 2 + litellm_params: + prompt_integration: dotprompt + ignore_prompt_manager_model: true + prompt_extension: x + prompt_info: + prompt_type: config +"#, + ); + let prompt = &config.prompts[0]; + let params = prompt.litellm_params.as_ref().unwrap(); + + assert_eq!(prompt.prompt_id.as_deref(), Some("greeting")); + assert_eq!(prompt.version, Some(Spelled::Value(2))); + assert_eq!(params.prompt_integration.as_deref(), Some("dotprompt")); + assert_eq!( + params.ignore_prompt_manager_model, + Some(Spelled::Value(true)) + ); + assert!(params.additional_fields.contains_key("prompt_extension")); + assert_eq!( + prompt.prompt_info.as_ref().unwrap().prompt_type.as_deref(), + Some("config") + ); + } + + #[rstest] + #[case::prompt_key( + r#" +prompts: + - litellm_params: {api_key: SENTINEL} +"# + )] + #[case::prompt_query_params( + r#" +prompts: + - litellm_params: {provider_specific_query_params: {token: SENTINEL}} +"# + )] + #[case::prompt_extra( + r#" +prompts: + - litellm_params: {vendor_secret: SENTINEL} +"# + )] + fn debug_never_prints_prompt_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/provider.rs b/litellm-rust/crates/config/src/provider.rs new file mode 100644 index 00000000000..e4107bebddf --- /dev/null +++ b/litellm-rust/crates/config/src/provider.rs @@ -0,0 +1,118 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Raw}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AssistantSettings { + pub custom_llm_provider: Option, + pub litellm_params: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ProviderSettings { + pub custom_llm_provider: Option, + pub api_key: Option, + pub api_base: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct VertexConfig { + pub vertex_project: Option, + pub vertex_location: Option, + pub vertex_credentials: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Worker { + pub worker_id: Option, + pub name: Option, + pub url: Option, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::Config; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + #[case::files("files_settings")] + #[case::finetune("finetune_settings")] + fn provider_settings_keep_the_provider_and_unknown_keys(#[case] section: &str) { + let config = parse(&format!( + "{section}:\n - custom_llm_provider: azure\n api_version: '2024-01-01'\n" + )); + let settings = config + .files_settings + .first() + .or(config.finetune_settings.first()) + .unwrap(); + + assert_eq!(settings.custom_llm_provider.as_deref(), Some("azure")); + assert!(settings.additional_fields.contains_key("api_version")); + } + + #[rstest] + fn single_provider_sections_land_in_typed_fields() { + let config = parse( + r#" +assistant_settings: + custom_llm_provider: openai + litellm_params: + api_key: os.environ/OPENAI_API_KEY +default_vertex_config: + vertex_project: proj + vertex_location: us-central1 +worker_registry: + - worker_id: eu + name: Europe + url: https://eu.example.test +"#, + ); + let assistant = config.assistant_settings.as_ref().unwrap(); + let vertex = config.default_vertex_config.as_ref().unwrap(); + let worker = &config.worker_registry[0]; + + assert_eq!(assistant.custom_llm_provider.as_deref(), Some("openai")); + assert!( + assistant + .litellm_params + .as_ref() + .unwrap() + .contains_key("api_key") + ); + assert_eq!(vertex.vertex_project.as_deref(), Some("proj")); + assert_eq!(vertex.vertex_location.as_deref(), Some("us-central1")); + assert_eq!(worker.worker_id.as_deref(), Some("eu")); + assert_eq!(worker.name.as_deref(), Some("Europe")); + assert_eq!(worker.url.as_deref(), Some("https://eu.example.test")); + } + + #[rstest] + #[case::vertex_credentials("default_vertex_config: {vertex_credentials: SENTINEL}")] + #[case::vertex_credentials_map( + "default_vertex_config: {vertex_credentials: {private_key: SENTINEL}}" + )] + #[case::assistant_params("assistant_settings: {litellm_params: {api_key: SENTINEL}}")] + #[case::files_key( + r#" +files_settings: + - api_key: SENTINEL +"# + )] + #[case::finetune_extra( + r#" +finetune_settings: + - vendor_secret: SENTINEL +"# + )] + fn debug_never_prints_provider_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/router.rs b/litellm-rust/crates/config/src/router.rs new file mode 100644 index 00000000000..cf1c1365211 --- /dev/null +++ b/litellm-rust/crates/config/src/router.rs @@ -0,0 +1,191 @@ +use std::collections::BTreeMap; + +use litellm_auth_types::SecretValue; + +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RouterSettings { + pub redis_url: Option, + pub redis_host: Option, + pub redis_port: Option>, + pub redis_password: Option, + pub redis_db: Option>, + pub cache_responses: Option>, + pub cache_kwargs: Option, + pub caching_groups: Option, + pub client_ttl: Option>, + pub polling_interval: Option>, + pub default_priority: Option, + pub num_retries: Option, + pub max_fallbacks: Option>, + pub timeout: Option, + pub stream_timeout: Option>, + pub default_litellm_params: Option, + pub default_max_parallel_requests: Option>, + pub set_verbose: Option>, + pub debug_level: Option, + pub default_fallbacks: Option>, + pub fallbacks: Option>, + pub context_window_fallbacks: Option>, + pub content_policy_fallbacks: Option>, + pub model_group_alias: Option, + pub enable_pre_call_checks: Option>, + pub enable_tag_filtering: Option, + pub tag_filtering_match_any: Option>, + pub tag_routing_prefix: Option, + pub plugins: Option>, + pub retry_after: Option, + pub retry_policy: Option, + pub model_group_retry_policy: Option, + pub allowed_fails: Option, + pub allowed_fails_policy: Option, + pub cooldown_time: Option, + pub disable_cooldowns: Option>, + pub fallback_on_unsupported_params: Option>, + pub routing_strategy: Option, + pub optional_pre_call_checks: Option>, + pub routing_strategy_args: Option, + pub routing_groups: Option>, + pub provider_budget_config: Option>, + pub alerting_config: Option, + pub deployment_affinity_ttl_seconds: Option>, + pub model_group_affinity_config: Option, + pub enable_health_check_routing: Option>, + pub health_check_staleness_threshold: Option>, + pub health_check_ignore_transient_errors: Option>, + pub background_health_check_model_groups: Option>, + pub enable_weighted_failover: Option>, + pub enable_mid_stream_fallback_continuation: Option>, + pub guardrail_list: Option>, + pub max_retries: Option, + pub weights: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[macro_rules_attribute::apply(crate::config_section)] +#[serde(rename_all = "PascalCase")] +pub struct AllowedFailsPolicy { + pub bad_request_error_allowed_fails: Option>, + pub authentication_error_allowed_fails: Option>, + pub timeout_error_allowed_fails: Option>, + pub rate_limit_error_allowed_fails: Option>, + pub content_policy_violation_error_allowed_fails: Option>, + pub internal_server_error_allowed_fails: Option>, + pub service_unavailable_error_allowed_fails: Option>, + pub bad_gateway_error_allowed_fails: Option>, + pub not_found_error_allowed_fails: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ProviderBudget { + pub budget_limit: Option>, + pub time_period: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AlertingConfig { + pub webhook_url: Option, + pub alerting_threshold: Option>, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, RouterSettings, Spelled}; + + fn router(yaml: &str) -> RouterSettings { + Config::from_yaml(&format!("router_settings:\n{yaml}")) + .unwrap() + .router_settings + } + + #[rstest] + #[case::int(" redis_port: 6379", |s: &RouterSettings| s.redis_port == Some(Spelled::Value(6379)))] + #[case::int_env( + " client_ttl: os.environ/CLIENT_TTL", + |s: &RouterSettings| s.client_ttl == Some(Spelled::Text("os.environ/CLIENT_TTL".to_string())) +)] + #[case::float_from_int(" stream_timeout: 30", |s: &RouterSettings| s.stream_timeout == Some(Spelled::Value(30.0)))] + #[case::float( + " health_check_staleness_threshold: 2.5", + |s: &RouterSettings| s.health_check_staleness_threshold == Some(Spelled::Value(2.5)) +)] + #[case::bool(" enable_weighted_failover: true", |s: &RouterSettings| s.enable_weighted_failover == Some(Spelled::Value(true)))] + #[case::bool_word( + " enable_mid_stream_fallback_continuation: \"true\"", + |s: &RouterSettings| s.enable_mid_stream_fallback_continuation == Some(Spelled::Text("true".to_string())) +)] + #[case::string(" redis_host: cache.internal", |s: &RouterSettings| s.redis_host.as_deref() == Some("cache.internal"))] + #[case::string_list( + " default_fallbacks: [gpt-small, gpt-large]", + |s: &RouterSettings| s.default_fallbacks.as_deref() == Some(&["gpt-small".to_string(), "gpt-large".to_string()][..]) +)] + #[case::mapping_list( + " content_policy_fallbacks: [{gpt: [claude]}]", + |s: &RouterSettings| s.content_policy_fallbacks.as_ref().is_some_and(|items| items[0].contains_key("gpt")) +)] + #[case::mapping( + " default_litellm_params: {temperature: 0}", + |s: &RouterSettings| s.default_litellm_params.as_ref().is_some_and(|params| params.contains_key("temperature")) +)] + #[case::non_integer_priority(" default_priority: 1.5", |s: &RouterSettings| s.default_priority.is_some())] + #[case::bool_priority(" default_priority: true", |s: &RouterSettings| s.default_priority.is_some())] + #[case::caching_groups( + " caching_groups: [[gpt, [gpt-a, gpt-b]]]", + |s: &RouterSettings| s.caching_groups.as_ref().is_some_and(|groups| groups.get().is_sequence()) +)] + #[case::allowed_fails_policy( + " allowed_fails_policy: {RateLimitErrorAllowedFails: \"3\", BadGatewayErrorAllowedFails: 2, FutureAllowedFails: 1}", + |s: &RouterSettings| s.allowed_fails_policy.as_ref().is_some_and(|policy| { + policy.rate_limit_error_allowed_fails == Some(Spelled::Text("3".to_string())) + && policy.bad_gateway_error_allowed_fails == Some(Spelled::Value(2)) + }) +)] + #[case::provider_budget_config( + " provider_budget_config: {openai: {budget_limit: 100, time_period: 1d, rpm_limit: 5}}", + |s: &RouterSettings| s.provider_budget_config.as_ref().is_some_and(|budgets| { + let openai = &budgets["openai"]; + openai.budget_limit == Some(Spelled::Value(100.0)) + && openai.time_period.as_deref() == Some("1d") + }) +)] + #[case::alerting_config( + " alerting_config: {webhook_url: https://hooks.example/a, alerting_threshold: 10, channel: ops}", + |s: &RouterSettings| s.alerting_config.as_ref().is_some_and(|alerting| { + alerting.webhook_url.as_ref().map(|url| url.expose()) == Some("https://hooks.example/a") + && alerting.alerting_threshold == Some(Spelled::Value(10.0)) + }) +)] + fn parses_python_router_spellings( + #[case] yaml: &str, + #[case] check: fn(&RouterSettings) -> bool, + ) { + assert!(check(&router(yaml))); + } + + #[rstest] + #[case::unknown("future_router_setting")] + #[case::moved_to_general_settings("health_check_interval")] + #[case::managed_outside_config("model_list")] + fn keeps_keys_outside_the_router_signature(#[case] key: &str) { + let settings = router(&format!(" {key}: 60")); + assert!(settings.additional_fields.contains_key(key)); + } + + #[rstest] + #[case::redis_password(" redis_password: router-sentinel")] + #[case::redis_url(" redis_url: redis://user:router-sentinel@cache:6379")] + #[case::webhook_url(" alerting_config: {webhook_url: https://hooks.example/router-sentinel}")] + #[case::cache_kwargs(" cache_kwargs: {password: router-sentinel}")] + #[case::default_priority(" default_priority: router-sentinel")] + #[case::caching_groups(" caching_groups: [[router-sentinel, [a]]]")] + #[case::guardrail_list(" guardrail_list: [{api_key: router-sentinel}]")] + #[case::nested_extension(" allowed_fails_policy: {FutureAllowedFails: router-sentinel}")] + fn debug_output_does_not_expose_router_values(#[case] yaml: &str) { + let config = Config::from_yaml(&format!("router_settings:\n{yaml}")).unwrap(); + assert!(!format!("{config:?}").contains("router-sentinel")); + } +} diff --git a/litellm-rust/crates/config/src/settings.rs b/litellm-rust/crates/config/src/settings.rs deleted file mode 100644 index 1094de1638e..00000000000 --- a/litellm-rust/crates/config/src/settings.rs +++ /dev/null @@ -1,268 +0,0 @@ -use std::fmt; - -use litellm_auth_types::SecretValue; -use serde::Deserialize; - -use crate::{AdditionalFields, Object, Spelled, Value, value::one_or_many}; - -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum TracingStoreKind { - Clickhouse, -} - -#[derive(Clone, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ClickHouseStoreSettings { - #[serde(rename = "type")] - pub kind: TracingStoreKind, - pub url: Option, - pub database: Option, - pub retention_days: Option>, -} - -impl fmt::Debug for ClickHouseStoreSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("ClickHouseStoreSettings") - .field("kind", &self.kind) - .field("database", &self.database) - .field("retention_days", &self.retention_days) - .finish() - } -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(untagged)] -pub enum TracingStoreSettings { - ClickHouse(ClickHouseStoreSettings), -} - -#[derive(Clone, Default, Debug, Deserialize)] -#[serde(default)] -pub struct TracingSettings { - pub store: Option, - #[serde(flatten)] - pub additional_fields: AdditionalFields, -} - -#[derive(Clone, Deserialize)] -#[serde(default)] -pub struct GeneralSettings { - pub completion_model: Option, - pub max_in_flight_requests_per_worker: Option, - pub max_queued_requests_per_worker: Option, - pub admission_queue_timeout_seconds: f64, - pub master_key: Option, - pub database_url: Option, - pub tracing: Option, - pub database_connection_pool_limit: Option, - pub database_connection_timeout: Option, - pub database_connect_timeout: Option, - pub database_socket_timeout: Option, - pub database_max_idle_connection_lifetime: Option, - pub max_parallel_requests: Option, - pub global_max_parallel_requests: Option, - pub max_request_size_mb: Option, - pub max_response_size_mb: Option, - pub proxy_config_reload_interval_seconds: u64, - pub background_health_checks: Option, - pub health_check_interval: u64, - pub health_check_concurrency: Option, - pub store_model_in_db: Option, - pub forward_client_headers_to_llm_api: Option, - pub cancel_on_disconnect: Option, - pub infer_model_from_keys: Option, - pub enable_public_model_hub: bool, - pub dangerously_permit_weak_or_unset_master_key: Option, - pub plugins: Option>, - pub coordination_redis: Option, - pub mcp_allowed_hosts: Option>, - pub mcp_allowed_origins: Box<[String]>, - #[serde(flatten)] - pub additional_fields: AdditionalFields, -} - -impl Default for GeneralSettings { - fn default() -> Self { - Self { - completion_model: None, - max_in_flight_requests_per_worker: None, - max_queued_requests_per_worker: None, - admission_queue_timeout_seconds: 1.0, - master_key: None, - database_url: None, - tracing: None, - database_connection_pool_limit: Some(10), - database_connection_timeout: Some(60.0), - database_connect_timeout: None, - database_socket_timeout: None, - database_max_idle_connection_lifetime: Some(60.0), - max_parallel_requests: None, - global_max_parallel_requests: None, - max_request_size_mb: None, - max_response_size_mb: None, - proxy_config_reload_interval_seconds: 30, - background_health_checks: None, - health_check_interval: 300, - health_check_concurrency: None, - store_model_in_db: None, - forward_client_headers_to_llm_api: None, - cancel_on_disconnect: None, - infer_model_from_keys: None, - enable_public_model_hub: false, - dangerously_permit_weak_or_unset_master_key: None, - plugins: None, - coordination_redis: None, - mcp_allowed_hosts: None, - mcp_allowed_origins: Box::default(), - additional_fields: AdditionalFields::new(), - } - } -} - -impl fmt::Debug for GeneralSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("GeneralSettings") - .field("completion_model", &self.completion_model) - .field( - "max_in_flight_requests_per_worker", - &self.max_in_flight_requests_per_worker, - ) - .field( - "max_queued_requests_per_worker", - &self.max_queued_requests_per_worker, - ) - .field( - "admission_queue_timeout_seconds", - &self.admission_queue_timeout_seconds, - ) - .field("master_key", &self.master_key) - .field("database_url", &self.database_url) - .field("tracing", &self.tracing) - .field("store_model_in_db", &self.store_model_in_db) - .field("additional_fields", &self.additional_fields.keys()) - .finish_non_exhaustive() - } -} - -#[derive(Clone, Default, Deserialize)] -#[serde(default)] -pub struct RouterSettings { - pub routing_strategy: Option, - pub routing_strategy_args: Option, - pub routing_groups: Option>, - pub retry_policy: Option, - pub model_group_retry_policy: Option, - pub model_group_affinity_config: Option, - pub allowed_fails: Option, - pub cooldown_time: Option, - pub num_retries: Option, - pub timeout: Option, - pub max_retries: Option, - pub retry_after: Option, - pub fallbacks: Option>, - pub context_window_fallbacks: Option>, - pub model_group_alias: Option, - pub enable_tag_filtering: Option, - pub weights: Option, - pub tag_routing_prefix: Option, - pub optional_pre_call_checks: Option>, - #[serde(flatten)] - pub additional_fields: AdditionalFields, -} - -impl fmt::Debug for RouterSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("RouterSettings") - .field("routing_strategy", &self.routing_strategy) - .field("routing_strategy_args", &self.routing_strategy_args) - .field("routing_groups", &self.routing_groups) - .field("retry_policy", &self.retry_policy) - .field("model_group_retry_policy", &self.model_group_retry_policy) - .field( - "model_group_affinity_config", - &self.model_group_affinity_config, - ) - .field("allowed_fails", &self.allowed_fails) - .field("cooldown_time", &self.cooldown_time) - .field("num_retries", &self.num_retries) - .field("timeout", &self.timeout) - .field("max_retries", &self.max_retries) - .field("retry_after", &self.retry_after) - .field("fallbacks", &self.fallbacks) - .field("context_window_fallbacks", &self.context_window_fallbacks) - .field("model_group_alias", &self.model_group_alias) - .field("enable_tag_filtering", &self.enable_tag_filtering) - .field("weights", &self.weights) - .field("tag_routing_prefix", &self.tag_routing_prefix) - .field("optional_pre_call_checks", &self.optional_pre_call_checks) - .field("additional_fields", &self.additional_fields.keys()) - .finish() - } -} - -#[derive(Clone, Default, Deserialize)] -#[serde(default)] -pub struct LiteLlmSettings { - pub ssl_verify: Option>, - pub ssl_certificate: Option, - pub ssl_security_level: Option, - pub ssl_ecdh_curve: Option, - pub force_ipv4: Option, - pub http2: Option, - pub aiohttp_trust_env: Option, - pub disable_aiohttp_trust_env: Option, - pub disable_aiohttp_transport: Option, - pub drop_params: Option>, - pub request_timeout: Option>, - pub num_retries: Option, - pub cache: Option, - pub cache_params: Option, - #[serde(deserialize_with = "one_or_many")] - pub callbacks: Option>, - #[serde(deserialize_with = "one_or_many")] - pub success_callback: Option>, - #[serde(deserialize_with = "one_or_many")] - pub failure_callback: Option>, - pub json_logs: Option, - pub set_verbose: Option, - #[serde(flatten)] - pub additional_fields: AdditionalFields, -} - -impl fmt::Debug for LiteLlmSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LiteLlmSettings") - .field("drop_params", &self.drop_params) - .field("request_timeout", &self.request_timeout) - .field("num_retries", &self.num_retries) - .field("cache", &self.cache) - .field("cache_params", &self.cache_params) - .field( - "callbacks", - &self.callbacks.as_ref().map(|callbacks| callbacks.len()), - ) - .field( - "success_callback", - &self - .success_callback - .as_ref() - .map(|callbacks| callbacks.len()), - ) - .field( - "failure_callback", - &self - .failure_callback - .as_ref() - .map(|callbacks| callbacks.len()), - ) - .field("json_logs", &self.json_logs) - .field("set_verbose", &self.set_verbose) - .field("additional_fields", &self.additional_fields.keys()) - .finish() - } -} diff --git a/litellm-rust/crates/config/src/tool.rs b/litellm-rust/crates/config/src/tool.rs new file mode 100644 index 00000000000..bc2e93819b9 --- /dev/null +++ b/litellm-rust/crates/config/src/tool.rs @@ -0,0 +1,128 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SearchTool { + pub search_tool_name: Option, + pub litellm_params: Option, + pub search_tool_info: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SearchToolParams { + pub search_provider: Option, + pub api_key: Option, + pub api_base: Option, + pub timeout: Option>, + pub max_retries: Option>, + pub input_cost_per_query: Option>, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SearchToolInfo { + pub description: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SandboxTool { + pub sandbox_tool_name: Option, + pub litellm_params: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SandboxToolParams { + pub sandbox_provider: Option, + pub api_key: Option, + pub api_base: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Spelled}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn search_tools_keep_typed_params_and_info() { + let config = parse( + r#" +search_tools: + - search_tool_name: web + litellm_params: + search_provider: perplexity + max_retries: 3 + timeout: 2.5 + region: eu + search_tool_info: + description: web search +"#, + ); + let tool = &config.search_tools[0]; + let params = tool.litellm_params.as_ref().unwrap(); + + assert_eq!(tool.search_tool_name.as_deref(), Some("web")); + assert_eq!(params.search_provider.as_deref(), Some("perplexity")); + assert_eq!(params.max_retries, Some(Spelled::Value(3))); + assert_eq!(params.timeout, Some(Spelled::Value(2.5))); + assert!(params.additional_fields.contains_key("region")); + assert_eq!( + tool.search_tool_info + .as_ref() + .unwrap() + .description + .as_deref(), + Some("web search") + ); + } + + #[rstest] + fn sandbox_tools_keep_typed_params() { + let config = parse( + "sandbox_tools:\n - sandbox_tool_name: e2b\n litellm_params: {sandbox_provider: e2b, template: base}\n", + ); + let tool = &config.sandbox_tools[0]; + let params = tool.litellm_params.as_ref().unwrap(); + + assert_eq!(tool.sandbox_tool_name.as_deref(), Some("e2b")); + assert_eq!(params.sandbox_provider.as_deref(), Some("e2b")); + assert!(params.additional_fields.contains_key("template")); + } + + #[rstest] + #[case::search_key( + r#" +search_tools: + - litellm_params: {api_key: SENTINEL} +"# + )] + #[case::search_extra( + r#" +search_tools: + - litellm_params: {vendor_secret: SENTINEL} +"# + )] + #[case::sandbox_key( + r#" +sandbox_tools: + - litellm_params: {api_key: SENTINEL} +"# + )] + #[case::sandbox_extra( + r#" +sandbox_tools: + - litellm_params: {vendor_secret: SENTINEL} +"# + )] + fn debug_never_prints_tool_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/value.rs b/litellm-rust/crates/config/src/value.rs index 5becdc87b70..d40b8730413 100644 --- a/litellm-rust/crates/config/src/value.rs +++ b/litellm-rust/crates/config/src/value.rs @@ -1,28 +1,13 @@ use std::{collections::BTreeMap, fmt, ops::Deref}; -use serde::{Deserialize, Deserializer}; +use serde::Deserialize; pub type Value = serde_yaml_ng::Value; -pub type AdditionalFields = BTreeMap; #[derive(Clone, Default, Deserialize)] #[serde(transparent)] pub struct Object(BTreeMap); -impl Object { - pub fn get(&self, key: &str) -> Option<&Value> { - self.0.get(key) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn len(&self) -> usize { - self.0.len() - } -} - impl Deref for Object { type Target = BTreeMap; @@ -40,14 +25,27 @@ impl fmt::Debug for Object { } } -/// Python's callback shorthand: one entry or a list of them, kept as the values written. -pub(crate) fn one_or_many<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result>, D::Error> { - Ok( - Option::::deserialize(deserializer)?.map(|value| match value { - Value::Sequence(values) => values, - one => vec![one], - }), - ) +#[derive(Clone, Deserialize)] +#[serde(transparent)] +pub struct Raw(Value); + +impl Raw { + pub fn get(&self) -> &Value { + &self.0 + } +} + +impl fmt::Debug for Raw { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let kind = match &self.0 { + Value::Null => "null", + Value::Bool(_) => "bool", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Sequence(_) => "sequence", + Value::Mapping(_) => "mapping", + Value::Tagged(_) => "tagged", + }; + formatter.debug_tuple("Raw").field(&kind).finish() + } } diff --git a/litellm-rust/crates/config/src/vector_store.rs b/litellm-rust/crates/config/src/vector_store.rs new file mode 100644 index 00000000000..067d7502aa6 --- /dev/null +++ b/litellm-rust/crates/config/src/vector_store.rs @@ -0,0 +1,63 @@ +use crate::{Object, Raw}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct VectorStore { + pub vector_store_name: Option, + pub litellm_params: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct VectorStoreParams { + pub vector_store_id: Option, + pub custom_llm_provider: Option, + pub vector_store_description: Option, + pub vector_store_metadata: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::Config; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn vector_store_entries_keep_typed_params_and_unknown_keys() { + let config = parse( + r#" +vector_store_registry: + - vector_store_name: kb + litellm_params: {vector_store_id: vs_123, custom_llm_provider: openai, region: eu} +"#, + ); + let store = &config.vector_store_registry[0]; + let params = store.litellm_params.as_ref().unwrap(); + + assert_eq!(store.vector_store_name.as_deref(), Some("kb")); + assert_eq!(params.vector_store_id.as_deref(), Some("vs_123")); + assert_eq!(params.custom_llm_provider.as_deref(), Some("openai")); + assert!(params.additional_fields.contains_key("region")); + } + + #[rstest] + #[case::vector_store_metadata( + r#" +vector_store_registry: + - litellm_params: {vector_store_metadata: SENTINEL} +"# + )] + #[case::vector_store_extra( + r#" +vector_store_registry: + - litellm_params: {api_key: SENTINEL} +"# + )] + fn debug_never_prints_vector_store_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/yaml.rs b/litellm-rust/crates/config/src/yaml.rs new file mode 100644 index 00000000000..2de71e00e06 --- /dev/null +++ b/litellm-rust/crates/config/src/yaml.rs @@ -0,0 +1,177 @@ +use std::fmt; + +use serde::{ + Deserialize, Deserializer, + de::{MapAccess, SeqAccess, Visitor}, +}; +use serde_yaml_ng::{Mapping, Value}; + +pub fn mapping(text: &str) -> Result { + serde_yaml_ng::from_value(serde_yaml_ng::from_str::(text)?.0) +} + +struct LastWins(Value); + +impl<'de> Deserialize<'de> for LastWins { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(LastWinsVisitor).map(Self) + } +} + +// PyYAML safe_load keeps the later value of a duplicate key, serde_yaml_ng rejects it +struct LastWinsVisitor; + +impl<'de> Visitor<'de> for LastWinsVisitor { + type Value = Value; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("any YAML value") + } + + fn visit_bool(self, value: bool) -> Result { + Ok(Value::Bool(value)) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(Value::Number(value.into())) + } + + fn visit_u64(self, value: u64) -> Result { + Ok(Value::Number(value.into())) + } + + fn visit_f64(self, value: f64) -> Result { + Ok(Value::Number(value.into())) + } + + fn visit_str(self, value: &str) -> Result { + Ok(Value::String(value.to_owned())) + } + + fn visit_string(self, value: String) -> Result { + Ok(Value::String(value)) + } + + fn visit_unit(self) -> Result { + Ok(Value::Null) + } + + fn visit_none(self) -> Result { + Ok(Value::Null) + } + + fn visit_some>(self, deserializer: D) -> Result { + LastWins::deserialize(deserializer).map(|value| value.0) + } + + fn visit_seq>(self, mut items: A) -> Result { + let mut sequence = Vec::new(); + while let Some(LastWins(item)) = items.next_element()? { + sequence.push(item); + } + Ok(Value::Sequence(sequence)) + } + + fn visit_map>(self, mut entries: A) -> Result { + let mut mapping = Mapping::new(); + while let Some((LastWins(key), LastWins(value))) = entries.next_entry()? { + mapping.insert(key, value); + } + Ok(Value::Mapping(mapping)) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn yaml(text: &str) -> Mapping { + serde_yaml_ng::from_str(text).unwrap() + } + + #[rstest] + #[case::top_level( + r#" +a: 1 +b: 2 +a: 3 +"#, + "{a: 3, b: 2}" + )] + #[case::sequence_value( + r#" +cb: [prometheus] +cb: [s3] +"#, + "cb: [s3]" + )] + #[case::nested("outer: {x: [1], y: 2, x: [3]}", "outer: {x: [3], y: 2}")] + #[case::deeply_nested("a: {b: {c: 1, c: 2}}", "a: {b: {c: 2}}")] + #[case::inside_sequence("list: [{k: old, k: new}]", "list: [{k: new}]")] + #[case::mapping_replaced_by_scalar( + r#" +a: {b: 1} +a: 2 +"#, + "a: 2" + )] + #[case::value_replaced_by_null( + r#" +a: 1 +a: +"#, + "a: null" + )] + #[case::three_occurrences( + r#" +a: 1 +a: 2 +a: 3 +"#, + "a: 3" + )] + #[case::non_string_keys( + r#" +1: a +true: b +1: c +"#, + "{1: c, true: b}" + )] + #[case::scalars_without_duplicates( + r#" +i: -1 +u: 18446744073709551615 +f: 1.5 +b: false +n: ~ +s: text +"#, + "{i: -1, u: 18446744073709551615, f: 1.5, b: false, n: ~, s: text}" + )] + #[case::empty_document("", "{}")] + fn later_duplicate_keys_override_earlier_ones(#[case] text: &str, #[case] expected: &str) { + assert_eq!(mapping(text).unwrap(), yaml(expected)); + } + + #[rstest] + fn duplicate_key_keeps_its_first_position() { + let keys: Vec<_> = mapping("a: 1\nb: 2\na: 3\n").unwrap().into_keys().collect(); + assert_eq!(keys, [Value::from("a"), Value::from("b")]); + } + + #[rstest] + #[case::sequence( + r#" +- a +- b +"# + )] + #[case::scalar("text")] + #[case::malformed("a: [")] + fn rejects_a_document_that_is_not_a_mapping(#[case] text: &str) { + assert!(mapping(text).is_err()); + } +} diff --git a/litellm-rust/crates/config/tests/config.rs b/litellm-rust/crates/config/tests/config.rs deleted file mode 100644 index d425b62efc2..00000000000 --- a/litellm-rust/crates/config/tests/config.rs +++ /dev/null @@ -1,436 +0,0 @@ -use litellm_config::{Config, Error, Spelled, TracingStoreSettings}; -use rstest::{fixture, rstest}; -use tempfile::TempDir; - -#[fixture] -fn directory() -> TempDir { - tempfile::tempdir().unwrap() -} - -#[fixture] -fn model_list_yaml() -> &'static str { - r#" -model_list: - - model_name: assistant - litellm_params: - model: anthropic/test-model - api_key: os.environ/ANTHROPIC_API_KEY - - model_name: local - litellm_params: - model: test-model - api_base: http://localhost:8000/v1 - custom_llm_provider: openai -"# -} - -#[rstest] -fn loads_model_list_from_file(directory: TempDir, model_list_yaml: &str) { - let path = directory.path().join("config.yaml"); - std::fs::write(&path, model_list_yaml).unwrap(); - - let config = Config::load(path).unwrap(); - assert_eq!(config.model_list.len(), 2); - let anthropic = &config.model_list[0]; - assert_eq!(anthropic.model_name, "assistant"); - assert_eq!(anthropic.litellm_params.model, "anthropic/test-model"); - assert_eq!( - anthropic.litellm_params.api_key.as_ref().unwrap().expose(), - "os.environ/ANTHROPIC_API_KEY" - ); - assert!(anthropic.litellm_params.api_base.is_none()); - assert!(anthropic.litellm_params.custom_llm_provider.is_none()); - let local = &config.model_list[1]; - assert_eq!(local.model_name, "local"); - assert_eq!(local.litellm_params.model, "test-model"); - assert!(local.litellm_params.api_key.is_none()); - assert_eq!( - local.litellm_params.api_base.as_deref(), - Some("http://localhost:8000/v1") - ); - assert_eq!( - local.litellm_params.custom_llm_provider.as_deref(), - Some("openai") - ); -} - -#[rstest] -fn config_debug_redacts_api_keys() { - let config = Config::from_yaml( - "model_list: [{model_name: assistant, litellm_params: {model: anthropic/test-model, api_key: secret-value}}]", - ) - .unwrap(); - assert_eq!( - config.model_list[0] - .litellm_params - .api_key - .as_ref() - .unwrap() - .expose(), - "secret-value" - ); - assert!(!format!("{config:?}").contains("secret-value")); -} - -#[rstest] -#[case::malformed_yaml("model_list: [")] -#[case::missing_params("model_list: [{model_name: assistant}]")] -#[case::missing_model("model_list: [{model_name: assistant, litellm_params: {api_key: key}}]")] -#[case::provider_param_of_the_wrong_type( - "model_list: [{model_name: assistant, litellm_params: {model: test, aws_region_name: [eu-central-1]}}]" -)] -fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) { - assert!(matches!(Config::from_yaml(yaml), Err(Error::Parse(_)))); -} - -#[rstest] -fn distinguishes_read_errors_from_parse_errors(directory: TempDir) { - assert!(matches!( - Config::load(directory.path().join("missing.yaml")), - Err(Error::Read(error)) if error.kind() == std::io::ErrorKind::NotFound - )); -} - -#[rstest] -#[case::literal("secret-master-key")] -#[case::reference("os.environ/LITELLM_MASTER_KEY")] -fn loads_and_redacts_the_master_key(#[case] key: &str) { - let config = Config::from_yaml(&format!( - "model_list: []\ngeneral_settings:\n master_key: {key}\n" - )) - .unwrap(); - assert_eq!( - config - .general_settings - .master_key - .as_ref() - .unwrap() - .expose(), - key - ); - assert!(!format!("{config:?}").contains(key)); -} - -#[rstest] -fn missing_general_settings_has_no_master_key() { - let config = Config::from_yaml("model_list: []").unwrap(); - assert!(config.general_settings.master_key.is_none()); -} - -#[test] -fn tracing_settings_are_typed_and_redact_the_url() { - let config = Config::from_yaml( - "general_settings:\n tracing:\n store:\n type: clickhouse\n url: https://writer:password@example.com\n database: analytics\n retention_days: 7\n", - ) - .unwrap(); - let tracing = config.general_settings.tracing.as_ref().unwrap(); - let Some(TracingStoreSettings::ClickHouse(store)) = tracing.store.as_ref() else { - panic!("expected ClickHouse tracing store") - }; - assert_eq!( - store.url.as_ref().unwrap().expose(), - "https://writer:password@example.com" - ); - assert_eq!(store.database.as_deref(), Some("analytics")); - assert_eq!(store.retention_days, Some(Spelled::Value(7.0))); - assert!(!format!("{config:?}").contains("password")); -} - -#[test] -fn tracing_settings_accept_environment_references() { - let config = Config::from_yaml( - "general_settings:\n tracing:\n store:\n type: clickhouse\n url: os.environ/CLICKHOUSE_URL\n retention_days: os.environ/RETENTION_DAYS\n", - ) - .unwrap(); - let Some(TracingStoreSettings::ClickHouse(store)) = - config.general_settings.tracing.unwrap().store - else { - panic!("expected ClickHouse tracing store") - }; - assert_eq!( - store.retention_days, - Some(Spelled::Text("os.environ/RETENTION_DAYS".to_owned())) - ); -} - -#[test] -fn tracing_settings_reject_string_store() { - assert!(Config::from_yaml("general_settings:\n tracing:\n store: clickhouse\n").is_err()); -} - -#[test] -fn tracing_settings_reject_removed_reader_configuration() { - assert!(Config::from_yaml("general_settings:\n tracing:\n store:\n type: clickhouse\n reader_url: http://localhost:8123\n").is_err()); -} - -#[rstest] -fn empty_config_matches_python_defaults() { - let config = Config::from_yaml("{}").unwrap(); - assert!(config.model_list.is_empty()); - assert_eq!(config.general_settings.admission_queue_timeout_seconds, 1.0); - assert_eq!( - config.general_settings.database_connection_pool_limit, - Some(10) - ); - assert_eq!( - config.general_settings.proxy_config_reload_interval_seconds, - 30 - ); - assert_eq!(config.general_settings.health_check_interval, 300); -} - -#[rstest] -fn parses_typed_settings_and_preserves_extension_fields() { - let config = Config::from_yaml( - r#" -model_list: - - model_name: assistant - litellm_params: - model: vertex_ai/test-model - timeout: os.environ/REQUEST_TIMEOUT - tpm: os.environ/TPM_LIMIT - rpm: 5 - drop_params: "true" - vertex_project: test-project - vertex_credentials: - type: service_account - future_param: 1 - model_info: - mode: chat - access_groups: [internal] -general_settings: - master_key: secret-master-key - store_model_in_db: true - custom_auth: auth.py -router_settings: - routing_strategy: simple-shuffle - allowed_fails: 2 - redis_host: cache.internal -litellm_settings: - drop_params: true - cache: true - custom_callback_name: audit -future_section: - enabled: true -"#, - ) - .unwrap(); - - let model = &config.model_list[0]; - assert_eq!( - model.litellm_params.timeout, - Some(Spelled::Text("os.environ/REQUEST_TIMEOUT".to_string())) - ); - assert_eq!( - model.litellm_params.tpm, - Some(Spelled::Text("os.environ/TPM_LIMIT".to_string())) - ); - assert_eq!(model.litellm_params.rpm, Some(Spelled::Value(5.0))); - assert_eq!( - model.litellm_params.drop_params, - Some(Spelled::Text("true".to_string())) - ); - assert_eq!( - model.litellm_params.vertex.vertex_project.as_deref(), - Some("test-project") - ); - assert_eq!( - model.litellm_params.vertex.vertex_credentials.as_deref(), - Some(r#"{"type":"service_account"}"#) - ); - assert!(model.litellm_params.extra.contains_key("future_param")); - assert!(model.additional_fields.contains_key("access_groups")); - assert_eq!(config.router_settings.allowed_fails, Some(2)); - assert!( - config - .router_settings - .additional_fields - .contains_key("redis_host") - ); - assert!( - config - .litellm_settings - .additional_fields - .contains_key("custom_callback_name") - ); - assert!(config.additional_fields.contains_key("future_section")); -} - -#[rstest] -fn parses_python_config_sections() { - let config = Config::from_yaml( - r#" -environment_variables: - REDIS_PORT: 6379 -callback_settings: - otel: - message_logging: false -assistant_settings: - custom_llm_provider: openai -credential_list: - - credential_name: bedrock - credential_values: - aws_region_name: us-east-1 -guardrails: - - guardrail_name: pii - litellm_params: - guardrail: presidio -prompts: - - prompt_id: support -sandbox_tools: - - sandbox_tool_name: e2b -search_tools: - - search_tool_name: web -files_settings: - - custom_llm_provider: openai -finetune_settings: - - custom_llm_provider: openai -mcp_tools: - - name: lookup -mcp_servers: - docs: - url: https://example.test/mcp -vector_store_registry: - - vector_store_name: docs -worker_registry: - - worker_id: regional -agents: - - agent_name: reviewer -agent_list: - - agent_name: legacy-reviewer -policies: - safe: - guardrails: - add: [pii] -policy_attachments: - - policy_id: safe -include: - - models.yaml -"#, - ) - .unwrap(); - - assert_eq!(config.environment_variables.len(), 1); - assert_eq!(config.credential_list.len(), 1); - assert_eq!(config.guardrails.len(), 1); - assert_eq!(config.prompts.len(), 1); - assert_eq!(config.sandbox_tools.len(), 1); - assert_eq!(config.search_tools.len(), 1); - assert_eq!(config.files_settings.len(), 1); - assert_eq!(config.finetune_settings.len(), 1); - assert_eq!(config.mcp_tools.len(), 1); - assert_eq!(config.mcp_servers.len(), 1); - assert_eq!(config.vector_store_registry.len(), 1); - assert_eq!(config.worker_registry.len(), 1); - assert_eq!(config.agents.len(), 1); - assert_eq!(config.agent_list.len(), 1); - assert!(config.policies.contains_key("safe")); - assert_eq!(config.policy_attachments.len(), 1); - assert_eq!(&*config.include, &["models.yaml"]); -} - -#[rstest] -#[case::policy_pipeline("../../../litellm/proxy/example_config_yaml/test_pipeline_config.yaml")] -#[case::gateway("../../../tests/e2e/gateway/stage_mirror_ci_config.yml")] -fn parses_representative_python_configs(#[case] relative_path: &str) { - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(relative_path); - let config = Config::load(path).unwrap(); - assert!(!config.model_list.is_empty()); -} - -#[rstest] -#[case::one("callbacks: custom_callbacks.logger", 1)] -#[case::many("callbacks: [prometheus, otel]", 2)] -fn accepts_python_callback_shorthand(#[case] setting: &str, #[case] expected_len: usize) { - let config = Config::from_yaml(&format!("litellm_settings:\n {setting}")).unwrap(); - assert_eq!( - config.litellm_settings.callbacks.as_ref().unwrap().len(), - expected_len - ); -} - -#[rstest] -#[case::api_key("api_key", "provider-secret")] -#[case::aws_secret("aws_secret_access_key", "aws-secret")] -#[case::vertex_credentials("vertex_credentials", "vertex-secret")] -#[case::provider_extension("azure_ad_token", "azure-secret")] -#[case::general_extension("custom_auth_secret", "auth-secret")] -#[case::router_extension("redis_password", "redis-secret")] -#[case::litellm_extension("callback_token", "callback-secret")] -#[case::root_extension("private_token", "root-secret")] -fn debug_output_does_not_expose_config_values(#[case] field: &str, #[case] secret: &str) { - let yaml = match field { - "api_key" => format!( - "model_list: [{{model_name: assistant, litellm_params: {{model: test, api_key: {secret}}}}}]" - ), - "aws_secret_access_key" => format!( - "model_list: [{{model_name: assistant, litellm_params: {{model: test, aws_secret_access_key: {secret}}}}}]" - ), - "vertex_credentials" => format!( - "model_list: [{{model_name: assistant, litellm_params: {{model: test, vertex_credentials: {secret}}}}}]" - ), - "azure_ad_token" => format!( - "model_list: [{{model_name: assistant, litellm_params: {{model: test, azure_ad_token: {secret}}}}}]" - ), - "custom_auth_secret" => format!("general_settings: {{{field}: {secret}}}"), - "redis_password" => format!("router_settings: {{{field}: {secret}}}"), - "callback_token" => format!("litellm_settings: {{{field}: {secret}}}"), - "private_token" => format!("{field}: {secret}"), - _ => unreachable!(), - }; - let config = Config::from_yaml(&yaml).unwrap(); - assert!(!format!("{config:?}").contains(secret)); -} - -#[rstest] -fn resolves_nested_includes_once_in_breadth_first_order() { - let directory = TempDir::new().unwrap(); - let root = directory.path(); - std::fs::create_dir(root.join("nested")).unwrap(); - std::fs::write(root.join("config.yaml"), "include: [nested/first.yaml, second.yaml]\nmodel_list: [{model_name: root, litellm_params: {model: root}}]\n").unwrap(); - std::fs::write(root.join("nested/first.yaml"), "include: [third.yaml]\nmodel_list: [{model_name: first, litellm_params: {model: first}}]\n").unwrap(); - std::fs::write(root.join("second.yaml"), "general_settings: {master_key: second}\nmodel_list: [{model_name: second, litellm_params: {model: second}}]\n").unwrap(); - std::fs::write(root.join("nested/third.yaml"), "include: [../config.yaml]\ngeneral_settings: {master_key: third}\nmodel_list: [{model_name: third, litellm_params: {model: third}}]\n").unwrap(); - - let config = Config::load(root.join("config.yaml")).unwrap(); - assert_eq!( - config - .model_list - .iter() - .map(|model| model.model_name.as_str()) - .collect::>(), - ["root", "first", "second", "third"] - ); - assert_eq!( - config.general_settings.master_key.unwrap().expose(), - "third" - ); - assert!(config.include.is_empty()); -} - -#[rstest] -fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() { - let config = Config::from_yaml("mcp_servers:\n docs:\n url: https://example.test/private-secret/mcp\n authentication_token: upstream-secret\n static_headers: {x-token: header-secret}\n env: {TOKEN: env-secret}\n args: [argument-secret]\n client_secret: oauth-secret\n allowed_tools: [search]\n").unwrap(); - let server = &config.mcp_servers["docs"]; - assert_eq!(server.allowed_tools.as_deref().unwrap(), ["search"]); - assert!(server.unsupported.contains_key("client_secret")); - let debug = format!("{config:?}"); - for secret in [ - "private-secret", - "upstream-secret", - "header-secret", - "env-secret", - "oauth-secret", - "argument-secret", - ] { - assert!(!debug.contains(secret)); - } -} - -#[rstest] -#[case::transport("transport: invalid")] -#[case::auth("auth_type: invalid")] -#[case::concurrency("max_concurrent_requests: -1")] -#[case::headers("static_headers: {x-token: [not, a, string]}")] -fn rejects_invalid_typed_mcp_settings(#[case] setting: &str) { - assert!(Config::from_yaml(&format!("mcp_servers:\n docs:\n {setting}\n")).is_err()); -} diff --git a/litellm-rust/crates/config/tests/mcp.rs b/litellm-rust/crates/config/tests/mcp.rs deleted file mode 100644 index 9ac6ad02bbb..00000000000 --- a/litellm-rust/crates/config/tests/mcp.rs +++ /dev/null @@ -1,39 +0,0 @@ -use litellm_config::{McpAuth, McpTransport}; -use rstest::rstest; -use serde_json::json; - -#[rstest] -#[case::http(McpTransport::Http, "http")] -#[case::sse(McpTransport::Sse, "sse")] -#[case::stdio(McpTransport::Stdio, "stdio")] -fn mcp_transport_as_str_matches_the_serde_spelling( - #[case] transport: McpTransport, - #[case] name: &str, -) { - assert_eq!(<&'static str>::from(transport), name); - assert_eq!( - serde_json::from_value::(json!(name)).unwrap(), - transport - ); -} - -#[rstest] -#[case::none(McpAuth::None, "none")] -#[case::api_key(McpAuth::ApiKey, "api_key")] -#[case::bearer_token(McpAuth::BearerToken, "bearer_token")] -#[case::basic(McpAuth::Basic, "basic")] -#[case::authorization(McpAuth::Authorization, "authorization")] -#[case::token(McpAuth::Token, "token")] -#[case::oauth2(McpAuth::Oauth2, "oauth2")] -#[case::aws_sigv4(McpAuth::AwsSigv4, "aws_sigv4")] -#[case::oauth2_token_exchange(McpAuth::Oauth2TokenExchange, "oauth2_token_exchange")] -#[case::oauth2_id_jag(McpAuth::Oauth2IdJag, "oauth2_id_jag")] -#[case::true_passthrough(McpAuth::TruePassthrough, "true_passthrough")] -#[case::oauth_delegate(McpAuth::OauthDelegate, "oauth_delegate")] -fn mcp_auth_as_str_matches_the_serde_spelling(#[case] auth: McpAuth, #[case] name: &str) { - assert_eq!(<&'static str>::from(auth), name); - assert_eq!( - serde_json::from_value::(json!(name)).unwrap(), - auth - ); -} diff --git a/litellm-rust/crates/gateway-mcp/src/configured.rs b/litellm-rust/crates/gateway-mcp/src/configured.rs index bb4f5594f1f..9514e0b7997 100644 --- a/litellm-rust/crates/gateway-mcp/src/configured.rs +++ b/litellm-rust/crates/gateway-mcp/src/configured.rs @@ -136,7 +136,7 @@ fn validate(name: &str, config: &McpServer) -> Result<(), ConnectError> { if config .command .as_ref() - .is_none_or(|command| command.trim().is_empty()) + .is_none_or(|command| command.expose().trim().is_empty()) || config.url.is_some() { return Err(invalid( @@ -377,11 +377,12 @@ async fn connect_service( .await?; let program = config .command - .as_deref() + .as_ref() + .map(SecretValue::expose) .ok_or_else(|| invalid(name, "stdio requires command"))?; let mut command = tokio::process::Command::new(program); command - .args(&config.args) + .args(config.args.iter().map(SecretValue::expose)) .envs(environment) .kill_on_drop(true); ().serve_with_ct(TokioChildProcess::new(command)?, shutdown) diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 387bcd43231..5fc17476af6 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -19,6 +19,7 @@ litellm-gateway-auth.workspace = true litellm-gateway-ui.workspace = true litellm-auth-types.workspace = true litellm-config.workspace = true +litellm-config-load.workspace = true litellm-http = { workspace = true, features = ["mcp"] } litellm-llms.workspace = true litellm-secrets.workspace = true diff --git a/litellm-rust/crates/gateway/README.md b/litellm-rust/crates/gateway/README.md index 125c2b07448..da8dc9cbc9a 100644 --- a/litellm-rust/crates/gateway/README.md +++ b/litellm-rust/crates/gateway/README.md @@ -32,7 +32,7 @@ MCP clients can connect to `/mcp` or the scoped `/math/mcp` endpoint with the sa ## Configuration behavior -`Config::load` resolves nested `include` files once in breadth-first order. Included lists append and other top-level values replace earlier values. Paths resolve relative to the declaring file, with the root config directory as a fallback +`litellm_config_load::load` resolves nested `include` files once in breadth-first order. Included lists append and other top-level values replace earlier values. Paths resolve relative to the declaring file, with the root config directory as a fallback `environment_variables` provides scalar overrides to the injected secret source and HTTP settings without changing process globals. Credentials, static headers, URLs, and stdio environment values can reference `os.environ/NAME`. Model providers and MCP share the host's HTTP settings, including TLS certificates and environment proxies. Explicit environment HTTP settings take precedence over `litellm_settings` diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 0b3aba81181..1218db083c0 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,5 +1,6 @@ mod auth; mod error; +mod mcp_settings; mod secrets; pub use error::Error; @@ -15,6 +16,7 @@ use axum::{ use http_body_util::BodyExt; use litellm_config::Config; +use litellm_config_load::{Environment, Transport}; use litellm_gateway_auth::Auth; use litellm_gateway_inference::{Gateway, ModelRouter}; use litellm_http::{ @@ -26,34 +28,18 @@ use litellm_secrets::source::EnvironmentSecrets; use litellm_tracing::ByteChunk; use uuid::Uuid; -pub fn build_inference(config: &Config) -> Result, Error> { +pub fn build_inference(config: &Config, process: &Environment) -> Result, Error> { let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))); let environment = secrets::environment_values(&config.environment_variables)?; - let lookup = |name: &str| { + let lookup = process.clone().overlaid( environment - .get(name) - .map(|value| value.expose().to_owned()) - .or_else(|| std::env::var(name).ok()) - }; - let settings = &config.litellm_settings; + .iter() + .map(|(name, value)| (name.clone(), value.expose().to_owned())), + ); + let transport = Transport::lower(&config.litellm_settings, &lookup); let http_settings = HttpSettings::from_layers([ - HttpSettingsLayer::from_environment(&lookup), - HttpSettingsLayer { - ssl_verify: settings.ssl_verify.as_ref().map(|value| match value { - litellm_config::Spelled::Value(true) => SslVerify::Enabled, - litellm_config::Spelled::Value(false) => SslVerify::Disabled, - litellm_config::Spelled::Text(value) => SslVerify::parse(value), - }), - ssl_certificate: settings.ssl_certificate.as_ref().map(Into::into), - ssl_security_level: settings.ssl_security_level.clone(), - ssl_ecdh_curve: settings.ssl_ecdh_curve.clone(), - force_ipv4: settings.force_ipv4, - http2: settings.http2, - aiohttp_trust_env: settings.aiohttp_trust_env, - disable_aiohttp_trust_env: settings.disable_aiohttp_trust_env, - disable_aiohttp_transport: settings.disable_aiohttp_transport, - ..Default::default() - }, + transport_layer(&transport), + HttpSettingsLayer::from_environment(&|name: &str| lookup.var(name).map(str::to_owned)), ]); let http = Resolution::from(&http_settings).config; let client = pool.client(&http, ClientVariant::Provider)?; @@ -70,6 +56,25 @@ pub fn build_inference(config: &Config) -> Result, Error> { )?)) } +fn transport_layer(transport: &Transport) -> HttpSettingsLayer { + HttpSettingsLayer { + ssl_verify: Some(match &transport.ssl_verify { + litellm_config_load::SslVerify::Enabled => SslVerify::Enabled, + litellm_config_load::SslVerify::Disabled => SslVerify::Disabled, + litellm_config_load::SslVerify::CaBundle(path) => SslVerify::CaBundle(path.clone()), + }), + ssl_certificate: transport.ssl_certificate.clone(), + ssl_security_level: transport.ssl_security_level.clone(), + ssl_ecdh_curve: transport.ssl_ecdh_curve.clone(), + force_ipv4: Some(transport.force_ipv4), + http2: Some(transport.http2), + aiohttp_trust_env: Some(transport.aiohttp_trust_env), + disable_aiohttp_trust_env: Some(transport.disable_aiohttp_trust_env), + disable_aiohttp_transport: Some(transport.disable_aiohttp_transport), + ..Default::default() + } +} + pub async fn build_mcp( config: &Config, secrets: Arc, @@ -83,14 +88,8 @@ pub async fn build_mcp( if config.mcp_servers.is_empty() { return Ok(None); } - if let Some(setting) = config - .general_settings - .additional_fields - .keys() - .chain(config.litellm_settings.additional_fields.keys()) - .find(|key| key.starts_with("mcp_")) - { - return Err(Error::McpSetting(setting.clone())); + if let Some(setting) = mcp_settings::first_unsupported(config) { + return Err(Error::McpSetting(setting.into())); } if !config.guardrails.is_empty() || !config.policies.is_empty() diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index f8150e6e860..dc3c25f1329 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -3,7 +3,7 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; -use litellm_config::Config; +use litellm_config_load::Environment; use litellm_gateway_auth::UiBackend; use litellm_tracing::{Level, Logger, Metadata, Record, Sink}; use serde_json::json; @@ -41,14 +41,15 @@ impl Sink for StderrSink { #[tokio::main] async fn main() -> Result<(), Box> { - let settings: Settings = settings::from_iter(std::env::vars_os())?; + let environment = Environment::from_os(std::env::vars_os()); + let settings: Settings = settings::from_environment(&environment)?; let level = settings.log_level(); Logger::new(StderrSink { level }).install_global()?; - let config = Config::load(settings.litellm_config)?; - let inference = litellm_gateway::build_inference(&config)?; - let ui = match std::env::var_os("LITELLM_UI_PATH") { + let config = litellm_config_load::load(settings.litellm_config)?; + let inference = litellm_gateway::build_inference(&config, &environment)?; + let ui = match environment.var("LITELLM_UI_PATH") { Some(directory) => { - let ui_settings: UiSettings = settings::from_iter(std::env::vars_os())?; + let ui_settings: UiSettings = settings::from_environment(&environment)?; let backend = UiBackend::new(ui_settings.ui_username, ui_settings.ui_password)?; Some( litellm_gateway_ui::router( @@ -85,11 +86,12 @@ async fn main() -> Result<(), Box> { litellm_gateway_mcp::HttpConfig { allowed_hosts: config .general_settings + .mcp .mcp_allowed_hosts .as_deref() .map(Vec::from) .unwrap_or(defaults.allowed_hosts), - allowed_origins: config.general_settings.mcp_allowed_origins.to_vec(), + allowed_origins: config.general_settings.mcp.mcp_allowed_origins.to_vec(), cancellation_token: shutdown.clone(), server_info: defaults.server_info, }, diff --git a/litellm-rust/crates/gateway/src/mcp_settings.rs b/litellm-rust/crates/gateway/src/mcp_settings.rs new file mode 100644 index 00000000000..7213e0c2530 --- /dev/null +++ b/litellm-rust/crates/gateway/src/mcp_settings.rs @@ -0,0 +1,62 @@ +use litellm_config::Config; + +pub(crate) fn first_unsupported(config: &Config) -> Option<&'static str> { + let general = &config.general_settings.mcp; + let litellm = &config.litellm_settings.mcp; + [ + ( + "mcp_advertised_versions", + general.mcp_advertised_versions.is_some(), + ), + ( + "mcp_disabled_capabilities", + general.mcp_disabled_capabilities.is_some(), + ), + ("mcp_allowed_clients", general.mcp_allowed_clients.is_some()), + ( + "mcp_require_explicit_server_scope", + general.mcp_require_explicit_server_scope.is_some(), + ), + ( + "mcp_client_id_header", + general.mcp_client_id_header.is_some(), + ), + ( + "mcp_client_side_auth_header_name", + general.mcp_client_side_auth_header_name.is_some(), + ), + ( + "mcp_allow_all_keys_respects_mcp_scope", + general.mcp_allow_all_keys_respects_mcp_scope.is_some(), + ), + ( + "mcp_prefer_client_id_metadata_document", + general.mcp_prefer_client_id_metadata_document.is_some(), + ), + ( + "mcp_internal_ip_ranges", + general.mcp_internal_ip_ranges.is_some(), + ), + ( + "mcp_trusted_proxy_ranges", + general.mcp_trusted_proxy_ranges.is_some(), + ), + ( + "mcp_xff_num_trusted_hops", + general.mcp_xff_num_trusted_hops.is_some(), + ), + ("mcp_required_fields", general.mcp_required_fields.is_some()), + ( + "mcp_session_token_signing", + general.mcp_session_token_signing.is_some(), + ), + ("mcp_aliases", litellm.mcp_aliases.is_some()), + ( + "mcp_semantic_tool_filter", + litellm.mcp_semantic_tool_filter.is_some(), + ), + ("mcp_tool_search", litellm.mcp_tool_search.is_some()), + ] + .into_iter() + .find_map(|(key, set)| set.then_some(key)) +} diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index 2dd8872065f..02b47d62c46 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -1,6 +1,5 @@ -use std::ffi::OsString; - use litellm_auth_types::SecretValue; +use litellm_config_load::Environment; use litellm_tracing::Level; use serde::{Deserialize, de::DeserializeOwned}; @@ -50,14 +49,8 @@ fn default_secure_cookies() -> bool { true } -pub(super) fn from_iter( - variables: impl IntoIterator, -) -> envy::Result { - envy::from_iter( - variables - .into_iter() - .filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))), - ) +pub(super) fn from_environment(environment: &Environment) -> envy::Result { + envy::from_iter(environment.variables()) } #[cfg(test)] @@ -67,10 +60,11 @@ mod tests { use super::*; fn parse(variables: &[(&str, &str)]) -> envy::Result { - from_iter( - variables + from_environment( + &variables .iter() - .map(|(key, value)| (OsString::from(key), OsString::from(value))), + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(), ) } @@ -156,13 +150,13 @@ mod tests { #[case::unrelated_variable("UNRELATED")] #[case::host_fallback("HOST")] fn non_unicode_environment_values_are_ignored(#[case] key: &str) { - use std::os::unix::ffi::OsStringExt; + use std::{ffi::OsString, os::unix::ffi::OsStringExt}; - let settings: Settings = from_iter([ + let settings: Settings = from_environment(&Environment::from_os([ (OsString::from(key), OsString::from_vec(vec![0xff])), (OsString::from_vec(vec![0xff]), OsString::from("value")), (OsString::from("PORT"), OsString::from("8080")), - ]) + ])) .unwrap(); assert_eq!(settings.host, "0.0.0.0"); diff --git a/litellm-rust/crates/gateway/tests/mcp.rs b/litellm-rust/crates/gateway/tests/mcp.rs index 593fbfeb129..7494fb9bdf4 100644 --- a/litellm-rust/crates/gateway/tests/mcp.rs +++ b/litellm-rust/crates/gateway/tests/mcp.rs @@ -5,7 +5,13 @@ use axum::{ body::{Body, to_bytes}, http::Request, }; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use tokio_util::sync::CancellationToken; +use tower::ServiceExt; + use litellm_config::Config; +use litellm_config_load::Environment; use litellm_gateway_mcp::{ HttpConfig, rmcp::{ @@ -17,10 +23,6 @@ use litellm_gateway_mcp::{ }, }, }; -use rstest::{fixture, rstest}; -use serde_json::{Value, json}; -use tokio_util::sync::CancellationToken; -use tower::ServiceExt; #[derive(Clone)] struct Upstream; @@ -126,7 +128,7 @@ mcp_servers: upstream.url )) .unwrap(); - let inference = litellm_gateway::build_inference(&config).unwrap(); + let inference = litellm_gateway::build_inference(&config, &Environment::default()).unwrap(); let shutdown = CancellationToken::new(); let mcp = litellm_gateway::build_mcp( &config, @@ -243,7 +245,7 @@ async fn rejects_unsupported_or_invalid_settings_before_serving( #[case] expected: &str, ) { let config = Config::from_yaml(&format!("general_settings: {{master_key: gateway-key}}\nmcp_servers:\n docs:\n url: http://127.0.0.1:1/mcp\n {settings}\n")).unwrap(); - let inference = litellm_gateway::build_inference(&config).unwrap(); + let inference = litellm_gateway::build_inference(&config, &Environment::default()).unwrap(); let result = litellm_gateway::build_mcp( &config, Arc::new(NoSecrets), @@ -261,7 +263,7 @@ async fn rejects_unsupported_or_invalid_settings_before_serving( #[tokio::test] async fn mcp_is_optional_and_requires_admission_credentials_when_enabled() { let empty = Config::from_yaml("{}").unwrap(); - let inference = litellm_gateway::build_inference(&empty).unwrap(); + let inference = litellm_gateway::build_inference(&empty, &Environment::default()).unwrap(); assert!( litellm_gateway::build_mcp( &empty, @@ -311,7 +313,9 @@ fn applies_configured_http_environment_before_building_clients() { missing.display(), )) .unwrap(); - let error = litellm_gateway::build_inference(&config).err().unwrap(); + let error = litellm_gateway::build_inference(&config, &Environment::default()) + .err() + .unwrap(); assert!(matches!( error, litellm_gateway::Error::Http(litellm_http::Error::Read { path, .. }) @@ -320,16 +324,51 @@ fn applies_configured_http_environment_before_building_clients() { } #[rstest] -#[case::client_policy("general_settings: {master_key: key, mcp_allowed_clients: [approved]}")] -#[case::guardrail("guardrails: [{guardrail_name: policy}]")] -#[case::local_tools("mcp_tools: [{name: custom}]")] +#[case::client_policy( + "general_settings: {master_key: key, mcp_allowed_clients: [approved]}", + "mcp_allowed_clients" +)] +#[case::aliases( + "general_settings: {master_key: key}\nlitellm_settings: {mcp_aliases: {d: docs}}", + "mcp_aliases" +)] +#[case::tool_search( + "general_settings: {master_key: key}\nlitellm_settings: {mcp_tool_search: {top_k: 3}}", + "mcp_tool_search" +)] +#[case::advertised_versions( + "general_settings: {mcp_advertised_versions: [2025-06-18]}", + "mcp_advertised_versions" +)] +#[case::spelled_bool( + "general_settings: {mcp_require_explicit_server_scope: 'true'}", + "mcp_require_explicit_server_scope" +)] +#[case::header( + "general_settings: {mcp_client_side_auth_header_name: x-auth}", + "mcp_client_side_auth_header_name" +)] +#[case::hops( + "general_settings: {mcp_xff_num_trusted_hops: 2}", + "mcp_xff_num_trusted_hops" +)] +#[case::signing( + "general_settings: {mcp_session_token_signing: {algorithm: RS256}}", + "mcp_session_token_signing" +)] +#[case::semantic_filter( + "litellm_settings: {mcp_semantic_tool_filter: {enabled: true}}", + "mcp_semantic_tool_filter" +)] +#[case::guardrail("guardrails: [{guardrail_name: policy}]", "guardrails or policies")] +#[case::local_tools("mcp_tools: [{name: custom}]", "mcp_tools")] #[tokio::test] -async fn refuses_unimplemented_global_mcp_policy(#[case] settings: &str) { +async fn refuses_unimplemented_global_mcp_policy(#[case] settings: &str, #[case] key: &str) { let config = Config::from_yaml(&format!( "{settings}\nmcp_servers: {{docs: {{url: 'http://127.0.0.1:1/mcp'}}}}\n" )) .unwrap(); - let inference = litellm_gateway::build_inference(&config).unwrap(); + let inference = litellm_gateway::build_inference(&config, &Environment::default()).unwrap(); let error = litellm_gateway::build_mcp( &config, Arc::new(NoSecrets), @@ -340,5 +379,31 @@ async fn refuses_unimplemented_global_mcp_policy(#[case] settings: &str) { .await .err() .unwrap(); - assert!(matches!(error, litellm_gateway::Error::McpSetting(_))); + assert!(matches!(error, litellm_gateway::Error::McpSetting(setting) if setting == key)); +} + +#[rstest] +#[case::general("general_settings: {mcp_whatever: 1}")] +#[case::litellm("litellm_settings: {mcp_whatever: 1}")] +#[case::null("general_settings: {mcp_trusted_proxy_ranges: null}")] +#[case::allowed_hosts("general_settings: {mcp_allowed_hosts: [localhost]}")] +#[case::allowed_origins("general_settings: {mcp_allowed_origins: ['https://app.test']}")] +#[tokio::test] +async fn unknown_null_or_supported_mcp_keys_do_not_block_mcp(#[case] settings: &str) { + let config = Config::from_yaml(&format!( + "{settings}\nmcp_servers: {{docs: {{url: 'http://127.0.0.1:1/mcp'}}}}\n" + )) + .unwrap(); + let inference = litellm_gateway::build_inference(&config, &Environment::default()).unwrap(); + let error = litellm_gateway::build_mcp( + &config, + Arc::new(NoSecrets), + CancellationToken::new(), + &inference.resources.pool, + &inference.http, + ) + .await + .err() + .unwrap(); + assert!(matches!(error, litellm_gateway::Error::Auth(_)), "{error}"); } diff --git a/litellm-rust/crates/gateway/tests/server.rs b/litellm-rust/crates/gateway/tests/server.rs index 71edd3c3650..32c1b148b88 100644 --- a/litellm-rust/crates/gateway/tests/server.rs +++ b/litellm-rust/crates/gateway/tests/server.rs @@ -5,15 +5,17 @@ use std::{ use axum::{body::Body, http::Request}; use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; -use litellm_config::Config; -use litellm_gateway_inference::{Error, Gateway}; -use litellm_http::ClientVariant; -use litellm_tracing::{Logger, Metadata, Record, Sink}; use rstest::{fixture, rstest}; use serde_json::{Value, json}; use tokio::{net::TcpListener, sync::oneshot, time::timeout}; use tower::ServiceExt; +use litellm_config::Config; +use litellm_config_load::Environment; +use litellm_gateway_inference::{Error, Gateway}; +use litellm_http::ClientVariant; +use litellm_tracing::{Logger, Metadata, Record, Sink}; + struct LogSink(mpsc::Sender); impl Sink for LogSink { @@ -30,7 +32,11 @@ impl Sink for LogSink { #[fixture] fn inference() -> Arc { - litellm_gateway::build_inference(&Config::from_yaml("model_list: []").unwrap()).unwrap() + litellm_gateway::build_inference( + &Config::from_yaml("model_list: []").unwrap(), + &Environment::default(), + ) + .unwrap() } #[rstest] diff --git a/litellm-rust/crates/python-compat/AGENTS.md b/litellm-rust/crates/python-compat/AGENTS.md index 55049daf612..ff0b34bc431 100644 --- a/litellm-rust/crates/python-compat/AGENTS.md +++ b/litellm-rust/crates/python-compat/AGENTS.md @@ -6,6 +6,7 @@ - `cache-disk`: diskcache's pickled values (`pickle`), falsy-is-miss (`truthy`) - Cache-key derivation: sha256 over `str(value)` must match Python byte for byte (`repr::to_str`) - Byte-identical writes where Python compares raw values (`json::dumps`, `repr`) +- `read` holds Python's config reading shapes (`Spelled`, `one_or_many`). No setting names or config keys here, those stay with the crate that reads them - Relation to [`py_literal`](https://docs.rs/py_literal/latest/py_literal/): replaced, do not reintroduce - Its pest grammar backtracks: parse time doubles per nested `[`/`{` (105 ms at depth 16). Ours is linear (19 µs at depth 128) - Its formatter is not `repr` (`2e-1`, always single quotes, escapes non-ASCII). It also lost `-0.0`, `(1+2j)`, `set()` @@ -20,4 +21,4 @@ - `scripts/verify_rust_pickles.py` checks CPython reads Rust pickles, with class resolution disabled. CI does not run Python - Decoders recurse, so they reject nesting beyond `MAX_DEPTH` for stack safety: deliberately stricter than CPython, whose parser takes ~200 levels and whose unpickler has no limit (pinned as `nested_150`) - Formatters (`repr`, `json`) are unbounded. Values from the decoders are already capped, a hand-built `Value` is the caller's responsibility - - `literal_eval` must stay linear in depth: `tests/limits.rs` times the deepest parse, `benches/formats.rs` measures the curve but is manual, since CI runs no Rust bench + - `literal_eval` must stay linear in depth: the tests in `src/literal.rs` time the deepest parse, `benches/formats.rs` measures the curve but is manual, since CI runs no Rust bench diff --git a/litellm-rust/crates/python-compat/src/json.rs b/litellm-rust/crates/python-compat/src/json.rs index d3436519ac4..00c96ae2a13 100644 --- a/litellm-rust/crates/python-compat/src/json.rs +++ b/litellm-rust/crates/python-compat/src/json.rs @@ -166,3 +166,36 @@ fn write_string(out: &mut String, text: &str) { } out.push('"'); } + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::{dumps, to_json}; + use crate::Value; + + #[rstest] + #[case::bytes(Value::Bytes(b"x".to_vec()), "Object of type bytes is not JSON serializable")] + #[case::set(Value::Set(vec![Value::from(1)]), "Object of type set is not JSON serializable")] + #[case::complex(Value::Complex { re: 1.0, im: 2.0 }, "Object of type complex is not JSON serializable")] + #[case::nan( + Value::Float(f64::NAN), + "Out of range float values are not JSON compliant" + )] + fn to_json_reports_what_python_json_dumps_would_reject( + #[case] value: Value, + #[case] message: &str, + ) { + let error = to_json(&value).expect_err("value has no serde_json form"); + assert_eq!(error.to_string(), message); + } + + /// `json.dumps` writes the non-finite floats that `to_json` cannot represent. + #[rstest] + #[case::nan(f64::NAN, "NaN")] + #[case::infinity(f64::INFINITY, "Infinity")] + #[case::negative_infinity(f64::NEG_INFINITY, "-Infinity")] + fn dumps_writes_non_finite_floats(#[case] value: f64, #[case] text: &str) { + assert_eq!(dumps(&Value::Float(value)).unwrap(), text); + } +} diff --git a/litellm-rust/crates/python-compat/src/lib.rs b/litellm-rust/crates/python-compat/src/lib.rs index b23aff409f2..b1388e2c117 100644 --- a/litellm-rust/crates/python-compat/src/lib.rs +++ b/litellm-rust/crates/python-compat/src/lib.rs @@ -10,6 +10,7 @@ //! | [`json`] | `json.dumps(value)`, and `json.loads(json.dumps(value))` as a JSON value | //! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data | //! | [`truthy`] | `bool(value)` | +//! | [`read`] | reading a config value the way Python's loaders do | //! //! [`Value`] is the closed data model these formats share. Live Python objects //! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the @@ -25,6 +26,7 @@ mod error; pub mod json; pub mod literal; pub mod pickle; +pub mod read; pub mod repr; pub mod truthy; mod value; diff --git a/litellm-rust/crates/python-compat/src/literal.rs b/litellm-rust/crates/python-compat/src/literal.rs index 12e9036572f..361ebbd1a68 100644 --- a/litellm-rust/crates/python-compat/src/literal.rs +++ b/litellm-rust/crates/python-compat/src/literal.rs @@ -743,3 +743,52 @@ impl UniqueValues { self.keys.into_iter().zip(self.values).collect() } } + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use rstest::rstest; + + use super::literal_eval; + use crate::{Error, MAX_DEPTH, Value}; + + fn nested_text(open: &str, close: &str, depth: usize) -> String { + format!("{}1{}", open.repeat(depth), close.repeat(depth)) + } + + #[rstest] + #[case::list("[", "]")] + #[case::dict("{'a': ", "}")] + #[case::tuple("(", ",)")] + fn literal_eval_accepts_the_limit_and_rejects_past_it(#[case] open: &str, #[case] close: &str) { + assert!(literal_eval(&nested_text(open, close, MAX_DEPTH)).is_ok()); + assert!(matches!( + literal_eval(&nested_text(open, close, MAX_DEPTH + 1)), + Err(Error::TooDeep) + )); + } + + /// A backtracking parser (the `py_literal` grammar this replaced) doubles per nested level + /// and takes minutes here; the bound is loose enough to survive a slow debug build. + #[rstest] + #[case::list("[", "]")] + #[case::dict("{'a': ", "}")] + #[case::tuple("(", ",)")] + fn literal_eval_stays_linear_in_depth(#[case] open: &str, #[case] close: &str) { + let text = nested_text(open, close, MAX_DEPTH); + let start = Instant::now(); + assert!(literal_eval(&text).is_ok()); + let elapsed = start.elapsed(); + assert!( + elapsed < Duration::from_millis(50), + "{open} nested {MAX_DEPTH} deep took {elapsed:?}" + ); + } + + #[rstest] + fn literal_eval_ignores_brackets_inside_strings() { + let text = format!("'{}'", "[".repeat(MAX_DEPTH + 1)); + assert!(matches!(literal_eval(&text), Ok(Value::Str(_)))); + } +} diff --git a/litellm-rust/crates/python-compat/src/pickle.rs b/litellm-rust/crates/python-compat/src/pickle.rs index bff7442dc6b..e19867cd46d 100644 --- a/litellm-rust/crates/python-compat/src/pickle.rs +++ b/litellm-rust/crates/python-compat/src/pickle.rs @@ -185,3 +185,62 @@ impl<'de> Visitor<'de> for Seed { Ok(Value::Dict(entries)) } } + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::{dumps, loads}; + use crate::{Error, MAX_DEPTH, Value, literal::literal_eval}; + + fn nested_list(depth: usize) -> Value { + (0..depth).fold(Value::from(1), |value, _| Value::List(vec![value])) + } + + /// A protocol 3 pickle of `depth` nested lists around `1`: `EMPTY_LIST` per level, then + /// `BININT1 1`, then `APPEND` per level. Written by hand because `dumps` refuses the depth. + fn nested_list_pickle(depth: usize) -> Vec { + let mut data = vec![0x80, 3]; + data.extend(std::iter::repeat_n(b']', depth)); + data.extend([b'K', 1]); + data.extend(std::iter::repeat_n(b'a', depth)); + data.push(b'.'); + data + } + + #[rstest] + fn loads_accepts_the_limit_and_rejects_past_it() { + assert_eq!( + loads(&nested_list_pickle(MAX_DEPTH)).unwrap(), + nested_list(MAX_DEPTH) + ); + assert!(matches!( + loads(&nested_list_pickle(MAX_DEPTH + 1)), + Err(Error::InvalidPickle(_)) + )); + } + + #[rstest] + fn dumps_accepts_the_limit_and_rejects_past_it() { + assert!(dumps(&nested_list(MAX_DEPTH)).is_ok()); + assert!(matches!( + dumps(&nested_list(MAX_DEPTH + 2)), + Err(Error::TooDeep) + )); + } + + #[rstest] + #[case::set("{1, 2}", "set")] + #[case::complex("1+2j", "complex")] + fn dumps_refuses_types_it_would_change(#[case] source: &str, #[case] type_name: &str) { + let value = literal_eval(source).expect("source is a literal"); + assert!(matches!(dumps(&value), Err(Error::NotPicklable(name)) if name == type_name)); + } + + #[rstest] + #[case::class_reference(b"\x80\x02c__builtin__\ncomplex\nq\x00.".to_vec())] + #[case::trailing_data({ let mut data = dumps(&Value::from(1)).unwrap(); data.push(b'.'); data })] + fn loads_rejects(#[case] data: Vec) { + assert!(matches!(loads(&data), Err(Error::InvalidPickle(_)))); + } +} diff --git a/litellm-rust/crates/python-compat/src/read.rs b/litellm-rust/crates/python-compat/src/read.rs new file mode 100644 index 00000000000..2d649af2aca --- /dev/null +++ b/litellm-rust/crates/python-compat/src/read.rs @@ -0,0 +1,78 @@ +use serde::{Deserialize, Deserializer}; + +/// A typed value, or the text a config spells in its place: an `os.environ/NAME` reference +/// the secrets layer resolves later, or a word Python's coercion reads the value from. +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum Spelled { + Value(T), + Text(String), +} + +/// `x if isinstance(x, list) else [x]`, for `#[serde(deserialize_with)]`. +pub fn one_or_many<'de, D: Deserializer<'de>, T: Deserialize<'de>>( + deserializer: D, +) -> Result>, D::Error> { + #[derive(Deserialize)] + #[serde(untagged)] + enum OneOrMany { + Many(Box<[T]>), + One(T), + } + + Ok( + Option::>::deserialize(deserializer)?.map(|value| match value { + OneOrMany::Many(values) => values, + OneOrMany::One(value) => Box::from([value]), + }), + ) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde::Deserialize; + use serde_json::{Value, json}; + + use super::{Spelled, one_or_many}; + + #[derive(Deserialize)] + struct Entries { + #[serde(default, deserialize_with = "one_or_many")] + entries: Option>, + } + + fn entries(document: Value) -> Option> { + serde_json::from_value::(document).unwrap().entries + } + + #[rstest] + #[case::one(json!({"entries": "langfuse"}), Some(vec![json!("langfuse")]))] + #[case::one_mapping(json!({"entries": {"name": "otel"}}), Some(vec![json!({"name": "otel"})]))] + #[case::many(json!({"entries": ["langfuse", "otel"]}), Some(vec![json!("langfuse"), json!("otel")]))] + #[case::many_keeps_nested_lists(json!({"entries": [["a"], "b"]}), Some(vec![json!(["a"]), json!("b")]))] + #[case::empty(json!({"entries": []}), Some(vec![]))] + #[case::null(json!({"entries": null}), None)] + #[case::absent(json!({}), None)] + fn one_or_many_wraps_a_single_entry_and_keeps_a_list( + #[case] document: Value, + #[case] expected: Option>, + ) { + assert_eq!(entries(document), expected.map(Vec::into_boxed_slice)); + } + + #[rstest] + #[case::value(json!(3.5), Spelled::Value(3.5))] + #[case::integer(json!(3), Spelled::Value(3.0))] + #[case::reference(json!("os.environ/TIMEOUT"), Spelled::Text("os.environ/TIMEOUT".into()))] + #[case::numeric_text(json!("3"), Spelled::Text("3".into()))] + fn spelled_keeps_text_that_is_not_the_value_type( + #[case] input: Value, + #[case] expected: Spelled, + ) { + assert_eq!( + serde_json::from_value::>(input).unwrap(), + expected + ); + } +} diff --git a/litellm-rust/crates/python-compat/tests/limits.rs b/litellm-rust/crates/python-compat/tests/limits.rs deleted file mode 100644 index c4d931034c4..00000000000 --- a/litellm-rust/crates/python-compat/tests/limits.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::time::{Duration, Instant}; - -use litellm_python_compat::{Error, MAX_DEPTH, Value, json, literal::literal_eval, pickle}; -use rstest::{fixture, rstest}; - -/// The bracket pair of one container shape, as `(open, close)`. -#[fixture] -fn shapes() -> [(&'static str, &'static str); 3] { - [("[", "]"), ("{'a': ", "}"), ("(", ",)")] -} - -fn nested_text(open: &str, close: &str, depth: usize) -> String { - format!("{}1{}", open.repeat(depth), close.repeat(depth)) -} - -fn nested_list(depth: usize) -> Value { - (0..depth).fold(Value::from(1), |value, _| Value::List(vec![value])) -} - -/// A protocol 3 pickle of `depth` nested lists around `1`: `EMPTY_LIST` per level, then -/// `BININT1 1`, then `APPEND` per level. Written by hand because `dumps` refuses the depth. -fn nested_list_pickle(depth: usize) -> Vec { - let mut data = vec![0x80, 3]; - data.extend(std::iter::repeat_n(b']', depth)); - data.extend([b'K', 1]); - data.extend(std::iter::repeat_n(b'a', depth)); - data.push(b'.'); - data -} - -#[rstest] -fn literal_eval_accepts_the_limit_and_rejects_past_it(shapes: [(&'static str, &'static str); 3]) { - for (open, close) in shapes { - assert!(literal_eval(&nested_text(open, close, MAX_DEPTH)).is_ok()); - assert!(matches!( - literal_eval(&nested_text(open, close, MAX_DEPTH + 1)), - Err(Error::TooDeep) - )); - } -} - -/// A backtracking parser (the `py_literal` grammar this replaced) doubles per nested level -/// and takes minutes here; the bound is loose enough to survive a slow debug build. -#[rstest] -fn literal_eval_stays_linear_in_depth(shapes: [(&'static str, &'static str); 3]) { - for (open, close) in shapes { - let text = nested_text(open, close, MAX_DEPTH); - let start = Instant::now(); - assert!(literal_eval(&text).is_ok()); - let elapsed = start.elapsed(); - assert!( - elapsed < Duration::from_millis(50), - "{open} nested {MAX_DEPTH} deep took {elapsed:?}" - ); - } -} - -#[rstest] -fn literal_eval_ignores_brackets_inside_strings() { - let text = format!("'{}'", "[".repeat(MAX_DEPTH + 1)); - assert!(matches!(literal_eval(&text), Ok(Value::Str(_)))); -} - -#[rstest] -fn pickle_nesting_is_bounded_in_both_directions() { - assert_eq!( - pickle::loads(&nested_list_pickle(MAX_DEPTH)).unwrap(), - nested_list(MAX_DEPTH) - ); - assert!(matches!( - pickle::loads(&nested_list_pickle(MAX_DEPTH + 1)), - Err(Error::InvalidPickle(_)) - )); - assert!(pickle::dumps(&nested_list(MAX_DEPTH)).is_ok()); - assert!(matches!( - pickle::dumps(&nested_list(MAX_DEPTH + 2)), - Err(Error::TooDeep) - )); -} - -#[rstest] -#[case("{1, 2}", "set")] -#[case("1+2j", "complex")] -fn pickle_dumps_refuses_types_it_would_change(#[case] source: &str, #[case] type_name: &str) { - let value = literal_eval(source).expect("source is a literal"); - assert!(matches!(pickle::dumps(&value), Err(Error::NotPicklable(name)) if name == type_name)); -} - -#[rstest] -#[case(b"\x80\x02c__builtin__\ncomplex\nq\x00.".to_vec(), "a class reference")] -#[case({ let mut data = pickle::dumps(&Value::from(1)).unwrap(); data.push(b'.'); data }, "trailing data")] -fn pickle_loads_rejects(#[case] data: Vec, #[case] what: &str) { - assert!( - matches!(pickle::loads(&data), Err(Error::InvalidPickle(_))), - "{what} must not decode" - ); -} - -#[rstest] -#[case(Value::Bytes(b"x".to_vec()), "Object of type bytes is not JSON serializable")] -#[case(Value::Set(vec![Value::from(1)]), "Object of type set is not JSON serializable")] -#[case(Value::Complex { re: 1.0, im: 2.0 }, "Object of type complex is not JSON serializable")] -#[case( - Value::Float(f64::NAN), - "Out of range float values are not JSON compliant" -)] -fn to_json_reports_what_python_json_dumps_would_reject( - #[case] value: Value, - #[case] message: &str, -) { - let error = json::to_json(&value).expect_err("value has no serde_json form"); - assert_eq!(error.to_string(), message); -} - -/// `json.dumps` writes the non-finite floats that `to_json` cannot represent. -#[rstest] -#[case(f64::NAN, "NaN")] -#[case(f64::INFINITY, "Infinity")] -#[case(f64::NEG_INFINITY, "-Infinity")] -fn json_dumps_writes_non_finite_floats(#[case] value: f64, #[case] text: &str) { - assert_eq!(json::dumps(&Value::Float(value)).unwrap(), text); -} diff --git a/litellm-rust/crates/router-types/Cargo.toml b/litellm-rust/crates/router-types/Cargo.toml index 2e9fec28685..c6a273fa010 100644 --- a/litellm-rust/crates/router-types/Cargo.toml +++ b/litellm-rust/crates/router-types/Cargo.toml @@ -8,6 +8,7 @@ description = "The types of litellm/types/router.py: a deployment's litellm_para [dependencies] litellm-auth-types.workspace = true +litellm-python-compat.workspace = true serde.workspace = true serde_json.workspace = true serde_with.workspace = true diff --git a/litellm-rust/crates/router-types/src/lib.rs b/litellm-rust/crates/router-types/src/lib.rs index 2235e40a891..83c2c683ad5 100644 --- a/litellm-rust/crates/router-types/src/lib.rs +++ b/litellm-rust/crates/router-types/src/lib.rs @@ -1,5 +1,3 @@ mod litellm_params; -mod value; pub use litellm_params::{ExtraParams, LitellmParams}; -pub use value::Spelled; diff --git a/litellm-rust/crates/router-types/src/litellm_params.rs b/litellm-rust/crates/router-types/src/litellm_params.rs index 186a1d98036..cc73a1c85bf 100644 --- a/litellm-rust/crates/router-types/src/litellm_params.rs +++ b/litellm-rust/crates/router-types/src/litellm_params.rs @@ -5,7 +5,7 @@ use serde::Deserialize; use serde_json::Value; use serde_with::{OneOrMany, formats::PreferMany, serde_as}; -use crate::Spelled; +use litellm_python_compat::read::Spelled; /// Python's `LiteLLM_Params`: a deployment's `litellm_params`, as a config spells them and as /// a caller passes them. Each credential family is one flattened group owned by its auth @@ -30,11 +30,11 @@ pub struct LitellmParams { pub custom_llm_provider: Option, pub timeout: Option>, pub stream_timeout: Option>, - pub max_retries: Option>, - pub tpm: Option>, - pub rpm: Option>, - pub itpm: Option>, - pub otpm: Option>, + pub max_retries: Option>, + pub tpm: Option>, + pub rpm: Option>, + pub itpm: Option>, + pub otpm: Option>, pub max_parallel_requests: Option, #[serde_as(as = "Option>")] pub organization: Option>, diff --git a/litellm-rust/crates/router-types/src/value.rs b/litellm-rust/crates/router-types/src/value.rs deleted file mode 100644 index e52cf60c0ef..00000000000 --- a/litellm-rust/crates/router-types/src/value.rs +++ /dev/null @@ -1,10 +0,0 @@ -use serde::Deserialize; - -/// A typed value, or the text a config spells in its place: an `os.environ/NAME` reference -/// the secrets layer resolves later, or a word Python's coercion reads the value from. -#[derive(Clone, Debug, Deserialize, PartialEq)] -#[serde(untagged)] -pub enum Spelled { - Value(T), - Text(String), -} diff --git a/litellm-rust/crates/router-types/tests/litellm_params.rs b/litellm-rust/crates/router-types/tests/litellm_params.rs index 7d021fd857a..186444b816d 100644 --- a/litellm-rust/crates/router-types/tests/litellm_params.rs +++ b/litellm-rust/crates/router-types/tests/litellm_params.rs @@ -1,8 +1,10 @@ -use litellm_auth_types::{AwsParams, SecretValue, VertexParams}; -use litellm_router_types::{LitellmParams, Spelled}; use rstest::rstest; use serde_json::{Map, Value, json}; +use litellm_auth_types::{AwsParams, SecretValue, VertexParams}; +use litellm_python_compat::read::Spelled; +use litellm_router_types::LitellmParams; + fn params(value: Value) -> LitellmParams { serde_json::from_value(value).unwrap() } @@ -55,7 +57,7 @@ fn params(value: Value) -> LitellmParams { LitellmParams { model: "m".into(), timeout: Some(Spelled::Text("os.environ/T".into())), - rpm: Some(Spelled::Value(5.0)), + rpm: Some(Spelled::Value(5)), drop_params: Some(Spelled::Text("true".into())), tags: Some(Box::from(["a".to_string()])), max_budget: Some(1.5), @@ -116,6 +118,9 @@ fn keys_no_field_names_land_in_extra_and_nothing_else_does() { #[case::vertex(json!({"model": "m", "vertex_project": ["p"]}))] #[case::vertex_credentials(json!({"model": "m", "vertex_credentials": 7}))] #[case::rate_limit(json!({"model": "m", "max_parallel_requests": "many"}))] +#[case::fractional_rpm(json!({"model": "m", "rpm": 1.5}))] +#[case::fractional_max_retries(json!({"model": "m", "max_retries": 2.5}))] +#[case::negative_tpm(json!({"model": "m", "tpm": -1}))] #[case::tags(json!({"model": "m", "tags": "not-a-list"}))] fn a_param_of_the_wrong_type_is_rejected(#[case] value: Value) { assert!(serde_json::from_value::(value).is_err());