From 6bbf2cc017de473f89b34c53cb738326278fabbb Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Thu, 22 Oct 2026 11:28:36 -0700 Subject: [PATCH 1/5] feat(config): type every config.yaml key and move loading into config-load litellm-config now models every top-level config.yaml key, one module per key, and only parses YAML with PyYAML's reading rules. Include resolution, env lookups and transport lowering move into the new litellm-config-load crate, which the gateway uses to load its config. Co-Authored-By: Claude Opus 5.5 --- litellm-rust/Cargo.lock | 15 + litellm-rust/Cargo.toml | 1 + litellm-rust/crates/config-load/AGENTS.md | 11 + litellm-rust/crates/config-load/Cargo.toml | 16 + litellm-rust/crates/config-load/build.rs | 3 + litellm-rust/crates/config-load/src/error.rs | 26 + .../config-load/src/include/documents.rs | 39 ++ .../crates/config-load/src/include/merge.rs | 90 +++ .../crates/config-load/src/include/mod.rs | 169 ++++++ .../crates/config-load/src/include/store.rs | 46 ++ litellm-rust/crates/config-load/src/lib.rs | 65 +++ .../crates/config-load/src/transport.rs | 166 ++++++ .../crates/config-load/tests/AGENTS.md | 5 + .../crates/config-load/tests/corpus.rs | 16 + .../config-load/tests/fixtures/AGENTS.md | 3 + .../invalid/config_with_missing_include.yaml | 5 + .../fixtures/invalid/langfuse_config.yaml | 7 + .../tests/fixtures/valid/aliases_config.yaml | 30 + .../tests/fixtures/valid/azure_config.yaml | 15 + .../tests/fixtures/valid/cache_no_params.yaml | 7 + .../fixtures/valid/cache_with_params.yaml | 11 + .../fixtures/valid/config_with_env_vars.yaml | 48 ++ .../fixtures/valid/config_with_include.yaml | 5 + .../valid/config_with_multiple_includes.yaml | 6 + .../tests/fixtures/valid/included_models.yaml | 4 + .../tests/fixtures/valid/load_balancer.yaml | 28 + .../tests/fixtures/valid/models_file_1.yaml | 4 + .../tests/fixtures/valid/models_file_2.yaml | 4 + .../fixtures/valid/opentelemetry_config.yaml | 7 + .../tests/fixtures/valid/simple_config.yaml | 4 + .../tests/fixtures/valid/test_bad_config.yaml | 20 + ...st_cloudflare_azure_with_cache_config.yaml | 17 + .../tests/fixtures/valid/test_config.yaml | 28 + .../valid/test_config_custom_auth.yaml | 11 + .../test_config_hosted_vllm_embedding.yaml | 6 + .../fixtures/valid/test_config_no_auth.yaml | 127 +++++ .../fixtures/valid/test_custom_logger.yaml | 21 + .../valid/test_guardrails_config.yaml | 32 ++ litellm-rust/crates/config-load/tests/load.rs | 111 ++++ litellm-rust/crates/config/AGENTS.md | 60 ++ litellm-rust/crates/config/Cargo.toml | 3 +- litellm-rust/crates/config/src/AGENTS.md | 32 ++ litellm-rust/crates/config/src/agent.rs | 75 +++ litellm-rust/crates/config/src/callback.rs | 77 +++ litellm-rust/crates/config/src/credential.rs | 65 +++ litellm-rust/crates/config/src/error.rs | 2 - .../crates/config/src/general/AGENTS.md | 14 + .../crates/config/src/general/alerting.rs | 39 ++ .../crates/config/src/general/auth.rs | 77 +++ .../crates/config/src/general/database.rs | 15 + .../crates/config/src/general/files.rs | 15 + .../crates/config/src/general/health.rs | 22 + .../crates/config/src/general/limits.rs | 11 + litellm-rust/crates/config/src/general/mcp.rs | 20 + litellm-rust/crates/config/src/general/mod.rs | 535 ++++++++++++++++++ .../crates/config/src/general/objects.rs | 26 + .../crates/config/src/general/requests.rs | 66 +++ .../crates/config/src/general/spend.rs | 61 ++ .../crates/config/src/general/tracing.rs | 84 +++ litellm-rust/crates/config/src/general/ui.rs | 50 ++ litellm-rust/crates/config/src/guardrail.rs | 102 ++++ litellm-rust/crates/config/src/includes.rs | 61 -- litellm-rust/crates/config/src/lib.rs | 391 ++++++++++--- .../crates/config/src/litellm/AGENTS.md | 9 + .../crates/config/src/litellm/budget.rs | 32 ++ .../crates/config/src/litellm/caching.rs | 137 +++++ .../crates/config/src/litellm/fallbacks.rs | 7 + .../crates/config/src/litellm/logging.rs | 24 + litellm-rust/crates/config/src/litellm/mcp.rs | 8 + litellm-rust/crates/config/src/litellm/mod.rs | 400 +++++++++++++ .../crates/config/src/litellm/transport.rs | 34 ++ litellm-rust/crates/config/src/mcp.rs | 92 ++- litellm-rust/crates/config/src/model.rs | 189 ++++++- litellm-rust/crates/config/src/policy.rs | 144 +++++ litellm-rust/crates/config/src/prompt.rs | 99 ++++ litellm-rust/crates/config/src/provider.rs | 118 ++++ litellm-rust/crates/config/src/router.rs | 191 +++++++ litellm-rust/crates/config/src/settings.rs | 268 --------- litellm-rust/crates/config/src/tool.rs | 128 +++++ litellm-rust/crates/config/src/value.rs | 50 +- .../crates/config/src/vector_store.rs | 63 +++ litellm-rust/crates/config/src/yaml.rs | 177 ++++++ litellm-rust/crates/config/tests/config.rs | 436 -------------- litellm-rust/crates/config/tests/mcp.rs | 39 -- .../crates/gateway-mcp/src/configured.rs | 7 +- litellm-rust/crates/gateway/Cargo.toml | 1 + litellm-rust/crates/gateway/README.md | 2 +- litellm-rust/crates/gateway/src/lib.rs | 63 +-- litellm-rust/crates/gateway/src/main.rs | 16 +- .../crates/gateway/src/mcp_settings.rs | 62 ++ litellm-rust/crates/gateway/src/settings.rs | 26 +- litellm-rust/crates/gateway/tests/mcp.rs | 93 ++- litellm-rust/crates/gateway/tests/server.rs | 16 +- litellm-rust/crates/python-compat/AGENTS.md | 3 +- litellm-rust/crates/python-compat/src/json.rs | 33 ++ litellm-rust/crates/python-compat/src/lib.rs | 2 + .../crates/python-compat/src/literal.rs | 49 ++ .../crates/python-compat/src/pickle.rs | 59 ++ litellm-rust/crates/python-compat/src/read.rs | 78 +++ .../crates/python-compat/tests/limits.rs | 122 ---- litellm-rust/crates/router-types/Cargo.toml | 1 + litellm-rust/crates/router-types/src/lib.rs | 2 - .../crates/router-types/src/litellm_params.rs | 12 +- litellm-rust/crates/router-types/src/value.rs | 10 - .../router-types/tests/litellm_params.rs | 11 +- 105 files changed, 5290 insertions(+), 1155 deletions(-) create mode 100644 litellm-rust/crates/config-load/AGENTS.md create mode 100644 litellm-rust/crates/config-load/Cargo.toml create mode 100644 litellm-rust/crates/config-load/build.rs create mode 100644 litellm-rust/crates/config-load/src/error.rs create mode 100644 litellm-rust/crates/config-load/src/include/documents.rs create mode 100644 litellm-rust/crates/config-load/src/include/merge.rs create mode 100644 litellm-rust/crates/config-load/src/include/mod.rs create mode 100644 litellm-rust/crates/config-load/src/include/store.rs create mode 100644 litellm-rust/crates/config-load/src/lib.rs create mode 100644 litellm-rust/crates/config-load/src/transport.rs create mode 100644 litellm-rust/crates/config-load/tests/AGENTS.md create mode 100644 litellm-rust/crates/config-load/tests/corpus.rs create mode 100644 litellm-rust/crates/config-load/tests/fixtures/AGENTS.md create mode 100644 litellm-rust/crates/config-load/tests/fixtures/invalid/config_with_missing_include.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/invalid/langfuse_config.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/aliases_config.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/azure_config.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/cache_no_params.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/cache_with_params.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/config_with_env_vars.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/config_with_include.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/config_with_multiple_includes.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/included_models.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/load_balancer.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/models_file_1.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/models_file_2.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/opentelemetry_config.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/simple_config.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/test_bad_config.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/test_cloudflare_azure_with_cache_config.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/test_config.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/test_config_custom_auth.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/test_config_hosted_vllm_embedding.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/test_config_no_auth.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/test_custom_logger.yaml create mode 100644 litellm-rust/crates/config-load/tests/fixtures/valid/test_guardrails_config.yaml create mode 100644 litellm-rust/crates/config-load/tests/load.rs create mode 100644 litellm-rust/crates/config/AGENTS.md create mode 100644 litellm-rust/crates/config/src/AGENTS.md create mode 100644 litellm-rust/crates/config/src/agent.rs create mode 100644 litellm-rust/crates/config/src/callback.rs create mode 100644 litellm-rust/crates/config/src/credential.rs create mode 100644 litellm-rust/crates/config/src/general/AGENTS.md create mode 100644 litellm-rust/crates/config/src/general/alerting.rs create mode 100644 litellm-rust/crates/config/src/general/auth.rs create mode 100644 litellm-rust/crates/config/src/general/database.rs create mode 100644 litellm-rust/crates/config/src/general/files.rs create mode 100644 litellm-rust/crates/config/src/general/health.rs create mode 100644 litellm-rust/crates/config/src/general/limits.rs create mode 100644 litellm-rust/crates/config/src/general/mcp.rs create mode 100644 litellm-rust/crates/config/src/general/mod.rs create mode 100644 litellm-rust/crates/config/src/general/objects.rs create mode 100644 litellm-rust/crates/config/src/general/requests.rs create mode 100644 litellm-rust/crates/config/src/general/spend.rs create mode 100644 litellm-rust/crates/config/src/general/tracing.rs create mode 100644 litellm-rust/crates/config/src/general/ui.rs create mode 100644 litellm-rust/crates/config/src/guardrail.rs delete mode 100644 litellm-rust/crates/config/src/includes.rs create mode 100644 litellm-rust/crates/config/src/litellm/AGENTS.md create mode 100644 litellm-rust/crates/config/src/litellm/budget.rs create mode 100644 litellm-rust/crates/config/src/litellm/caching.rs create mode 100644 litellm-rust/crates/config/src/litellm/fallbacks.rs create mode 100644 litellm-rust/crates/config/src/litellm/logging.rs create mode 100644 litellm-rust/crates/config/src/litellm/mcp.rs create mode 100644 litellm-rust/crates/config/src/litellm/mod.rs create mode 100644 litellm-rust/crates/config/src/litellm/transport.rs create mode 100644 litellm-rust/crates/config/src/policy.rs create mode 100644 litellm-rust/crates/config/src/prompt.rs create mode 100644 litellm-rust/crates/config/src/provider.rs create mode 100644 litellm-rust/crates/config/src/router.rs delete mode 100644 litellm-rust/crates/config/src/settings.rs create mode 100644 litellm-rust/crates/config/src/tool.rs create mode 100644 litellm-rust/crates/config/src/vector_store.rs create mode 100644 litellm-rust/crates/config/src/yaml.rs delete mode 100644 litellm-rust/crates/config/tests/config.rs delete mode 100644 litellm-rust/crates/config/tests/mcp.rs create mode 100644 litellm-rust/crates/gateway/src/mcp_settings.rs create mode 100644 litellm-rust/crates/python-compat/src/read.rs delete mode 100644 litellm-rust/crates/python-compat/tests/limits.rs delete mode 100644 litellm-rust/crates/router-types/src/value.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index c868d3e4b8a..6632fd279fd 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3647,12 +3647,25 @@ name = "litellm-config" version = "0.1.0" dependencies = [ "litellm-auth-types", + "litellm-python-compat", "litellm-router-types", + "macro_rules_attribute", "rstest", "serde", "serde_json", "serde_yaml_ng", "strum", + "thiserror 2.0.19", +] + +[[package]] +name = "litellm-config-load" +version = "0.1.0" +dependencies = [ + "litellm-config", + "litellm-core-utils", + "rstest", + "serde_yaml_ng", "tempfile", "thiserror 2.0.19", ] @@ -3737,6 +3750,7 @@ dependencies = [ "http-body-util", "litellm-auth-types", "litellm-config", + "litellm-config-load", "litellm-gateway-auth", "litellm-gateway-inference", "litellm-gateway-mcp", @@ -4291,6 +4305,7 @@ name = "litellm-router-types" version = "0.1.0" dependencies = [ "litellm-auth-types", + "litellm-python-compat", "rstest", "serde", "serde_json", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 9e5e0a96b73..ce86003de28 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -10,6 +10,7 @@ repository = "https://github.com/BerriAI/litellm" [workspace.dependencies] litellm-config = { path = "crates/config" } +litellm-config-load = { path = "crates/config-load" } litellm-router = { path = "crates/router" } litellm-router-types = { path = "crates/router-types" } litellm-tracing = { path = "crates/tracing" } diff --git a/litellm-rust/crates/config-load/AGENTS.md b/litellm-rust/crates/config-load/AGENTS.md new file mode 100644 index 00000000000..bd4892d8d40 --- /dev/null +++ b/litellm-rust/crates/config-load/AGENTS.md @@ -0,0 +1,11 @@ +## Scope + +- This crate is the gateway's config loader. It takes the types `litellm-config` parses and turns them into the values the gateway runs on, so env lookups, the DB overlay, reference substitution, defaults and the domain types live here, never in `litellm-config` +- It never reads process state. The caller reads the process environment once and passes it in as an `Environment`, so tests pass their own variables +- Python never uses this crate. It shares only the `litellm-config` types, so behavior here may differ from Python where Python is wrong, and each difference is listed in the config compat doc + +- Fixture and corpus rules live in `tests/AGENTS.md` + +## Precedence + +- Each lowered value takes the env, the file and then the default in the order Python uses for that key today, until that order is normalized diff --git a/litellm-rust/crates/config-load/Cargo.toml b/litellm-rust/crates/config-load/Cargo.toml new file mode 100644 index 00000000000..5810cccb77b --- /dev/null +++ b/litellm-rust/crates/config-load/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "litellm-config-load" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-config.workspace = true +litellm-core-utils.workspace = true +serde_yaml_ng = "0.10.0" +thiserror.workspace = true + +[dev-dependencies] +rstest.workspace = true +tempfile.workspace = true diff --git a/litellm-rust/crates/config-load/build.rs b/litellm-rust/crates/config-load/build.rs new file mode 100644 index 00000000000..74f67a8473c --- /dev/null +++ b/litellm-rust/crates/config-load/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo::rerun-if-changed=tests/fixtures"); +} diff --git a/litellm-rust/crates/config-load/src/error.rs b/litellm-rust/crates/config-load/src/error.rs new file mode 100644 index 00000000000..b10aff338c8 --- /dev/null +++ b/litellm-rust/crates/config-load/src/error.rs @@ -0,0 +1,26 @@ +use std::fmt; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("could not read config")] + Read(#[from] std::io::Error), + #[error("invalid YAML config")] + Parse(#[from] serde_yaml_ng::Error), + #[error("invalid config")] + Config(#[from] litellm_config::Error), + #[error("could not load config {key}")] + Document { + key: String, + #[source] + source: Box, + }, +} + +impl Error { + pub(crate) fn document(key: &impl fmt::Debug, source: Error) -> Self { + Self::Document { + key: format!("{key:?}"), + source: Box::new(source), + } + } +} diff --git a/litellm-rust/crates/config-load/src/include/documents.rs b/litellm-rust/crates/config-load/src/include/documents.rs new file mode 100644 index 00000000000..6dad6b39e65 --- /dev/null +++ b/litellm-rust/crates/config-load/src/include/documents.rs @@ -0,0 +1,39 @@ +use std::collections::{BTreeSet, VecDeque}; + +use serde_yaml_ng::{Mapping, Value}; + +use super::Store; +use crate::Error; + +#[derive(Debug)] +pub struct Document { + pub key: K, + pub body: Mapping, +} + +pub fn documents(store: &S, root: S::Key) -> Result>, Error> { + let mut pending = VecDeque::from([root.clone()]); + let mut loaded = BTreeSet::new(); + let mut documents = Vec::new(); + while let Some(key) = pending.pop_front() { + if !loaded.insert(key.clone()) { + continue; + } + let (body, includes) = read(store, &key).map_err(|error| Error::document(&key, error))?; + for entry in includes { + pending.push_back(store.locate(&entry, &key, &root)?); + } + documents.push(Document { key, body }); + } + Ok(documents) +} + +fn read(store: &S, key: &S::Key) -> Result<(Mapping, Vec), Error> { + let mut body = litellm_config::yaml::mapping(&store.read(key)?)?; + let includes = body + .remove(Value::from("include")) + .map(serde_yaml_ng::from_value) + .transpose()? + .unwrap_or_default(); + Ok((body, includes)) +} diff --git a/litellm-rust/crates/config-load/src/include/merge.rs b/litellm-rust/crates/config-load/src/include/merge.rs new file mode 100644 index 00000000000..b319b6e7558 --- /dev/null +++ b/litellm-rust/crates/config-load/src/include/merge.rs @@ -0,0 +1,90 @@ +use std::collections::BTreeMap; + +use serde_yaml_ng::{Mapping, Value}; + +use super::Document; + +#[derive(Debug)] +pub struct Merged { + pub document: Mapping, + origins: BTreeMap>, +} + +impl Merged { + /// Maps item `index` of the merged top-level `field` back to its document and its index there. + /// A value that is not a list is item 0. + pub fn locate(&self, field: &str, index: usize) -> Option<(&K, usize)> { + self.origins + .get(field)? + .iter() + .scan(0, |start, (key, len)| { + let run = (key, *start, *len); + *start += len; + Some(run) + }) + .find(|(_, start, len)| (*start..start + len).contains(&index)) + .map(|(key, start, _)| (key, index - start)) + } +} + +pub fn merge(documents: &[Document]) -> Merged { + let fields = documents + .iter() + .flat_map(|document| document.body.keys()) + .fold(Vec::new(), |mut fields, field| { + if !fields.contains(&field) { + fields.push(field); + } + fields + }); + let (document, origins) = fields + .into_iter() + .map(|field| { + let layers: Vec<_> = documents + .iter() + .filter_map(|document| Some((&document.key, document.body.get(field)?))) + .collect(); + (field, combine(&layers)) + }) + .map(|(field, (value, runs))| { + ( + (field.clone(), value), + (field.as_str().map(str::to_owned), runs), + ) + }) + .unzip::<_, _, Mapping, Vec<_>>(); + Merged { + document, + origins: origins + .into_iter() + .filter_map(|(field, runs)| Some((field?, runs))) + .collect(), + } +} + +/// Python's rule (config L16): a list extends a list, anything else replaces. So the merged value is the +/// lists after the last non-list concatenated, or that non-list alone when nothing follows it. +fn combine(layers: &[(&K, &Value)]) -> (Value, Vec<(K, usize)>) { + let tail = layers + .iter() + .rposition(|(_, value)| !value.is_sequence()) + .map_or(0, |last| last + 1); + match &layers[tail..] { + [] => { + let (key, value) = layers[layers.len() - 1]; + (value.clone(), vec![(key.clone(), 1)]) + } + lists => ( + Value::Sequence( + lists + .iter() + .flat_map(|(_, value)| value.as_sequence().into_iter().flatten().cloned()) + .collect(), + ), + lists + .iter() + .map(|(key, value)| ((*key).clone(), value.as_sequence().map_or(0, Vec::len))) + .collect(), + ), + } +} diff --git a/litellm-rust/crates/config-load/src/include/mod.rs b/litellm-rust/crates/config-load/src/include/mod.rs new file mode 100644 index 00000000000..8eec7c42e86 --- /dev/null +++ b/litellm-rust/crates/config-load/src/include/mod.rs @@ -0,0 +1,169 @@ +mod documents; +mod merge; +mod store; + +pub use documents::{Document, documents}; +pub use merge::{Merged, merge}; +pub use store::{Files, Store}; + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use rstest::rstest; + use serde_yaml_ng::Value; + + use super::*; + use crate::Error; + + struct Memory(BTreeMap<&'static str, &'static str>); + + impl From<[(&'static str, &'static str); N]> for Memory { + fn from(files: [(&'static str, &'static str); N]) -> Self { + Self(files.into()) + } + } + + impl Store for Memory { + type Key = &'static str; + + fn locate( + &self, + entry: &str, + _: &&'static str, + _: &&'static str, + ) -> Result<&'static str, Error> { + self.0 + .get_key_value(entry) + .map(|(key, _)| *key) + .ok_or_else(|| { + Error::document( + &entry, + std::io::Error::from(std::io::ErrorKind::NotFound).into(), + ) + }) + } + + fn read(&self, key: &&'static str) -> Result { + Ok(self.0[key].to_owned()) + } + } + + fn load(files: impl Into) -> Result, Error> { + documents(&files.into(), "root").map(|documents| merge(&documents)) + } + + fn yaml(text: &str) -> Value { + serde_yaml_ng::from_str(text).unwrap() + } + + #[rstest] + fn reads_each_document_once_in_breadth_first_order() { + let order = documents( + &Memory::from([ + ("root", "include: [a, b]"), + ("a", "include: [c, root]"), + ("b", "include: [a]"), + ("c", "{}"), + ]), + "root", + ) + .unwrap() + .into_iter() + .map(|document| document.key) + .collect::>(); + assert_eq!(order, ["root", "a", "b", "c"]); + } + + #[rstest] + #[case::lists_concatenate("x: [1]", "x: [2]", "[1, 2]")] + #[case::mappings_replace_whole("x: {a: 1, b: 2}", "x: {c: 3}", "{c: 3}")] + #[case::scalar_replaces_list("x: [1]", "x: 2", "2")] + #[case::list_replaces_scalar("x: 1", "x: [2]", "[2]")] + fn merges_top_level_keys_like_python( + #[case] root: &'static str, + #[case] other: &'static str, + #[case] expected: &str, + ) { + let root = Box::leak(format!("include: [other]\n{root}").into_boxed_str()); + let merged = load([("root", &*root), ("other", other)]).unwrap(); + assert_eq!(merged.document.get("x"), Some(&yaml(expected))); + assert!(!merged.document.contains_key("include")); + } + + #[rstest] + fn keeps_concatenating_after_a_replace() { + let merged = load([ + ("root", "include: [a, b, c]\nx: [1]"), + ("a", "x: 2"), + ("b", "x: [3]"), + ("c", "x: [4, 5]"), + ]) + .unwrap(); + assert_eq!(merged.document.get("x"), Some(&yaml("[3, 4, 5]"))); + assert_eq!(merged.locate("x", 0), Some((&"b", 0))); + assert_eq!(merged.locate("x", 2), Some((&"c", 1))); + assert_eq!(merged.locate("x", 3), None); + } + + #[rstest] + fn locates_list_items_and_replaced_values_in_their_documents() { + let merged = load([ + ( + "root", + "include: [a]\nmodel_list: [r0, r1]\ngeneral_settings: {k: 1}\nonly_root: 1", + ), + ("a", "model_list: [a0]\ngeneral_settings: {k: 2}"), + ]) + .unwrap(); + assert_eq!(merged.locate("model_list", 1), Some((&"root", 1))); + assert_eq!(merged.locate("model_list", 2), Some((&"a", 0))); + assert_eq!(merged.locate("general_settings", 0), Some((&"a", 0))); + assert_eq!(merged.locate("only_root", 0), Some((&"root", 0))); + assert_eq!(merged.locate("missing", 0), None); + } + + #[rstest] + fn names_the_missing_include() { + let error = load([("root", "include: [gone.yaml]")]).unwrap_err(); + assert!( + matches!(error, Error::Document { ref key, .. } if key.contains("gone.yaml")), + "{error:?}" + ); + } + + #[rstest] + #[case::bare_string("include: a.yaml")] + #[case::number_item("include: [1]")] + fn rejects_include_that_is_not_a_list_of_paths(#[case] root: &'static str) { + assert!(load([("root", root)]).is_err()); + } + + #[rstest] + #[case::declaring_directory_wins(&["nested/x.yaml", "x.yaml"], "nested/x.yaml")] + #[case::falls_back_to_root_directory(&["x.yaml"], "x.yaml")] + #[case::reports_declaring_path_when_neither_exists(&[], "nested/x.yaml")] + fn files_resolve_against_declaring_then_root_directory( + #[case] existing: &[&str], + #[case] expected: &str, + ) { + let directory = tempfile::TempDir::new().unwrap(); + let root = directory.path().canonicalize().unwrap(); + std::fs::create_dir(root.join("nested")).unwrap(); + for file in ["config.yaml", "nested/first.yaml"].iter().chain(existing) { + std::fs::write(root.join(file), "{}").unwrap(); + } + let located = Files.locate( + "x.yaml", + &root.join("nested/first.yaml"), + &root.join("config.yaml"), + ); + match located { + Ok(path) => assert_eq!(path, root.join(expected)), + Err(Error::Document { key, .. }) if !existing.contains(&expected) => { + assert!(key.contains(&*root.join(expected).to_string_lossy())) + } + Err(error) => panic!("{error:?}"), + } + } +} diff --git a/litellm-rust/crates/config-load/src/include/store.rs b/litellm-rust/crates/config-load/src/include/store.rs new file mode 100644 index 00000000000..af4d5a01b51 --- /dev/null +++ b/litellm-rust/crates/config-load/src/include/store.rs @@ -0,0 +1,46 @@ +use std::{ + fmt, + path::{Path, PathBuf}, +}; + +use crate::Error; + +pub trait Store { + type Key: Ord + Clone + fmt::Debug; + + fn locate( + &self, + entry: &str, + declaring: &Self::Key, + root: &Self::Key, + ) -> Result; + + fn read(&self, key: &Self::Key) -> Result; +} + +pub struct Files; + +impl Store for Files { + type Key = PathBuf; + + fn locate(&self, entry: &str, declaring: &PathBuf, root: &PathBuf) -> Result { + let declared = sibling(declaring, entry); + let fallback = sibling(root, entry); + let location = if declared.exists() || !fallback.exists() { + declared + } else { + fallback + }; + location + .canonicalize() + .map_err(|error| Error::document(&location, error.into())) + } + + fn read(&self, key: &PathBuf) -> Result { + Ok(std::fs::read_to_string(key)?) + } +} + +fn sibling(file: &Path, entry: &str) -> PathBuf { + file.parent().unwrap_or(Path::new(".")).join(entry) +} diff --git a/litellm-rust/crates/config-load/src/lib.rs b/litellm-rust/crates/config-load/src/lib.rs new file mode 100644 index 00000000000..62f1ed876b5 --- /dev/null +++ b/litellm-rust/crates/config-load/src/lib.rs @@ -0,0 +1,65 @@ +mod error; +pub mod include; +mod transport; + +use std::{collections::BTreeMap, ffi::OsString, path::Path}; + +use litellm_config::Config; +use litellm_core_utils::settings::Lookup; + +pub use self::{ + error::Error, + transport::{SslVerify, Transport}, +}; + +pub fn load(path: impl AsRef) -> Result { + let path = path.as_ref(); + let root = path + .canonicalize() + .map_err(|error| Error::document(&path, error.into()))?; + load_from(&include::Files, root) +} + +pub fn load_from(store: &S, root: S::Key) -> Result { + Ok(Config::from_mapping( + include::merge(&include::documents(store, root)?).document, + )?) +} + +#[derive(Clone, Default)] +pub struct Environment(BTreeMap); + +impl Environment { + pub fn from_os(variables: impl IntoIterator) -> Self { + variables + .into_iter() + .filter_map(|(name, value)| Some((name.into_string().ok()?, value.into_string().ok()?))) + .collect() + } + + pub fn overlaid(self, higher: impl IntoIterator) -> Self { + Self(self.0.into_iter().chain(higher).collect()) + } + + pub fn var(&self, name: &str) -> Option<&str> { + self.0.get(name).map(String::as_str) + } + + pub fn variables(&self) -> impl Iterator + '_ { + self.0 + .iter() + .map(|(name, value)| (name.clone(), value.clone())) + } +} + +impl FromIterator<(String, String)> for Environment { + fn from_iter>(variables: I) -> Self { + Self(variables.into_iter().collect()) + } +} + +impl Lookup for Environment { + fn get(&self, name: &str) -> Option { + self.0.get(name).cloned() + } +} diff --git a/litellm-rust/crates/config-load/src/transport.rs b/litellm-rust/crates/config-load/src/transport.rs new file mode 100644 index 00000000000..ba1f95b8296 --- /dev/null +++ b/litellm-rust/crates/config-load/src/transport.rs @@ -0,0 +1,166 @@ +use std::path::PathBuf; + +use litellm_core_utils::{serde_compat::parse_str_bool, settings::Lookup}; + +use litellm_config::{LiteLlmSettings, Spelled}; + +use crate::Environment; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum SslVerify { + #[default] + Enabled, + Disabled, + CaBundle(PathBuf), +} + +impl SslVerify { + fn parse(value: &str) -> Self { + match parse_str_bool(value) { + Some(true) => Self::Enabled, + Some(false) => Self::Disabled, + None => Self::CaBundle(PathBuf::from(value)), + } + } + + fn written(value: &Spelled) -> Self { + match value { + Spelled::Value(true) => Self::Enabled, + Spelled::Value(false) => Self::Disabled, + Spelled::Text(text) => Self::parse(text), + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Transport { + pub ssl_verify: SslVerify, + pub ssl_certificate: Option, + pub ssl_security_level: Option, + pub ssl_ecdh_curve: Option, + pub force_ipv4: bool, + pub http2: bool, + pub aiohttp_trust_env: bool, + pub disable_aiohttp_trust_env: bool, + pub disable_aiohttp_transport: bool, +} + +impl Transport { + pub fn lower(settings: &LiteLlmSettings, environment: &Environment) -> Self { + let text = |name: &str, written: &Option| { + environment + .get(name) + .or_else(|| written.clone()) + .filter(|value| !value.is_empty()) + }; + let flag = |name: &str, written: Option| { + environment.enabled(name).or(written).unwrap_or(false) + }; + Self { + ssl_verify: environment + .get("SSL_VERIFY") + .map(|value| SslVerify::parse(&value)) + .or_else(|| settings.tls.ssl_verify.as_ref().map(SslVerify::written)) + .unwrap_or_default(), + ssl_certificate: environment + .get("SSL_CERTIFICATE") + .or_else(|| settings.client_certificate.ssl_certificate.clone()) + .map(PathBuf::from), + ssl_security_level: text("SSL_SECURITY_LEVEL", &settings.tls.ssl_security_level), + ssl_ecdh_curve: text("SSL_ECDH_CURVE", &settings.tls.ssl_ecdh_curve), + force_ipv4: settings.socket.force_ipv4.unwrap_or(false), + http2: flag("LITELLM_HTTP2", settings.http2.http2), + aiohttp_trust_env: flag("AIOHTTP_TRUST_ENV", settings.aiohttp.aiohttp_trust_env), + disable_aiohttp_trust_env: flag( + "DISABLE_AIOHTTP_TRUST_ENV", + settings.aiohttp.disable_aiohttp_trust_env, + ), + disable_aiohttp_transport: flag( + "DISABLE_AIOHTTP_TRANSPORT", + settings.aiohttp.disable_aiohttp_transport, + ), + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use rstest::rstest; + + use super::{SslVerify, Transport}; + use litellm_config::Config; + + use crate::Environment; + + fn lowered(yaml: &str, variables: &[(&str, &str)]) -> Transport { + let config = Config::from_yaml(yaml).unwrap(); + let environment = variables + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + Transport::lower(&config.litellm_settings, &environment) + } + + #[rstest] + fn absent_everywhere_is_the_default() { + assert_eq!(lowered("{}", &[]), Transport::default()); + } + + #[rstest] + #[case::default("{}", &[], SslVerify::Enabled)] + #[case::written("litellm_settings: {ssl_verify: false}", &[], SslVerify::Disabled)] + #[case::written_bundle("litellm_settings: {ssl_verify: /etc/roots.pem}", &[], SslVerify::CaBundle(PathBuf::from("/etc/roots.pem")))] + #[case::environment("{}", &[("SSL_VERIFY", "False")], SslVerify::Disabled)] + #[case::environment_over_file("litellm_settings: {ssl_verify: false}", &[("SSL_VERIFY", "true")], SslVerify::Enabled)] + fn ssl_verify_takes_environment_then_file_then_default( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: SslVerify, + ) { + assert_eq!(lowered(yaml, variables).ssl_verify, expected); + } + + #[rstest] + #[case::file("litellm_settings: {http2: true}", &[], true)] + #[case::environment("{}", &[("LITELLM_HTTP2", "TRUE")], true)] + #[case::environment_false_does_not_override_file("litellm_settings: {http2: true}", &[("LITELLM_HTTP2", "false")], true)] + fn flags_are_only_turned_on_by_the_environment( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: bool, + ) { + assert_eq!(lowered(yaml, variables).http2, expected); + } + + #[rstest] + #[case::file("litellm_settings: {ssl_security_level: '1'}", &[], Some("1"))] + #[case::environment_over_file("litellm_settings: {ssl_security_level: '1'}", &[("SSL_SECURITY_LEVEL", "2")], Some("2"))] + #[case::empty_is_unset("{}", &[("SSL_SECURITY_LEVEL", "")], None)] + fn text_settings_skip_empty_values( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: Option<&str>, + ) { + assert_eq!( + lowered(yaml, variables).ssl_security_level.as_deref(), + expected + ); + } + + #[rstest] + fn later_overlay_wins_and_unrelated_variables_are_ignored() { + let environment = Environment::from_iter([("SSL_VERIFY".to_owned(), "true".to_owned())]) + .overlaid([ + ("SSL_VERIFY".to_owned(), "false".to_owned()), + ("HTTP2".to_owned(), "true".to_owned()), + ]); + let transport = Transport::lower( + &Config::from_yaml("{}").unwrap().litellm_settings, + &environment, + ); + assert_eq!(transport.ssl_verify, SslVerify::Disabled); + assert!(!transport.http2); + } +} diff --git a/litellm-rust/crates/config-load/tests/AGENTS.md b/litellm-rust/crates/config-load/tests/AGENTS.md new file mode 100644 index 00000000000..e6484b746c9 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/AGENTS.md @@ -0,0 +1,5 @@ +## Fixtures + +- `corpus.rs` requires every config in `fixtures/valid/` to load and every config in `fixtures/invalid/` to fail. There is no known-gap list, so a valid config that fails to load is fixed in `src/` +- A config goes in `invalid/` only when Python rejects it too, either by raising on read or by crashing when the proxy loads it +- `#[files]` globs at compile time, so `build.rs` reruns the build when `fixtures/` changes diff --git a/litellm-rust/crates/config-load/tests/corpus.rs b/litellm-rust/crates/config-load/tests/corpus.rs new file mode 100644 index 00000000000..ffaf8864c05 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/corpus.rs @@ -0,0 +1,16 @@ +use std::path::PathBuf; + +use litellm_config_load::load; +use rstest::rstest; + +#[rstest] +fn loads_every_valid_fixture(#[files("tests/fixtures/valid/*.yaml")] path: PathBuf) { + if let Err(error) = load(&path) { + panic!("{}: {error:?}", path.display()); + } +} + +#[rstest] +fn rejects_every_invalid_fixture(#[files("tests/fixtures/invalid/*.yaml")] path: PathBuf) { + assert!(load(&path).is_err(), "{} loads", path.display()); +} diff --git a/litellm-rust/crates/config-load/tests/fixtures/AGENTS.md b/litellm-rust/crates/config-load/tests/fixtures/AGENTS.md new file mode 100644 index 00000000000..6ad1ed3978d --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/AGENTS.md @@ -0,0 +1,3 @@ +These configs are copied as-is from `tests/unit/proxy/example_config_yaml` and `tests/unit/proxy/test_configs` for now. Keep each of `valid/` and `invalid/` flat so relative `include` paths still resolve + +`invalid/config_with_missing_include.yaml` is in Python's `INVALID_FILES`. `invalid/langfuse_config.yaml` passes Python's read-only `get_config` test, but its model has no `litellm_params`, so Python's `load_config` raises `KeyError` on it diff --git a/litellm-rust/crates/config-load/tests/fixtures/invalid/config_with_missing_include.yaml b/litellm-rust/crates/config-load/tests/fixtures/invalid/config_with_missing_include.yaml new file mode 100644 index 00000000000..40d3e9e7f8d --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/invalid/config_with_missing_include.yaml @@ -0,0 +1,5 @@ +include: + - non-existent-file.yaml + +litellm_settings: + callbacks: ["prometheus"] \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/invalid/langfuse_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/invalid/langfuse_config.yaml new file mode 100644 index 00000000000..c2a77b5adc4 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/invalid/langfuse_config.yaml @@ -0,0 +1,7 @@ +model_list: + - model_name: gpt-3.5-turbo + +litellm_settings: + drop_params: True + success_callback: ["langfuse"] # https://docs.litellm.ai/docs/observability/langfuse_integration + diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/aliases_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/aliases_config.yaml new file mode 100644 index 00000000000..43681f64ba0 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/aliases_config.yaml @@ -0,0 +1,30 @@ +model_list: + - model_name: gpt-3.5-turbo-instruct + litellm_params: + model: ollama/zephyr + - model_name: gpt-4 + litellm_params: + model: ollama/llama2 + - model_name: gpt-3.5-turbo + litellm_params: + model: ollama/llama2 + temperature: 0.1 + max_tokens: 20 + + +# request to gpt-4, response from ollama/llama2 +# curl --location 'http://0.0.0.0:8000/chat/completions' \ +# --header 'Content-Type: application/json' \ +# --data ' { +# "model": "gpt-4", +# "messages": [ +# { +# "role": "user", +# "content": "what llm are you" +# } +# ], +# } +# ' +# + +# {"id":"chatcmpl-27c85cf0-ab09-4bcf-8cb1-0ee950520743","choices":[{"finish_reason":"stop","index":0,"message":{"content":" Hello! I'm just an AI, I don't have personal experiences or emotions like humans do. However, I can help you with any questions or tasks you may have! Is there something specific you'd like to know or discuss?","role":"assistant","_logprobs":null}}],"created":1700094955.373751,"model":"ollama/llama2","object":"chat.completion","system_fingerprint":null,"usage":{"prompt_tokens":12,"completion_tokens":47,"total_tokens":59},"_response_ms":8028.017999999999}% \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/azure_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/azure_config.yaml new file mode 100644 index 00000000000..05ba0c9bf54 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/azure_config.yaml @@ -0,0 +1,15 @@ +model_list: + - model_name: gpt-4-team1 + litellm_params: + model: azure/gpt-4.1-mini + api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ + api_version: "2023-05-15" + api_key: os.environ/AZURE_AI_API_KEY + tpm: 20_000 + - model_name: gpt-4-team2 + litellm_params: + model: azure/gpt-4 + api_key: os.environ/AZURE_AI_API_KEY + api_base: https://openai-gpt-4-test-v-2.openai.azure.com/ + tpm: 100_000 + diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/cache_no_params.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/cache_no_params.yaml new file mode 100644 index 00000000000..20ed919dd56 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/cache_no_params.yaml @@ -0,0 +1,7 @@ +model_list: + - model_name: "openai-model" + litellm_params: + model: "gpt-3.5-turbo" + +litellm_settings: + cache: True diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/cache_with_params.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/cache_with_params.yaml new file mode 100644 index 00000000000..068e2cc4a2b --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/cache_with_params.yaml @@ -0,0 +1,11 @@ +model_list: + - model_name: "openai-model" + litellm_params: + model: "gpt-3.5-turbo" + +litellm_settings: + cache: True + cache_params: + type: "redis" + supported_call_types: ["embedding", "aembedding"] + host: "os.environ/REDIS_HOST" \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_env_vars.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_env_vars.yaml new file mode 100644 index 00000000000..bae738c735d --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_env_vars.yaml @@ -0,0 +1,48 @@ +model_list: + ################################################################################ + # Azure + - model_name: gpt-4o-mini + litellm_params: + model: azure/gpt-4o-mini + api_base: https://amazin-prod.openai.azure.com + api_key: "os.environ/AZURE_GPT_4O" + deployment_id: gpt-4o-mini + - model_name: gpt-4o + litellm_params: + model: azure/gpt-4o + api_base: https://very-cool-prod.openai.azure.com + api_key: "os.environ/AZURE_GPT_4O" + deployment_id: gpt-4o + + ################################################################################ + # Fireworks + - model_name: fireworks-llama-v3p1-405b-instruct + litellm_params: + model: fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct + api_key: "os.environ/FIREWORKS" + - model_name: fireworks-llama-v3p1-70b-instruct + litellm_params: + model: fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct + api_key: "os.environ/FIREWORKS" + +general_settings: + alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+ +litellm_settings: # module level litellm settings - https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py + success_callback: ["prometheus"] + service_callback: ["prometheus_system"] + drop_params: False # Raise an exception if the openai param being passed in isn't supported. + cache: false + default_internal_user_params: + user_role: os.environ/DEFAULT_USER_ROLE + + success_callback: ["s3"] + s3_callback_params: + s3_bucket_name: logs-bucket-litellm # AWS Bucket Name for S3 + s3_region_name: us-west-2 # AWS Region Name for S3 + s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/ to pass environment variables. This is AWS Access Key ID for S3 + s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3 + s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to + s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets + +router_settings: + routing_strategy: simple-shuffle # "simple-shuffle" shown to result in highest throughput. https://docs.litellm.ai/docs/proxy/configs#load-balancing diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_include.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_include.yaml new file mode 100644 index 00000000000..0a0c9434b07 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_include.yaml @@ -0,0 +1,5 @@ +include: + - included_models.yaml + +litellm_settings: + callbacks: ["prometheus"] \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_multiple_includes.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_multiple_includes.yaml new file mode 100644 index 00000000000..c46adacd737 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/config_with_multiple_includes.yaml @@ -0,0 +1,6 @@ +include: + - models_file_1.yaml + - models_file_2.yaml + +litellm_settings: + callbacks: ["prometheus"] \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/included_models.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/included_models.yaml new file mode 100644 index 00000000000..c1526b203a6 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/included_models.yaml @@ -0,0 +1,4 @@ +model_list: + - model_name: included-model + litellm_params: + model: gpt-4 \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/load_balancer.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/load_balancer.yaml new file mode 100644 index 00000000000..502b90ff92a --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/load_balancer.yaml @@ -0,0 +1,28 @@ +litellm_settings: + drop_params: True + +# Model-specific settings +model_list: # use the same model_name for using the litellm router. LiteLLM will use the router between gpt-3.5-turbo + - model_name: gpt-3.5-turbo # litellm will + litellm_params: + model: gpt-3.5-turbo + api_key: sk-uj6F + tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm + rpm: 3 # [OPTIONAL] REPLACE with your openai rpm + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + api_key: sk-Imn + tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm + rpm: 3 # [OPTIONAL] REPLACE with your openai rpm + - model_name: gpt-3.5-turbo + litellm_params: + model: openrouter/gpt-3.5-turbo + - model_name: mistral-7b-instruct + litellm_params: + model: mistralai/mistral-7b-instruct + +environment_variables: + REDIS_HOST: localhost + REDIS_PASSWORD: + REDIS_PORT: \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_1.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_1.yaml new file mode 100644 index 00000000000..344f67128fb --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_1.yaml @@ -0,0 +1,4 @@ +model_list: + - model_name: included-model-1 + litellm_params: + model: gpt-4 \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_2.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_2.yaml new file mode 100644 index 00000000000..56bc3b1aa08 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/models_file_2.yaml @@ -0,0 +1,4 @@ +model_list: + - model_name: included-model-2 + litellm_params: + model: gpt-3.5-turbo \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/opentelemetry_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/opentelemetry_config.yaml new file mode 100644 index 00000000000..92d3454d7d5 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/opentelemetry_config.yaml @@ -0,0 +1,7 @@ +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + +general_settings: + otel: True # OpenTelemetry Logger this logs OTEL data to your collector diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/simple_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/simple_config.yaml new file mode 100644 index 00000000000..14b39a12518 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/simple_config.yaml @@ -0,0 +1,4 @@ +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_bad_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_bad_config.yaml new file mode 100644 index 00000000000..72ceefcda2d --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_bad_config.yaml @@ -0,0 +1,20 @@ +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + api_key: bad-key + model: gpt-3.5-turbo + - model_name: working-azure-gpt-3.5-turbo + litellm_params: + model: azure/gpt-4.1-mini + api_base: https://azure-unit.example.com + api_key: bad-key + - model_name: azure-gpt-3.5-turbo + litellm_params: + model: azure/gpt-4.1-mini + api_base: https://azure-unit.example.com + api_key: bad-key + - model_name: azure-embedding + litellm_params: + model: azure/text-embedding-ada-002 + api_base: https://azure-unit.example.com + api_key: bad-key diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_cloudflare_azure_with_cache_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_cloudflare_azure_with_cache_config.yaml new file mode 100644 index 00000000000..24240008fe2 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_cloudflare_azure_with_cache_config.yaml @@ -0,0 +1,17 @@ +model_list: + - model_name: azure-cloudflare + litellm_params: + model: azure/gpt-4.1-mini + api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1 + api_key: os.environ/AZURE_AI_API_KEY + api_version: 2023-07-01-preview + +litellm_settings: + set_verbose: True + cache: True # set cache responses to True + cache_params: # set cache params for s3 + type: s3 + s3_bucket_name: litellm-my-test-bucket-2 # AWS Bucket Name for S3 + s3_region_name: us-east-1 # AWS Region Name for S3 + s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # AWS Access Key ID for S3 + s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3 \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config.yaml new file mode 100644 index 00000000000..a711b65ea03 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config.yaml @@ -0,0 +1,28 @@ +general_settings: + database_url: os.environ/DATABASE_URL + master_key: os.environ/PROXY_MASTER_KEY +litellm_settings: + drop_params: true + success_callback: ["langfuse"] + +model_list: +- litellm_params: + api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ + api_key: os.environ/AZURE_EUROPE_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://my-endpoint-canada-berri992.openai.azure.com + api_key: os.environ/AZURE_CANADA_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://openai-france-1234.openai.azure.com + api_key: os.environ/AZURE_FRANCE_API_KEY + model: azure/gpt-turbo + model_name: azure-model +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + model_name: test_openai_models diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_custom_auth.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_custom_auth.yaml new file mode 100644 index 00000000000..33088bd1ce1 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_custom_auth.yaml @@ -0,0 +1,11 @@ +model_list: + - model_name: "openai-model" + litellm_params: + model: "gpt-3.5-turbo" + +litellm_settings: + drop_params: True + set_verbose: True + +general_settings: + custom_auth: custom_auth.user_api_key_auth \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_hosted_vllm_embedding.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_hosted_vllm_embedding.yaml new file mode 100644 index 00000000000..1b6b9ad198e --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_hosted_vllm_embedding.yaml @@ -0,0 +1,6 @@ +model_list: +- litellm_params: + model: hosted_vllm/embed_model + model_info: + description: this is a test embedding hosted_vllm model + model_name: vllm_embed_model diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_no_auth.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_no_auth.yaml new file mode 100644 index 00000000000..f4896217049 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_config_no_auth.yaml @@ -0,0 +1,127 @@ +model_list: +- litellm_params: + api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ + api_key: os.environ/AZURE_EUROPE_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://my-endpoint-canada-berri992.openai.azure.com + api_key: os.environ/AZURE_CANADA_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1 + api_key: os.environ/AZURE_AI_API_KEY + model: azure/gpt-4.1-mini + model_name: azure-cloudflare-model +- litellm_params: + api_base: https://openai-france-1234.openai.azure.com + api_key: os.environ/AZURE_FRANCE_API_KEY + model: azure/gpt-turbo + model_name: azure-model +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 56f1bd94-3b54-4b67-9ea2-7c70e9a3a709 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 4d1ee26c-abca-450c-8744-8e87fd6755e9 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 00e19c0f-b63d-42bb-88e9-016fb0c60764 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 79fc75bf-8e1b-47d5-8d24-9365a854af03 + model_name: test_openai_models +- litellm_params: + api_base: os.environ/AZURE_AI_API_BASE + api_key: os.environ/AZURE_AI_API_KEY + api_version: 2023-07-01-preview + model: azure/text-embedding-ada-002 + model_info: + mode: embedding + model_name: azure-embedding-model +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 55848c55-4162-40f9-a6e2-9a722b9ef404 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 34339b1e-e030-4bcc-a531-c48559f10ce4 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: f6f74e14-ac64-4403-9365-319e584dcdc5 + model_name: test_openai_models +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 9b1ef341-322c-410a-8992-903987fef439 + model_name: test_openai_models +- litellm_params: + model: dall-e-3 + model_info: + mode: image_generation + model_name: dall-e-3 +- litellm_params: + api_base: os.environ/AZURE_SWEDEN_API_BASE + api_key: os.environ/AZURE_SWEDEN_API_KEY + api_version: 2023-12-01-preview + model: azure/dall-e-3-test + model_info: + mode: image_generation + model_name: dall-e-3 +- litellm_params: + api_base: os.environ/AZURE_AI_API_BASE + api_key: os.environ/AZURE_AI_API_KEY + api_version: 2023-06-01-preview + model: azure/ + model_info: + mode: image_generation + model_name: dall-e-2 +- litellm_params: + api_base: os.environ/AZURE_AI_API_BASE + api_key: os.environ/AZURE_AI_API_KEY + api_version: 2023-07-01-preview + model: azure/text-embedding-ada-002 + model_info: + base_model: text-embedding-ada-002 + mode: embedding + model_name: text-embedding-ada-002 +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 34cb2419-7c63-44ae-a189-53f1d1ce5953 + model_name: test_openai_models +- litellm_params: + model: amazon.titan-embed-text-v1 + model_name: amazon-embeddings +- litellm_params: + model: gpt-3.5-turbo + model_info: + description: this is a test openai model + id: 753dca9a-898d-4ff7-9961-5acf7cdf38cf + model_name: test_openai_models diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_custom_logger.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_custom_logger.yaml new file mode 100644 index 00000000000..c58df01b602 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_custom_logger.yaml @@ -0,0 +1,21 @@ +model_list: + - model_name: Azure OpenAI GPT-4 Canada + litellm_params: + model: gpt-4.1-nano + api_key: sk-unit-fake + model_info: + mode: chat + input_cost_per_token: 0.0002 + id: gm + - model_name: azure-embedding-model + litellm_params: + model: text-embedding-ada-002 + api_key: sk-unit-fake + model_info: + mode: embedding + input_cost_per_token: 0.002 + id: hello + +litellm_settings: + drop_params: True + callbacks: custom_callbacks.my_custom_logger diff --git a/litellm-rust/crates/config-load/tests/fixtures/valid/test_guardrails_config.yaml b/litellm-rust/crates/config-load/tests/fixtures/valid/test_guardrails_config.yaml new file mode 100644 index 00000000000..f09ff9d1bc4 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/fixtures/valid/test_guardrails_config.yaml @@ -0,0 +1,32 @@ + + +model_list: +- litellm_params: + api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ + api_key: os.environ/AZURE_EUROPE_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://my-endpoint-canada-berri992.openai.azure.com + api_key: os.environ/AZURE_CANADA_API_KEY + model: azure/gpt-35-turbo + model_name: azure-model +- litellm_params: + api_base: https://openai-france-1234.openai.azure.com + api_key: os.environ/AZURE_FRANCE_API_KEY + model: azure/gpt-turbo + model_name: azure-model + + + +litellm_settings: + guardrails: + - prompt_injection: + callbacks: [lakera_prompt_injection, detect_prompt_injection] + default_on: true + - hide_secrets: + callbacks: [hide_secrets] + default_on: true + - moderations: + callbacks: [openai_moderations] + default_on: false \ No newline at end of file diff --git a/litellm-rust/crates/config-load/tests/load.rs b/litellm-rust/crates/config-load/tests/load.rs new file mode 100644 index 00000000000..60772507861 --- /dev/null +++ b/litellm-rust/crates/config-load/tests/load.rs @@ -0,0 +1,111 @@ +use std::fs; + +use litellm_config_load::{Error, load}; +use rstest::{fixture, rstest}; +use tempfile::TempDir; + +#[fixture] +fn directory() -> TempDir { + tempfile::tempdir().unwrap() +} + +#[rstest] +fn loads_model_list_from_file(directory: TempDir) { + let path = directory.path().join("config.yaml"); + fs::write( + &path, + r#" +model_list: + - model_name: assistant + litellm_params: + model: anthropic/test-model + - model_name: local + litellm_params: + model: test-model +"#, + ) + .unwrap(); + + let config = load(path).unwrap(); + assert_eq!( + config + .model_list + .iter() + .map(|model| model.model_name.as_str()) + .collect::>(), + ["assistant", "local"] + ); +} + +#[rstest] +#[case::missing_file(None, |error: &Error| matches!(error, Error::Read(error) if error.kind() == std::io::ErrorKind::NotFound))] +#[case::malformed_yaml(Some("model_list: ["), |error: &Error| matches!(error, Error::Parse(_)))] +fn distinguishes_read_errors_from_parse_errors( + directory: TempDir, + #[case] contents: Option<&str>, + #[case] expected: fn(&Error) -> bool, +) { + let path = directory.path().join("config.yaml"); + if let Some(contents) = contents { + fs::write(&path, contents).unwrap(); + } + + let error = load(&path).unwrap_err(); + let Error::Document { source, .. } = &error else { + panic!("{error:?}"); + }; + assert!(expected(source), "{error:?}"); +} + +#[rstest] +fn resolves_nested_includes_once_in_breadth_first_order(directory: TempDir) { + let root = directory.path(); + fs::create_dir(root.join("nested")).unwrap(); + fs::write( + root.join("config.yaml"), + r#" +include: [nested/first.yaml, second.yaml] +model_list: [{model_name: root, litellm_params: {model: root}}] +"#, + ) + .unwrap(); + fs::write( + root.join("nested/first.yaml"), + r#" +include: [third.yaml] +model_list: [{model_name: first, litellm_params: {model: first}}] +"#, + ) + .unwrap(); + fs::write( + root.join("second.yaml"), + r#" +general_settings: {master_key: second} +model_list: [{model_name: second, litellm_params: {model: second}}] +"#, + ) + .unwrap(); + fs::write( + root.join("nested/third.yaml"), + r#" +include: [../config.yaml] +general_settings: {master_key: third} +model_list: [{model_name: third, litellm_params: {model: third}}] +"#, + ) + .unwrap(); + + let config = load(root.join("config.yaml")).unwrap(); + assert_eq!( + config + .model_list + .iter() + .map(|model| model.model_name.as_str()) + .collect::>(), + ["root", "first", "second", "third"] + ); + assert_eq!( + config.general_settings.master_key.unwrap().expose(), + "third" + ); +} diff --git a/litellm-rust/crates/config/AGENTS.md b/litellm-rust/crates/config/AGENTS.md new file mode 100644 index 00000000000..310c97bb566 --- /dev/null +++ b/litellm-rust/crates/config/AGENTS.md @@ -0,0 +1,60 @@ +## Scope + +- The crate is the type definition of `config.yaml`, what a user writes and nothing more. Parsing one YAML text into those types (`Config::from_yaml`, `Config::from_mapping`, the PyYAML reading rules in `yaml.rs`) is all it does +- Everything that turns those types into what the gateway runs on lives in `crates/config-load`: resolving `include`, overlaying DB rows, reading the environment, substituting references, applying defaults and building the domain types. So this crate never reads files, the environment or a secret manager +- It knows nothing about which settings a consumer implements or rolls out, so checks like "refuse setting X because it is not supported yet" live in the consuming crate (e.g. `gateway`), never here + +## Defaults + +- The value the existing proxy uses when a key is absent (`health_check_interval` is 300, `database_connection_pool_limit` is 10) is part of what `config.yaml` means, but it is applied in `crates/config-load`, never here. Parsing never fills a default in, so every key here keeps whether the file set it. The proxy treats a key set in the file as owned by the file, so the DB value and Admin UI edits for it are ignored +- Other sections have no DB overlay, so a key there whose default is empty (an empty list or map) can be the plain collection, filled in by `#[serde(default)]` + +## Module layout + +Each top-level `config.yaml` key belongs to exactly one module under `src/`. A module is a single `.rs` file until it has flattened sub-groups, then it becomes a `/` folder with one file per sub-group. This only applies one level down, so groups flattened inside a sub-group (like the cache backends in `litellm/caching.rs`) stay in that sub-group's file. Keys typed as a plain `Object` have no module and stay on `Config` in `lib.rs`. Keep this list in sync when adding a key + +- `model.rs`: `model_list` +- `general/`: `general_settings` +- `router.rs`: `router_settings` +- `litellm/`: `litellm_settings` +- `callback.rs`: `callback_settings` +- `provider.rs`: `assistant_settings`, `default_vertex_config`, `files_settings`, `finetune_settings`, `worker_registry` +- `mcp.rs`: `mcp_servers` +- `credential.rs`: `credential_list` +- `guardrail.rs`: `guardrails` +- `prompt.rs`: `prompts` +- `tool.rs`: `sandbox_tools`, `search_tools` +- `vector_store.rs`: `vector_store_registry` +- `agent.rs`: `agents`, `agent_list` +- `policy.rs`: `policies`, `policy_attachments` +- `lib.rs`: `environment_variables`, `mcp_tools`, `include` + +`error.rs`, `value.rs` and `yaml.rs` are crate infrastructure, not config keys + +## Serde flatten + +Sections split one YAML mapping into several structs with nested `#[serde(flatten)]` (`LitellmSettings` flattens `CachingSettings`, `GeneralSettings` flattens `AuthSettings`, and so on). Serde hands keys to flattened fields like this + +- A flattened struct without a flatten field of its own takes only the keys it declares, and later siblings and the catch-all never see them +- A flattened struct or map that has its own flatten field is read as a map and claims nothing. It sees every key still unclaimed, including keys a later sibling declares, and those keys also reach the outer catch-all +- Fields claim keys in declaration order, so when two flattened structs declare the same key only the first one gets it + +Which gives these rules + +- Each mapping has exactly one `#[serde(flatten)] additional_fields: Object`, declared last on the outermost struct of that mapping. Flattened sub-structs stay plain, with no flatten field inside +- A key belongs to exactly one struct. Which one is decided by who reads it, see `src/AGENTS.md` +- Never combine `deny_unknown_fields` with flatten, on either the outer or the flattened struct. It is fine on a struct that is never flattened and has no flatten fields + +## Inline YAML + +- Write inline YAML as a multi-line raw string (`r#"..."#`), not as a one-line string with `\n` escapes like `"key:\n value"` +- This includes `#[case(...)]` arguments. Open the raw string on the attribute line and start the YAML at column 0, e.g. `#[case::name(r#"` then the YAML lines then `"#)]` +- YAML that fits on one line, like flow-style `key: {a: 1}`, stays a plain string with no trailing `\n` + +## Debug and redaction + +Every struct derives `Debug`, and redaction lives on the field types, so no struct has a hand-written `Debug` + +- Anything that is or can carry a secret (keys, tokens, passwords, URLs that may embed credentials, MCP `command` and `args`) is `SecretValue`, which prints `[REDACTED]` +- YAML we don't model is `Object` (prints its keys) or `Raw` (prints its kind). They are the only hand-written `Debug` impls. Never put a bare `Value` in a field +- Redaction tests put a unique `SENTINEL` under the slot and assert it never shows up in `format!("{config:?}")`. Words like `password` also match field names such as `redis_password` diff --git a/litellm-rust/crates/config/Cargo.toml b/litellm-rust/crates/config/Cargo.toml index 23d53923362..cda0bf53b45 100644 --- a/litellm-rust/crates/config/Cargo.toml +++ b/litellm-rust/crates/config/Cargo.toml @@ -7,7 +7,9 @@ repository.workspace = true [dependencies] litellm-auth-types.workspace = true +litellm-python-compat.workspace = true litellm-router-types.workspace = true +macro_rules_attribute.workspace = true serde.workspace = true serde_yaml_ng = "0.10.0" strum.workspace = true @@ -16,4 +18,3 @@ thiserror.workspace = true [dev-dependencies] rstest.workspace = true serde_json.workspace = true -tempfile.workspace = true diff --git a/litellm-rust/crates/config/src/AGENTS.md b/litellm-rust/crates/config/src/AGENTS.md new file mode 100644 index 00000000000..9c967d7fa1f --- /dev/null +++ b/litellm-rust/crates/config/src/AGENTS.md @@ -0,0 +1,32 @@ +## Grouping lives in the types + +The YAML shape is Python's `config.yaml` and we never change it, so a mapping stays flat even when its keys serve different consumers. The grouping is encoded in the types instead, with flattened structs (see the "Serde flatten" rules in `../AGENTS.md`) + +### Rules + +- Every distinct set of readers gets its own flattened struct, even when only one key has that set. `namespace` in `cache_params` is read only by `redis`, so it gets its own struct and does not sit with `host` and `port`, which `redis-semantic` and `valkey-semantic` also read +- Only keys that every reader of the mapping uses stay on the parent struct. A key that some readers ignore never stays on the parent just because no group fits yet. Make the group +- Once a mapping has keys with different readers, all of them move into groups, never only some. `router_settings` typed as one flat struct, or a parent that keeps keys an existing group already covers, breaks this + +### Finding the readers + +- A reader is the code that receives or branches on the value, found in the Python that consumes the key. The key's name does not tell you. `default_in_redis_ttl` looks Redis-only but Valkey reads it too, and `default_redis_ttl` sits under `cache_params` yet configures the proxy's internal caches, not the response cache +- Follow the value to where it is used, not just to where it is passed. A constructor argument read in one `if type == ...` arm has that arm as its reader. A value applied behind an `isinstance` guard is read by that class and its subclasses only (`namespace` reaches `RedisCache` and `RedisClusterCache`, not `RedisSemanticCache`) +- A key read by the shared code path for every variant (e.g. `ttl` and `mode`, read by `Cache` itself) is read by all of them and stays on the parent + +### Naming + +- Name each struct for its reader set (`RedisServerCacheParams`, `SemanticCacheParams`), and name the parent's flattened field for the same set (`redis_server`, `semantic`) +- When the name alone does not say who reads the keys, add a one-line doc comment on the struct that lists the readers, like the one on `RedisServerCacheParams` + +### Tests + +- Each mapping has one rstest with a case per group, putting one key of that group in the YAML and asserting it lands in that group's field and not in `additional_fields` +- When a key's readers were easy to get wrong (the examples above), add a case for that key too, so moving it back into a broader group fails a test + +## Lists are boxed slices + +Config is parsed once and only read after that, so every list field on a config type is `Box<[T]>` (or `Option>`), never `Vec`. The type then says the list can't grow, and serde deserializes into it directly + +- `Vec` is only for values that get built up and thrown away, like the layers and origins in `include/`. Those never become config fields +- Code that builds a field by hand collects into a `Vec` and converts at the end (`Box::from`, `.into_boxed_slice()`), and never keeps the `Vec` around diff --git a/litellm-rust/crates/config/src/agent.rs b/litellm-rust/crates/config/src/agent.rs new file mode 100644 index 00000000000..c2de9654557 --- /dev/null +++ b/litellm-rust/crates/config/src/agent.rs @@ -0,0 +1,75 @@ +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Agent { + pub agent_name: Option, + pub agent_card_params: Option, + pub litellm_params: Option, + pub identity: Option, + pub enabled: Option>, + pub execution_mode: Option, + pub object_permission: Option, + pub tpm_limit: Option>, + pub rpm_limit: Option>, + pub session_tpm_limit: Option>, + pub session_rpm_limit: Option>, + pub static_headers: Option, + pub extra_headers: Option>, + pub access_group_ids: Option>, + pub kill_switch: Option, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Spelled}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + #[case::agents("agents")] + #[case::agent_list("agent_list")] + fn agent_entries_keep_typed_limits_and_flags(#[case] section: &str) { + let config = parse(&format!( + "{section}:\n - {{agent_name: reviewer, agent_card_params: {{name: x}}, tpm_limit: 100, enabled: false}}\n" + )); + let agent = config.agents.first().or(config.agent_list.first()).unwrap(); + + assert_eq!(agent.agent_name.as_deref(), Some("reviewer")); + assert!( + agent + .agent_card_params + .as_ref() + .unwrap() + .contains_key("name") + ); + assert_eq!(agent.tpm_limit, Some(Spelled::Value(100))); + assert_eq!(agent.enabled, Some(Spelled::Value(false))); + } + + #[rstest] + #[case::agent_headers( + r#" +agents: + - static_headers: {Authorization: SENTINEL} +"# + )] + #[case::agent_params( + r#" +agents: + - litellm_params: {api_key: SENTINEL} +"# + )] + #[case::agent_identity( + r#" +agent_list: + - identity: {token: SENTINEL} +"# + )] + fn debug_never_prints_agent_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/callback.rs b/litellm-rust/crates/config/src/callback.rs new file mode 100644 index 00000000000..5a6c0ab57f4 --- /dev/null +++ b/litellm-rust/crates/config/src/callback.rs @@ -0,0 +1,77 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct CallbackSettings { + pub callback_type: Option, + pub endpoint: Option, + pub headers: Option, + pub event_types: Option>, + pub log_format: Option, + pub max_retries: Option>, + pub retry_delay: Option>, + pub timeout: Option>, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Spelled}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn callback_settings_keep_typed_fields_and_unknown_keys() { + let config = parse( + "callback_settings:\n hook:\n callback_type: generic_api\n retry_delay: 0.5\n batch_size: 10\n", + ); + let hook = &config.callback_settings["hook"]; + + assert_eq!(hook.callback_type.as_deref(), Some("generic_api")); + assert_eq!(hook.retry_delay, Some(Spelled::Value(0.5))); + assert!(hook.additional_fields.contains_key("batch_size")); + } + + #[rstest] + #[case::env_reference("os.environ/RETRIES", Spelled::Text("os.environ/RETRIES".to_owned()))] + #[case::number("4", Spelled::Value(4))] + fn spelled_int_fields_keep_references_as_text( + #[case] spelled: &str, + #[case] expected: Spelled, + ) { + let config = parse(&format!( + "callback_settings:\n hook:\n max_retries: {spelled}\n" + )); + + assert_eq!(config.callback_settings["hook"].max_retries, Some(expected)); + } + + #[rstest] + #[case::callback_endpoint( + r#" +callback_settings: + hook: {endpoint: SENTINEL} +"# + )] + #[case::callback_headers( + r#" +callback_settings: + hook: {headers: {Authorization: SENTINEL}} +"# + )] + #[case::callback_extra( + r#" +callback_settings: + hook: {vendor_secret: SENTINEL} +"# + )] + fn debug_never_prints_callback_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/credential.rs b/litellm-rust/crates/config/src/credential.rs new file mode 100644 index 00000000000..9db78717e88 --- /dev/null +++ b/litellm-rust/crates/config/src/credential.rs @@ -0,0 +1,65 @@ +use crate::Object; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Credential { + pub credential_name: Option, + pub credential_info: Option, + pub credential_values: Option, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::Config; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn credential_entries_keep_name_info_and_values() { + let config = parse( + r#" +credential_list: + - credential_name: azure-prod + credential_info: {description: prod} + credential_values: {api_key: os.environ/AZURE_API_KEY} +"#, + ); + let credential = &config.credential_list[0]; + + assert_eq!(credential.credential_name.as_deref(), Some("azure-prod")); + assert!( + credential + .credential_info + .as_ref() + .unwrap() + .contains_key("description") + ); + assert!( + credential + .credential_values + .as_ref() + .unwrap() + .contains_key("api_key") + ); + } + + #[rstest] + #[case::credential_values( + r#" +credential_list: + - credential_values: {api_key: SENTINEL} +"# + )] + #[case::credential_info( + r#" +credential_list: + - credential_info: {owner: SENTINEL} +"# + )] + fn debug_never_prints_credential_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/error.rs b/litellm-rust/crates/config/src/error.rs index 61d19491abc..e3db06bfdd2 100644 --- a/litellm-rust/crates/config/src/error.rs +++ b/litellm-rust/crates/config/src/error.rs @@ -1,7 +1,5 @@ #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("could not read config")] - Read(#[from] std::io::Error), #[error("invalid YAML config")] Parse(#[from] serde_yaml_ng::Error), } diff --git a/litellm-rust/crates/config/src/general/AGENTS.md b/litellm-rust/crates/config/src/general/AGENTS.md new file mode 100644 index 00000000000..42578db1ed4 --- /dev/null +++ b/litellm-rust/crates/config/src/general/AGENTS.md @@ -0,0 +1,14 @@ +## Docs + +- https://github.com/BerriAI/litellm-docs/blob/main/docs/proxy/config_settings.md + +## Python + +- `ConfigGeneralSettings` in `litellm/proxy/_types.py` is the Python model but is incomplete. What the proxy reads is `general_settings.get(...)`, mostly in `proxy_server.py` +- `general_settings` is one of the sections the proxy persists to the DB (`_CONFIG_PERSISTED_SECTIONS`) and the Admin UI edits, so at runtime the YAML is only one source of the effective values. Merging the DB overlay is the consumer's job, not this crate's + +## Fields that need care + +- `master_key`, `database_url` and `drain_endpoint_token` fall back to `LITELLM_MASTER_KEY`, `DATABASE_URL` and `DRAIN_ENDPOINT_TOKEN`. That fallback belongs to the consumer, so the fields stay optional here +- `custom_auth`, `custom_key_generate`, `custom_key_update`, `custom_key_policy`, `custom_team_metadata_validate`, `custom_sso`, `custom_ui_sso_sign_in_handler`, `litellm_jwtauth.custom_validate` and `pass_through_endpoints[].target` are Python module paths that the proxy imports. Type them as plain strings and never resolve or validate them here +- `key_management_system` and `key_management_settings` configure the secret manager itself, so they can only use `os.environ/` references, not secret manager ones diff --git a/litellm-rust/crates/config/src/general/alerting.rs b/litellm-rust/crates/config/src/general/alerting.rs new file mode 100644 index 00000000000..66aed7441e4 --- /dev/null +++ b/litellm-rust/crates/config/src/general/alerting.rs @@ -0,0 +1,39 @@ +use serde::Deserialize; + +use crate::{Object, Spelled}; + +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum Alerting { + List(Box<[String]>), + One(String), +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AlertingSettings { + pub alerting: Option, + pub alerting_threshold: Option>, + pub alert_types: Option>, + pub alert_to_webhook_url: Option, + pub alerting_args: Option, + pub alert_type_config: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AlertingArgs { + pub daily_report_frequency: Option>, + pub report_check_interval: Option>, + pub budget_alert_ttl: Option>, + pub outage_alert_ttl: Option>, + pub region_outage_alert_ttl: Option>, + pub minor_outage_alert_threshold: Option>, + pub major_outage_alert_threshold: Option>, + pub max_outage_alert_list_size: Option>, + pub log_to_console: Option>, + pub daily_spend_per_user_threshold: Option>, + pub monthly_spend_per_user_threshold: Option>, + pub spend_anomaly_multiplier: Option>, + pub spend_anomaly_baseline_days: Option>, + pub spend_anomaly_min_spend: Option>, + pub user_spend_check_interval: Option>, +} diff --git a/litellm-rust/crates/config/src/general/auth.rs b/litellm-rust/crates/config/src/general/auth.rs new file mode 100644 index 00000000000..8b62febc152 --- /dev/null +++ b/litellm-rust/crates/config/src/general/auth.rs @@ -0,0 +1,77 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AuthSettings { + pub litellm_key_header_name: Option, + pub enable_jwt_auth: Option>, + pub litellm_jwtauth: Option, + pub enable_oauth2_auth: Option>, + pub enable_oauth2_proxy_auth: Option>, + pub oauth2_config_mappings: Option, + pub custom_auth: Option, + pub custom_auth_run_common_checks: Option>, + pub allow_user_auth: Option>, + pub allowed_routes: Option>, + pub admin_only_routes: Option>, + pub public_routes: Option>, + pub allowed_ips: Option>, + pub use_x_forwarded_for: Option>, + pub trusted_proxy_ranges: Option>, + pub user_header_name: Option, + pub user_header_mappings: Option>, + pub role_permissions: Option>, + pub enforce_rbac: Option>, + pub allow_client_side_credentials: Option>, + pub enforce_user_param: Option>, + pub reject_clientside_metadata_tags: Option>, + pub enforced_params: Option>, + pub service_account_settings: Option, + pub litellm_license: Option, + pub control_plane_url: Option, + pub key_management_system: Option, + pub key_management_settings: Option, + pub use_google_kms: Option>, + pub use_azure_key_vault: Option>, + pub encryption_algorithm: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct UserHeaderMapping { + pub header_name: Option, + pub litellm_user_role: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RoleBasedPermissions { + pub role: Option, + pub models: Option>, + pub routes: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ServiceAccountSettings { + pub enforced_params: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct KeyManagementSettings { + pub hosted_keys: Option>, + pub store_virtual_keys: Option>, + pub prefix_for_stored_virtual_keys: Option, + pub access_mode: Option, + pub primary_secret_name: Option, + pub description: Option, + pub tags: Option, + pub kms_key_id: Option, + pub custom_secret_manager: Option, + pub aws_region_name: Option, + pub aws_role_name: Option, + pub aws_session_name: Option, + pub aws_external_id: Option, + pub aws_profile_name: Option, + pub aws_web_identity_token: Option, + pub aws_sts_endpoint: Option, + pub replica_regions: Option>, +} diff --git a/litellm-rust/crates/config/src/general/database.rs b/litellm-rust/crates/config/src/general/database.rs new file mode 100644 index 00000000000..e3f823d93d5 --- /dev/null +++ b/litellm-rust/crates/config/src/general/database.rs @@ -0,0 +1,15 @@ +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct DatabaseSettings { + pub database_connection_pool_timeout: Option>, + pub database_extra_connection_params: Option, + pub database_disable_prepared_statements: Option>, + pub database_statement_timeout: Option>, + pub database_lock_timeout: Option>, + pub database_idle_in_transaction_session_timeout: Option>, + pub disable_prisma_schema_update: Option>, + pub allow_requests_on_db_unavailable: Option>, + pub supported_db_objects: Option>, + pub use_redis_transaction_buffer: Option>, +} diff --git a/litellm-rust/crates/config/src/general/files.rs b/litellm-rust/crates/config/src/general/files.rs new file mode 100644 index 00000000000..e54aec83e90 --- /dev/null +++ b/litellm-rust/crates/config/src/general/files.rs @@ -0,0 +1,15 @@ +use crate::Spelled; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct FilesSettings { + pub max_file_size_mb: Option>, + pub max_batch_file_size_mb: Option>, + pub max_batch_file_records: Option>, + pub max_batch_file_uploads_per_day: Option>, + pub max_file_downloads_per_minute: Option>, + pub allowed_file_extensions: Option>, + pub blocked_file_extensions: Option>, + pub disable_batch_input_file_rate_limiting: Option>, + pub skip_batch_input_file_rate_limiting_for_providers: Option>, + pub skip_batch_input_file_rate_limiting_for_models: Option>, +} diff --git a/litellm-rust/crates/config/src/general/health.rs b/litellm-rust/crates/config/src/general/health.rs new file mode 100644 index 00000000000..169be606700 --- /dev/null +++ b/litellm-rust/crates/config/src/general/health.rs @@ -0,0 +1,22 @@ +use litellm_auth_types::SecretValue; + +use crate::Spelled; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct HealthSettings { + pub use_shared_health_check: Option>, + pub health_check_details: Option>, + pub enable_health_check_routing: Option>, + pub health_check_staleness_threshold: Option>, + pub health_check_ignore_transient_errors: Option>, + pub health_check_skip_disabled_background_models: Option>, + pub background_health_check_model_groups: Option>, + pub model_list_healthy_only: Option>, + pub allow_public_health_readiness_details: Option>, + pub enable_drain_endpoint: Option>, + pub drain_endpoint_token: Option, + pub disable_model_info_refresh: Option>, + pub use_team_public_model_name: Option>, + pub image_generation_model: Option, + pub moderation_model: Option, +} diff --git a/litellm-rust/crates/config/src/general/limits.rs b/litellm-rust/crates/config/src/general/limits.rs new file mode 100644 index 00000000000..d1522903e2c --- /dev/null +++ b/litellm-rust/crates/config/src/general/limits.rs @@ -0,0 +1,11 @@ +use crate::Spelled; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct LimitsSettings { + pub fail_closed_rate_limit_enforcement: Option>, + pub token_rate_limit_type: Option, + pub disable_retry_on_max_parallel_request_limit_error: Option>, + pub disable_fallbacks_on_per_model_rate_limits: Option>, + pub user_api_key_cache_ttl: Option>, + pub user_api_key_cache_max_size: Option>, +} diff --git a/litellm-rust/crates/config/src/general/mcp.rs b/litellm-rust/crates/config/src/general/mcp.rs new file mode 100644 index 00000000000..c60e7665c3c --- /dev/null +++ b/litellm-rust/crates/config/src/general/mcp.rs @@ -0,0 +1,20 @@ +use crate::{Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct McpSettings { + pub mcp_allowed_hosts: Option>, + pub mcp_allowed_origins: Box<[String]>, + pub mcp_advertised_versions: Option, + pub mcp_disabled_capabilities: Option, + pub mcp_allowed_clients: Option, + pub mcp_require_explicit_server_scope: Option>, + pub mcp_client_id_header: Option, + pub mcp_client_side_auth_header_name: Option, + pub mcp_allow_all_keys_respects_mcp_scope: Option>, + pub mcp_prefer_client_id_metadata_document: Option>, + pub mcp_internal_ip_ranges: Option, + pub mcp_trusted_proxy_ranges: Option, + pub mcp_xff_num_trusted_hops: Option>, + pub mcp_required_fields: Option, + pub mcp_session_token_signing: Option, +} diff --git a/litellm-rust/crates/config/src/general/mod.rs b/litellm-rust/crates/config/src/general/mod.rs new file mode 100644 index 00000000000..72871437f00 --- /dev/null +++ b/litellm-rust/crates/config/src/general/mod.rs @@ -0,0 +1,535 @@ +mod alerting; +mod auth; +mod database; +mod files; +mod health; +mod limits; +mod mcp; +mod objects; +mod requests; +mod spend; +mod tracing; +mod ui; + +use litellm_auth_types::SecretValue; +use serde::Deserialize; + +pub use self::{ + alerting::{Alerting, AlertingArgs, AlertingSettings}, + auth::{ + AuthSettings, KeyManagementSettings, RoleBasedPermissions, ServiceAccountSettings, + UserHeaderMapping, + }, + database::DatabaseSettings, + files::FilesSettings, + health::HealthSettings, + limits::LimitsSettings, + mcp::McpSettings, + objects::{ObjectsSettings, TeamMetadataField}, + requests::{PassThroughEndpoint, RequestTagHeader, RequestTagMapping, RequestsSettings}, + spend::{ + ScheduledJobStaggerSettings, SpendCaptureRateCheckSettings, SpendLogsMetadataFields, + SpendSettings, + }, + tracing::{ClickHouseStoreSettings, TracingSettings, TracingStoreSettings}, + ui::{UiAccessMode, UiAccessSsoGroup, UiSettings}, +}; +use crate::Object; + +#[derive(Clone, Debug, Deserialize)] +#[serde(default)] +pub struct GeneralSettings { + pub completion_model: Option, + pub max_in_flight_requests_per_worker: Option, + pub max_queued_requests_per_worker: Option, + pub admission_queue_timeout_seconds: f64, + pub master_key: Option, + pub database_url: Option, + pub tracing: Option, + pub database_connection_pool_limit: Option, + pub database_connection_timeout: Option, + pub database_connect_timeout: Option, + pub database_socket_timeout: Option, + pub database_max_idle_connection_lifetime: Option, + pub max_parallel_requests: Option, + pub global_max_parallel_requests: Option, + pub max_request_size_mb: Option, + pub max_response_size_mb: Option, + pub proxy_config_reload_interval_seconds: u64, + pub background_health_checks: Option, + pub health_check_interval: u64, + pub health_check_concurrency: Option, + pub store_model_in_db: Option, + pub forward_client_headers_to_llm_api: Option, + pub cancel_on_disconnect: Option, + pub infer_model_from_keys: Option, + pub enable_public_model_hub: bool, + pub dangerously_permit_weak_or_unset_master_key: Option, + pub plugins: Option>, + pub coordination_redis: Option, + #[serde(flatten)] + pub auth: AuthSettings, + #[serde(flatten)] + pub database: DatabaseSettings, + #[serde(flatten)] + pub spend: SpendSettings, + #[serde(flatten)] + pub limits: LimitsSettings, + #[serde(flatten)] + pub health: HealthSettings, + #[serde(flatten)] + pub alerting: AlertingSettings, + #[serde(flatten)] + pub ui: UiSettings, + #[serde(flatten)] + pub objects: ObjectsSettings, + #[serde(flatten)] + pub requests: RequestsSettings, + #[serde(flatten)] + pub files: FilesSettings, + #[serde(flatten)] + pub mcp: McpSettings, + #[serde(flatten)] + pub additional_fields: Object, +} + +impl Default for GeneralSettings { + fn default() -> Self { + Self { + completion_model: None, + max_in_flight_requests_per_worker: None, + max_queued_requests_per_worker: None, + admission_queue_timeout_seconds: 1.0, + master_key: None, + database_url: None, + tracing: None, + database_connection_pool_limit: Some(10), + database_connection_timeout: Some(60.0), + database_connect_timeout: None, + database_socket_timeout: None, + database_max_idle_connection_lifetime: Some(60.0), + max_parallel_requests: None, + global_max_parallel_requests: None, + max_request_size_mb: None, + max_response_size_mb: None, + proxy_config_reload_interval_seconds: 30, + background_health_checks: None, + health_check_interval: 300, + health_check_concurrency: None, + store_model_in_db: None, + forward_client_headers_to_llm_api: None, + cancel_on_disconnect: None, + infer_model_from_keys: None, + enable_public_model_hub: false, + dangerously_permit_weak_or_unset_master_key: None, + plugins: None, + coordination_redis: None, + auth: AuthSettings::default(), + database: DatabaseSettings::default(), + spend: SpendSettings::default(), + limits: LimitsSettings::default(), + health: HealthSettings::default(), + alerting: AlertingSettings::default(), + ui: UiSettings::default(), + objects: ObjectsSettings::default(), + requests: RequestsSettings::default(), + files: FilesSettings::default(), + mcp: McpSettings::default(), + additional_fields: Object::default(), + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Alerting, Config, GeneralSettings, Spelled, UiAccessMode}; + + fn general(yaml: &str) -> GeneralSettings { + Config::from_yaml(&format!("general_settings:\n{yaml}")) + .unwrap() + .general_settings + } + + fn text(value: &str) -> Option> { + Some(Spelled::Text(value.to_string())) + } + + #[rstest] + #[case::yaml_bool("true", Some(Spelled::Value(true)))] + #[case::yaml_1_1_word("yes", text("yes"))] + #[case::quoted("'false'", text("false"))] + #[case::environment_reference("os.environ/ENABLE_JWT", text("os.environ/ENABLE_JWT"))] + fn bool_settings_keep_every_python_spelling( + #[case] spelling: &str, + #[case] expected: Option>, + ) { + let settings = general(&format!( + " enable_jwt_auth: {spelling}\n disable_spend_logs: {spelling}\n use_redis_transaction_buffer: {spelling}\n model_list_healthy_only: {spelling}\n" + )); + assert_eq!(settings.auth.enable_jwt_auth, expected); + assert_eq!(settings.spend.disable_spend_logs, expected); + assert_eq!(settings.database.use_redis_transaction_buffer, expected); + assert_eq!(settings.health.model_list_healthy_only, expected); + } + + #[rstest] + #[case::bare_days("7", Some(Spelled::Value(7)))] + #[case::duration("'60d'", text("60d"))] + #[case::environment_reference("os.environ/RETENTION", text("os.environ/RETENTION"))] + fn retention_periods_accept_days_or_a_duration( + #[case] spelling: &str, + #[case] expected: Option>, + ) { + let settings = general(&format!( + " maximum_spend_logs_retention_period: {spelling}\n" + )); + assert_eq!(settings.spend.maximum_spend_logs_retention_period, expected); + } + + #[rstest] + #[case::integer("5", Some(Spelled::Value(5.0)))] + #[case::fraction("0.5", Some(Spelled::Value(0.5)))] + #[case::environment_reference("os.environ/TTL", text("os.environ/TTL"))] + fn number_settings_accept_integers_fractions_and_references( + #[case] spelling: &str, + #[case] expected: Option>, + ) { + let settings = general(&format!( + " user_api_key_cache_ttl: {spelling}\n database_statement_timeout: {spelling}\n health_check_staleness_threshold: {spelling}\n" + )); + assert_eq!(settings.limits.user_api_key_cache_ttl, expected); + assert_eq!(settings.database.database_statement_timeout, expected); + assert_eq!(settings.health.health_check_staleness_threshold, expected); + } + + #[test] + fn nested_settings_are_typed_and_keep_unknown_keys() { + let settings = general( + r#" + allowed_routes: [/chat/completions, /models] + custom_auth: custom_auth.user_api_key_auth + user_header_mappings: + - header_name: X-OpenWebUI-User-Id + litellm_user_role: internal_user + role_permissions: + - role: team + models: [gpt] + routes: [/chat/completions] + service_account_settings: + enforced_params: [user] + key_management_settings: + access_mode: read_and_write + store_virtual_keys: true + future_option: 1 + spend_logs_metadata_fields: + include: [user_api_key_alias] + spend_capture_rate_check: + threshold: 0.8 + lookback_days: 3 + scheduled_job_stagger: + window_seconds: 120 + offsets: {reset_budget_job: 0} +"#, + ); + assert_eq!( + settings.auth.allowed_routes.as_deref(), + Some(["/chat/completions".to_string(), "/models".to_string()].as_slice()) + ); + assert_eq!( + settings.auth.custom_auth.as_deref(), + Some("custom_auth.user_api_key_auth") + ); + let mapping = &settings.auth.user_header_mappings.as_deref().unwrap()[0]; + assert_eq!(mapping.header_name.as_deref(), Some("X-OpenWebUI-User-Id")); + assert_eq!(mapping.litellm_user_role.as_deref(), Some("internal_user")); + let permission = &settings.auth.role_permissions.as_deref().unwrap()[0]; + assert_eq!(permission.role.as_deref(), Some("team")); + assert_eq!( + permission.routes.as_deref(), + Some(["/chat/completions".to_string()].as_slice()) + ); + assert_eq!( + settings + .auth + .service_account_settings + .as_ref() + .unwrap() + .enforced_params + .as_deref(), + Some(["user".to_string()].as_slice()) + ); + let key_management = settings.auth.key_management_settings.as_ref().unwrap(); + assert_eq!( + key_management.access_mode.as_deref(), + Some("read_and_write") + ); + assert_eq!( + key_management.store_virtual_keys, + Some(Spelled::Value(true)) + ); + assert_eq!( + settings + .spend + .spend_logs_metadata_fields + .as_ref() + .unwrap() + .include + .as_deref(), + Some(["user_api_key_alias".to_string()].as_slice()) + ); + let capture = settings.spend.spend_capture_rate_check.as_ref().unwrap(); + assert_eq!(capture.threshold, Some(Spelled::Value(0.8))); + assert_eq!(capture.lookback_days, Some(Spelled::Value(3))); + let stagger = settings.spend.scheduled_job_stagger.as_ref().unwrap(); + assert_eq!(stagger.window_seconds, Some(Spelled::Value(120))); + assert!( + stagger + .offsets + .as_ref() + .unwrap() + .contains_key("reset_budget_job") + ); + assert!(settings.additional_fields.is_empty()); + } + + #[rstest] + #[case::license("litellm_license: SENTINEL")] + #[case::drain_token("drain_endpoint_token: SENTINEL")] + #[case::jwt_auth_object("litellm_jwtauth: {team_id_jwt_field: SENTINEL}")] + #[case::oauth2_mappings("oauth2_config_mappings: {X-Header: SENTINEL}")] + #[case::database_params("database_extra_connection_params: {sslpassword: SENTINEL}")] + #[case::key_management_token("key_management_settings: {aws_web_identity_token: SENTINEL}")] + #[case::key_management_external_id("key_management_settings: {aws_external_id: SENTINEL}")] + #[case::key_management_hosted_keys("key_management_settings: {hosted_keys: [SENTINEL]}")] + #[case::key_management_tags("key_management_settings: {tags: {owner: SENTINEL}}")] + #[case::mcp_session_signing("mcp_session_token_signing: {private_key: SENTINEL}")] + #[case::master_key("master_key: SENTINEL")] + #[case::database_url("database_url: postgres://user:SENTINEL@db/litellm")] + #[case::coordination_redis("coordination_redis: {password: SENTINEL}")] + #[case::plugins("plugins: [{token: SENTINEL}]")] + #[case::unknown_setting("future_setting: SENTINEL")] + fn debug_output_does_not_expose_setting_values(#[case] setting: &str) { + let config = Config::from_yaml(&format!("general_settings:\n {setting}\n")).unwrap(); + let printed = format!("{config:?}"); + assert!(!printed.contains("SENTINEL"), "{printed}"); + } + + #[test] + fn unknown_keys_still_land_in_additional_fields() { + let settings = general(" future_setting: 1\n custom_auth: auth.py\n"); + assert!(settings.additional_fields.contains_key("future_setting")); + assert!(!settings.additional_fields.contains_key("custom_auth")); + } + + #[rstest] + #[case("true", Spelled::Value(true))] + #[case("\"false\"", Spelled::Text("false".to_string()))] + #[case("yes", Spelled::Text("yes".to_string()))] + #[case("os.environ/GATEWAY_FLAG", Spelled::Text("os.environ/GATEWAY_FLAG".to_string()))] + fn bool_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled) { + let settings = general(&format!(" enable_claude_code_gateway: {value}\n")); + assert_eq!(settings.requests.enable_claude_code_gateway, Some(expected)); + } + + #[rstest] + #[case("100", Spelled::Value(100))] + #[case("os.environ/MAX_FILE_MB", Spelled::Text("os.environ/MAX_FILE_MB".to_string()))] + fn int_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled) { + let settings = general(&format!(" max_file_size_mb: {value}\n")); + assert_eq!(settings.files.max_file_size_mb, Some(expected)); + } + + #[rstest] + #[case("600", Spelled::Value(600.0))] + #[case("2.5", Spelled::Value(2.5))] + #[case("os.environ/ALERT_THRESHOLD", Spelled::Text("os.environ/ALERT_THRESHOLD".to_string()))] + fn float_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled) { + let settings = general(&format!(" alerting_threshold: {value}\n")); + assert_eq!(settings.alerting.alerting_threshold, Some(expected)); + } + + #[rstest] + fn nested_mappings_land_in_typed_structs() { + let settings = general( + r#" + alerting: [slack, email] + alerting_args: + daily_report_frequency: 3600 + log_to_console: true + spend_anomaly_multiplier: 2.5 + future_arg: 1 + team_metadata_schema: + - key: cost_center + label: Cost center + request_tag_mappings: + - header: baggage + key: tenant + tag_prefix: tenant- + required: true + pass_through_endpoints: + - path: /bria + target: https://engine.prod.bria-api.com + include_subpath: true + cost_per_request: 0.5 + methods: [GET, POST] + custom_handler: handler.fn + allowed_file_extensions: [jsonl, csv] +"#, + ); + + assert_eq!( + settings.alerting.alerting, + Some(Alerting::List( + ["slack".to_string(), "email".to_string()].into() + )) + ); + let args = settings.alerting.alerting_args.unwrap(); + assert_eq!(args.daily_report_frequency, Some(Spelled::Value(3600))); + assert_eq!(args.log_to_console, Some(Spelled::Value(true))); + assert_eq!(args.spend_anomaly_multiplier, Some(Spelled::Value(2.5))); + + let schema = settings.objects.team_metadata_schema.unwrap(); + assert_eq!(schema[0].key.as_deref(), Some("cost_center")); + assert_eq!(schema[0].label.as_deref(), Some("Cost center")); + + let mapping = &settings.requests.request_tag_mappings.unwrap()[0]; + let header: &'static str = mapping.header.unwrap().into(); + assert_eq!(header, "baggage"); + assert_eq!(mapping.key.as_deref(), Some("tenant")); + assert_eq!(mapping.tag_prefix.as_deref(), Some("tenant-")); + assert_eq!(mapping.required, Some(Spelled::Value(true))); + + let endpoint = &settings.requests.pass_through_endpoints.unwrap()[0]; + assert_eq!(endpoint.path.as_deref(), Some("/bria")); + assert_eq!( + endpoint.target.as_ref().unwrap().expose(), + "https://engine.prod.bria-api.com" + ); + assert_eq!(endpoint.include_subpath, Some(Spelled::Value(true))); + assert_eq!(endpoint.cost_per_request, Some(Spelled::Value(0.5))); + assert_eq!( + endpoint.methods.as_deref(), + Some(&["GET".to_string(), "POST".to_string()][..]) + ); + + assert_eq!( + settings.files.allowed_file_extensions.as_deref(), + Some(&["jsonl".to_string(), "csv".to_string()][..]) + ); + } + + #[rstest] + fn ui_access_mode_accepts_a_mode_string() { + let settings = general(" ui_access_mode: admin_only\n"); + assert!( + matches!(settings.ui.ui_access_mode, Some(UiAccessMode::Mode(mode)) if mode == "admin_only") + ); + } + + #[rstest] + fn ui_access_mode_accepts_an_sso_group_mapping() { + let settings = general( + " ui_access_mode: {type: restricted_sso_group, restricted_sso_group: ai-admins, sso_group_jwt_field: groups}\n", + ); + let Some(UiAccessMode::SsoGroup(group)) = settings.ui.ui_access_mode else { + panic!("expected an SSO group mapping") + }; + assert_eq!(group.kind.as_deref(), Some("restricted_sso_group")); + assert_eq!(group.restricted_sso_group.as_deref(), Some("ai-admins")); + assert_eq!(group.sso_group_jwt_field.as_deref(), Some("groups")); + } + + #[rstest] + fn alerting_accepts_a_single_name() { + let settings = general(" alerting: slack\n"); + assert_eq!( + settings.alerting.alerting, + Some(Alerting::One("slack".to_string())) + ); + } + + #[rstest] + fn rejects_request_tag_header_python_rejects_at_boot() { + assert!( + Config::from_yaml( + "general_settings:\n request_tag_mappings: [{header: x-tenant, key: tenant, tag: t}]\n" + ) + .is_err() + ); + } + + #[rstest] + #[case("alert_to_webhook_url: {budget_alerts: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))] + #[case("alert_type_config: {budget_alerts: {webhook: SENTINEL}}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))] + #[case("alerting_args: {webhook: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))] + #[case("pass_through_endpoints: [{path: /x, target: SENTINEL}]", |settings: &GeneralSettings| format!("{:?}", settings.requests))] + #[case("pass_through_endpoints: [{path: /x, target: t, headers: {Authorization: SENTINEL}}]", |settings: &GeneralSettings| format!("{:?}", settings.requests))] + #[case("claude_code_gateway_managed_settings: {token: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.requests))] + #[case("search_tools: [{litellm_params: {api_key: SENTINEL}}]", |settings: &GeneralSettings| format!("{:?}", settings.objects))] + #[case("max_failed_login_attempts_per_source_overrides: {10.0.0.1: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.ui))] + fn debug_hides_values_that_can_carry_secrets( + #[case] entry: &str, + #[case] render: fn(&GeneralSettings) -> String, + ) { + let settings = general(&format!(" {entry}\n")); + assert!(!render(&settings).contains("SENTINEL")); + } + + #[rstest] + fn unknown_key_still_lands_in_additional_fields() { + let settings = general(" future_setting: 1\n forward_openai_org_id: true\n"); + assert!(settings.additional_fields.contains_key("future_setting")); + assert!( + !settings + .additional_fields + .contains_key("forward_openai_org_id") + ); + assert_eq!( + settings.requests.forward_openai_org_id, + Some(Spelled::Value(true)) + ); + } + + #[rstest] + fn mcp_allowed_hosts_and_origins_land_typed() { + let settings = general( + " mcp_allowed_hosts: [localhost]\n mcp_allowed_origins: ['https://app.test']\n", + ); + assert_eq!( + settings.mcp.mcp_allowed_hosts.as_deref(), + Some(&["localhost".to_string()][..]) + ); + assert_eq!( + settings.mcp.mcp_allowed_origins.as_ref(), + ["https://app.test".to_string()] + ); + } + + #[rstest] + #[case::literal("secret-master-key")] + #[case::reference("os.environ/LITELLM_MASTER_KEY")] + fn loads_and_redacts_the_master_key(#[case] key: &str) { + let config = Config::from_yaml(&format!( + "model_list: []\ngeneral_settings:\n master_key: {key}\n" + )) + .unwrap(); + assert_eq!( + config + .general_settings + .master_key + .as_ref() + .unwrap() + .expose(), + key + ); + assert!(!format!("{config:?}").contains(key)); + } + + #[rstest] + fn missing_general_settings_has_no_master_key() { + let config = Config::from_yaml("model_list: []").unwrap(); + assert!(config.general_settings.master_key.is_none()); + } +} diff --git a/litellm-rust/crates/config/src/general/objects.rs b/litellm-rust/crates/config/src/general/objects.rs new file mode 100644 index 00000000000..d3cb7bed753 --- /dev/null +++ b/litellm-rust/crates/config/src/general/objects.rs @@ -0,0 +1,26 @@ +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ObjectsSettings { + pub custom_key_generate: Option, + pub custom_key_update: Option, + pub custom_key_policy: Option, + pub custom_team_metadata_validate: Option, + pub team_metadata_schema: Option>, + pub team_metadata_validation_timeout: Option>, + pub team_metadata_validation_error_message: Option, + pub disable_auto_add_proxy_admin_to_teams: Option>, + pub self_serve_budget_policy: Option, + pub enforce_rpm_tpm_on_model_add: Option>, + pub agent_access_default_deny: Option>, + pub vector_store_deny_by_default: Option>, + pub search_tool_deny_by_default: Option>, + pub search_tools: Option>, + pub enforce_fallback_model_access: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct TeamMetadataField { + pub key: Option, + pub label: Option, +} diff --git a/litellm-rust/crates/config/src/general/requests.rs b/litellm-rust/crates/config/src/general/requests.rs new file mode 100644 index 00000000000..11b5782ae2f --- /dev/null +++ b/litellm-rust/crates/config/src/general/requests.rs @@ -0,0 +1,66 @@ +use litellm_auth_types::SecretValue; +use serde::Deserialize; + +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RequestsSettings { + pub forward_llm_provider_auth_headers: Option>, + pub forward_openai_org_id: Option>, + pub always_include_stream_usage: Option>, + pub expose_fallback_errors_to_caller: Option>, + pub include_call_id_in_error_body: Option>, + pub request_tag_mappings: Option>, + pub pass_through_endpoints: Option>, + pub pass_through_request_timeout: Option>, + pub passthrough_managed_object_ids: Option>, + pub disable_bedrock_agent_runtime_passthrough: Option>, + pub enable_openai_websocket_passthrough: Option>, + pub vertex_ai_pass_through_drop_anthropic_betas: Option>, + pub transcribe_media_buckets: Option>, + pub responses_websocket_session_limit_seconds: Option>, + pub disable_responses_id_security: Option>, + pub allow_unmanaged_response_ids: Option>, + pub dangerously_allow_mock_testing_request_params: Option>, + pub enable_claude_code_gateway: Option>, + pub claude_code_gateway_managed_settings: Option, + pub context_management_summary_model: Option, + pub context_management_summary_max_tokens: Option>, + pub safeguards_classifier_model: Option, + pub user_url_validation: Option>, + pub user_url_allowed_hosts: Option>, + pub provider_url_destination_allowed_hosts: Option>, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, strum::IntoStaticStr)] +#[serde(rename_all = "lowercase")] +pub enum RequestTagHeader { + #[strum(serialize = "baggage")] + Baggage, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RequestTagMapping { + pub header: Option, + pub key: Option, + pub value: Option, + pub tag: Option, + pub tag_prefix: Option, + pub required: Option>, + pub require_budgeted_tag: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PassThroughEndpoint { + pub id: Option, + pub path: Option, + pub target: Option, + pub headers: Option, + pub default_query_params: Option, + pub include_subpath: Option>, + pub cost_per_request: Option>, + pub timeout: Option>, + pub auth: Option>, + pub guardrails: Option, + pub methods: Option>, +} diff --git a/litellm-rust/crates/config/src/general/spend.rs b/litellm-rust/crates/config/src/general/spend.rs new file mode 100644 index 00000000000..a86e3ebe756 --- /dev/null +++ b/litellm-rust/crates/config/src/general/spend.rs @@ -0,0 +1,61 @@ +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SpendSettings { + pub disable_spend_logs: Option>, + pub disable_spend_updates: Option>, + pub disable_reset_budget: Option>, + pub disable_error_logs: Option>, + pub store_prompts_in_spend_logs: Option>, + pub disable_requester_ip_address_logging: Option>, + pub spend_logs_metadata_fields: Option, + pub proxy_batch_write_at: Option>, + pub proxy_batch_polling_interval: Option>, + pub proxy_budget_rescheduler_min_time: Option>, + pub proxy_budget_rescheduler_max_time: Option>, + pub disable_budget_reservation: Option>, + pub fail_closed_budget_enforcement: Option>, + pub apply_user_budget_to_team_keys: Option>, + pub enforce_fallback_budget: Option>, + pub track_unmanaged_batch_cost: Option>, + pub spend_report_frequency: Option, + pub spend_capture_rate_check: Option, + pub maximum_spend_logs_retention_period: Option>, + pub maximum_autorouter_session_retention_period: Option>, + pub maximum_health_check_retention_period: Option>, + pub maximum_daily_tag_spend_retention_period: Option>, + pub maximum_spend_logs_size: Option>, + pub maximum_spend_logs_cleanup_target_size: Option>, + pub maximum_spend_logs_size_check_interval: Option, + pub maximum_spend_logs_retention_interval: Option, + pub maximum_spend_logs_cleanup_cron: Option, + pub maximum_spend_logs_cleanup_batch_size: Option>, + pub maximum_spend_logs_cleanup_max_batches: Option>, + pub maximum_spend_logs_cleanup_run_budget: Option>, + pub maximum_spend_logs_cleanup_batch_timeout: Option>, + pub use_spend_logs_partitioning: Option>, + pub scheduled_job_stagger: Option, + pub missing_session_id: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SpendLogsMetadataFields { + pub include: Option>, + pub exclude: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SpendCaptureRateCheckSettings { + pub providers: Option>, + pub threshold: Option>, + pub lookback_days: Option>, + pub openai_project_ids: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ScheduledJobStaggerSettings { + pub enabled: Option>, + pub window_seconds: Option>, + pub identity: Option, + pub offsets: Option, +} diff --git a/litellm-rust/crates/config/src/general/tracing.rs b/litellm-rust/crates/config/src/general/tracing.rs new file mode 100644 index 00000000000..b6bf1a4de2b --- /dev/null +++ b/litellm-rust/crates/config/src/general/tracing.rs @@ -0,0 +1,84 @@ +use litellm_auth_types::SecretValue; +use serde::Deserialize; + +use crate::Spelled; + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum TracingStoreKind { + Clickhouse, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClickHouseStoreSettings { + #[serde(rename = "type")] + pub kind: TracingStoreKind, + pub url: Option, + pub database: Option, + pub retention_days: Option>, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum TracingStoreSettings { + ClickHouse(ClickHouseStoreSettings), +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct TracingSettings { + pub store: Option, +} + +#[cfg(test)] +mod tests { + use crate::{Config, Spelled, TracingStoreSettings}; + + #[test] + fn tracing_settings_are_typed_and_redact_the_url() { + let config = Config::from_yaml( + "general_settings:\n tracing:\n store:\n type: clickhouse\n url: https://writer:SENTINEL@example.com\n database: analytics\n retention_days: 7\n", + ) + .unwrap(); + let tracing = config.general_settings.tracing.as_ref().unwrap(); + let Some(TracingStoreSettings::ClickHouse(store)) = tracing.store.as_ref() else { + panic!("expected ClickHouse tracing store") + }; + assert_eq!( + store.url.as_ref().unwrap().expose(), + "https://writer:SENTINEL@example.com" + ); + assert_eq!(store.database.as_deref(), Some("analytics")); + assert_eq!(store.retention_days, Some(Spelled::Value(7.0))); + assert!(!format!("{config:?}").contains("SENTINEL")); + } + + #[test] + fn tracing_settings_accept_environment_references() { + let config = Config::from_yaml( + "general_settings:\n tracing:\n store:\n type: clickhouse\n url: os.environ/CLICKHOUSE_URL\n retention_days: os.environ/RETENTION_DAYS\n", + ) + .unwrap(); + let Some(TracingStoreSettings::ClickHouse(store)) = + config.general_settings.tracing.unwrap().store + else { + panic!("expected ClickHouse tracing store") + }; + assert_eq!( + store.retention_days, + Some(Spelled::Text("os.environ/RETENTION_DAYS".to_owned())) + ); + } + + #[test] + fn tracing_settings_reject_string_store() { + assert!( + Config::from_yaml("general_settings:\n tracing:\n store: clickhouse\n").is_err() + ); + } + + #[test] + fn tracing_settings_reject_removed_reader_configuration() { + assert!(Config::from_yaml("general_settings:\n tracing:\n store:\n type: clickhouse\n reader_url: http://localhost:8123\n").is_err()); + } +} diff --git a/litellm-rust/crates/config/src/general/ui.rs b/litellm-rust/crates/config/src/general/ui.rs new file mode 100644 index 00000000000..5129d8f829c --- /dev/null +++ b/litellm-rust/crates/config/src/general/ui.rs @@ -0,0 +1,50 @@ +use serde::Deserialize; + +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct UiSettings { + pub ui_access_mode: Option, + pub custom_sso: Option, + pub custom_ui_sso_sign_in_handler: Option, + pub auto_redirect_ui_login_to_sso: Option>, + pub default_team_disabled: Option>, + pub allow_cli_sso_verification_uri_complete: Option>, + pub disable_env_credential_login: Option>, + pub disable_password_login_when_sso_enabled: Option>, + pub password_policy_min_length: Option>, + pub password_policy_require_uppercase: Option>, + pub password_policy_require_lowercase: Option>, + pub password_policy_require_numbers: Option>, + pub password_policy_require_special_characters: Option>, + pub password_policy_check_breached_passwords: Option>, + pub max_failed_login_attempts_per_source: Option>, + pub failed_login_window_seconds: Option>, + pub failed_login_block_seconds: Option>, + pub max_failed_login_attempts_per_source_overrides: Option, + pub hide_default_credentials_hint: Option>, + pub disable_master_key_return: Option>, + pub disable_custom_api_keys: Option>, + pub disable_key_generate_for_org_admin: Option>, + pub team_admin_editable_team_fields: Option>, + pub disable_model_add_for_internal_users: Option>, + pub disable_agents_for_internal_users: Option>, + pub allow_agents_for_team_admins: Option>, + pub disable_vector_stores_for_internal_users: Option>, + pub allow_vector_stores_for_team_admins: Option>, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum UiAccessMode { + Mode(String), + SsoGroup(UiAccessSsoGroup), +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct UiAccessSsoGroup { + #[serde(rename = "type")] + pub kind: Option, + pub restricted_sso_group: Option, + pub sso_group_jwt_field: Option, +} diff --git a/litellm-rust/crates/config/src/guardrail.rs b/litellm-rust/crates/config/src/guardrail.rs new file mode 100644 index 00000000000..387418cd039 --- /dev/null +++ b/litellm-rust/crates/config/src/guardrail.rs @@ -0,0 +1,102 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Guardrail { + pub guardrail_id: Option, + pub guardrail_name: Option, + pub litellm_params: Option, + pub guardrail_info: Option, + pub policy_template: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct GuardrailParams { + pub guardrail: Option, + pub mode: Option, + pub default_on: Option>, + pub api_key: Option, + pub api_base: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Spelled, Value}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn guardrail_entries_keep_typed_params_and_unknown_keys() { + let config = parse( + r#" +guardrails: + - guardrail_name: pii + litellm_params: + guardrail: presidio + mode: [pre_call, post_call] + default_on: "true" + api_key: os.environ/PRESIDIO_KEY + presidio_language: en + guardrail_info: + owner: security +"#, + ); + let guardrail = &config.guardrails[0]; + let params = guardrail.litellm_params.as_ref().unwrap(); + + assert_eq!(guardrail.guardrail_name.as_deref(), Some("pii")); + assert_eq!(params.guardrail.as_deref(), Some("presidio")); + assert!( + matches!(params.mode.as_ref().unwrap().get(), Value::Sequence(modes) if modes.len() == 2) + ); + assert_eq!(params.default_on, Some(Spelled::Text("true".to_owned()))); + assert_eq!( + params.api_key.as_ref().unwrap().expose(), + "os.environ/PRESIDIO_KEY" + ); + assert!(params.additional_fields.contains_key("presidio_language")); + assert!( + guardrail + .guardrail_info + .as_ref() + .unwrap() + .contains_key("owner") + ); + } + + #[rstest] + #[case::guardrail_key( + r#" +guardrails: + - litellm_params: {api_key: SENTINEL} +"# + )] + #[case::guardrail_extra( + r#" +guardrails: + - litellm_params: {vendor_secret: SENTINEL} +"# + )] + #[case::guardrail_info( + r#" +guardrails: + - guardrail_info: {owner: SENTINEL} +"# + )] + #[case::guardrail_mode( + r#" +guardrails: + - litellm_params: {mode: SENTINEL} +"# + )] + fn debug_never_prints_guardrail_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/includes.rs b/litellm-rust/crates/config/src/includes.rs deleted file mode 100644 index 78df8e9a33a..00000000000 --- a/litellm-rust/crates/config/src/includes.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::{ - collections::{BTreeSet, VecDeque}, - path::{Path, PathBuf}, -}; - -use serde::Deserialize; -use serde_yaml_ng::{Mapping, Value}; - -use crate::Error; - -#[derive(Deserialize)] -struct Includes { - #[serde(default)] - include: Vec, -} - -fn read(path: &Path) -> Result { - Ok(serde_yaml_ng::from_str(&std::fs::read_to_string(path)?)?) -} - -fn entries(config: &Mapping, path: &Path) -> Result, Error> { - let includes: Includes = serde_yaml_ng::from_value(Value::Mapping(config.clone()))?; - Ok(includes - .include - .into_iter() - .map(|entry| (entry, path.to_owned())) - .collect()) -} - -pub(super) fn load(path: &Path) -> Result { - let root = path.canonicalize()?; - let mut merged = read(&root)?; - let mut pending: VecDeque<_> = entries(&merged, &root)?.into(); - let mut loaded = BTreeSet::from([root.clone()]); - merged.remove(Value::String("include".into())); - while let Some((entry, declaring)) = pending.pop_front() { - let declared = declaring.parent().unwrap_or(Path::new(".")).join(&entry); - let fallback = root.parent().unwrap_or(Path::new(".")).join(&entry); - let location = if declared.exists() { - declared - } else { - fallback - } - .canonicalize()?; - if !loaded.insert(location.clone()) { - continue; - } - let mut included = read(&location)?; - pending.extend(entries(&included, &location)?); - included.remove(Value::String("include".into())); - for (key, value) in included { - match (merged.get_mut(&key), value) { - (Some(Value::Sequence(base)), Value::Sequence(extra)) => base.extend(extra), - (_, value) => { - merged.insert(key, value); - } - } - } - } - Ok(Value::Mapping(merged)) -} diff --git a/litellm-rust/crates/config/src/lib.rs b/litellm-rust/crates/config/src/lib.rs index 8603ef3279c..f831dbd710a 100644 --- a/litellm-rust/crates/config/src/lib.rs +++ b/litellm-rust/crates/config/src/lib.rs @@ -1,94 +1,349 @@ +macro_rules_attribute::attribute_alias! { + #[apply(config_section)] = + #[derive(Clone, Debug, Default, ::serde::Deserialize)] + #[serde(default)]; +} + +mod agent; +mod callback; +mod credential; mod error; -mod includes; +mod general; +mod guardrail; +mod litellm; mod mcp; mod model; -mod settings; +mod policy; +mod prompt; +mod provider; +mod router; +mod tool; mod value; +mod vector_store; +pub mod yaml; -use std::{fmt, path::Path}; - -use serde::Deserialize; +use std::collections::BTreeMap; +pub use agent::Agent; +pub use callback::CallbackSettings; +pub use credential::Credential; pub use error::Error; -pub use litellm_router_types::{LitellmParams, Spelled}; -pub use mcp::{McpAuth, McpServer, McpTransport}; -pub use model::Model; -pub use settings::{ - ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings, - TracingStoreSettings, +pub use general::{ + Alerting, AlertingArgs, AlertingSettings, AuthSettings, ClickHouseStoreSettings, + DatabaseSettings, FilesSettings, GeneralSettings, HealthSettings, KeyManagementSettings, + LimitsSettings, McpSettings, ObjectsSettings, PassThroughEndpoint, RequestTagHeader, + RequestTagMapping, RequestsSettings, RoleBasedPermissions, ScheduledJobStaggerSettings, + ServiceAccountSettings, SpendCaptureRateCheckSettings, SpendLogsMetadataFields, SpendSettings, + TeamMetadataField, TracingSettings, TracingStoreSettings, UiAccessMode, UiAccessSsoGroup, + UiSettings, UserHeaderMapping, }; -pub use value::{AdditionalFields, Object, Value}; +pub use guardrail::{Guardrail, GuardrailParams}; +pub use litellm::{ + AiohttpSettings, AzureBlobCacheParams, BackgroundMode, BudgetSettings, CacheParams, + CachingSettings, CallerIdentity, ClientCertificateSettings, CustomProvider, DiskCacheParams, + FallbackSettings, GcsCacheParams, Http2Settings, LiteLlmSettings, LoggingSettings, + McpToolSettings, ModelGroupSettings, PriorityReservationSettings, ProxyCacheTtlParams, + QdrantCacheParams, RedisCacheParams, RedisServerCacheParams, ResponsesSettings, S3CacheParams, + SemanticCacheParams, SocketSettings, TeamDefaultSettings, TlsSettings, + UpperboundKeyGenerateParams, ValkeyCacheParams, +}; +pub use litellm_python_compat::read::Spelled; +pub use litellm_router_types::LitellmParams; +pub use mcp::{McpAuth, McpServer, McpTransport}; +pub use model::{AccessWindow, Model, ModelId, ModelInfo}; +pub use policy::{ + Policy, PolicyAttachment, PolicyCondition, PolicyGuardrailChanges, PolicyGuardrails, + PolicyPipeline, PolicyPipelineStep, +}; +pub use prompt::{Prompt, PromptInfo, PromptParams}; +pub use provider::{AssistantSettings, ProviderSettings, VertexConfig, Worker}; +pub use router::{AlertingConfig, AllowedFailsPolicy, ProviderBudget, RouterSettings}; +pub use tool::{SandboxTool, SandboxToolParams, SearchTool, SearchToolInfo, SearchToolParams}; +pub use value::{Object, Raw, Value}; +pub use vector_store::{VectorStore, VectorStoreParams}; -#[derive(Clone, Default, Deserialize)] -#[serde(default)] +#[macro_rules_attribute::apply(config_section)] pub struct Config { pub model_list: Box<[Model]>, pub general_settings: GeneralSettings, pub router_settings: RouterSettings, pub litellm_settings: LiteLlmSettings, pub environment_variables: Object, - pub callback_settings: Object, - pub assistant_settings: Object, - pub default_vertex_config: Object, - pub mcp_servers: std::collections::BTreeMap, - pub credential_list: Box<[Object]>, - pub guardrails: Box<[Object]>, - pub prompts: Box<[Object]>, - pub sandbox_tools: Box<[Object]>, - pub search_tools: Box<[Object]>, - pub files_settings: Box<[Object]>, - pub finetune_settings: Box<[Object]>, + pub callback_settings: BTreeMap, + pub assistant_settings: Option, + pub default_vertex_config: Option, + pub mcp_servers: BTreeMap, + pub credential_list: Box<[Credential]>, + pub guardrails: Box<[Guardrail]>, + pub prompts: Box<[Prompt]>, + pub sandbox_tools: Box<[SandboxTool]>, + pub search_tools: Box<[SearchTool]>, + pub files_settings: Box<[ProviderSettings]>, + pub finetune_settings: Box<[ProviderSettings]>, pub mcp_tools: Box<[Object]>, - pub vector_store_registry: Box<[Object]>, - pub worker_registry: Box<[Object]>, - pub agents: Box<[Object]>, - pub agent_list: Box<[Object]>, - pub policies: Object, - pub policy_attachments: Box<[Object]>, + pub vector_store_registry: Box<[VectorStore]>, + pub worker_registry: Box<[Worker]>, + pub agents: Box<[Agent]>, + pub agent_list: Box<[Agent]>, + pub policies: BTreeMap, + pub policy_attachments: Box<[PolicyAttachment]>, pub include: Box<[String]>, #[serde(flatten)] - pub additional_fields: AdditionalFields, -} - -impl fmt::Debug for Config { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("Config") - .field("model_list", &self.model_list) - .field("general_settings", &self.general_settings) - .field("router_settings", &self.router_settings) - .field("litellm_settings", &self.litellm_settings) - .field("environment_variables", &self.environment_variables) - .field("callback_settings", &self.callback_settings) - .field("assistant_settings", &self.assistant_settings) - .field("default_vertex_config", &self.default_vertex_config) - .field("mcp_servers", &self.mcp_servers) - .field("credential_list", &self.credential_list) - .field("guardrails", &self.guardrails) - .field("prompts", &self.prompts) - .field("sandbox_tools", &self.sandbox_tools) - .field("search_tools", &self.search_tools) - .field("files_settings", &self.files_settings) - .field("finetune_settings", &self.finetune_settings) - .field("mcp_tools", &self.mcp_tools) - .field("vector_store_registry", &self.vector_store_registry) - .field("worker_registry", &self.worker_registry) - .field("agents", &self.agents) - .field("agent_list", &self.agent_list) - .field("policies", &self.policies) - .field("policy_attachments", &self.policy_attachments) - .field("include", &self.include) - .field("additional_fields", &self.additional_fields.keys()) - .finish() - } + pub additional_fields: Object, } impl Config { pub fn from_yaml(yaml: &str) -> Result { - Ok(serde_yaml_ng::from_str(yaml)?) + Self::from_mapping(yaml::mapping(yaml)?) } - pub fn load(path: impl AsRef) -> Result { - Ok(serde_yaml_ng::from_value(includes::load(path.as_ref())?)?) + pub fn from_mapping(document: serde_yaml_ng::Mapping) -> Result { + Ok(serde_yaml_ng::from_value(serde_yaml_ng::Value::Mapping( + document, + ))?) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Error, Spelled}; + + #[rstest] + fn config_debug_redacts_api_keys() { + let config = Config::from_yaml( + "model_list: [{model_name: assistant, litellm_params: {model: anthropic/test-model, api_key: secret-value}}]", + ) + .unwrap(); + assert_eq!( + config.model_list[0] + .litellm_params + .api_key + .as_ref() + .unwrap() + .expose(), + "secret-value" + ); + assert!(!format!("{config:?}").contains("secret-value")); + } + + #[rstest] + #[case::malformed_yaml("model_list: [")] + #[case::missing_params("model_list: [{model_name: assistant}]")] + #[case::missing_model("model_list: [{model_name: assistant, litellm_params: {api_key: key}}]")] + #[case::provider_param_of_the_wrong_type( + "model_list: [{model_name: assistant, litellm_params: {model: test, aws_region_name: [eu-central-1]}}]" + )] + #[case::negative_model_rpm( + "model_list: [{model_name: assistant, rpm: -1, litellm_params: {model: test}}]" + )] + fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) { + let error = Config::from_yaml(yaml).unwrap_err(); + assert!(matches!(error, Error::Parse(_)), "{error:?}"); + } + + #[rstest] + fn empty_config_matches_python_defaults() { + let config = Config::from_yaml("{}").unwrap(); + assert!(config.model_list.is_empty()); + assert_eq!(config.general_settings.admission_queue_timeout_seconds, 1.0); + assert_eq!( + config.general_settings.database_connection_pool_limit, + Some(10) + ); + assert_eq!( + config.general_settings.proxy_config_reload_interval_seconds, + 30 + ); + assert_eq!(config.general_settings.health_check_interval, 300); + } + + #[rstest] + fn parses_typed_settings_and_preserves_extension_fields() { + let config = Config::from_yaml( + r#" +model_list: + - model_name: assistant + litellm_params: + model: vertex_ai/test-model + timeout: os.environ/REQUEST_TIMEOUT + tpm: os.environ/TPM_LIMIT + rpm: 5 + drop_params: "true" + vertex_project: test-project + vertex_credentials: + type: service_account + future_param: 1 + model_info: + mode: chat + access_groups: [internal] +general_settings: + master_key: secret-master-key + store_model_in_db: true + custom_auth: auth.py +router_settings: + routing_strategy: simple-shuffle + allowed_fails: 2 + future_router_setting: cache.internal +litellm_settings: + drop_params: true + cache: true + custom_callback_name: audit +future_section: + enabled: true +"#, + ) + .unwrap(); + + let model = &config.model_list[0]; + assert_eq!( + model.litellm_params.timeout, + Some(Spelled::Text("os.environ/REQUEST_TIMEOUT".to_string())) + ); + assert_eq!( + model.litellm_params.tpm, + Some(Spelled::Text("os.environ/TPM_LIMIT".to_string())) + ); + assert_eq!(model.litellm_params.rpm, Some(Spelled::Value(5))); + assert_eq!( + model.litellm_params.drop_params, + Some(Spelled::Text("true".to_string())) + ); + assert_eq!( + model.litellm_params.vertex.vertex_project.as_deref(), + Some("test-project") + ); + assert_eq!( + model.litellm_params.vertex.vertex_credentials.as_deref(), + Some(r#"{"type":"service_account"}"#) + ); + assert!(model.litellm_params.extra.contains_key("future_param")); + assert!(model.additional_fields.contains_key("access_groups")); + assert_eq!(config.router_settings.allowed_fails, Some(2)); + assert!( + config + .router_settings + .additional_fields + .contains_key("future_router_setting") + ); + assert!( + config + .litellm_settings + .additional_fields + .contains_key("custom_callback_name") + ); + assert!(config.additional_fields.contains_key("future_section")); + } + + #[rstest] + fn parses_python_config_sections() { + let config = Config::from_yaml( + r#" +environment_variables: + REDIS_PORT: 6379 +callback_settings: + otel: + message_logging: false +assistant_settings: + custom_llm_provider: openai +credential_list: + - credential_name: bedrock + credential_values: + aws_region_name: us-east-1 +guardrails: + - guardrail_name: pii + litellm_params: + guardrail: presidio +prompts: + - prompt_id: support +sandbox_tools: + - sandbox_tool_name: e2b +search_tools: + - search_tool_name: web +files_settings: + - custom_llm_provider: openai +finetune_settings: + - custom_llm_provider: openai +mcp_tools: + - name: lookup +mcp_servers: + docs: + url: https://example.test/mcp +vector_store_registry: + - vector_store_name: docs +worker_registry: + - worker_id: regional +agents: + - agent_name: reviewer +agent_list: + - agent_name: legacy-reviewer +policies: + safe: + guardrails: + add: [pii] +policy_attachments: + - policy_id: safe +"#, + ) + .unwrap(); + + assert_eq!(config.environment_variables.len(), 1); + assert_eq!(config.credential_list.len(), 1); + assert_eq!(config.guardrails.len(), 1); + assert_eq!(config.prompts.len(), 1); + assert_eq!(config.sandbox_tools.len(), 1); + assert_eq!(config.search_tools.len(), 1); + assert_eq!(config.files_settings.len(), 1); + assert_eq!(config.finetune_settings.len(), 1); + assert_eq!(config.mcp_tools.len(), 1); + assert_eq!(config.mcp_servers.len(), 1); + assert_eq!(config.vector_store_registry.len(), 1); + assert_eq!(config.worker_registry.len(), 1); + assert_eq!(config.agents.len(), 1); + assert_eq!(config.agent_list.len(), 1); + assert!(config.policies.contains_key("safe")); + assert_eq!(config.policy_attachments.len(), 1); + } + + #[rstest] + #[case::api_key("api_key", "provider-secret")] + #[case::aws_secret("aws_secret_access_key", "aws-secret")] + #[case::vertex_credentials("vertex_credentials", "vertex-secret")] + #[case::provider_extension("azure_ad_token", "azure-secret")] + #[case::general_extension("custom_auth_secret", "auth-secret")] + #[case::router_extension("future_router_token", "router-secret")] + #[case::litellm_extension("callback_token", "callback-secret")] + #[case::root_extension("private_token", "root-secret")] + fn debug_output_does_not_expose_config_values(#[case] field: &str, #[case] secret: &str) { + let yaml = match field { + "api_key" => format!( + "model_list: [{{model_name: assistant, litellm_params: {{model: test, api_key: {secret}}}}}]" + ), + "aws_secret_access_key" => format!( + "model_list: [{{model_name: assistant, litellm_params: {{model: test, aws_secret_access_key: {secret}}}}}]" + ), + "vertex_credentials" => format!( + "model_list: [{{model_name: assistant, litellm_params: {{model: test, vertex_credentials: {secret}}}}}]" + ), + "azure_ad_token" => format!( + "model_list: [{{model_name: assistant, litellm_params: {{model: test, azure_ad_token: {secret}}}}}]" + ), + "custom_auth_secret" => format!("general_settings: {{{field}: {secret}}}"), + "future_router_token" => format!("router_settings: {{{field}: {secret}}}"), + "callback_token" => format!("litellm_settings: {{{field}: {secret}}}"), + "private_token" => format!("{field}: {secret}"), + _ => unreachable!(), + }; + let config = Config::from_yaml(&yaml).unwrap(); + assert!(!format!("{config:?}").contains(secret)); + } + + #[rstest] + fn unknown_top_level_key_lands_in_additional_fields() { + let config = Config::from_yaml("future_section:\n a: 1\nenable_jwt_auth: true\n").unwrap(); + + assert!(config.additional_fields.contains_key("future_section")); + assert!(config.additional_fields.contains_key("enable_jwt_auth")); } } diff --git a/litellm-rust/crates/config/src/litellm/AGENTS.md b/litellm-rust/crates/config/src/litellm/AGENTS.md new file mode 100644 index 00000000000..e19b6c87bbc --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/AGENTS.md @@ -0,0 +1,9 @@ +## Docs + +- https://github.com/BerriAI/litellm-docs/blob/main/docs/proxy/config_settings.md + +## Python + +- There is no Python model. `ConfigYAML.litellm_settings` in `litellm/proxy/_types.py` is a plain `dict` +- `ProxyConfig.load_config` in `proxy_server.py` handles a few keys specially (`cache`, `cache_params`, `callbacks`, `success_callback`, `guardrails`, `drop_params` and others) and does `setattr(litellm, key, value)` for everything else. So any module-level attribute in `litellm/__init__.py` is a valid key, and that is the real key set +- `litellm_settings` is one of the sections the proxy persists to the DB (`_CONFIG_PERSISTED_SECTIONS`) and the Admin UI edits, so at runtime the YAML is only one source of the effective values. Merging the DB overlay is the consumer's job, not this crate's diff --git a/litellm-rust/crates/config/src/litellm/budget.rs b/litellm-rust/crates/config/src/litellm/budget.rs new file mode 100644 index 00000000000..12acac2c1c2 --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/budget.rs @@ -0,0 +1,32 @@ +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct BudgetSettings { + pub max_budget: Option>, + pub max_internal_user_budget: Option>, + pub default_max_internal_user_budget: Option>, + pub default_internal_user_params: Option, + pub max_ui_session_budget: Option>, + pub default_team_settings: Option>, + pub upperbound_key_generate_params: Option, + pub key_alias_pattern: Option, + pub budget_reset_time: Option, + pub disable_end_user_cost_tracking: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct TeamDefaultSettings { + pub team_id: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct UpperboundKeyGenerateParams { + pub max_budget: Option>, + pub budget_duration: Option, + pub duration: Option, + pub max_parallel_requests: Option>, + pub tpm_limit: Option>, + pub rpm_limit: Option>, +} diff --git a/litellm-rust/crates/config/src/litellm/caching.rs b/litellm-rust/crates/config/src/litellm/caching.rs new file mode 100644 index 00000000000..d7ce2e3028f --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/caching.rs @@ -0,0 +1,137 @@ +use serde::{Deserialize, Deserializer}; + +use litellm_auth_types::SecretValue; + +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct CachingSettings { + pub cache: Option, + pub cache_params: Option, + pub enable_redis_auth_cache: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct CacheParams { + #[serde(rename = "type")] + pub cache_type: Option, + pub mode: Option, + pub namespace: Option, + pub ttl: Option>, + pub supported_call_types: Option>, + #[serde(deserialize_with = "present")] + pub max_messages: Option>>, + #[serde(flatten)] + pub redis_server: RedisServerCacheParams, + #[serde(flatten)] + pub semantic: SemanticCacheParams, + #[serde(flatten)] + pub proxy_ttl: ProxyCacheTtlParams, + #[serde(flatten)] + pub redis: RedisCacheParams, + #[serde(flatten)] + pub valkey: ValkeyCacheParams, + #[serde(flatten)] + pub s3: S3CacheParams, + #[serde(flatten)] + pub gcs: GcsCacheParams, + #[serde(flatten)] + pub azure_blob: AzureBlobCacheParams, + #[serde(flatten)] + pub qdrant: QdrantCacheParams, + #[serde(flatten)] + pub disk: DiskCacheParams, + #[serde(flatten)] + pub additional_fields: Object, +} + +/// Read by every backend that talks to a Redis-protocol server: `redis`, `redis-semantic` and `valkey-semantic`. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RedisServerCacheParams { + pub host: Option, + pub port: Option>, + pub password: Option, + pub redis_startup_nodes: Option, + pub default_in_redis_ttl: Option>, +} + +/// Read by the semantic backends: `redis-semantic`, `valkey-semantic` and `qdrant-semantic`. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SemanticCacheParams { + pub similarity_threshold: Option>, + pub semantic_cache_embedding_max_input_tokens: Option>, + pub semantic_cache_embedding_timeout: Option>, + pub semantic_cache_scope: Option, +} + +/// Not response-cache settings. The proxy copies them onto `litellm.default_*_ttl` for its own internal caches. +/// `default_in_memory_ttl` also replaces `ttl` when `type` is `local`. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ProxyCacheTtlParams { + pub default_in_memory_ttl: Option>, + pub default_redis_ttl: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RedisCacheParams { + pub redis_semantic_cache_embedding_model: Option, + pub redis_semantic_cache_index_name: Option, + pub redis_flush_size: Option>, + pub gcp_service_account: Option, + pub gcp_ssl_ca_certs: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ValkeyCacheParams { + pub valkey_semantic_cache_embedding_model: Option, + pub valkey_semantic_cache_index_name: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct S3CacheParams { + pub s3_bucket_name: Option, + pub s3_region_name: Option, + pub s3_api_version: Option, + pub s3_use_ssl: Option>, + pub s3_verify: Option>, + pub s3_endpoint_url: Option, + pub s3_aws_access_key_id: Option, + pub s3_aws_secret_access_key: Option, + pub s3_aws_session_token: Option, + pub s3_config: Option, + pub s3_path: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct GcsCacheParams { + pub gcs_bucket_name: Option, + pub gcs_path_service_account: Option, + pub gcs_path: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AzureBlobCacheParams { + pub azure_account_url: Option, + pub azure_blob_container: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct QdrantCacheParams { + pub qdrant_api_base: Option, + pub qdrant_api_key: Option, + pub qdrant_collection_name: Option, + pub qdrant_quantization_config: Option, + pub qdrant_semantic_cache_embedding_model: Option, + pub qdrant_semantic_cache_vector_size: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct DiskCacheParams { + pub disk_cache_dir: Option, +} + +fn present<'de, D: Deserializer<'de>, T: Deserialize<'de>>( + deserializer: D, +) -> Result>, D::Error> { + Ok(Some(Option::deserialize(deserializer)?)) +} diff --git a/litellm-rust/crates/config/src/litellm/fallbacks.rs b/litellm-rust/crates/config/src/litellm/fallbacks.rs new file mode 100644 index 00000000000..f930ffaa208 --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/fallbacks.rs @@ -0,0 +1,7 @@ +use crate::Raw; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct FallbackSettings { + pub default_fallbacks: Option, + pub context_window_fallbacks: Option, +} diff --git a/litellm-rust/crates/config/src/litellm/logging.rs b/litellm-rust/crates/config/src/litellm/logging.rs new file mode 100644 index 00000000000..46f74c17ab0 --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/logging.rs @@ -0,0 +1,24 @@ +use litellm_python_compat::read::one_or_many; + +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct LoggingSettings { + #[serde(deserialize_with = "one_or_many")] + pub callbacks: Option>, + #[serde(deserialize_with = "one_or_many")] + pub success_callback: Option>, + #[serde(deserialize_with = "one_or_many")] + pub failure_callback: Option>, + pub service_callback: Option>, + pub audit_log_callbacks: Option>, + pub s3_callback_params: Option, + pub s3_audit_callback_params: Option, + pub store_audit_logs: Option>, + pub json_logs: Option, + pub set_verbose: Option, + pub turn_off_message_logging: Option>, + pub redact_messages_in_exceptions: Option>, + pub log_raw_request_response: Option>, + pub presidio_logging_only: Option>, +} diff --git a/litellm-rust/crates/config/src/litellm/mcp.rs b/litellm-rust/crates/config/src/litellm/mcp.rs new file mode 100644 index 00000000000..43245638835 --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/mcp.rs @@ -0,0 +1,8 @@ +use crate::Raw; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct McpToolSettings { + pub mcp_aliases: Option, + pub mcp_semantic_tool_filter: Option, + pub mcp_tool_search: Option, +} diff --git a/litellm-rust/crates/config/src/litellm/mod.rs b/litellm-rust/crates/config/src/litellm/mod.rs new file mode 100644 index 00000000000..a5d3ee3ea1a --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/mod.rs @@ -0,0 +1,400 @@ +mod budget; +mod caching; +mod fallbacks; +mod logging; +mod mcp; +mod transport; + +use serde::Deserialize; + +pub use self::{ + budget::{BudgetSettings, TeamDefaultSettings, UpperboundKeyGenerateParams}, + caching::{ + AzureBlobCacheParams, CacheParams, CachingSettings, DiskCacheParams, GcsCacheParams, + ProxyCacheTtlParams, QdrantCacheParams, RedisCacheParams, RedisServerCacheParams, + S3CacheParams, SemanticCacheParams, ValkeyCacheParams, + }, + fallbacks::FallbackSettings, + logging::LoggingSettings, + mcp::McpToolSettings, + transport::{ + AiohttpSettings, ClientCertificateSettings, Http2Settings, SocketSettings, TlsSettings, + }, +}; +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct LiteLlmSettings { + pub drop_params: Option>, + pub request_timeout: Option>, + pub num_retries: Option, + pub prometheus_deployment_and_latency_caller_identity: Option, + pub guardrails: Option>, + pub prompt_injection_params: Option, + pub azure_content_safety_params: Option, + pub global_prompt_directory: Option, + pub global_bitbucket_config: Option, + pub global_gitlab_config: Option, + pub priority_reservation_settings: Option, + pub model_group_settings: Option, + pub post_call_rules: Option, + pub custom_provider_map: Option>, + pub responses: Option, + pub enable_preview_features: Option>, + pub public_model_groups: Option>, + pub timezone: Option, + #[serde(flatten)] + pub tls: TlsSettings, + #[serde(flatten)] + pub client_certificate: ClientCertificateSettings, + #[serde(flatten)] + pub socket: SocketSettings, + #[serde(flatten)] + pub http2: Http2Settings, + #[serde(flatten)] + pub aiohttp: AiohttpSettings, + #[serde(flatten)] + pub caching: CachingSettings, + #[serde(flatten)] + pub budget: BudgetSettings, + #[serde(flatten)] + pub logging: LoggingSettings, + #[serde(flatten)] + pub fallbacks: FallbackSettings, + #[serde(flatten)] + pub mcp: McpToolSettings, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, strum::IntoStaticStr)] +#[serde(rename_all = "snake_case")] +pub enum CallerIdentity { + #[strum(serialize = "api_key_alias")] + ApiKeyAlias, + #[strum(serialize = "user_email")] + UserEmail, + #[strum(serialize = "both")] + Both, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PriorityReservationSettings { + pub default_priority: Option>, + pub saturation_threshold: Option>, + pub saturation_check_cache_ttl: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ModelGroupSettings { + pub forward_client_headers_to_llm_api: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct CustomProvider { + pub provider: Option, + pub custom_handler: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ResponsesSettings { + pub background_mode: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct BackgroundMode { + pub polling_via_cache: Option, + pub native_background_mode: Option>, + pub ttl: Option>, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{CallerIdentity, Config, LiteLlmSettings, Spelled}; + + fn settings(yaml: &str) -> LiteLlmSettings { + Config::from_yaml(&format!("litellm_settings:\n{yaml}")) + .unwrap() + .litellm_settings + } + + #[rstest] + #[case::bool_word(" turn_off_message_logging: 'false'", Spelled::Text("false".into()))] + #[case::bool_value(" turn_off_message_logging: true", Spelled::Value(true))] + #[case::env_reference(" turn_off_message_logging: os.environ/REDACT", Spelled::Text("os.environ/REDACT".into()))] + fn bool_settings_keep_python_spellings(#[case] yaml: &str, #[case] expected: Spelled) { + assert_eq!( + settings(yaml).logging.turn_off_message_logging, + Some(expected) + ); + } + + #[rstest] + #[case::number(" max_budget: 10", Spelled::Value(10.0))] + #[case::string(" max_budget: '10'", Spelled::Text("10".into()))] + #[case::env_reference(" max_budget: os.environ/MAX_BUDGET", Spelled::Text("os.environ/MAX_BUDGET".into()))] + fn budget_accepts_number_or_string(#[case] yaml: &str, #[case] expected: Spelled) { + assert_eq!(settings(yaml).budget.max_budget, Some(expected)); + } + + #[rstest] + #[case::api_key_alias("api_key_alias", CallerIdentity::ApiKeyAlias)] + #[case::user_email("user_email", CallerIdentity::UserEmail)] + #[case::both("both", CallerIdentity::Both)] + fn caller_identity_accepts_python_values( + #[case] value: &str, + #[case] expected: CallerIdentity, + ) { + assert_eq!( + settings(&format!( + " prometheus_deployment_and_latency_caller_identity: {value}" + )) + .prometheus_deployment_and_latency_caller_identity, + Some(expected) + ); + } + + #[rstest] + fn caller_identity_rejects_values_python_refuses() { + assert!( + Config::from_yaml( + "litellm_settings:\n prometheus_deployment_and_latency_caller_identity: team" + ) + .is_err() + ); + } + + #[rstest] + fn cache_params_reads_cache_keywords() { + let cache = settings( + " cache_params:\n type: redis\n port: 6379\n password: os.environ/REDIS_PASSWORD\n ttl: 600\n supported_call_types: [acompletion]\n namespace_extra: x", + ) + .caching + .cache_params + .unwrap(); + assert_eq!(cache.cache_type.as_deref(), Some("redis")); + assert_eq!(cache.redis_server.port, Some(Spelled::Value(6379))); + assert_eq!( + cache.redis_server.password.unwrap().expose(), + "os.environ/REDIS_PASSWORD" + ); + assert_eq!(cache.ttl, Some(Spelled::Value(600.0))); + assert_eq!( + cache.supported_call_types.as_deref(), + Some(&["acompletion".to_owned()][..]) + ); + assert!(cache.additional_fields.contains_key("namespace_extra")); + } + + #[rstest] + fn cache_params_route_each_key_to_its_backend_group() { + let cache = settings( + r#" cache_params: + type: s3 + namespace: shared + port: 6379 + similarity_threshold: 0.8 + default_redis_ttl: 30 + redis_flush_size: 10 + valkey_semantic_cache_index_name: valkey-index + s3_bucket_name: bucket + gcs_path: gcs/path + azure_blob_container: container + qdrant_collection_name: collection + disk_cache_dir: /tmp/cache + namespace_extra: x"#, + ) + .caching + .cache_params + .unwrap(); + assert_eq!(cache.namespace.as_deref(), Some("shared")); + assert_eq!(cache.redis_server.port, Some(Spelled::Value(6379))); + assert_eq!( + cache.semantic.similarity_threshold, + Some(Spelled::Value(0.8)) + ); + assert_eq!( + cache.proxy_ttl.default_redis_ttl, + Some(Spelled::Value(30.0)) + ); + assert_eq!(cache.redis.redis_flush_size, Some(Spelled::Value(10))); + assert_eq!( + cache.valkey.valkey_semantic_cache_index_name.as_deref(), + Some("valkey-index") + ); + assert_eq!(cache.s3.s3_bucket_name.as_deref(), Some("bucket")); + assert_eq!(cache.gcs.gcs_path.as_deref(), Some("gcs/path")); + assert_eq!( + cache.azure_blob.azure_blob_container.as_deref(), + Some("container") + ); + assert_eq!( + cache.qdrant.qdrant_collection_name.as_deref(), + Some("collection") + ); + assert_eq!(cache.disk.disk_cache_dir.as_deref(), Some("/tmp/cache")); + assert_eq!( + cache.additional_fields.keys().collect::>(), + ["namespace_extra"] + ); + } + + #[rstest] + #[case::absent(" ttl: 1", None)] + #[case::explicit_null(" max_messages: null", Some(None))] + #[case::limit(" max_messages: 8", Some(Some(Spelled::Value(8))))] + fn cache_max_messages_tells_null_from_absent( + #[case] yaml: &str, + #[case] expected: Option>>, + ) { + assert_eq!( + settings(&format!(" cache_params:\n{yaml}")) + .caching + .cache_params + .unwrap() + .max_messages, + expected + ); + } + + #[rstest] + fn nested_settings_land_in_typed_fields() { + let settings = settings( + " priority_reservation_settings: {default_priority: 0.5, saturation_check_cache_ttl: 30} + model_group_settings: {forward_client_headers_to_llm_api: [gpt]} + custom_provider_map: [{provider: my-llm, custom_handler: handler.instance}] + responses: {background_mode: {polling_via_cache: all, native_background_mode: [o3], ttl: 60}} + default_team_settings: [{team_id: team-1, success_callback: [langfuse]}] + upperbound_key_generate_params: {max_budget: 100, duration: 30d, rpm_limit: os.environ/RPM}", + ); + let priority = settings.priority_reservation_settings.unwrap(); + assert_eq!(priority.default_priority, Some(Spelled::Value(0.5))); + assert_eq!( + priority.saturation_check_cache_ttl, + Some(Spelled::Value(30)) + ); + assert_eq!( + settings + .model_group_settings + .unwrap() + .forward_client_headers_to_llm_api + .as_deref(), + Some(&["gpt".to_owned()][..]) + ); + let provider = &settings.custom_provider_map.unwrap()[0]; + assert_eq!(provider.provider.as_deref(), Some("my-llm")); + assert_eq!(provider.custom_handler.as_deref(), Some("handler.instance")); + let background = settings.responses.unwrap().background_mode.unwrap(); + assert_eq!( + background.polling_via_cache.unwrap().get().as_str(), + Some("all") + ); + assert_eq!( + background.native_background_mode.as_deref(), + Some(&["o3".to_owned()][..]) + ); + assert_eq!(background.ttl, Some(Spelled::Value(60))); + let team = &settings.budget.default_team_settings.unwrap()[0]; + assert_eq!(team.team_id.as_deref(), Some("team-1")); + assert!(team.additional_fields.contains_key("success_callback")); + let upperbound = settings.budget.upperbound_key_generate_params.unwrap(); + assert_eq!(upperbound.max_budget, Some(Spelled::Value(100.0))); + assert_eq!(upperbound.duration.as_deref(), Some("30d")); + assert_eq!( + upperbound.rpm_limit, + Some(Spelled::Text("os.environ/RPM".into())) + ); + } + + #[rstest] + #[case::cache_password( + r#" + cache: true + cache_params: {password: SENTINEL} +"# + )] + #[case::cache_endpoint(" cache_params: {s3_endpoint_url: 'https://user:SENTINEL@s3.local'}")] + #[case::cache_raw(" cache_params: {s3_config: {secret: SENTINEL}}")] + #[case::gitlab_object(" global_gitlab_config: {access_token: SENTINEL}")] + #[case::s3_callback_object(" s3_callback_params: {s3_aws_secret_access_key: SENTINEL}")] + #[case::team_object(" default_team_settings: [{team_id: t, langfuse_secret: SENTINEL}]")] + #[case::fallbacks_raw(" default_fallbacks: [SENTINEL]")] + #[case::user_params_raw(" default_internal_user_params: {models: [SENTINEL]}")] + #[case::callback_entry(" callbacks: [SENTINEL]")] + #[case::success_callback_map(" success_callback: {name: SENTINEL}")] + #[case::failure_callback_one(" failure_callback: SENTINEL")] + #[case::audit_object(" s3_audit_callback_params: {secret: SENTINEL}")] + #[case::context_window_raw(" context_window_fallbacks: [{gpt: [SENTINEL]}]")] + #[case::mcp_raw(" mcp_aliases: {docs: SENTINEL}")] + #[case::injection_object(" prompt_injection_params: {key: SENTINEL}")] + #[case::unknown_setting(" future_setting: SENTINEL")] + fn debug_output_does_not_expose_settings_values(#[case] yaml: &str) { + let config = Config::from_yaml(&format!("litellm_settings:\n{yaml}")).unwrap(); + assert!(!format!("{config:?}").contains("SENTINEL")); + } + + #[rstest] + fn unknown_setting_stays_in_additional_fields() { + assert!( + settings(" future_setting: 1") + .additional_fields + .contains_key("future_setting") + ); + } + + #[rstest] + fn transport_keys_land_in_their_reader_groups() { + let settings = settings( + r#" ssl_verify: false + ssl_security_level: '1' + ssl_certificate: /etc/client.pem + force_ipv4: true + http2: true + aiohttp_trust_env: true"#, + ); + assert_eq!(settings.tls.ssl_verify, Some(Spelled::Value(false))); + assert_eq!(settings.tls.ssl_security_level.as_deref(), Some("1")); + assert_eq!( + settings.client_certificate.ssl_certificate.as_deref(), + Some("/etc/client.pem") + ); + assert_eq!(settings.socket.force_ipv4, Some(true)); + assert_eq!(settings.http2.http2, Some(true)); + assert_eq!(settings.aiohttp.aiohttp_trust_env, Some(true)); + assert!(settings.additional_fields.is_empty()); + } + + #[rstest] + #[case::tls(" ssl_verify: false", "ssl_verify")] + #[case::client_certificate(" ssl_certificate: /etc/client.pem", "ssl_certificate")] + #[case::socket(" force_ipv4: true", "force_ipv4")] + #[case::http2(" http2: true", "http2")] + #[case::aiohttp(" disable_aiohttp_transport: true", "disable_aiohttp_transport")] + #[case::caching(" cache: true", "cache")] + #[case::budget(" max_budget: 10", "max_budget")] + #[case::logging(" json_logs: true", "json_logs")] + #[case::fallbacks(" default_fallbacks: [gpt]", "default_fallbacks")] + #[case::mcp(" mcp_aliases: {docs: docs_server}", "mcp_aliases")] + fn grouped_settings_stay_out_of_additional_fields(#[case] yaml: &str, #[case] key: &str) { + assert!(!settings(yaml).additional_fields.contains_key(key)); + } + + #[rstest] + #[case::one("callbacks: custom_callbacks.logger", 1)] + #[case::many("callbacks: [prometheus, otel]", 2)] + fn accepts_python_callback_shorthand(#[case] setting: &str, #[case] expected_len: usize) { + let config = Config::from_yaml(&format!("litellm_settings:\n {setting}")).unwrap(); + assert_eq!( + config + .litellm_settings + .logging + .callbacks + .as_ref() + .unwrap() + .len(), + expected_len + ); + } +} diff --git a/litellm-rust/crates/config/src/litellm/transport.rs b/litellm-rust/crates/config/src/litellm/transport.rs new file mode 100644 index 00000000000..bd1a055b97f --- /dev/null +++ b/litellm-rust/crates/config/src/litellm/transport.rs @@ -0,0 +1,34 @@ +use crate::Spelled; + +/// Read by the SSL context builder, `get_ssl_configuration`. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct TlsSettings { + pub ssl_verify: Option>, + pub ssl_security_level: Option, + pub ssl_ecdh_curve: Option, +} + +/// Read by the sync and async HTTP clients, which send it as the client certificate. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ClientCertificateSettings { + pub ssl_certificate: Option, +} + +/// Read by the httpx and aiohttp transports when they bind the local socket. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SocketSettings { + pub force_ipv4: Option, +} + +/// Read by the httpx client. +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Http2Settings { + pub http2: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AiohttpSettings { + pub aiohttp_trust_env: Option, + pub disable_aiohttp_trust_env: Option, + pub disable_aiohttp_transport: Option, +} diff --git a/litellm-rust/crates/config/src/mcp.rs b/litellm-rust/crates/config/src/mcp.rs index cbe992f9956..c2ecb7824fc 100644 --- a/litellm-rust/crates/config/src/mcp.rs +++ b/litellm-rust/crates/config/src/mcp.rs @@ -1,11 +1,11 @@ -use std::{collections::BTreeMap, fmt}; +use std::collections::BTreeMap; use litellm_auth_types::SecretValue; use serde::Deserialize; use crate::Object; -#[derive(Clone, Default, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize)] #[serde(default)] pub struct McpServer { pub server_id: Option, @@ -14,8 +14,8 @@ pub struct McpServer { pub mcp_info: Object, pub transport: McpTransport, pub url: Option, - pub command: Option, - pub args: Box<[String]>, + pub command: Option, + pub args: Box<[SecretValue]>, pub env: BTreeMap, pub auth_type: Option, #[serde(alias = "auth_value")] @@ -29,18 +29,6 @@ pub struct McpServer { pub unsupported: Object, } -impl fmt::Debug for McpServer { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("McpServer") - .field("transport", &self.transport) - .field("auth_type", &self.auth_type) - .field("timeout", &self.timeout) - .field("max_concurrent_requests", &self.max_concurrent_requests) - .finish_non_exhaustive() - } -} - #[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, strum::IntoStaticStr)] #[serde(rename_all = "snake_case")] pub enum McpTransport { @@ -81,3 +69,75 @@ pub enum McpAuth { #[strum(serialize = "oauth_delegate")] OauthDelegate, } + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use crate::{Config, McpAuth, McpTransport}; + + #[rstest] + #[case::http(McpTransport::Http, "http")] + #[case::sse(McpTransport::Sse, "sse")] + #[case::stdio(McpTransport::Stdio, "stdio")] + fn mcp_transport_as_str_matches_the_serde_spelling( + #[case] transport: McpTransport, + #[case] name: &str, + ) { + assert_eq!(<&'static str>::from(transport), name); + assert_eq!( + serde_json::from_value::(json!(name)).unwrap(), + transport + ); + } + + #[rstest] + #[case::none(McpAuth::None, "none")] + #[case::api_key(McpAuth::ApiKey, "api_key")] + #[case::bearer_token(McpAuth::BearerToken, "bearer_token")] + #[case::basic(McpAuth::Basic, "basic")] + #[case::authorization(McpAuth::Authorization, "authorization")] + #[case::token(McpAuth::Token, "token")] + #[case::oauth2(McpAuth::Oauth2, "oauth2")] + #[case::aws_sigv4(McpAuth::AwsSigv4, "aws_sigv4")] + #[case::oauth2_token_exchange(McpAuth::Oauth2TokenExchange, "oauth2_token_exchange")] + #[case::oauth2_id_jag(McpAuth::Oauth2IdJag, "oauth2_id_jag")] + #[case::true_passthrough(McpAuth::TruePassthrough, "true_passthrough")] + #[case::oauth_delegate(McpAuth::OauthDelegate, "oauth_delegate")] + fn mcp_auth_as_str_matches_the_serde_spelling(#[case] auth: McpAuth, #[case] name: &str) { + assert_eq!(<&'static str>::from(auth), name); + assert_eq!( + serde_json::from_value::(json!(name)).unwrap(), + auth + ); + } + + #[rstest] + fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() { + let config = Config::from_yaml("mcp_servers:\n docs:\n url: https://example.test/private-secret/mcp\n authentication_token: upstream-secret\n static_headers: {x-token: header-secret}\n env: {TOKEN: env-secret}\n args: [argument-secret]\n client_secret: oauth-secret\n allowed_tools: [search]\n").unwrap(); + let server = &config.mcp_servers["docs"]; + assert_eq!(server.allowed_tools.as_deref().unwrap(), ["search"]); + assert!(server.unsupported.contains_key("client_secret")); + let debug = format!("{config:?}"); + for secret in [ + "private-secret", + "upstream-secret", + "header-secret", + "env-secret", + "oauth-secret", + "argument-secret", + ] { + assert!(!debug.contains(secret)); + } + } + + #[rstest] + #[case::transport("transport: invalid")] + #[case::auth("auth_type: invalid")] + #[case::concurrency("max_concurrent_requests: -1")] + #[case::headers("static_headers: {x-token: [not, a, string]}")] + fn rejects_invalid_typed_mcp_settings(#[case] setting: &str) { + assert!(Config::from_yaml(&format!("mcp_servers:\n docs:\n {setting}\n")).is_err()); + } +} diff --git a/litellm-rust/crates/config/src/model.rs b/litellm-rust/crates/config/src/model.rs index c0f8516103d..8ec24f354f0 100644 --- a/litellm-rust/crates/config/src/model.rs +++ b/litellm-rust/crates/config/src/model.rs @@ -1,30 +1,187 @@ -use std::fmt; - use litellm_router_types::LitellmParams; use serde::Deserialize; -use crate::{AdditionalFields, Object}; +use crate::{Object, Raw, Spelled}; -#[derive(Clone, Deserialize)] +#[derive(Clone, Debug, Deserialize)] pub struct Model { pub model_name: String, pub litellm_params: LitellmParams, #[serde(default)] - pub model_info: Object, + pub model_info: ModelInfo, pub blocked: Option, + pub rpm: Option>, + pub tpm: Option>, #[serde(flatten)] - pub additional_fields: AdditionalFields, + pub additional_fields: Object, } -impl fmt::Debug for Model { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("Model") - .field("model_name", &self.model_name) - .field("litellm_params", &self.litellm_params) - .field("model_info", &self.model_info) - .field("blocked", &self.blocked) - .field("additional_fields", &self.additional_fields.keys()) - .finish() +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ModelInfo { + pub id: Option, + pub db_model: Option>, + pub updated_at: Option, + pub created_at: Option, + pub updated_by: Option, + pub created_by: Option, + pub base_model: Option, + pub pricing_provider: Option, + pub tier: Option, + pub team_id: Option, + pub team_public_model_name: Option, + pub member_auto_router: Option>, + pub blocked: Option>, + pub discoverable: Option>, + pub access_windows: Option>, + pub ptu_count: Option>, + pub cost_per_ptu_per_hour: Option>, + pub ptu_effective_from: Option, + pub ptu_effective_to: Option, + pub ptu_shares: Option, + pub allow_fail_open: Option>, + pub enable_tag_filtering: Option>, + pub internal_router_model: Option>, + pub max_embedding_batch_size: Option>, + pub input_cost_per_token: Option>, + pub output_cost_per_token: Option>, + pub input_cost_per_character: Option>, + pub output_cost_per_character: Option>, + pub cache_read_input_token_cost: Option>, + pub explicit_cache_read_input_token_cost: Option>, + pub cache_creation_input_token_cost: Option>, + pub cache_creation_input_token_cost_above_1hr: Option>, + pub tiered_pricing: Option>, + pub mode: Option, + pub access_groups: Option>, + pub supported_environments: Option>, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum ModelId { + Text(String), + Number(i64), +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AccessWindow { + pub start: Option, + pub end: Option, + pub timezone: Option, + pub team_ids: Option>, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Model, ModelId, Spelled}; + + fn model(entry: &str) -> Model { + let config = Config::from_yaml(&format!( + "model_list:\n - model_name: assistant\n litellm_params: {{model: openai/test}}\n{entry}" + )) + .unwrap(); + config.model_list[0].clone() + } + + #[rstest] + #[case::text(" model_info: {id: primary}", Some(ModelId::Text("primary".to_string())))] + #[case::number(" model_info: {id: 7}", Some(ModelId::Number(7)))] + #[case::null(" model_info: {id: null}", None)] + #[case::absent(" model_info: {mode: chat}", None)] + fn model_info_id_accepts_what_python_accepts(#[case] entry: &str, #[case] id: Option) { + assert_eq!(model(entry).model_info.id, id); + } + + #[rstest] + #[case::missing("")] + #[case::null(" model_info:")] + fn model_info_may_be_missing_or_null(#[case] entry: &str) { + let model = model(entry); + assert!(model.model_info.id.is_none()); + assert!(model.model_info.additional_fields.is_empty()); + } + + #[rstest] + fn model_info_fields_land_typed() { + let model = model( + r#" model_info: + mode: embedding + base_model: azure/test-base + team_id: team-1 + access_groups: [internal, beta] + db_model: "1" + blocked: true + ptu_count: 4 + cost_per_ptu_per_hour: 2 + input_cost_per_token: os.environ/INPUT_COST + tiered_pricing: [{range: [0, 1000]}] + access_windows: + - {start: "09:00", end: "17:00", timezone: UTC, team_ids: [team-1]} + supports_vision: true +"#, + ); + let info = &model.model_info; + assert_eq!(info.mode.as_deref(), Some("embedding")); + assert_eq!(info.base_model.as_deref(), Some("azure/test-base")); + assert_eq!(info.team_id.as_deref(), Some("team-1")); + assert_eq!( + info.access_groups.as_deref(), + Some(&["internal".to_string(), "beta".to_string()][..]) + ); + assert_eq!(info.db_model, Some(Spelled::Text("1".to_string()))); + assert_eq!(info.blocked, Some(Spelled::Value(true))); + assert_eq!(info.ptu_count, Some(Spelled::Value(4))); + assert_eq!(info.cost_per_ptu_per_hour, Some(Spelled::Value(2.0))); + assert_eq!( + info.input_cost_per_token, + Some(Spelled::Text("os.environ/INPUT_COST".to_string())) + ); + assert_eq!( + info.tiered_pricing.as_ref().map(|tiers| tiers.len()), + Some(1) + ); + let window = &info.access_windows.as_ref().unwrap()[0]; + assert_eq!(window.timezone.as_deref(), Some("UTC")); + assert_eq!(window.start.as_ref().unwrap().get().as_str(), Some("09:00")); + assert_eq!( + window.team_ids.as_deref(), + Some(&["team-1".to_string()][..]) + ); + assert!(info.additional_fields.contains_key("supports_vision")); + } + + #[rstest] + fn top_level_limits_and_unknown_keys_land_on_the_model() { + let model = model(" rpm: 100\n tpm: os.environ/TPM\n team: research\n"); + assert_eq!(model.rpm, Some(Spelled::Value(100))); + assert_eq!(model.tpm, Some(Spelled::Text("os.environ/TPM".to_string()))); + assert!(model.additional_fields.contains_key("team")); + assert!(!model.additional_fields.contains_key("rpm")); + } + + #[rstest] + #[case::ptu_shares( + " model_info: {ptu_shares: {team-1: sentinel-share-value}}", + "sentinel-share-value" + )] + #[case::created_at( + " model_info: {created_at: sentinel-created-at}", + "sentinel-created-at" + )] + #[case::tiered_pricing( + " model_info: {tiered_pricing: [{label: sentinel-tier-value}]}", + "sentinel-tier-value" + )] + #[case::window_start( + " model_info: {access_windows: [{start: sentinel-window-start, end: '17:00', timezone: UTC, team_ids: [t]}]}", + "sentinel-window-start" + )] + #[case::model_extension(" team_token: sentinel-model-extension", "sentinel-model-extension")] + fn opaque_model_info_values_stay_out_of_debug(#[case] entry: &str, #[case] sentinel: &str) { + assert!(!format!("{:?}", model(entry)).contains(sentinel)); } } diff --git a/litellm-rust/crates/config/src/policy.rs b/litellm-rust/crates/config/src/policy.rs new file mode 100644 index 00000000000..4c68e8d1dce --- /dev/null +++ b/litellm-rust/crates/config/src/policy.rs @@ -0,0 +1,144 @@ +use serde::Deserialize; + +use crate::{Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Policy { + pub guardrails: Option, + pub condition: Option, + pub pipeline: Option, + pub inherit: Option, + pub description: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum PolicyGuardrails { + Added(Box<[String]>), + Changes(PolicyGuardrailChanges), +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PolicyGuardrailChanges { + pub add: Option>, + pub remove: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PolicyCondition { + pub model: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PolicyPipeline { + pub mode: Option, + pub steps: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PolicyPipelineStep { + pub guardrail: Option, + pub on_fail: Option, + pub on_pass: Option, + pub on_error: Option, + pub pass_data: Option>, + pub modify_response_message: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PolicyAttachment { + pub policy: Option, + pub scope: Option, + pub teams: Option>, + pub keys: Option>, + pub models: Option>, + pub tags: Option>, + pub priority: Option>, + pub default: Option>, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, PolicyGuardrails, Spelled, Value}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + #[case::legacy_list("guardrails: [pii, toxicity]", &["pii", "toxicity"], &[])] + #[case::add_remove("guardrails: {add: [pii], remove: [toxicity]}", &["pii"], &["toxicity"])] + fn policy_guardrails_accept_both_python_spellings( + #[case] guardrails: &str, + #[case] added: &[&str], + #[case] removed: &[&str], + ) { + let config = parse(&format!("policies:\n safe:\n {guardrails}\n")); + let (add, remove) = match config.policies["safe"].guardrails.as_ref().unwrap() { + PolicyGuardrails::Added(add) => (add.to_vec(), Vec::new()), + PolicyGuardrails::Changes(changes) => ( + changes.add.as_deref().unwrap_or_default().to_vec(), + changes.remove.as_deref().unwrap_or_default().to_vec(), + ), + }; + + assert_eq!(add, added); + assert_eq!(remove, removed); + } + + #[rstest] + fn policies_and_attachments_parse_pipeline_condition_and_scope() { + let config = parse( + r#" +policies: + safe: + inherit: base + condition: + model: [gpt-a, gpt-b] + pipeline: + mode: pre_call + steps: + - guardrail: pii + on_fail: next + pass_data: true +policy_attachments: + - policy: safe + teams: [research] + priority: 10 + default: true +"#, + ); + let policy = &config.policies["safe"]; + let step = &policy.pipeline.as_ref().unwrap().steps.as_ref().unwrap()[0]; + let attachment = &config.policy_attachments[0]; + + assert_eq!(policy.inherit.as_deref(), Some("base")); + assert!(matches!( + policy.condition.as_ref().unwrap().model.as_ref().unwrap().get(), + Value::Sequence(models) if models.len() == 2 + )); + assert_eq!(step.guardrail.as_deref(), Some("pii")); + assert_eq!(step.on_fail.as_deref(), Some("next")); + assert_eq!(step.pass_data, Some(Spelled::Value(true))); + assert_eq!(attachment.policy.as_deref(), Some("safe")); + assert_eq!( + attachment.teams.as_deref(), + Some(&["research".to_owned()][..]) + ); + assert_eq!(attachment.priority, Some(Spelled::Value(10))); + assert_eq!(attachment.default, Some(Spelled::Value(true))); + } + + #[rstest] + #[case::policy_model( + r#" +policies: + safe: {condition: {model: SENTINEL}} +"# + )] + fn debug_never_prints_policy_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/prompt.rs b/litellm-rust/crates/config/src/prompt.rs new file mode 100644 index 00000000000..5a298a9efda --- /dev/null +++ b/litellm-rust/crates/config/src/prompt.rs @@ -0,0 +1,99 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Prompt { + pub prompt_id: Option, + pub litellm_params: Option, + pub prompt_info: Option, + pub version: Option>, + pub environment: Option, + pub created_by: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PromptParams { + pub prompt_id: Option, + pub prompt_integration: Option, + pub api_base: Option, + pub api_key: Option, + pub provider_specific_query_params: Option, + pub ignore_prompt_manager_model: Option>, + pub ignore_prompt_manager_optional_params: Option>, + pub dotprompt_content: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct PromptInfo { + pub prompt_type: Option, + pub environment: Option, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Spelled}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn prompt_entries_keep_typed_params_and_info() { + let config = parse( + r#" +prompts: + - prompt_id: greeting + version: 2 + litellm_params: + prompt_integration: dotprompt + ignore_prompt_manager_model: true + prompt_extension: x + prompt_info: + prompt_type: config +"#, + ); + let prompt = &config.prompts[0]; + let params = prompt.litellm_params.as_ref().unwrap(); + + assert_eq!(prompt.prompt_id.as_deref(), Some("greeting")); + assert_eq!(prompt.version, Some(Spelled::Value(2))); + assert_eq!(params.prompt_integration.as_deref(), Some("dotprompt")); + assert_eq!( + params.ignore_prompt_manager_model, + Some(Spelled::Value(true)) + ); + assert!(params.additional_fields.contains_key("prompt_extension")); + assert_eq!( + prompt.prompt_info.as_ref().unwrap().prompt_type.as_deref(), + Some("config") + ); + } + + #[rstest] + #[case::prompt_key( + r#" +prompts: + - litellm_params: {api_key: SENTINEL} +"# + )] + #[case::prompt_query_params( + r#" +prompts: + - litellm_params: {provider_specific_query_params: {token: SENTINEL}} +"# + )] + #[case::prompt_extra( + r#" +prompts: + - litellm_params: {vendor_secret: SENTINEL} +"# + )] + fn debug_never_prints_prompt_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/provider.rs b/litellm-rust/crates/config/src/provider.rs new file mode 100644 index 00000000000..e4107bebddf --- /dev/null +++ b/litellm-rust/crates/config/src/provider.rs @@ -0,0 +1,118 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Raw}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AssistantSettings { + pub custom_llm_provider: Option, + pub litellm_params: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ProviderSettings { + pub custom_llm_provider: Option, + pub api_key: Option, + pub api_base: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct VertexConfig { + pub vertex_project: Option, + pub vertex_location: Option, + pub vertex_credentials: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct Worker { + pub worker_id: Option, + pub name: Option, + pub url: Option, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::Config; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + #[case::files("files_settings")] + #[case::finetune("finetune_settings")] + fn provider_settings_keep_the_provider_and_unknown_keys(#[case] section: &str) { + let config = parse(&format!( + "{section}:\n - custom_llm_provider: azure\n api_version: '2024-01-01'\n" + )); + let settings = config + .files_settings + .first() + .or(config.finetune_settings.first()) + .unwrap(); + + assert_eq!(settings.custom_llm_provider.as_deref(), Some("azure")); + assert!(settings.additional_fields.contains_key("api_version")); + } + + #[rstest] + fn single_provider_sections_land_in_typed_fields() { + let config = parse( + r#" +assistant_settings: + custom_llm_provider: openai + litellm_params: + api_key: os.environ/OPENAI_API_KEY +default_vertex_config: + vertex_project: proj + vertex_location: us-central1 +worker_registry: + - worker_id: eu + name: Europe + url: https://eu.example.test +"#, + ); + let assistant = config.assistant_settings.as_ref().unwrap(); + let vertex = config.default_vertex_config.as_ref().unwrap(); + let worker = &config.worker_registry[0]; + + assert_eq!(assistant.custom_llm_provider.as_deref(), Some("openai")); + assert!( + assistant + .litellm_params + .as_ref() + .unwrap() + .contains_key("api_key") + ); + assert_eq!(vertex.vertex_project.as_deref(), Some("proj")); + assert_eq!(vertex.vertex_location.as_deref(), Some("us-central1")); + assert_eq!(worker.worker_id.as_deref(), Some("eu")); + assert_eq!(worker.name.as_deref(), Some("Europe")); + assert_eq!(worker.url.as_deref(), Some("https://eu.example.test")); + } + + #[rstest] + #[case::vertex_credentials("default_vertex_config: {vertex_credentials: SENTINEL}")] + #[case::vertex_credentials_map( + "default_vertex_config: {vertex_credentials: {private_key: SENTINEL}}" + )] + #[case::assistant_params("assistant_settings: {litellm_params: {api_key: SENTINEL}}")] + #[case::files_key( + r#" +files_settings: + - api_key: SENTINEL +"# + )] + #[case::finetune_extra( + r#" +finetune_settings: + - vendor_secret: SENTINEL +"# + )] + fn debug_never_prints_provider_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/router.rs b/litellm-rust/crates/config/src/router.rs new file mode 100644 index 00000000000..cf1c1365211 --- /dev/null +++ b/litellm-rust/crates/config/src/router.rs @@ -0,0 +1,191 @@ +use std::collections::BTreeMap; + +use litellm_auth_types::SecretValue; + +use crate::{Object, Raw, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct RouterSettings { + pub redis_url: Option, + pub redis_host: Option, + pub redis_port: Option>, + pub redis_password: Option, + pub redis_db: Option>, + pub cache_responses: Option>, + pub cache_kwargs: Option, + pub caching_groups: Option, + pub client_ttl: Option>, + pub polling_interval: Option>, + pub default_priority: Option, + pub num_retries: Option, + pub max_fallbacks: Option>, + pub timeout: Option, + pub stream_timeout: Option>, + pub default_litellm_params: Option, + pub default_max_parallel_requests: Option>, + pub set_verbose: Option>, + pub debug_level: Option, + pub default_fallbacks: Option>, + pub fallbacks: Option>, + pub context_window_fallbacks: Option>, + pub content_policy_fallbacks: Option>, + pub model_group_alias: Option, + pub enable_pre_call_checks: Option>, + pub enable_tag_filtering: Option, + pub tag_filtering_match_any: Option>, + pub tag_routing_prefix: Option, + pub plugins: Option>, + pub retry_after: Option, + pub retry_policy: Option, + pub model_group_retry_policy: Option, + pub allowed_fails: Option, + pub allowed_fails_policy: Option, + pub cooldown_time: Option, + pub disable_cooldowns: Option>, + pub fallback_on_unsupported_params: Option>, + pub routing_strategy: Option, + pub optional_pre_call_checks: Option>, + pub routing_strategy_args: Option, + pub routing_groups: Option>, + pub provider_budget_config: Option>, + pub alerting_config: Option, + pub deployment_affinity_ttl_seconds: Option>, + pub model_group_affinity_config: Option, + pub enable_health_check_routing: Option>, + pub health_check_staleness_threshold: Option>, + pub health_check_ignore_transient_errors: Option>, + pub background_health_check_model_groups: Option>, + pub enable_weighted_failover: Option>, + pub enable_mid_stream_fallback_continuation: Option>, + pub guardrail_list: Option>, + pub max_retries: Option, + pub weights: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[macro_rules_attribute::apply(crate::config_section)] +#[serde(rename_all = "PascalCase")] +pub struct AllowedFailsPolicy { + pub bad_request_error_allowed_fails: Option>, + pub authentication_error_allowed_fails: Option>, + pub timeout_error_allowed_fails: Option>, + pub rate_limit_error_allowed_fails: Option>, + pub content_policy_violation_error_allowed_fails: Option>, + pub internal_server_error_allowed_fails: Option>, + pub service_unavailable_error_allowed_fails: Option>, + pub bad_gateway_error_allowed_fails: Option>, + pub not_found_error_allowed_fails: Option>, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct ProviderBudget { + pub budget_limit: Option>, + pub time_period: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct AlertingConfig { + pub webhook_url: Option, + pub alerting_threshold: Option>, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, RouterSettings, Spelled}; + + fn router(yaml: &str) -> RouterSettings { + Config::from_yaml(&format!("router_settings:\n{yaml}")) + .unwrap() + .router_settings + } + + #[rstest] + #[case::int(" redis_port: 6379", |s: &RouterSettings| s.redis_port == Some(Spelled::Value(6379)))] + #[case::int_env( + " client_ttl: os.environ/CLIENT_TTL", + |s: &RouterSettings| s.client_ttl == Some(Spelled::Text("os.environ/CLIENT_TTL".to_string())) +)] + #[case::float_from_int(" stream_timeout: 30", |s: &RouterSettings| s.stream_timeout == Some(Spelled::Value(30.0)))] + #[case::float( + " health_check_staleness_threshold: 2.5", + |s: &RouterSettings| s.health_check_staleness_threshold == Some(Spelled::Value(2.5)) +)] + #[case::bool(" enable_weighted_failover: true", |s: &RouterSettings| s.enable_weighted_failover == Some(Spelled::Value(true)))] + #[case::bool_word( + " enable_mid_stream_fallback_continuation: \"true\"", + |s: &RouterSettings| s.enable_mid_stream_fallback_continuation == Some(Spelled::Text("true".to_string())) +)] + #[case::string(" redis_host: cache.internal", |s: &RouterSettings| s.redis_host.as_deref() == Some("cache.internal"))] + #[case::string_list( + " default_fallbacks: [gpt-small, gpt-large]", + |s: &RouterSettings| s.default_fallbacks.as_deref() == Some(&["gpt-small".to_string(), "gpt-large".to_string()][..]) +)] + #[case::mapping_list( + " content_policy_fallbacks: [{gpt: [claude]}]", + |s: &RouterSettings| s.content_policy_fallbacks.as_ref().is_some_and(|items| items[0].contains_key("gpt")) +)] + #[case::mapping( + " default_litellm_params: {temperature: 0}", + |s: &RouterSettings| s.default_litellm_params.as_ref().is_some_and(|params| params.contains_key("temperature")) +)] + #[case::non_integer_priority(" default_priority: 1.5", |s: &RouterSettings| s.default_priority.is_some())] + #[case::bool_priority(" default_priority: true", |s: &RouterSettings| s.default_priority.is_some())] + #[case::caching_groups( + " caching_groups: [[gpt, [gpt-a, gpt-b]]]", + |s: &RouterSettings| s.caching_groups.as_ref().is_some_and(|groups| groups.get().is_sequence()) +)] + #[case::allowed_fails_policy( + " allowed_fails_policy: {RateLimitErrorAllowedFails: \"3\", BadGatewayErrorAllowedFails: 2, FutureAllowedFails: 1}", + |s: &RouterSettings| s.allowed_fails_policy.as_ref().is_some_and(|policy| { + policy.rate_limit_error_allowed_fails == Some(Spelled::Text("3".to_string())) + && policy.bad_gateway_error_allowed_fails == Some(Spelled::Value(2)) + }) +)] + #[case::provider_budget_config( + " provider_budget_config: {openai: {budget_limit: 100, time_period: 1d, rpm_limit: 5}}", + |s: &RouterSettings| s.provider_budget_config.as_ref().is_some_and(|budgets| { + let openai = &budgets["openai"]; + openai.budget_limit == Some(Spelled::Value(100.0)) + && openai.time_period.as_deref() == Some("1d") + }) +)] + #[case::alerting_config( + " alerting_config: {webhook_url: https://hooks.example/a, alerting_threshold: 10, channel: ops}", + |s: &RouterSettings| s.alerting_config.as_ref().is_some_and(|alerting| { + alerting.webhook_url.as_ref().map(|url| url.expose()) == Some("https://hooks.example/a") + && alerting.alerting_threshold == Some(Spelled::Value(10.0)) + }) +)] + fn parses_python_router_spellings( + #[case] yaml: &str, + #[case] check: fn(&RouterSettings) -> bool, + ) { + assert!(check(&router(yaml))); + } + + #[rstest] + #[case::unknown("future_router_setting")] + #[case::moved_to_general_settings("health_check_interval")] + #[case::managed_outside_config("model_list")] + fn keeps_keys_outside_the_router_signature(#[case] key: &str) { + let settings = router(&format!(" {key}: 60")); + assert!(settings.additional_fields.contains_key(key)); + } + + #[rstest] + #[case::redis_password(" redis_password: router-sentinel")] + #[case::redis_url(" redis_url: redis://user:router-sentinel@cache:6379")] + #[case::webhook_url(" alerting_config: {webhook_url: https://hooks.example/router-sentinel}")] + #[case::cache_kwargs(" cache_kwargs: {password: router-sentinel}")] + #[case::default_priority(" default_priority: router-sentinel")] + #[case::caching_groups(" caching_groups: [[router-sentinel, [a]]]")] + #[case::guardrail_list(" guardrail_list: [{api_key: router-sentinel}]")] + #[case::nested_extension(" allowed_fails_policy: {FutureAllowedFails: router-sentinel}")] + fn debug_output_does_not_expose_router_values(#[case] yaml: &str) { + let config = Config::from_yaml(&format!("router_settings:\n{yaml}")).unwrap(); + assert!(!format!("{config:?}").contains("router-sentinel")); + } +} diff --git a/litellm-rust/crates/config/src/settings.rs b/litellm-rust/crates/config/src/settings.rs deleted file mode 100644 index 1094de1638e..00000000000 --- a/litellm-rust/crates/config/src/settings.rs +++ /dev/null @@ -1,268 +0,0 @@ -use std::fmt; - -use litellm_auth_types::SecretValue; -use serde::Deserialize; - -use crate::{AdditionalFields, Object, Spelled, Value, value::one_or_many}; - -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum TracingStoreKind { - Clickhouse, -} - -#[derive(Clone, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ClickHouseStoreSettings { - #[serde(rename = "type")] - pub kind: TracingStoreKind, - pub url: Option, - pub database: Option, - pub retention_days: Option>, -} - -impl fmt::Debug for ClickHouseStoreSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("ClickHouseStoreSettings") - .field("kind", &self.kind) - .field("database", &self.database) - .field("retention_days", &self.retention_days) - .finish() - } -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(untagged)] -pub enum TracingStoreSettings { - ClickHouse(ClickHouseStoreSettings), -} - -#[derive(Clone, Default, Debug, Deserialize)] -#[serde(default)] -pub struct TracingSettings { - pub store: Option, - #[serde(flatten)] - pub additional_fields: AdditionalFields, -} - -#[derive(Clone, Deserialize)] -#[serde(default)] -pub struct GeneralSettings { - pub completion_model: Option, - pub max_in_flight_requests_per_worker: Option, - pub max_queued_requests_per_worker: Option, - pub admission_queue_timeout_seconds: f64, - pub master_key: Option, - pub database_url: Option, - pub tracing: Option, - pub database_connection_pool_limit: Option, - pub database_connection_timeout: Option, - pub database_connect_timeout: Option, - pub database_socket_timeout: Option, - pub database_max_idle_connection_lifetime: Option, - pub max_parallel_requests: Option, - pub global_max_parallel_requests: Option, - pub max_request_size_mb: Option, - pub max_response_size_mb: Option, - pub proxy_config_reload_interval_seconds: u64, - pub background_health_checks: Option, - pub health_check_interval: u64, - pub health_check_concurrency: Option, - pub store_model_in_db: Option, - pub forward_client_headers_to_llm_api: Option, - pub cancel_on_disconnect: Option, - pub infer_model_from_keys: Option, - pub enable_public_model_hub: bool, - pub dangerously_permit_weak_or_unset_master_key: Option, - pub plugins: Option>, - pub coordination_redis: Option, - pub mcp_allowed_hosts: Option>, - pub mcp_allowed_origins: Box<[String]>, - #[serde(flatten)] - pub additional_fields: AdditionalFields, -} - -impl Default for GeneralSettings { - fn default() -> Self { - Self { - completion_model: None, - max_in_flight_requests_per_worker: None, - max_queued_requests_per_worker: None, - admission_queue_timeout_seconds: 1.0, - master_key: None, - database_url: None, - tracing: None, - database_connection_pool_limit: Some(10), - database_connection_timeout: Some(60.0), - database_connect_timeout: None, - database_socket_timeout: None, - database_max_idle_connection_lifetime: Some(60.0), - max_parallel_requests: None, - global_max_parallel_requests: None, - max_request_size_mb: None, - max_response_size_mb: None, - proxy_config_reload_interval_seconds: 30, - background_health_checks: None, - health_check_interval: 300, - health_check_concurrency: None, - store_model_in_db: None, - forward_client_headers_to_llm_api: None, - cancel_on_disconnect: None, - infer_model_from_keys: None, - enable_public_model_hub: false, - dangerously_permit_weak_or_unset_master_key: None, - plugins: None, - coordination_redis: None, - mcp_allowed_hosts: None, - mcp_allowed_origins: Box::default(), - additional_fields: AdditionalFields::new(), - } - } -} - -impl fmt::Debug for GeneralSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("GeneralSettings") - .field("completion_model", &self.completion_model) - .field( - "max_in_flight_requests_per_worker", - &self.max_in_flight_requests_per_worker, - ) - .field( - "max_queued_requests_per_worker", - &self.max_queued_requests_per_worker, - ) - .field( - "admission_queue_timeout_seconds", - &self.admission_queue_timeout_seconds, - ) - .field("master_key", &self.master_key) - .field("database_url", &self.database_url) - .field("tracing", &self.tracing) - .field("store_model_in_db", &self.store_model_in_db) - .field("additional_fields", &self.additional_fields.keys()) - .finish_non_exhaustive() - } -} - -#[derive(Clone, Default, Deserialize)] -#[serde(default)] -pub struct RouterSettings { - pub routing_strategy: Option, - pub routing_strategy_args: Option, - pub routing_groups: Option>, - pub retry_policy: Option, - pub model_group_retry_policy: Option, - pub model_group_affinity_config: Option, - pub allowed_fails: Option, - pub cooldown_time: Option, - pub num_retries: Option, - pub timeout: Option, - pub max_retries: Option, - pub retry_after: Option, - pub fallbacks: Option>, - pub context_window_fallbacks: Option>, - pub model_group_alias: Option, - pub enable_tag_filtering: Option, - pub weights: Option, - pub tag_routing_prefix: Option, - pub optional_pre_call_checks: Option>, - #[serde(flatten)] - pub additional_fields: AdditionalFields, -} - -impl fmt::Debug for RouterSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("RouterSettings") - .field("routing_strategy", &self.routing_strategy) - .field("routing_strategy_args", &self.routing_strategy_args) - .field("routing_groups", &self.routing_groups) - .field("retry_policy", &self.retry_policy) - .field("model_group_retry_policy", &self.model_group_retry_policy) - .field( - "model_group_affinity_config", - &self.model_group_affinity_config, - ) - .field("allowed_fails", &self.allowed_fails) - .field("cooldown_time", &self.cooldown_time) - .field("num_retries", &self.num_retries) - .field("timeout", &self.timeout) - .field("max_retries", &self.max_retries) - .field("retry_after", &self.retry_after) - .field("fallbacks", &self.fallbacks) - .field("context_window_fallbacks", &self.context_window_fallbacks) - .field("model_group_alias", &self.model_group_alias) - .field("enable_tag_filtering", &self.enable_tag_filtering) - .field("weights", &self.weights) - .field("tag_routing_prefix", &self.tag_routing_prefix) - .field("optional_pre_call_checks", &self.optional_pre_call_checks) - .field("additional_fields", &self.additional_fields.keys()) - .finish() - } -} - -#[derive(Clone, Default, Deserialize)] -#[serde(default)] -pub struct LiteLlmSettings { - pub ssl_verify: Option>, - pub ssl_certificate: Option, - pub ssl_security_level: Option, - pub ssl_ecdh_curve: Option, - pub force_ipv4: Option, - pub http2: Option, - pub aiohttp_trust_env: Option, - pub disable_aiohttp_trust_env: Option, - pub disable_aiohttp_transport: Option, - pub drop_params: Option>, - pub request_timeout: Option>, - pub num_retries: Option, - pub cache: Option, - pub cache_params: Option, - #[serde(deserialize_with = "one_or_many")] - pub callbacks: Option>, - #[serde(deserialize_with = "one_or_many")] - pub success_callback: Option>, - #[serde(deserialize_with = "one_or_many")] - pub failure_callback: Option>, - pub json_logs: Option, - pub set_verbose: Option, - #[serde(flatten)] - pub additional_fields: AdditionalFields, -} - -impl fmt::Debug for LiteLlmSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LiteLlmSettings") - .field("drop_params", &self.drop_params) - .field("request_timeout", &self.request_timeout) - .field("num_retries", &self.num_retries) - .field("cache", &self.cache) - .field("cache_params", &self.cache_params) - .field( - "callbacks", - &self.callbacks.as_ref().map(|callbacks| callbacks.len()), - ) - .field( - "success_callback", - &self - .success_callback - .as_ref() - .map(|callbacks| callbacks.len()), - ) - .field( - "failure_callback", - &self - .failure_callback - .as_ref() - .map(|callbacks| callbacks.len()), - ) - .field("json_logs", &self.json_logs) - .field("set_verbose", &self.set_verbose) - .field("additional_fields", &self.additional_fields.keys()) - .finish() - } -} diff --git a/litellm-rust/crates/config/src/tool.rs b/litellm-rust/crates/config/src/tool.rs new file mode 100644 index 00000000000..bc2e93819b9 --- /dev/null +++ b/litellm-rust/crates/config/src/tool.rs @@ -0,0 +1,128 @@ +use litellm_auth_types::SecretValue; + +use crate::{Object, Spelled}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SearchTool { + pub search_tool_name: Option, + pub litellm_params: Option, + pub search_tool_info: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SearchToolParams { + pub search_provider: Option, + pub api_key: Option, + pub api_base: Option, + pub timeout: Option>, + pub max_retries: Option>, + pub input_cost_per_query: Option>, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SearchToolInfo { + pub description: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SandboxTool { + pub sandbox_tool_name: Option, + pub litellm_params: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct SandboxToolParams { + pub sandbox_provider: Option, + pub api_key: Option, + pub api_base: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{Config, Spelled}; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn search_tools_keep_typed_params_and_info() { + let config = parse( + r#" +search_tools: + - search_tool_name: web + litellm_params: + search_provider: perplexity + max_retries: 3 + timeout: 2.5 + region: eu + search_tool_info: + description: web search +"#, + ); + let tool = &config.search_tools[0]; + let params = tool.litellm_params.as_ref().unwrap(); + + assert_eq!(tool.search_tool_name.as_deref(), Some("web")); + assert_eq!(params.search_provider.as_deref(), Some("perplexity")); + assert_eq!(params.max_retries, Some(Spelled::Value(3))); + assert_eq!(params.timeout, Some(Spelled::Value(2.5))); + assert!(params.additional_fields.contains_key("region")); + assert_eq!( + tool.search_tool_info + .as_ref() + .unwrap() + .description + .as_deref(), + Some("web search") + ); + } + + #[rstest] + fn sandbox_tools_keep_typed_params() { + let config = parse( + "sandbox_tools:\n - sandbox_tool_name: e2b\n litellm_params: {sandbox_provider: e2b, template: base}\n", + ); + let tool = &config.sandbox_tools[0]; + let params = tool.litellm_params.as_ref().unwrap(); + + assert_eq!(tool.sandbox_tool_name.as_deref(), Some("e2b")); + assert_eq!(params.sandbox_provider.as_deref(), Some("e2b")); + assert!(params.additional_fields.contains_key("template")); + } + + #[rstest] + #[case::search_key( + r#" +search_tools: + - litellm_params: {api_key: SENTINEL} +"# + )] + #[case::search_extra( + r#" +search_tools: + - litellm_params: {vendor_secret: SENTINEL} +"# + )] + #[case::sandbox_key( + r#" +sandbox_tools: + - litellm_params: {api_key: SENTINEL} +"# + )] + #[case::sandbox_extra( + r#" +sandbox_tools: + - litellm_params: {vendor_secret: SENTINEL} +"# + )] + fn debug_never_prints_tool_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/value.rs b/litellm-rust/crates/config/src/value.rs index 5becdc87b70..d40b8730413 100644 --- a/litellm-rust/crates/config/src/value.rs +++ b/litellm-rust/crates/config/src/value.rs @@ -1,28 +1,13 @@ use std::{collections::BTreeMap, fmt, ops::Deref}; -use serde::{Deserialize, Deserializer}; +use serde::Deserialize; pub type Value = serde_yaml_ng::Value; -pub type AdditionalFields = BTreeMap; #[derive(Clone, Default, Deserialize)] #[serde(transparent)] pub struct Object(BTreeMap); -impl Object { - pub fn get(&self, key: &str) -> Option<&Value> { - self.0.get(key) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn len(&self) -> usize { - self.0.len() - } -} - impl Deref for Object { type Target = BTreeMap; @@ -40,14 +25,27 @@ impl fmt::Debug for Object { } } -/// Python's callback shorthand: one entry or a list of them, kept as the values written. -pub(crate) fn one_or_many<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result>, D::Error> { - Ok( - Option::::deserialize(deserializer)?.map(|value| match value { - Value::Sequence(values) => values, - one => vec![one], - }), - ) +#[derive(Clone, Deserialize)] +#[serde(transparent)] +pub struct Raw(Value); + +impl Raw { + pub fn get(&self) -> &Value { + &self.0 + } +} + +impl fmt::Debug for Raw { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let kind = match &self.0 { + Value::Null => "null", + Value::Bool(_) => "bool", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Sequence(_) => "sequence", + Value::Mapping(_) => "mapping", + Value::Tagged(_) => "tagged", + }; + formatter.debug_tuple("Raw").field(&kind).finish() + } } diff --git a/litellm-rust/crates/config/src/vector_store.rs b/litellm-rust/crates/config/src/vector_store.rs new file mode 100644 index 00000000000..067d7502aa6 --- /dev/null +++ b/litellm-rust/crates/config/src/vector_store.rs @@ -0,0 +1,63 @@ +use crate::{Object, Raw}; + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct VectorStore { + pub vector_store_name: Option, + pub litellm_params: Option, +} + +#[macro_rules_attribute::apply(crate::config_section)] +pub struct VectorStoreParams { + pub vector_store_id: Option, + pub custom_llm_provider: Option, + pub vector_store_description: Option, + pub vector_store_metadata: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::Config; + + fn parse(yaml: &str) -> Config { + Config::from_yaml(yaml).unwrap() + } + + #[rstest] + fn vector_store_entries_keep_typed_params_and_unknown_keys() { + let config = parse( + r#" +vector_store_registry: + - vector_store_name: kb + litellm_params: {vector_store_id: vs_123, custom_llm_provider: openai, region: eu} +"#, + ); + let store = &config.vector_store_registry[0]; + let params = store.litellm_params.as_ref().unwrap(); + + assert_eq!(store.vector_store_name.as_deref(), Some("kb")); + assert_eq!(params.vector_store_id.as_deref(), Some("vs_123")); + assert_eq!(params.custom_llm_provider.as_deref(), Some("openai")); + assert!(params.additional_fields.contains_key("region")); + } + + #[rstest] + #[case::vector_store_metadata( + r#" +vector_store_registry: + - litellm_params: {vector_store_metadata: SENTINEL} +"# + )] + #[case::vector_store_extra( + r#" +vector_store_registry: + - litellm_params: {api_key: SENTINEL} +"# + )] + fn debug_never_prints_vector_store_values(#[case] yaml: &str) { + assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL")); + } +} diff --git a/litellm-rust/crates/config/src/yaml.rs b/litellm-rust/crates/config/src/yaml.rs new file mode 100644 index 00000000000..2de71e00e06 --- /dev/null +++ b/litellm-rust/crates/config/src/yaml.rs @@ -0,0 +1,177 @@ +use std::fmt; + +use serde::{ + Deserialize, Deserializer, + de::{MapAccess, SeqAccess, Visitor}, +}; +use serde_yaml_ng::{Mapping, Value}; + +pub fn mapping(text: &str) -> Result { + serde_yaml_ng::from_value(serde_yaml_ng::from_str::(text)?.0) +} + +struct LastWins(Value); + +impl<'de> Deserialize<'de> for LastWins { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(LastWinsVisitor).map(Self) + } +} + +// PyYAML safe_load keeps the later value of a duplicate key, serde_yaml_ng rejects it +struct LastWinsVisitor; + +impl<'de> Visitor<'de> for LastWinsVisitor { + type Value = Value; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("any YAML value") + } + + fn visit_bool(self, value: bool) -> Result { + Ok(Value::Bool(value)) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(Value::Number(value.into())) + } + + fn visit_u64(self, value: u64) -> Result { + Ok(Value::Number(value.into())) + } + + fn visit_f64(self, value: f64) -> Result { + Ok(Value::Number(value.into())) + } + + fn visit_str(self, value: &str) -> Result { + Ok(Value::String(value.to_owned())) + } + + fn visit_string(self, value: String) -> Result { + Ok(Value::String(value)) + } + + fn visit_unit(self) -> Result { + Ok(Value::Null) + } + + fn visit_none(self) -> Result { + Ok(Value::Null) + } + + fn visit_some>(self, deserializer: D) -> Result { + LastWins::deserialize(deserializer).map(|value| value.0) + } + + fn visit_seq>(self, mut items: A) -> Result { + let mut sequence = Vec::new(); + while let Some(LastWins(item)) = items.next_element()? { + sequence.push(item); + } + Ok(Value::Sequence(sequence)) + } + + fn visit_map>(self, mut entries: A) -> Result { + let mut mapping = Mapping::new(); + while let Some((LastWins(key), LastWins(value))) = entries.next_entry()? { + mapping.insert(key, value); + } + Ok(Value::Mapping(mapping)) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn yaml(text: &str) -> Mapping { + serde_yaml_ng::from_str(text).unwrap() + } + + #[rstest] + #[case::top_level( + r#" +a: 1 +b: 2 +a: 3 +"#, + "{a: 3, b: 2}" + )] + #[case::sequence_value( + r#" +cb: [prometheus] +cb: [s3] +"#, + "cb: [s3]" + )] + #[case::nested("outer: {x: [1], y: 2, x: [3]}", "outer: {x: [3], y: 2}")] + #[case::deeply_nested("a: {b: {c: 1, c: 2}}", "a: {b: {c: 2}}")] + #[case::inside_sequence("list: [{k: old, k: new}]", "list: [{k: new}]")] + #[case::mapping_replaced_by_scalar( + r#" +a: {b: 1} +a: 2 +"#, + "a: 2" + )] + #[case::value_replaced_by_null( + r#" +a: 1 +a: +"#, + "a: null" + )] + #[case::three_occurrences( + r#" +a: 1 +a: 2 +a: 3 +"#, + "a: 3" + )] + #[case::non_string_keys( + r#" +1: a +true: b +1: c +"#, + "{1: c, true: b}" + )] + #[case::scalars_without_duplicates( + r#" +i: -1 +u: 18446744073709551615 +f: 1.5 +b: false +n: ~ +s: text +"#, + "{i: -1, u: 18446744073709551615, f: 1.5, b: false, n: ~, s: text}" + )] + #[case::empty_document("", "{}")] + fn later_duplicate_keys_override_earlier_ones(#[case] text: &str, #[case] expected: &str) { + assert_eq!(mapping(text).unwrap(), yaml(expected)); + } + + #[rstest] + fn duplicate_key_keeps_its_first_position() { + let keys: Vec<_> = mapping("a: 1\nb: 2\na: 3\n").unwrap().into_keys().collect(); + assert_eq!(keys, [Value::from("a"), Value::from("b")]); + } + + #[rstest] + #[case::sequence( + r#" +- a +- b +"# + )] + #[case::scalar("text")] + #[case::malformed("a: [")] + fn rejects_a_document_that_is_not_a_mapping(#[case] text: &str) { + assert!(mapping(text).is_err()); + } +} diff --git a/litellm-rust/crates/config/tests/config.rs b/litellm-rust/crates/config/tests/config.rs deleted file mode 100644 index d425b62efc2..00000000000 --- a/litellm-rust/crates/config/tests/config.rs +++ /dev/null @@ -1,436 +0,0 @@ -use litellm_config::{Config, Error, Spelled, TracingStoreSettings}; -use rstest::{fixture, rstest}; -use tempfile::TempDir; - -#[fixture] -fn directory() -> TempDir { - tempfile::tempdir().unwrap() -} - -#[fixture] -fn model_list_yaml() -> &'static str { - r#" -model_list: - - model_name: assistant - litellm_params: - model: anthropic/test-model - api_key: os.environ/ANTHROPIC_API_KEY - - model_name: local - litellm_params: - model: test-model - api_base: http://localhost:8000/v1 - custom_llm_provider: openai -"# -} - -#[rstest] -fn loads_model_list_from_file(directory: TempDir, model_list_yaml: &str) { - let path = directory.path().join("config.yaml"); - std::fs::write(&path, model_list_yaml).unwrap(); - - let config = Config::load(path).unwrap(); - assert_eq!(config.model_list.len(), 2); - let anthropic = &config.model_list[0]; - assert_eq!(anthropic.model_name, "assistant"); - assert_eq!(anthropic.litellm_params.model, "anthropic/test-model"); - assert_eq!( - anthropic.litellm_params.api_key.as_ref().unwrap().expose(), - "os.environ/ANTHROPIC_API_KEY" - ); - assert!(anthropic.litellm_params.api_base.is_none()); - assert!(anthropic.litellm_params.custom_llm_provider.is_none()); - let local = &config.model_list[1]; - assert_eq!(local.model_name, "local"); - assert_eq!(local.litellm_params.model, "test-model"); - assert!(local.litellm_params.api_key.is_none()); - assert_eq!( - local.litellm_params.api_base.as_deref(), - Some("http://localhost:8000/v1") - ); - assert_eq!( - local.litellm_params.custom_llm_provider.as_deref(), - Some("openai") - ); -} - -#[rstest] -fn config_debug_redacts_api_keys() { - let config = Config::from_yaml( - "model_list: [{model_name: assistant, litellm_params: {model: anthropic/test-model, api_key: secret-value}}]", - ) - .unwrap(); - assert_eq!( - config.model_list[0] - .litellm_params - .api_key - .as_ref() - .unwrap() - .expose(), - "secret-value" - ); - assert!(!format!("{config:?}").contains("secret-value")); -} - -#[rstest] -#[case::malformed_yaml("model_list: [")] -#[case::missing_params("model_list: [{model_name: assistant}]")] -#[case::missing_model("model_list: [{model_name: assistant, litellm_params: {api_key: key}}]")] -#[case::provider_param_of_the_wrong_type( - "model_list: [{model_name: assistant, litellm_params: {model: test, aws_region_name: [eu-central-1]}}]" -)] -fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) { - assert!(matches!(Config::from_yaml(yaml), Err(Error::Parse(_)))); -} - -#[rstest] -fn distinguishes_read_errors_from_parse_errors(directory: TempDir) { - assert!(matches!( - Config::load(directory.path().join("missing.yaml")), - Err(Error::Read(error)) if error.kind() == std::io::ErrorKind::NotFound - )); -} - -#[rstest] -#[case::literal("secret-master-key")] -#[case::reference("os.environ/LITELLM_MASTER_KEY")] -fn loads_and_redacts_the_master_key(#[case] key: &str) { - let config = Config::from_yaml(&format!( - "model_list: []\ngeneral_settings:\n master_key: {key}\n" - )) - .unwrap(); - assert_eq!( - config - .general_settings - .master_key - .as_ref() - .unwrap() - .expose(), - key - ); - assert!(!format!("{config:?}").contains(key)); -} - -#[rstest] -fn missing_general_settings_has_no_master_key() { - let config = Config::from_yaml("model_list: []").unwrap(); - assert!(config.general_settings.master_key.is_none()); -} - -#[test] -fn tracing_settings_are_typed_and_redact_the_url() { - let config = Config::from_yaml( - "general_settings:\n tracing:\n store:\n type: clickhouse\n url: https://writer:password@example.com\n database: analytics\n retention_days: 7\n", - ) - .unwrap(); - let tracing = config.general_settings.tracing.as_ref().unwrap(); - let Some(TracingStoreSettings::ClickHouse(store)) = tracing.store.as_ref() else { - panic!("expected ClickHouse tracing store") - }; - assert_eq!( - store.url.as_ref().unwrap().expose(), - "https://writer:password@example.com" - ); - assert_eq!(store.database.as_deref(), Some("analytics")); - assert_eq!(store.retention_days, Some(Spelled::Value(7.0))); - assert!(!format!("{config:?}").contains("password")); -} - -#[test] -fn tracing_settings_accept_environment_references() { - let config = Config::from_yaml( - "general_settings:\n tracing:\n store:\n type: clickhouse\n url: os.environ/CLICKHOUSE_URL\n retention_days: os.environ/RETENTION_DAYS\n", - ) - .unwrap(); - let Some(TracingStoreSettings::ClickHouse(store)) = - config.general_settings.tracing.unwrap().store - else { - panic!("expected ClickHouse tracing store") - }; - assert_eq!( - store.retention_days, - Some(Spelled::Text("os.environ/RETENTION_DAYS".to_owned())) - ); -} - -#[test] -fn tracing_settings_reject_string_store() { - assert!(Config::from_yaml("general_settings:\n tracing:\n store: clickhouse\n").is_err()); -} - -#[test] -fn tracing_settings_reject_removed_reader_configuration() { - assert!(Config::from_yaml("general_settings:\n tracing:\n store:\n type: clickhouse\n reader_url: http://localhost:8123\n").is_err()); -} - -#[rstest] -fn empty_config_matches_python_defaults() { - let config = Config::from_yaml("{}").unwrap(); - assert!(config.model_list.is_empty()); - assert_eq!(config.general_settings.admission_queue_timeout_seconds, 1.0); - assert_eq!( - config.general_settings.database_connection_pool_limit, - Some(10) - ); - assert_eq!( - config.general_settings.proxy_config_reload_interval_seconds, - 30 - ); - assert_eq!(config.general_settings.health_check_interval, 300); -} - -#[rstest] -fn parses_typed_settings_and_preserves_extension_fields() { - let config = Config::from_yaml( - r#" -model_list: - - model_name: assistant - litellm_params: - model: vertex_ai/test-model - timeout: os.environ/REQUEST_TIMEOUT - tpm: os.environ/TPM_LIMIT - rpm: 5 - drop_params: "true" - vertex_project: test-project - vertex_credentials: - type: service_account - future_param: 1 - model_info: - mode: chat - access_groups: [internal] -general_settings: - master_key: secret-master-key - store_model_in_db: true - custom_auth: auth.py -router_settings: - routing_strategy: simple-shuffle - allowed_fails: 2 - redis_host: cache.internal -litellm_settings: - drop_params: true - cache: true - custom_callback_name: audit -future_section: - enabled: true -"#, - ) - .unwrap(); - - let model = &config.model_list[0]; - assert_eq!( - model.litellm_params.timeout, - Some(Spelled::Text("os.environ/REQUEST_TIMEOUT".to_string())) - ); - assert_eq!( - model.litellm_params.tpm, - Some(Spelled::Text("os.environ/TPM_LIMIT".to_string())) - ); - assert_eq!(model.litellm_params.rpm, Some(Spelled::Value(5.0))); - assert_eq!( - model.litellm_params.drop_params, - Some(Spelled::Text("true".to_string())) - ); - assert_eq!( - model.litellm_params.vertex.vertex_project.as_deref(), - Some("test-project") - ); - assert_eq!( - model.litellm_params.vertex.vertex_credentials.as_deref(), - Some(r#"{"type":"service_account"}"#) - ); - assert!(model.litellm_params.extra.contains_key("future_param")); - assert!(model.additional_fields.contains_key("access_groups")); - assert_eq!(config.router_settings.allowed_fails, Some(2)); - assert!( - config - .router_settings - .additional_fields - .contains_key("redis_host") - ); - assert!( - config - .litellm_settings - .additional_fields - .contains_key("custom_callback_name") - ); - assert!(config.additional_fields.contains_key("future_section")); -} - -#[rstest] -fn parses_python_config_sections() { - let config = Config::from_yaml( - r#" -environment_variables: - REDIS_PORT: 6379 -callback_settings: - otel: - message_logging: false -assistant_settings: - custom_llm_provider: openai -credential_list: - - credential_name: bedrock - credential_values: - aws_region_name: us-east-1 -guardrails: - - guardrail_name: pii - litellm_params: - guardrail: presidio -prompts: - - prompt_id: support -sandbox_tools: - - sandbox_tool_name: e2b -search_tools: - - search_tool_name: web -files_settings: - - custom_llm_provider: openai -finetune_settings: - - custom_llm_provider: openai -mcp_tools: - - name: lookup -mcp_servers: - docs: - url: https://example.test/mcp -vector_store_registry: - - vector_store_name: docs -worker_registry: - - worker_id: regional -agents: - - agent_name: reviewer -agent_list: - - agent_name: legacy-reviewer -policies: - safe: - guardrails: - add: [pii] -policy_attachments: - - policy_id: safe -include: - - models.yaml -"#, - ) - .unwrap(); - - assert_eq!(config.environment_variables.len(), 1); - assert_eq!(config.credential_list.len(), 1); - assert_eq!(config.guardrails.len(), 1); - assert_eq!(config.prompts.len(), 1); - assert_eq!(config.sandbox_tools.len(), 1); - assert_eq!(config.search_tools.len(), 1); - assert_eq!(config.files_settings.len(), 1); - assert_eq!(config.finetune_settings.len(), 1); - assert_eq!(config.mcp_tools.len(), 1); - assert_eq!(config.mcp_servers.len(), 1); - assert_eq!(config.vector_store_registry.len(), 1); - assert_eq!(config.worker_registry.len(), 1); - assert_eq!(config.agents.len(), 1); - assert_eq!(config.agent_list.len(), 1); - assert!(config.policies.contains_key("safe")); - assert_eq!(config.policy_attachments.len(), 1); - assert_eq!(&*config.include, &["models.yaml"]); -} - -#[rstest] -#[case::policy_pipeline("../../../litellm/proxy/example_config_yaml/test_pipeline_config.yaml")] -#[case::gateway("../../../tests/e2e/gateway/stage_mirror_ci_config.yml")] -fn parses_representative_python_configs(#[case] relative_path: &str) { - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(relative_path); - let config = Config::load(path).unwrap(); - assert!(!config.model_list.is_empty()); -} - -#[rstest] -#[case::one("callbacks: custom_callbacks.logger", 1)] -#[case::many("callbacks: [prometheus, otel]", 2)] -fn accepts_python_callback_shorthand(#[case] setting: &str, #[case] expected_len: usize) { - let config = Config::from_yaml(&format!("litellm_settings:\n {setting}")).unwrap(); - assert_eq!( - config.litellm_settings.callbacks.as_ref().unwrap().len(), - expected_len - ); -} - -#[rstest] -#[case::api_key("api_key", "provider-secret")] -#[case::aws_secret("aws_secret_access_key", "aws-secret")] -#[case::vertex_credentials("vertex_credentials", "vertex-secret")] -#[case::provider_extension("azure_ad_token", "azure-secret")] -#[case::general_extension("custom_auth_secret", "auth-secret")] -#[case::router_extension("redis_password", "redis-secret")] -#[case::litellm_extension("callback_token", "callback-secret")] -#[case::root_extension("private_token", "root-secret")] -fn debug_output_does_not_expose_config_values(#[case] field: &str, #[case] secret: &str) { - let yaml = match field { - "api_key" => format!( - "model_list: [{{model_name: assistant, litellm_params: {{model: test, api_key: {secret}}}}}]" - ), - "aws_secret_access_key" => format!( - "model_list: [{{model_name: assistant, litellm_params: {{model: test, aws_secret_access_key: {secret}}}}}]" - ), - "vertex_credentials" => format!( - "model_list: [{{model_name: assistant, litellm_params: {{model: test, vertex_credentials: {secret}}}}}]" - ), - "azure_ad_token" => format!( - "model_list: [{{model_name: assistant, litellm_params: {{model: test, azure_ad_token: {secret}}}}}]" - ), - "custom_auth_secret" => format!("general_settings: {{{field}: {secret}}}"), - "redis_password" => format!("router_settings: {{{field}: {secret}}}"), - "callback_token" => format!("litellm_settings: {{{field}: {secret}}}"), - "private_token" => format!("{field}: {secret}"), - _ => unreachable!(), - }; - let config = Config::from_yaml(&yaml).unwrap(); - assert!(!format!("{config:?}").contains(secret)); -} - -#[rstest] -fn resolves_nested_includes_once_in_breadth_first_order() { - let directory = TempDir::new().unwrap(); - let root = directory.path(); - std::fs::create_dir(root.join("nested")).unwrap(); - std::fs::write(root.join("config.yaml"), "include: [nested/first.yaml, second.yaml]\nmodel_list: [{model_name: root, litellm_params: {model: root}}]\n").unwrap(); - std::fs::write(root.join("nested/first.yaml"), "include: [third.yaml]\nmodel_list: [{model_name: first, litellm_params: {model: first}}]\n").unwrap(); - std::fs::write(root.join("second.yaml"), "general_settings: {master_key: second}\nmodel_list: [{model_name: second, litellm_params: {model: second}}]\n").unwrap(); - std::fs::write(root.join("nested/third.yaml"), "include: [../config.yaml]\ngeneral_settings: {master_key: third}\nmodel_list: [{model_name: third, litellm_params: {model: third}}]\n").unwrap(); - - let config = Config::load(root.join("config.yaml")).unwrap(); - assert_eq!( - config - .model_list - .iter() - .map(|model| model.model_name.as_str()) - .collect::>(), - ["root", "first", "second", "third"] - ); - assert_eq!( - config.general_settings.master_key.unwrap().expose(), - "third" - ); - assert!(config.include.is_empty()); -} - -#[rstest] -fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() { - let config = Config::from_yaml("mcp_servers:\n docs:\n url: https://example.test/private-secret/mcp\n authentication_token: upstream-secret\n static_headers: {x-token: header-secret}\n env: {TOKEN: env-secret}\n args: [argument-secret]\n client_secret: oauth-secret\n allowed_tools: [search]\n").unwrap(); - let server = &config.mcp_servers["docs"]; - assert_eq!(server.allowed_tools.as_deref().unwrap(), ["search"]); - assert!(server.unsupported.contains_key("client_secret")); - let debug = format!("{config:?}"); - for secret in [ - "private-secret", - "upstream-secret", - "header-secret", - "env-secret", - "oauth-secret", - "argument-secret", - ] { - assert!(!debug.contains(secret)); - } -} - -#[rstest] -#[case::transport("transport: invalid")] -#[case::auth("auth_type: invalid")] -#[case::concurrency("max_concurrent_requests: -1")] -#[case::headers("static_headers: {x-token: [not, a, string]}")] -fn rejects_invalid_typed_mcp_settings(#[case] setting: &str) { - assert!(Config::from_yaml(&format!("mcp_servers:\n docs:\n {setting}\n")).is_err()); -} diff --git a/litellm-rust/crates/config/tests/mcp.rs b/litellm-rust/crates/config/tests/mcp.rs deleted file mode 100644 index 9ac6ad02bbb..00000000000 --- a/litellm-rust/crates/config/tests/mcp.rs +++ /dev/null @@ -1,39 +0,0 @@ -use litellm_config::{McpAuth, McpTransport}; -use rstest::rstest; -use serde_json::json; - -#[rstest] -#[case::http(McpTransport::Http, "http")] -#[case::sse(McpTransport::Sse, "sse")] -#[case::stdio(McpTransport::Stdio, "stdio")] -fn mcp_transport_as_str_matches_the_serde_spelling( - #[case] transport: McpTransport, - #[case] name: &str, -) { - assert_eq!(<&'static str>::from(transport), name); - assert_eq!( - serde_json::from_value::(json!(name)).unwrap(), - transport - ); -} - -#[rstest] -#[case::none(McpAuth::None, "none")] -#[case::api_key(McpAuth::ApiKey, "api_key")] -#[case::bearer_token(McpAuth::BearerToken, "bearer_token")] -#[case::basic(McpAuth::Basic, "basic")] -#[case::authorization(McpAuth::Authorization, "authorization")] -#[case::token(McpAuth::Token, "token")] -#[case::oauth2(McpAuth::Oauth2, "oauth2")] -#[case::aws_sigv4(McpAuth::AwsSigv4, "aws_sigv4")] -#[case::oauth2_token_exchange(McpAuth::Oauth2TokenExchange, "oauth2_token_exchange")] -#[case::oauth2_id_jag(McpAuth::Oauth2IdJag, "oauth2_id_jag")] -#[case::true_passthrough(McpAuth::TruePassthrough, "true_passthrough")] -#[case::oauth_delegate(McpAuth::OauthDelegate, "oauth_delegate")] -fn mcp_auth_as_str_matches_the_serde_spelling(#[case] auth: McpAuth, #[case] name: &str) { - assert_eq!(<&'static str>::from(auth), name); - assert_eq!( - serde_json::from_value::(json!(name)).unwrap(), - auth - ); -} diff --git a/litellm-rust/crates/gateway-mcp/src/configured.rs b/litellm-rust/crates/gateway-mcp/src/configured.rs index bb4f5594f1f..9514e0b7997 100644 --- a/litellm-rust/crates/gateway-mcp/src/configured.rs +++ b/litellm-rust/crates/gateway-mcp/src/configured.rs @@ -136,7 +136,7 @@ fn validate(name: &str, config: &McpServer) -> Result<(), ConnectError> { if config .command .as_ref() - .is_none_or(|command| command.trim().is_empty()) + .is_none_or(|command| command.expose().trim().is_empty()) || config.url.is_some() { return Err(invalid( @@ -377,11 +377,12 @@ async fn connect_service( .await?; let program = config .command - .as_deref() + .as_ref() + .map(SecretValue::expose) .ok_or_else(|| invalid(name, "stdio requires command"))?; let mut command = tokio::process::Command::new(program); command - .args(&config.args) + .args(config.args.iter().map(SecretValue::expose)) .envs(environment) .kill_on_drop(true); ().serve_with_ct(TokioChildProcess::new(command)?, shutdown) diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 387bcd43231..5fc17476af6 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -19,6 +19,7 @@ litellm-gateway-auth.workspace = true litellm-gateway-ui.workspace = true litellm-auth-types.workspace = true litellm-config.workspace = true +litellm-config-load.workspace = true litellm-http = { workspace = true, features = ["mcp"] } litellm-llms.workspace = true litellm-secrets.workspace = true diff --git a/litellm-rust/crates/gateway/README.md b/litellm-rust/crates/gateway/README.md index 125c2b07448..da8dc9cbc9a 100644 --- a/litellm-rust/crates/gateway/README.md +++ b/litellm-rust/crates/gateway/README.md @@ -32,7 +32,7 @@ MCP clients can connect to `/mcp` or the scoped `/math/mcp` endpoint with the sa ## Configuration behavior -`Config::load` resolves nested `include` files once in breadth-first order. Included lists append and other top-level values replace earlier values. Paths resolve relative to the declaring file, with the root config directory as a fallback +`litellm_config_load::load` resolves nested `include` files once in breadth-first order. Included lists append and other top-level values replace earlier values. Paths resolve relative to the declaring file, with the root config directory as a fallback `environment_variables` provides scalar overrides to the injected secret source and HTTP settings without changing process globals. Credentials, static headers, URLs, and stdio environment values can reference `os.environ/NAME`. Model providers and MCP share the host's HTTP settings, including TLS certificates and environment proxies. Explicit environment HTTP settings take precedence over `litellm_settings` diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 0b3aba81181..1218db083c0 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,5 +1,6 @@ mod auth; mod error; +mod mcp_settings; mod secrets; pub use error::Error; @@ -15,6 +16,7 @@ use axum::{ use http_body_util::BodyExt; use litellm_config::Config; +use litellm_config_load::{Environment, Transport}; use litellm_gateway_auth::Auth; use litellm_gateway_inference::{Gateway, ModelRouter}; use litellm_http::{ @@ -26,34 +28,18 @@ use litellm_secrets::source::EnvironmentSecrets; use litellm_tracing::ByteChunk; use uuid::Uuid; -pub fn build_inference(config: &Config) -> Result, Error> { +pub fn build_inference(config: &Config, process: &Environment) -> Result, Error> { let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))); let environment = secrets::environment_values(&config.environment_variables)?; - let lookup = |name: &str| { + let lookup = process.clone().overlaid( environment - .get(name) - .map(|value| value.expose().to_owned()) - .or_else(|| std::env::var(name).ok()) - }; - let settings = &config.litellm_settings; + .iter() + .map(|(name, value)| (name.clone(), value.expose().to_owned())), + ); + let transport = Transport::lower(&config.litellm_settings, &lookup); let http_settings = HttpSettings::from_layers([ - HttpSettingsLayer::from_environment(&lookup), - HttpSettingsLayer { - ssl_verify: settings.ssl_verify.as_ref().map(|value| match value { - litellm_config::Spelled::Value(true) => SslVerify::Enabled, - litellm_config::Spelled::Value(false) => SslVerify::Disabled, - litellm_config::Spelled::Text(value) => SslVerify::parse(value), - }), - ssl_certificate: settings.ssl_certificate.as_ref().map(Into::into), - ssl_security_level: settings.ssl_security_level.clone(), - ssl_ecdh_curve: settings.ssl_ecdh_curve.clone(), - force_ipv4: settings.force_ipv4, - http2: settings.http2, - aiohttp_trust_env: settings.aiohttp_trust_env, - disable_aiohttp_trust_env: settings.disable_aiohttp_trust_env, - disable_aiohttp_transport: settings.disable_aiohttp_transport, - ..Default::default() - }, + transport_layer(&transport), + HttpSettingsLayer::from_environment(&|name: &str| lookup.var(name).map(str::to_owned)), ]); let http = Resolution::from(&http_settings).config; let client = pool.client(&http, ClientVariant::Provider)?; @@ -70,6 +56,25 @@ pub fn build_inference(config: &Config) -> Result, Error> { )?)) } +fn transport_layer(transport: &Transport) -> HttpSettingsLayer { + HttpSettingsLayer { + ssl_verify: Some(match &transport.ssl_verify { + litellm_config_load::SslVerify::Enabled => SslVerify::Enabled, + litellm_config_load::SslVerify::Disabled => SslVerify::Disabled, + litellm_config_load::SslVerify::CaBundle(path) => SslVerify::CaBundle(path.clone()), + }), + ssl_certificate: transport.ssl_certificate.clone(), + ssl_security_level: transport.ssl_security_level.clone(), + ssl_ecdh_curve: transport.ssl_ecdh_curve.clone(), + force_ipv4: Some(transport.force_ipv4), + http2: Some(transport.http2), + aiohttp_trust_env: Some(transport.aiohttp_trust_env), + disable_aiohttp_trust_env: Some(transport.disable_aiohttp_trust_env), + disable_aiohttp_transport: Some(transport.disable_aiohttp_transport), + ..Default::default() + } +} + pub async fn build_mcp( config: &Config, secrets: Arc, @@ -83,14 +88,8 @@ pub async fn build_mcp( if config.mcp_servers.is_empty() { return Ok(None); } - if let Some(setting) = config - .general_settings - .additional_fields - .keys() - .chain(config.litellm_settings.additional_fields.keys()) - .find(|key| key.starts_with("mcp_")) - { - return Err(Error::McpSetting(setting.clone())); + if let Some(setting) = mcp_settings::first_unsupported(config) { + return Err(Error::McpSetting(setting.into())); } if !config.guardrails.is_empty() || !config.policies.is_empty() diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index f8150e6e860..dc3c25f1329 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -3,7 +3,7 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; -use litellm_config::Config; +use litellm_config_load::Environment; use litellm_gateway_auth::UiBackend; use litellm_tracing::{Level, Logger, Metadata, Record, Sink}; use serde_json::json; @@ -41,14 +41,15 @@ impl Sink for StderrSink { #[tokio::main] async fn main() -> Result<(), Box> { - let settings: Settings = settings::from_iter(std::env::vars_os())?; + let environment = Environment::from_os(std::env::vars_os()); + let settings: Settings = settings::from_environment(&environment)?; let level = settings.log_level(); Logger::new(StderrSink { level }).install_global()?; - let config = Config::load(settings.litellm_config)?; - let inference = litellm_gateway::build_inference(&config)?; - let ui = match std::env::var_os("LITELLM_UI_PATH") { + let config = litellm_config_load::load(settings.litellm_config)?; + let inference = litellm_gateway::build_inference(&config, &environment)?; + let ui = match environment.var("LITELLM_UI_PATH") { Some(directory) => { - let ui_settings: UiSettings = settings::from_iter(std::env::vars_os())?; + let ui_settings: UiSettings = settings::from_environment(&environment)?; let backend = UiBackend::new(ui_settings.ui_username, ui_settings.ui_password)?; Some( litellm_gateway_ui::router( @@ -85,11 +86,12 @@ async fn main() -> Result<(), Box> { litellm_gateway_mcp::HttpConfig { allowed_hosts: config .general_settings + .mcp .mcp_allowed_hosts .as_deref() .map(Vec::from) .unwrap_or(defaults.allowed_hosts), - allowed_origins: config.general_settings.mcp_allowed_origins.to_vec(), + allowed_origins: config.general_settings.mcp.mcp_allowed_origins.to_vec(), cancellation_token: shutdown.clone(), server_info: defaults.server_info, }, diff --git a/litellm-rust/crates/gateway/src/mcp_settings.rs b/litellm-rust/crates/gateway/src/mcp_settings.rs new file mode 100644 index 00000000000..7213e0c2530 --- /dev/null +++ b/litellm-rust/crates/gateway/src/mcp_settings.rs @@ -0,0 +1,62 @@ +use litellm_config::Config; + +pub(crate) fn first_unsupported(config: &Config) -> Option<&'static str> { + let general = &config.general_settings.mcp; + let litellm = &config.litellm_settings.mcp; + [ + ( + "mcp_advertised_versions", + general.mcp_advertised_versions.is_some(), + ), + ( + "mcp_disabled_capabilities", + general.mcp_disabled_capabilities.is_some(), + ), + ("mcp_allowed_clients", general.mcp_allowed_clients.is_some()), + ( + "mcp_require_explicit_server_scope", + general.mcp_require_explicit_server_scope.is_some(), + ), + ( + "mcp_client_id_header", + general.mcp_client_id_header.is_some(), + ), + ( + "mcp_client_side_auth_header_name", + general.mcp_client_side_auth_header_name.is_some(), + ), + ( + "mcp_allow_all_keys_respects_mcp_scope", + general.mcp_allow_all_keys_respects_mcp_scope.is_some(), + ), + ( + "mcp_prefer_client_id_metadata_document", + general.mcp_prefer_client_id_metadata_document.is_some(), + ), + ( + "mcp_internal_ip_ranges", + general.mcp_internal_ip_ranges.is_some(), + ), + ( + "mcp_trusted_proxy_ranges", + general.mcp_trusted_proxy_ranges.is_some(), + ), + ( + "mcp_xff_num_trusted_hops", + general.mcp_xff_num_trusted_hops.is_some(), + ), + ("mcp_required_fields", general.mcp_required_fields.is_some()), + ( + "mcp_session_token_signing", + general.mcp_session_token_signing.is_some(), + ), + ("mcp_aliases", litellm.mcp_aliases.is_some()), + ( + "mcp_semantic_tool_filter", + litellm.mcp_semantic_tool_filter.is_some(), + ), + ("mcp_tool_search", litellm.mcp_tool_search.is_some()), + ] + .into_iter() + .find_map(|(key, set)| set.then_some(key)) +} diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index 2dd8872065f..02b47d62c46 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -1,6 +1,5 @@ -use std::ffi::OsString; - use litellm_auth_types::SecretValue; +use litellm_config_load::Environment; use litellm_tracing::Level; use serde::{Deserialize, de::DeserializeOwned}; @@ -50,14 +49,8 @@ fn default_secure_cookies() -> bool { true } -pub(super) fn from_iter( - variables: impl IntoIterator, -) -> envy::Result { - envy::from_iter( - variables - .into_iter() - .filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))), - ) +pub(super) fn from_environment(environment: &Environment) -> envy::Result { + envy::from_iter(environment.variables()) } #[cfg(test)] @@ -67,10 +60,11 @@ mod tests { use super::*; fn parse(variables: &[(&str, &str)]) -> envy::Result { - from_iter( - variables + from_environment( + &variables .iter() - .map(|(key, value)| (OsString::from(key), OsString::from(value))), + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(), ) } @@ -156,13 +150,13 @@ mod tests { #[case::unrelated_variable("UNRELATED")] #[case::host_fallback("HOST")] fn non_unicode_environment_values_are_ignored(#[case] key: &str) { - use std::os::unix::ffi::OsStringExt; + use std::{ffi::OsString, os::unix::ffi::OsStringExt}; - let settings: Settings = from_iter([ + let settings: Settings = from_environment(&Environment::from_os([ (OsString::from(key), OsString::from_vec(vec![0xff])), (OsString::from_vec(vec![0xff]), OsString::from("value")), (OsString::from("PORT"), OsString::from("8080")), - ]) + ])) .unwrap(); assert_eq!(settings.host, "0.0.0.0"); diff --git a/litellm-rust/crates/gateway/tests/mcp.rs b/litellm-rust/crates/gateway/tests/mcp.rs index 593fbfeb129..7494fb9bdf4 100644 --- a/litellm-rust/crates/gateway/tests/mcp.rs +++ b/litellm-rust/crates/gateway/tests/mcp.rs @@ -5,7 +5,13 @@ use axum::{ body::{Body, to_bytes}, http::Request, }; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use tokio_util::sync::CancellationToken; +use tower::ServiceExt; + use litellm_config::Config; +use litellm_config_load::Environment; use litellm_gateway_mcp::{ HttpConfig, rmcp::{ @@ -17,10 +23,6 @@ use litellm_gateway_mcp::{ }, }, }; -use rstest::{fixture, rstest}; -use serde_json::{Value, json}; -use tokio_util::sync::CancellationToken; -use tower::ServiceExt; #[derive(Clone)] struct Upstream; @@ -126,7 +128,7 @@ mcp_servers: upstream.url )) .unwrap(); - let inference = litellm_gateway::build_inference(&config).unwrap(); + let inference = litellm_gateway::build_inference(&config, &Environment::default()).unwrap(); let shutdown = CancellationToken::new(); let mcp = litellm_gateway::build_mcp( &config, @@ -243,7 +245,7 @@ async fn rejects_unsupported_or_invalid_settings_before_serving( #[case] expected: &str, ) { let config = Config::from_yaml(&format!("general_settings: {{master_key: gateway-key}}\nmcp_servers:\n docs:\n url: http://127.0.0.1:1/mcp\n {settings}\n")).unwrap(); - let inference = litellm_gateway::build_inference(&config).unwrap(); + let inference = litellm_gateway::build_inference(&config, &Environment::default()).unwrap(); let result = litellm_gateway::build_mcp( &config, Arc::new(NoSecrets), @@ -261,7 +263,7 @@ async fn rejects_unsupported_or_invalid_settings_before_serving( #[tokio::test] async fn mcp_is_optional_and_requires_admission_credentials_when_enabled() { let empty = Config::from_yaml("{}").unwrap(); - let inference = litellm_gateway::build_inference(&empty).unwrap(); + let inference = litellm_gateway::build_inference(&empty, &Environment::default()).unwrap(); assert!( litellm_gateway::build_mcp( &empty, @@ -311,7 +313,9 @@ fn applies_configured_http_environment_before_building_clients() { missing.display(), )) .unwrap(); - let error = litellm_gateway::build_inference(&config).err().unwrap(); + let error = litellm_gateway::build_inference(&config, &Environment::default()) + .err() + .unwrap(); assert!(matches!( error, litellm_gateway::Error::Http(litellm_http::Error::Read { path, .. }) @@ -320,16 +324,51 @@ fn applies_configured_http_environment_before_building_clients() { } #[rstest] -#[case::client_policy("general_settings: {master_key: key, mcp_allowed_clients: [approved]}")] -#[case::guardrail("guardrails: [{guardrail_name: policy}]")] -#[case::local_tools("mcp_tools: [{name: custom}]")] +#[case::client_policy( + "general_settings: {master_key: key, mcp_allowed_clients: [approved]}", + "mcp_allowed_clients" +)] +#[case::aliases( + "general_settings: {master_key: key}\nlitellm_settings: {mcp_aliases: {d: docs}}", + "mcp_aliases" +)] +#[case::tool_search( + "general_settings: {master_key: key}\nlitellm_settings: {mcp_tool_search: {top_k: 3}}", + "mcp_tool_search" +)] +#[case::advertised_versions( + "general_settings: {mcp_advertised_versions: [2025-06-18]}", + "mcp_advertised_versions" +)] +#[case::spelled_bool( + "general_settings: {mcp_require_explicit_server_scope: 'true'}", + "mcp_require_explicit_server_scope" +)] +#[case::header( + "general_settings: {mcp_client_side_auth_header_name: x-auth}", + "mcp_client_side_auth_header_name" +)] +#[case::hops( + "general_settings: {mcp_xff_num_trusted_hops: 2}", + "mcp_xff_num_trusted_hops" +)] +#[case::signing( + "general_settings: {mcp_session_token_signing: {algorithm: RS256}}", + "mcp_session_token_signing" +)] +#[case::semantic_filter( + "litellm_settings: {mcp_semantic_tool_filter: {enabled: true}}", + "mcp_semantic_tool_filter" +)] +#[case::guardrail("guardrails: [{guardrail_name: policy}]", "guardrails or policies")] +#[case::local_tools("mcp_tools: [{name: custom}]", "mcp_tools")] #[tokio::test] -async fn refuses_unimplemented_global_mcp_policy(#[case] settings: &str) { +async fn refuses_unimplemented_global_mcp_policy(#[case] settings: &str, #[case] key: &str) { let config = Config::from_yaml(&format!( "{settings}\nmcp_servers: {{docs: {{url: 'http://127.0.0.1:1/mcp'}}}}\n" )) .unwrap(); - let inference = litellm_gateway::build_inference(&config).unwrap(); + let inference = litellm_gateway::build_inference(&config, &Environment::default()).unwrap(); let error = litellm_gateway::build_mcp( &config, Arc::new(NoSecrets), @@ -340,5 +379,31 @@ async fn refuses_unimplemented_global_mcp_policy(#[case] settings: &str) { .await .err() .unwrap(); - assert!(matches!(error, litellm_gateway::Error::McpSetting(_))); + assert!(matches!(error, litellm_gateway::Error::McpSetting(setting) if setting == key)); +} + +#[rstest] +#[case::general("general_settings: {mcp_whatever: 1}")] +#[case::litellm("litellm_settings: {mcp_whatever: 1}")] +#[case::null("general_settings: {mcp_trusted_proxy_ranges: null}")] +#[case::allowed_hosts("general_settings: {mcp_allowed_hosts: [localhost]}")] +#[case::allowed_origins("general_settings: {mcp_allowed_origins: ['https://app.test']}")] +#[tokio::test] +async fn unknown_null_or_supported_mcp_keys_do_not_block_mcp(#[case] settings: &str) { + let config = Config::from_yaml(&format!( + "{settings}\nmcp_servers: {{docs: {{url: 'http://127.0.0.1:1/mcp'}}}}\n" + )) + .unwrap(); + let inference = litellm_gateway::build_inference(&config, &Environment::default()).unwrap(); + let error = litellm_gateway::build_mcp( + &config, + Arc::new(NoSecrets), + CancellationToken::new(), + &inference.resources.pool, + &inference.http, + ) + .await + .err() + .unwrap(); + assert!(matches!(error, litellm_gateway::Error::Auth(_)), "{error}"); } diff --git a/litellm-rust/crates/gateway/tests/server.rs b/litellm-rust/crates/gateway/tests/server.rs index 71edd3c3650..32c1b148b88 100644 --- a/litellm-rust/crates/gateway/tests/server.rs +++ b/litellm-rust/crates/gateway/tests/server.rs @@ -5,15 +5,17 @@ use std::{ use axum::{body::Body, http::Request}; use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; -use litellm_config::Config; -use litellm_gateway_inference::{Error, Gateway}; -use litellm_http::ClientVariant; -use litellm_tracing::{Logger, Metadata, Record, Sink}; use rstest::{fixture, rstest}; use serde_json::{Value, json}; use tokio::{net::TcpListener, sync::oneshot, time::timeout}; use tower::ServiceExt; +use litellm_config::Config; +use litellm_config_load::Environment; +use litellm_gateway_inference::{Error, Gateway}; +use litellm_http::ClientVariant; +use litellm_tracing::{Logger, Metadata, Record, Sink}; + struct LogSink(mpsc::Sender); impl Sink for LogSink { @@ -30,7 +32,11 @@ impl Sink for LogSink { #[fixture] fn inference() -> Arc { - litellm_gateway::build_inference(&Config::from_yaml("model_list: []").unwrap()).unwrap() + litellm_gateway::build_inference( + &Config::from_yaml("model_list: []").unwrap(), + &Environment::default(), + ) + .unwrap() } #[rstest] diff --git a/litellm-rust/crates/python-compat/AGENTS.md b/litellm-rust/crates/python-compat/AGENTS.md index 55049daf612..ff0b34bc431 100644 --- a/litellm-rust/crates/python-compat/AGENTS.md +++ b/litellm-rust/crates/python-compat/AGENTS.md @@ -6,6 +6,7 @@ - `cache-disk`: diskcache's pickled values (`pickle`), falsy-is-miss (`truthy`) - Cache-key derivation: sha256 over `str(value)` must match Python byte for byte (`repr::to_str`) - Byte-identical writes where Python compares raw values (`json::dumps`, `repr`) +- `read` holds Python's config reading shapes (`Spelled`, `one_or_many`). No setting names or config keys here, those stay with the crate that reads them - Relation to [`py_literal`](https://docs.rs/py_literal/latest/py_literal/): replaced, do not reintroduce - Its pest grammar backtracks: parse time doubles per nested `[`/`{` (105 ms at depth 16). Ours is linear (19 µs at depth 128) - Its formatter is not `repr` (`2e-1`, always single quotes, escapes non-ASCII). It also lost `-0.0`, `(1+2j)`, `set()` @@ -20,4 +21,4 @@ - `scripts/verify_rust_pickles.py` checks CPython reads Rust pickles, with class resolution disabled. CI does not run Python - Decoders recurse, so they reject nesting beyond `MAX_DEPTH` for stack safety: deliberately stricter than CPython, whose parser takes ~200 levels and whose unpickler has no limit (pinned as `nested_150`) - Formatters (`repr`, `json`) are unbounded. Values from the decoders are already capped, a hand-built `Value` is the caller's responsibility - - `literal_eval` must stay linear in depth: `tests/limits.rs` times the deepest parse, `benches/formats.rs` measures the curve but is manual, since CI runs no Rust bench + - `literal_eval` must stay linear in depth: the tests in `src/literal.rs` time the deepest parse, `benches/formats.rs` measures the curve but is manual, since CI runs no Rust bench diff --git a/litellm-rust/crates/python-compat/src/json.rs b/litellm-rust/crates/python-compat/src/json.rs index d3436519ac4..00c96ae2a13 100644 --- a/litellm-rust/crates/python-compat/src/json.rs +++ b/litellm-rust/crates/python-compat/src/json.rs @@ -166,3 +166,36 @@ fn write_string(out: &mut String, text: &str) { } out.push('"'); } + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::{dumps, to_json}; + use crate::Value; + + #[rstest] + #[case::bytes(Value::Bytes(b"x".to_vec()), "Object of type bytes is not JSON serializable")] + #[case::set(Value::Set(vec![Value::from(1)]), "Object of type set is not JSON serializable")] + #[case::complex(Value::Complex { re: 1.0, im: 2.0 }, "Object of type complex is not JSON serializable")] + #[case::nan( + Value::Float(f64::NAN), + "Out of range float values are not JSON compliant" + )] + fn to_json_reports_what_python_json_dumps_would_reject( + #[case] value: Value, + #[case] message: &str, + ) { + let error = to_json(&value).expect_err("value has no serde_json form"); + assert_eq!(error.to_string(), message); + } + + /// `json.dumps` writes the non-finite floats that `to_json` cannot represent. + #[rstest] + #[case::nan(f64::NAN, "NaN")] + #[case::infinity(f64::INFINITY, "Infinity")] + #[case::negative_infinity(f64::NEG_INFINITY, "-Infinity")] + fn dumps_writes_non_finite_floats(#[case] value: f64, #[case] text: &str) { + assert_eq!(dumps(&Value::Float(value)).unwrap(), text); + } +} diff --git a/litellm-rust/crates/python-compat/src/lib.rs b/litellm-rust/crates/python-compat/src/lib.rs index b23aff409f2..b1388e2c117 100644 --- a/litellm-rust/crates/python-compat/src/lib.rs +++ b/litellm-rust/crates/python-compat/src/lib.rs @@ -10,6 +10,7 @@ //! | [`json`] | `json.dumps(value)`, and `json.loads(json.dumps(value))` as a JSON value | //! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data | //! | [`truthy`] | `bool(value)` | +//! | [`read`] | reading a config value the way Python's loaders do | //! //! [`Value`] is the closed data model these formats share. Live Python objects //! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the @@ -25,6 +26,7 @@ mod error; pub mod json; pub mod literal; pub mod pickle; +pub mod read; pub mod repr; pub mod truthy; mod value; diff --git a/litellm-rust/crates/python-compat/src/literal.rs b/litellm-rust/crates/python-compat/src/literal.rs index 12e9036572f..361ebbd1a68 100644 --- a/litellm-rust/crates/python-compat/src/literal.rs +++ b/litellm-rust/crates/python-compat/src/literal.rs @@ -743,3 +743,52 @@ impl UniqueValues { self.keys.into_iter().zip(self.values).collect() } } + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use rstest::rstest; + + use super::literal_eval; + use crate::{Error, MAX_DEPTH, Value}; + + fn nested_text(open: &str, close: &str, depth: usize) -> String { + format!("{}1{}", open.repeat(depth), close.repeat(depth)) + } + + #[rstest] + #[case::list("[", "]")] + #[case::dict("{'a': ", "}")] + #[case::tuple("(", ",)")] + fn literal_eval_accepts_the_limit_and_rejects_past_it(#[case] open: &str, #[case] close: &str) { + assert!(literal_eval(&nested_text(open, close, MAX_DEPTH)).is_ok()); + assert!(matches!( + literal_eval(&nested_text(open, close, MAX_DEPTH + 1)), + Err(Error::TooDeep) + )); + } + + /// A backtracking parser (the `py_literal` grammar this replaced) doubles per nested level + /// and takes minutes here; the bound is loose enough to survive a slow debug build. + #[rstest] + #[case::list("[", "]")] + #[case::dict("{'a': ", "}")] + #[case::tuple("(", ",)")] + fn literal_eval_stays_linear_in_depth(#[case] open: &str, #[case] close: &str) { + let text = nested_text(open, close, MAX_DEPTH); + let start = Instant::now(); + assert!(literal_eval(&text).is_ok()); + let elapsed = start.elapsed(); + assert!( + elapsed < Duration::from_millis(50), + "{open} nested {MAX_DEPTH} deep took {elapsed:?}" + ); + } + + #[rstest] + fn literal_eval_ignores_brackets_inside_strings() { + let text = format!("'{}'", "[".repeat(MAX_DEPTH + 1)); + assert!(matches!(literal_eval(&text), Ok(Value::Str(_)))); + } +} diff --git a/litellm-rust/crates/python-compat/src/pickle.rs b/litellm-rust/crates/python-compat/src/pickle.rs index bff7442dc6b..e19867cd46d 100644 --- a/litellm-rust/crates/python-compat/src/pickle.rs +++ b/litellm-rust/crates/python-compat/src/pickle.rs @@ -185,3 +185,62 @@ impl<'de> Visitor<'de> for Seed { Ok(Value::Dict(entries)) } } + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::{dumps, loads}; + use crate::{Error, MAX_DEPTH, Value, literal::literal_eval}; + + fn nested_list(depth: usize) -> Value { + (0..depth).fold(Value::from(1), |value, _| Value::List(vec![value])) + } + + /// A protocol 3 pickle of `depth` nested lists around `1`: `EMPTY_LIST` per level, then + /// `BININT1 1`, then `APPEND` per level. Written by hand because `dumps` refuses the depth. + fn nested_list_pickle(depth: usize) -> Vec { + let mut data = vec![0x80, 3]; + data.extend(std::iter::repeat_n(b']', depth)); + data.extend([b'K', 1]); + data.extend(std::iter::repeat_n(b'a', depth)); + data.push(b'.'); + data + } + + #[rstest] + fn loads_accepts_the_limit_and_rejects_past_it() { + assert_eq!( + loads(&nested_list_pickle(MAX_DEPTH)).unwrap(), + nested_list(MAX_DEPTH) + ); + assert!(matches!( + loads(&nested_list_pickle(MAX_DEPTH + 1)), + Err(Error::InvalidPickle(_)) + )); + } + + #[rstest] + fn dumps_accepts_the_limit_and_rejects_past_it() { + assert!(dumps(&nested_list(MAX_DEPTH)).is_ok()); + assert!(matches!( + dumps(&nested_list(MAX_DEPTH + 2)), + Err(Error::TooDeep) + )); + } + + #[rstest] + #[case::set("{1, 2}", "set")] + #[case::complex("1+2j", "complex")] + fn dumps_refuses_types_it_would_change(#[case] source: &str, #[case] type_name: &str) { + let value = literal_eval(source).expect("source is a literal"); + assert!(matches!(dumps(&value), Err(Error::NotPicklable(name)) if name == type_name)); + } + + #[rstest] + #[case::class_reference(b"\x80\x02c__builtin__\ncomplex\nq\x00.".to_vec())] + #[case::trailing_data({ let mut data = dumps(&Value::from(1)).unwrap(); data.push(b'.'); data })] + fn loads_rejects(#[case] data: Vec) { + assert!(matches!(loads(&data), Err(Error::InvalidPickle(_)))); + } +} diff --git a/litellm-rust/crates/python-compat/src/read.rs b/litellm-rust/crates/python-compat/src/read.rs new file mode 100644 index 00000000000..2d649af2aca --- /dev/null +++ b/litellm-rust/crates/python-compat/src/read.rs @@ -0,0 +1,78 @@ +use serde::{Deserialize, Deserializer}; + +/// A typed value, or the text a config spells in its place: an `os.environ/NAME` reference +/// the secrets layer resolves later, or a word Python's coercion reads the value from. +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum Spelled { + Value(T), + Text(String), +} + +/// `x if isinstance(x, list) else [x]`, for `#[serde(deserialize_with)]`. +pub fn one_or_many<'de, D: Deserializer<'de>, T: Deserialize<'de>>( + deserializer: D, +) -> Result>, D::Error> { + #[derive(Deserialize)] + #[serde(untagged)] + enum OneOrMany { + Many(Box<[T]>), + One(T), + } + + Ok( + Option::>::deserialize(deserializer)?.map(|value| match value { + OneOrMany::Many(values) => values, + OneOrMany::One(value) => Box::from([value]), + }), + ) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde::Deserialize; + use serde_json::{Value, json}; + + use super::{Spelled, one_or_many}; + + #[derive(Deserialize)] + struct Entries { + #[serde(default, deserialize_with = "one_or_many")] + entries: Option>, + } + + fn entries(document: Value) -> Option> { + serde_json::from_value::(document).unwrap().entries + } + + #[rstest] + #[case::one(json!({"entries": "langfuse"}), Some(vec![json!("langfuse")]))] + #[case::one_mapping(json!({"entries": {"name": "otel"}}), Some(vec![json!({"name": "otel"})]))] + #[case::many(json!({"entries": ["langfuse", "otel"]}), Some(vec![json!("langfuse"), json!("otel")]))] + #[case::many_keeps_nested_lists(json!({"entries": [["a"], "b"]}), Some(vec![json!(["a"]), json!("b")]))] + #[case::empty(json!({"entries": []}), Some(vec![]))] + #[case::null(json!({"entries": null}), None)] + #[case::absent(json!({}), None)] + fn one_or_many_wraps_a_single_entry_and_keeps_a_list( + #[case] document: Value, + #[case] expected: Option>, + ) { + assert_eq!(entries(document), expected.map(Vec::into_boxed_slice)); + } + + #[rstest] + #[case::value(json!(3.5), Spelled::Value(3.5))] + #[case::integer(json!(3), Spelled::Value(3.0))] + #[case::reference(json!("os.environ/TIMEOUT"), Spelled::Text("os.environ/TIMEOUT".into()))] + #[case::numeric_text(json!("3"), Spelled::Text("3".into()))] + fn spelled_keeps_text_that_is_not_the_value_type( + #[case] input: Value, + #[case] expected: Spelled, + ) { + assert_eq!( + serde_json::from_value::>(input).unwrap(), + expected + ); + } +} diff --git a/litellm-rust/crates/python-compat/tests/limits.rs b/litellm-rust/crates/python-compat/tests/limits.rs deleted file mode 100644 index c4d931034c4..00000000000 --- a/litellm-rust/crates/python-compat/tests/limits.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::time::{Duration, Instant}; - -use litellm_python_compat::{Error, MAX_DEPTH, Value, json, literal::literal_eval, pickle}; -use rstest::{fixture, rstest}; - -/// The bracket pair of one container shape, as `(open, close)`. -#[fixture] -fn shapes() -> [(&'static str, &'static str); 3] { - [("[", "]"), ("{'a': ", "}"), ("(", ",)")] -} - -fn nested_text(open: &str, close: &str, depth: usize) -> String { - format!("{}1{}", open.repeat(depth), close.repeat(depth)) -} - -fn nested_list(depth: usize) -> Value { - (0..depth).fold(Value::from(1), |value, _| Value::List(vec![value])) -} - -/// A protocol 3 pickle of `depth` nested lists around `1`: `EMPTY_LIST` per level, then -/// `BININT1 1`, then `APPEND` per level. Written by hand because `dumps` refuses the depth. -fn nested_list_pickle(depth: usize) -> Vec { - let mut data = vec![0x80, 3]; - data.extend(std::iter::repeat_n(b']', depth)); - data.extend([b'K', 1]); - data.extend(std::iter::repeat_n(b'a', depth)); - data.push(b'.'); - data -} - -#[rstest] -fn literal_eval_accepts_the_limit_and_rejects_past_it(shapes: [(&'static str, &'static str); 3]) { - for (open, close) in shapes { - assert!(literal_eval(&nested_text(open, close, MAX_DEPTH)).is_ok()); - assert!(matches!( - literal_eval(&nested_text(open, close, MAX_DEPTH + 1)), - Err(Error::TooDeep) - )); - } -} - -/// A backtracking parser (the `py_literal` grammar this replaced) doubles per nested level -/// and takes minutes here; the bound is loose enough to survive a slow debug build. -#[rstest] -fn literal_eval_stays_linear_in_depth(shapes: [(&'static str, &'static str); 3]) { - for (open, close) in shapes { - let text = nested_text(open, close, MAX_DEPTH); - let start = Instant::now(); - assert!(literal_eval(&text).is_ok()); - let elapsed = start.elapsed(); - assert!( - elapsed < Duration::from_millis(50), - "{open} nested {MAX_DEPTH} deep took {elapsed:?}" - ); - } -} - -#[rstest] -fn literal_eval_ignores_brackets_inside_strings() { - let text = format!("'{}'", "[".repeat(MAX_DEPTH + 1)); - assert!(matches!(literal_eval(&text), Ok(Value::Str(_)))); -} - -#[rstest] -fn pickle_nesting_is_bounded_in_both_directions() { - assert_eq!( - pickle::loads(&nested_list_pickle(MAX_DEPTH)).unwrap(), - nested_list(MAX_DEPTH) - ); - assert!(matches!( - pickle::loads(&nested_list_pickle(MAX_DEPTH + 1)), - Err(Error::InvalidPickle(_)) - )); - assert!(pickle::dumps(&nested_list(MAX_DEPTH)).is_ok()); - assert!(matches!( - pickle::dumps(&nested_list(MAX_DEPTH + 2)), - Err(Error::TooDeep) - )); -} - -#[rstest] -#[case("{1, 2}", "set")] -#[case("1+2j", "complex")] -fn pickle_dumps_refuses_types_it_would_change(#[case] source: &str, #[case] type_name: &str) { - let value = literal_eval(source).expect("source is a literal"); - assert!(matches!(pickle::dumps(&value), Err(Error::NotPicklable(name)) if name == type_name)); -} - -#[rstest] -#[case(b"\x80\x02c__builtin__\ncomplex\nq\x00.".to_vec(), "a class reference")] -#[case({ let mut data = pickle::dumps(&Value::from(1)).unwrap(); data.push(b'.'); data }, "trailing data")] -fn pickle_loads_rejects(#[case] data: Vec, #[case] what: &str) { - assert!( - matches!(pickle::loads(&data), Err(Error::InvalidPickle(_))), - "{what} must not decode" - ); -} - -#[rstest] -#[case(Value::Bytes(b"x".to_vec()), "Object of type bytes is not JSON serializable")] -#[case(Value::Set(vec![Value::from(1)]), "Object of type set is not JSON serializable")] -#[case(Value::Complex { re: 1.0, im: 2.0 }, "Object of type complex is not JSON serializable")] -#[case( - Value::Float(f64::NAN), - "Out of range float values are not JSON compliant" -)] -fn to_json_reports_what_python_json_dumps_would_reject( - #[case] value: Value, - #[case] message: &str, -) { - let error = json::to_json(&value).expect_err("value has no serde_json form"); - assert_eq!(error.to_string(), message); -} - -/// `json.dumps` writes the non-finite floats that `to_json` cannot represent. -#[rstest] -#[case(f64::NAN, "NaN")] -#[case(f64::INFINITY, "Infinity")] -#[case(f64::NEG_INFINITY, "-Infinity")] -fn json_dumps_writes_non_finite_floats(#[case] value: f64, #[case] text: &str) { - assert_eq!(json::dumps(&Value::Float(value)).unwrap(), text); -} diff --git a/litellm-rust/crates/router-types/Cargo.toml b/litellm-rust/crates/router-types/Cargo.toml index 2e9fec28685..c6a273fa010 100644 --- a/litellm-rust/crates/router-types/Cargo.toml +++ b/litellm-rust/crates/router-types/Cargo.toml @@ -8,6 +8,7 @@ description = "The types of litellm/types/router.py: a deployment's litellm_para [dependencies] litellm-auth-types.workspace = true +litellm-python-compat.workspace = true serde.workspace = true serde_json.workspace = true serde_with.workspace = true diff --git a/litellm-rust/crates/router-types/src/lib.rs b/litellm-rust/crates/router-types/src/lib.rs index 2235e40a891..83c2c683ad5 100644 --- a/litellm-rust/crates/router-types/src/lib.rs +++ b/litellm-rust/crates/router-types/src/lib.rs @@ -1,5 +1,3 @@ mod litellm_params; -mod value; pub use litellm_params::{ExtraParams, LitellmParams}; -pub use value::Spelled; diff --git a/litellm-rust/crates/router-types/src/litellm_params.rs b/litellm-rust/crates/router-types/src/litellm_params.rs index 186a1d98036..cc73a1c85bf 100644 --- a/litellm-rust/crates/router-types/src/litellm_params.rs +++ b/litellm-rust/crates/router-types/src/litellm_params.rs @@ -5,7 +5,7 @@ use serde::Deserialize; use serde_json::Value; use serde_with::{OneOrMany, formats::PreferMany, serde_as}; -use crate::Spelled; +use litellm_python_compat::read::Spelled; /// Python's `LiteLLM_Params`: a deployment's `litellm_params`, as a config spells them and as /// a caller passes them. Each credential family is one flattened group owned by its auth @@ -30,11 +30,11 @@ pub struct LitellmParams { pub custom_llm_provider: Option, pub timeout: Option>, pub stream_timeout: Option>, - pub max_retries: Option>, - pub tpm: Option>, - pub rpm: Option>, - pub itpm: Option>, - pub otpm: Option>, + pub max_retries: Option>, + pub tpm: Option>, + pub rpm: Option>, + pub itpm: Option>, + pub otpm: Option>, pub max_parallel_requests: Option, #[serde_as(as = "Option>")] pub organization: Option>, diff --git a/litellm-rust/crates/router-types/src/value.rs b/litellm-rust/crates/router-types/src/value.rs deleted file mode 100644 index e52cf60c0ef..00000000000 --- a/litellm-rust/crates/router-types/src/value.rs +++ /dev/null @@ -1,10 +0,0 @@ -use serde::Deserialize; - -/// A typed value, or the text a config spells in its place: an `os.environ/NAME` reference -/// the secrets layer resolves later, or a word Python's coercion reads the value from. -#[derive(Clone, Debug, Deserialize, PartialEq)] -#[serde(untagged)] -pub enum Spelled { - Value(T), - Text(String), -} diff --git a/litellm-rust/crates/router-types/tests/litellm_params.rs b/litellm-rust/crates/router-types/tests/litellm_params.rs index 7d021fd857a..186444b816d 100644 --- a/litellm-rust/crates/router-types/tests/litellm_params.rs +++ b/litellm-rust/crates/router-types/tests/litellm_params.rs @@ -1,8 +1,10 @@ -use litellm_auth_types::{AwsParams, SecretValue, VertexParams}; -use litellm_router_types::{LitellmParams, Spelled}; use rstest::rstest; use serde_json::{Map, Value, json}; +use litellm_auth_types::{AwsParams, SecretValue, VertexParams}; +use litellm_python_compat::read::Spelled; +use litellm_router_types::LitellmParams; + fn params(value: Value) -> LitellmParams { serde_json::from_value(value).unwrap() } @@ -55,7 +57,7 @@ fn params(value: Value) -> LitellmParams { LitellmParams { model: "m".into(), timeout: Some(Spelled::Text("os.environ/T".into())), - rpm: Some(Spelled::Value(5.0)), + rpm: Some(Spelled::Value(5)), drop_params: Some(Spelled::Text("true".into())), tags: Some(Box::from(["a".to_string()])), max_budget: Some(1.5), @@ -116,6 +118,9 @@ fn keys_no_field_names_land_in_extra_and_nothing_else_does() { #[case::vertex(json!({"model": "m", "vertex_project": ["p"]}))] #[case::vertex_credentials(json!({"model": "m", "vertex_credentials": 7}))] #[case::rate_limit(json!({"model": "m", "max_parallel_requests": "many"}))] +#[case::fractional_rpm(json!({"model": "m", "rpm": 1.5}))] +#[case::fractional_max_retries(json!({"model": "m", "max_retries": 2.5}))] +#[case::negative_tpm(json!({"model": "m", "tpm": -1}))] #[case::tags(json!({"model": "m", "tags": "not-a-list"}))] fn a_param_of_the_wrong_type_is_rejected(#[case] value: Value) { assert!(serde_json::from_value::(value).is_err()); From e979df54a4b07760fddc35cc69f352ba3131daee Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Thu, 22 Oct 2026 11:41:00 -0700 Subject: [PATCH 2/5] refactor(gateway): drop the MCP unsupported-setting gate Co-Authored-By: Claude Opus 5.5 --- litellm-rust/crates/gateway/src/error.rs | 2 - litellm-rust/crates/gateway/src/lib.rs | 13 --- .../crates/gateway/src/mcp_settings.rs | 62 -------------- litellm-rust/crates/gateway/tests/mcp.rs | 85 ------------------- 4 files changed, 162 deletions(-) delete mode 100644 litellm-rust/crates/gateway/src/mcp_settings.rs diff --git a/litellm-rust/crates/gateway/src/error.rs b/litellm-rust/crates/gateway/src/error.rs index 75dc04d179d..a29fce3e64f 100644 --- a/litellm-rust/crates/gateway/src/error.rs +++ b/litellm-rust/crates/gateway/src/error.rs @@ -4,8 +4,6 @@ pub enum Error { Http(#[from] litellm_http::Error), #[error("environment_variables values must be strings, numbers or booleans")] Environment, - #[error("MCP host does not yet support configured setting {0}")] - McpSetting(String), #[error(transparent)] Mcp(#[from] litellm_gateway_mcp::ConnectError), #[error("MCP requires a configured master key")] diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 1218db083c0..a495c95238b 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,6 +1,5 @@ mod auth; mod error; -mod mcp_settings; mod secrets; pub use error::Error; @@ -82,21 +81,9 @@ pub async fn build_mcp( pool: &HttpClientPool, http: &litellm_http::HttpClientConfig, ) -> Result, Error> { - if !config.mcp_tools.is_empty() { - return Err(Error::McpSetting("mcp_tools".into())); - } if config.mcp_servers.is_empty() { return Ok(None); } - if let Some(setting) = mcp_settings::first_unsupported(config) { - return Err(Error::McpSetting(setting.into())); - } - if !config.guardrails.is_empty() - || !config.policies.is_empty() - || !config.policy_attachments.is_empty() - { - return Err(Error::McpSetting("guardrails or policies".into())); - } Auth::from_config(config, secrets.clone()) .validate() .await?; diff --git a/litellm-rust/crates/gateway/src/mcp_settings.rs b/litellm-rust/crates/gateway/src/mcp_settings.rs deleted file mode 100644 index 7213e0c2530..00000000000 --- a/litellm-rust/crates/gateway/src/mcp_settings.rs +++ /dev/null @@ -1,62 +0,0 @@ -use litellm_config::Config; - -pub(crate) fn first_unsupported(config: &Config) -> Option<&'static str> { - let general = &config.general_settings.mcp; - let litellm = &config.litellm_settings.mcp; - [ - ( - "mcp_advertised_versions", - general.mcp_advertised_versions.is_some(), - ), - ( - "mcp_disabled_capabilities", - general.mcp_disabled_capabilities.is_some(), - ), - ("mcp_allowed_clients", general.mcp_allowed_clients.is_some()), - ( - "mcp_require_explicit_server_scope", - general.mcp_require_explicit_server_scope.is_some(), - ), - ( - "mcp_client_id_header", - general.mcp_client_id_header.is_some(), - ), - ( - "mcp_client_side_auth_header_name", - general.mcp_client_side_auth_header_name.is_some(), - ), - ( - "mcp_allow_all_keys_respects_mcp_scope", - general.mcp_allow_all_keys_respects_mcp_scope.is_some(), - ), - ( - "mcp_prefer_client_id_metadata_document", - general.mcp_prefer_client_id_metadata_document.is_some(), - ), - ( - "mcp_internal_ip_ranges", - general.mcp_internal_ip_ranges.is_some(), - ), - ( - "mcp_trusted_proxy_ranges", - general.mcp_trusted_proxy_ranges.is_some(), - ), - ( - "mcp_xff_num_trusted_hops", - general.mcp_xff_num_trusted_hops.is_some(), - ), - ("mcp_required_fields", general.mcp_required_fields.is_some()), - ( - "mcp_session_token_signing", - general.mcp_session_token_signing.is_some(), - ), - ("mcp_aliases", litellm.mcp_aliases.is_some()), - ( - "mcp_semantic_tool_filter", - litellm.mcp_semantic_tool_filter.is_some(), - ), - ("mcp_tool_search", litellm.mcp_tool_search.is_some()), - ] - .into_iter() - .find_map(|(key, set)| set.then_some(key)) -} diff --git a/litellm-rust/crates/gateway/tests/mcp.rs b/litellm-rust/crates/gateway/tests/mcp.rs index 7494fb9bdf4..9e52a73f191 100644 --- a/litellm-rust/crates/gateway/tests/mcp.rs +++ b/litellm-rust/crates/gateway/tests/mcp.rs @@ -322,88 +322,3 @@ fn applies_configured_http_environment_before_building_clients() { if path == missing )); } - -#[rstest] -#[case::client_policy( - "general_settings: {master_key: key, mcp_allowed_clients: [approved]}", - "mcp_allowed_clients" -)] -#[case::aliases( - "general_settings: {master_key: key}\nlitellm_settings: {mcp_aliases: {d: docs}}", - "mcp_aliases" -)] -#[case::tool_search( - "general_settings: {master_key: key}\nlitellm_settings: {mcp_tool_search: {top_k: 3}}", - "mcp_tool_search" -)] -#[case::advertised_versions( - "general_settings: {mcp_advertised_versions: [2025-06-18]}", - "mcp_advertised_versions" -)] -#[case::spelled_bool( - "general_settings: {mcp_require_explicit_server_scope: 'true'}", - "mcp_require_explicit_server_scope" -)] -#[case::header( - "general_settings: {mcp_client_side_auth_header_name: x-auth}", - "mcp_client_side_auth_header_name" -)] -#[case::hops( - "general_settings: {mcp_xff_num_trusted_hops: 2}", - "mcp_xff_num_trusted_hops" -)] -#[case::signing( - "general_settings: {mcp_session_token_signing: {algorithm: RS256}}", - "mcp_session_token_signing" -)] -#[case::semantic_filter( - "litellm_settings: {mcp_semantic_tool_filter: {enabled: true}}", - "mcp_semantic_tool_filter" -)] -#[case::guardrail("guardrails: [{guardrail_name: policy}]", "guardrails or policies")] -#[case::local_tools("mcp_tools: [{name: custom}]", "mcp_tools")] -#[tokio::test] -async fn refuses_unimplemented_global_mcp_policy(#[case] settings: &str, #[case] key: &str) { - let config = Config::from_yaml(&format!( - "{settings}\nmcp_servers: {{docs: {{url: 'http://127.0.0.1:1/mcp'}}}}\n" - )) - .unwrap(); - let inference = litellm_gateway::build_inference(&config, &Environment::default()).unwrap(); - let error = litellm_gateway::build_mcp( - &config, - Arc::new(NoSecrets), - CancellationToken::new(), - &inference.resources.pool, - &inference.http, - ) - .await - .err() - .unwrap(); - assert!(matches!(error, litellm_gateway::Error::McpSetting(setting) if setting == key)); -} - -#[rstest] -#[case::general("general_settings: {mcp_whatever: 1}")] -#[case::litellm("litellm_settings: {mcp_whatever: 1}")] -#[case::null("general_settings: {mcp_trusted_proxy_ranges: null}")] -#[case::allowed_hosts("general_settings: {mcp_allowed_hosts: [localhost]}")] -#[case::allowed_origins("general_settings: {mcp_allowed_origins: ['https://app.test']}")] -#[tokio::test] -async fn unknown_null_or_supported_mcp_keys_do_not_block_mcp(#[case] settings: &str) { - let config = Config::from_yaml(&format!( - "{settings}\nmcp_servers: {{docs: {{url: 'http://127.0.0.1:1/mcp'}}}}\n" - )) - .unwrap(); - let inference = litellm_gateway::build_inference(&config, &Environment::default()).unwrap(); - let error = litellm_gateway::build_mcp( - &config, - Arc::new(NoSecrets), - CancellationToken::new(), - &inference.resources.pool, - &inference.http, - ) - .await - .err() - .unwrap(); - assert!(matches!(error, litellm_gateway::Error::Auth(_)), "{error}"); -} From 626f4d77242034c26fab8555fef3308e282c087b Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Thu, 22 Oct 2026 12:57:17 -0700 Subject: [PATCH 3/5] refactor(http): split user-facing HTTP settings into http-types HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive and EnvironmentProxies move to a new litellm-http-types crate that links neither reqwest nor rustls, so config-load can lower http settings without a TLS stack. UiCredentials moves to auth-types so gateway-auth and gateway-ui can share it, and the MCP auth glue moves from gateway into gateway-mcp Co-Authored-By: Claude Opus 5.5 --- litellm-rust/Cargo.lock | 12 ++ litellm-rust/Cargo.toml | 1 + litellm-rust/crates/auth-types/AGENTS.md | 35 ++++ litellm-rust/crates/auth-types/src/lib.rs | 2 + litellm-rust/crates/auth-types/src/ui.rs | 9 + litellm-rust/crates/config-load/AGENTS.md | 1 + litellm-rust/crates/config-load/Cargo.toml | 1 + litellm-rust/crates/config-load/src/http.rs | 156 ++++++++++++++++ litellm-rust/crates/config-load/src/lib.rs | 7 +- .../crates/config-load/src/transport.rs | 166 ----------------- litellm-rust/crates/gateway-auth/src/lib.rs | 2 +- litellm-rust/crates/gateway-auth/src/ui.rs | 9 +- litellm-rust/crates/gateway-auth/tests/ui.rs | 4 +- litellm-rust/crates/gateway-mcp/Cargo.toml | 6 +- .../{gateway => gateway-mcp}/src/auth.rs | 30 +-- litellm-rust/crates/gateway-mcp/src/error.rs | 9 +- litellm-rust/crates/gateway-mcp/src/lib.rs | 2 + litellm-rust/crates/gateway-ui/Cargo.toml | 2 +- litellm-rust/crates/gateway-ui/src/session.rs | 3 +- litellm-rust/crates/gateway/src/lib.rs | 41 +--- litellm-rust/crates/http-types/AGENTS.md | 21 +++ litellm-rust/crates/http-types/Cargo.toml | 13 ++ litellm-rust/crates/http-types/src/lib.rs | 7 + litellm-rust/crates/http-types/src/proxy.rs | 126 +++++++++++++ .../{http => http-types}/src/settings.rs | 2 +- litellm-rust/crates/http/AGENTS.md | 43 ++++- litellm-rust/crates/http/Cargo.toml | 1 + litellm-rust/crates/http/src/config.rs | 8 +- litellm-rust/crates/http/src/lib.rs | 6 +- litellm-rust/crates/http/src/mcp.rs | 4 +- litellm-rust/crates/http/src/media.rs | 6 +- litellm-rust/crates/http/src/pool.rs | 8 +- litellm-rust/crates/http/src/proxy.rs | 175 +++++------------- litellm-rust/crates/http/src/tls.rs | 4 +- 34 files changed, 527 insertions(+), 395 deletions(-) create mode 100644 litellm-rust/crates/auth-types/AGENTS.md create mode 100644 litellm-rust/crates/auth-types/src/ui.rs create mode 100644 litellm-rust/crates/config-load/src/http.rs delete mode 100644 litellm-rust/crates/config-load/src/transport.rs rename litellm-rust/crates/{gateway => gateway-mcp}/src/auth.rs (72%) create mode 100644 litellm-rust/crates/http-types/AGENTS.md create mode 100644 litellm-rust/crates/http-types/Cargo.toml create mode 100644 litellm-rust/crates/http-types/src/lib.rs create mode 100644 litellm-rust/crates/http-types/src/proxy.rs rename litellm-rust/crates/{http => http-types}/src/settings.rs (99%) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 6632fd279fd..ff2232ed0e7 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3664,6 +3664,7 @@ version = "0.1.0" dependencies = [ "litellm-config", "litellm-core-utils", + "litellm-http-types", "rstest", "serde_yaml_ng", "tempfile", @@ -3851,6 +3852,7 @@ dependencies = [ "http 1.4.2", "litellm-auth-types", "litellm-config", + "litellm-gateway-auth", "litellm-secrets", "moka", "rmcp", @@ -3958,6 +3960,7 @@ dependencies = [ "http 1.4.2", "hyper-util", "litellm-core-utils", + "litellm-http-types", "rcgen", "reqwest 0.12.28", "reqwest 0.13.5", @@ -3977,6 +3980,15 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "litellm-http-types" +version = "0.1.0" +dependencies = [ + "litellm-core-utils", + "rstest", + "veil", +] + [[package]] name = "litellm-inference" version = "0.1.0" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index ce86003de28..d290ca4ce7d 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -48,6 +48,7 @@ litellm-secrets-hashicorp = { path = "crates/secrets-hashicorp" } litellm-secrets-azure = { path = "crates/secrets-azure" } litellm-secrets-cyberark = { path = "crates/secrets-cyberark" } litellm-http = { path = "crates/http" } +litellm-http-types = { path = "crates/http-types" } litellm-llms = { path = "crates/llms" } litellm-llms-types = { path = "crates/llms-types" } litellm-core-utils = { path = "crates/core-utils" } diff --git a/litellm-rust/crates/auth-types/AGENTS.md b/litellm-rust/crates/auth-types/AGENTS.md new file mode 100644 index 00000000000..60998e4d399 --- /dev/null +++ b/litellm-rust/crates/auth-types/AGENTS.md @@ -0,0 +1,35 @@ +## Scope + +- This crate is the vocabulary for authenticating LiteLLM to a provider: how a credential is named before it is known (`CredentialRef`, `CredentialPlan`, `ParamSpec`), how it is obtained (`CredentialResolver`, `TokenProvider`, `ResolvedCredential`), how it is carried (`SecretValue`, `Sourced`), how it lands on a request (`CredentialPlacement`, `ProviderAuthPolicy`) and how that fails (`Error`) +- Outbound only. Who is calling the gateway is `gateway-auth`. The one exception is `UiCredentials`, which is here because `gateway-auth` and `gateway-ui` both read it and neither may depend on the other +- No I/O and no clock. Resolving a reference, fetching a token or reading a file is a `CredentialResolver` or `TokenProvider` implementation in `auth-aws`, `auth-azure`, `auth-gcp` or a host. This crate only defines the traits and runs the plan over an injected handle +- Depends on `serde`, `serde_json`, `subtle`, `thiserror` and `veil` only. Nothing here may pull in an HTTP client, a cloud SDK or a provider crate + +## Layering + +- `auth-types` (contracts) is below `auth-aws`, `auth-azure` and `auth-gcp` (one cloud's acquisition each), which are below `auth` (the `AuthServices` facade that bundles them behind features). `llms`, `inference-*`, `secrets-*` and `cache-*` take the types from here and the services from `auth` +- Import every type from `litellm_auth_types`. `auth` still glob re-exports this crate and `secrets-types` re-exports `SecretValue`, both against the workspace rule in `../inference/AGENTS.md`. Do not add a third path, and move consumers off those two when touching them +- A provider's own credential shape (`AwsParams`, `VertexParams`) lives in `src/params/` only because routing, config and the provider all read it. A field or env name used by one provider stays in that provider's crate + +## Contracts + +- `SecretValue` is the only way a secret moves between crates. `Debug` is redacted through `veil`, equality is constant time, and `expose()` is the single read. Never add `Display`, `Serialize` or an `AsRef` to it +- `Sourced` records whether a value came from the request, the deployment or the environment. Precedence between those is decided by whoever builds the `Sourced`, never by this crate +- `CredentialPlan::resolve` has three outcomes and they are not interchangeable. `Resolved` carries a credential, `Unavailable` means the caller may fall back to the next plan, and `Err` is terminal. A resolver that cannot answer returns `Declined` or `Missing`, it never errors to mean "not mine" +- A `CredentialRef::Explicit` resolves without consulting the resolver. Everything else, including `Host`, is resolved at acquisition time so rotated secrets are read fresh +- `apply_credential` prepends the credential header and refuses an empty credential or a header already present. Whether an existing header is preserved or rejected is `ProviderAuthPolicy::existing_header_behavior`, chosen per provider, never a default here +- `ParamSpec` lists every place Python reads one connection param, in order: wire names, the `litellm.` module global, then environment names. `resolve` walks that order and treats blank as absent. The static tables in `src/params/` are the one copy of those names + +## Errors + +- `Error` is the single credential error every provider nests. `auth-aws` maps its SDK failures into it with `From`, `auth-azure` and `auth-gcp` build the variants directly. Either way a consumer never sees a cloud SDK error type +- `MissingApiKey`, `MissingApiBase` and `MissingParam` are what the end user reads when a setting is absent. The message names what to pass or set, and `ParamSpec::guidance` derives that from the spec so the two never drift +- `ErrorDetail::Failed` wraps a source error by `Arc`, so equality is pointer identity. Tests compare the `Message`, `Empty`, `InvalidType` and `DuplicateHeader` variants and only match on `Failed` + +## Known gaps + +- `ProviderAuthPolicy` and `CredentialPlan` have no production consumer yet, `auth/tests/facade.rs` is the only user. Wire them through `llms/src/base_llm/auth.rs` or delete them, do not grow them unused + +## Tests + +- Test what crosses the boundary: redaction in `Debug` output, which header a placement writes, which plan outcome a resolver answer produces. Never test that a type derives something diff --git a/litellm-rust/crates/auth-types/src/lib.rs b/litellm-rust/crates/auth-types/src/lib.rs index c875bda7979..3e807431610 100644 --- a/litellm-rust/crates/auth-types/src/lib.rs +++ b/litellm-rust/crates/auth-types/src/lib.rs @@ -7,6 +7,7 @@ mod params; mod policy; mod secret; mod token; +mod ui; use serde::{Deserialize, Serialize}; @@ -57,3 +58,4 @@ pub use params::{AwsParams, ParamSpec, VertexParams}; pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; pub use secret::SecretValue; pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; +pub use ui::UiCredentials; diff --git a/litellm-rust/crates/auth-types/src/ui.rs b/litellm-rust/crates/auth-types/src/ui.rs new file mode 100644 index 00000000000..9e8aaa6d1b7 --- /dev/null +++ b/litellm-rust/crates/auth-types/src/ui.rs @@ -0,0 +1,9 @@ +use serde::Deserialize; + +use crate::SecretValue; + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct UiCredentials { + pub username: String, + pub password: SecretValue, +} diff --git a/litellm-rust/crates/config-load/AGENTS.md b/litellm-rust/crates/config-load/AGENTS.md index bd4892d8d40..bfd68815a9a 100644 --- a/litellm-rust/crates/config-load/AGENTS.md +++ b/litellm-rust/crates/config-load/AGENTS.md @@ -9,3 +9,4 @@ ## Precedence - Each lowered value takes the env, the file and then the default in the order Python uses for that key today, until that order is normalized +- HTTP client settings are lowered into `litellm-http-types` layers (`http_settings`), one layer per source, so this crate never defines its own transport types diff --git a/litellm-rust/crates/config-load/Cargo.toml b/litellm-rust/crates/config-load/Cargo.toml index 5810cccb77b..49343a46353 100644 --- a/litellm-rust/crates/config-load/Cargo.toml +++ b/litellm-rust/crates/config-load/Cargo.toml @@ -8,6 +8,7 @@ repository.workspace = true [dependencies] litellm-config.workspace = true litellm-core-utils.workspace = true +litellm-http-types.workspace = true serde_yaml_ng = "0.10.0" thiserror.workspace = true diff --git a/litellm-rust/crates/config-load/src/http.rs b/litellm-rust/crates/config-load/src/http.rs new file mode 100644 index 00000000000..f672cea4ae8 --- /dev/null +++ b/litellm-rust/crates/config-load/src/http.rs @@ -0,0 +1,156 @@ +use litellm_config::{LiteLlmSettings, Spelled}; +use litellm_http_types::{HttpSettings, HttpSettingsLayer, SslVerify}; + +use crate::Environment; + +pub fn http_settings(settings: &LiteLlmSettings, environment: &Environment) -> HttpSettings { + HttpSettings::from_layers([ + HttpSettingsLayer::from_environment(environment), + written(settings), + ]) +} + +fn written(settings: &LiteLlmSettings) -> HttpSettingsLayer { + HttpSettingsLayer { + ssl_verify: settings.tls.ssl_verify.as_ref().map(ssl_verify), + ssl_certificate: settings + .client_certificate + .ssl_certificate + .clone() + .map(Into::into), + ssl_security_level: settings.tls.ssl_security_level.clone(), + ssl_ecdh_curve: settings.tls.ssl_ecdh_curve.clone(), + force_ipv4: settings.socket.force_ipv4, + http2: settings.http2.http2, + aiohttp_trust_env: settings.aiohttp.aiohttp_trust_env, + disable_aiohttp_trust_env: settings.aiohttp.disable_aiohttp_trust_env, + disable_aiohttp_transport: settings.aiohttp.disable_aiohttp_transport, + ..HttpSettingsLayer::default() + } +} + +fn ssl_verify(value: &Spelled) -> SslVerify { + match value { + Spelled::Value(true) => SslVerify::Enabled, + Spelled::Value(false) => SslVerify::Disabled, + Spelled::Text(text) => SslVerify::parse(text), + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use rstest::rstest; + + use litellm_config::Config; + use litellm_http_types::{HttpSettings, SslVerify}; + + use super::http_settings; + use crate::Environment; + + fn lowered(yaml: &str, variables: &[(&str, &str)]) -> HttpSettings { + let config = Config::from_yaml(yaml).unwrap(); + let environment = variables + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + http_settings(&config.litellm_settings, &environment) + } + + #[test] + fn absent_everywhere_is_the_default() { + assert_eq!(lowered("{}", &[]), HttpSettings::default()); + } + + #[rstest] + #[case::default("{}", &[], None)] + #[case::written("litellm_settings: {ssl_verify: false}", &[], Some(SslVerify::Disabled))] + #[case::written_text("litellm_settings: {ssl_verify: 'False'}", &[], Some(SslVerify::Disabled))] + #[case::written_bundle("litellm_settings: {ssl_verify: /etc/roots.pem}", &[], Some(SslVerify::CaBundle(PathBuf::from("/etc/roots.pem"))))] + #[case::environment("{}", &[("SSL_VERIFY", "False")], Some(SslVerify::Disabled))] + #[case::environment_over_file("litellm_settings: {ssl_verify: false}", &[("SSL_VERIFY", "true")], Some(SslVerify::Enabled))] + fn ssl_verify_takes_environment_then_file( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: Option, + ) { + assert_eq!(lowered(yaml, variables).ssl_verify, expected); + } + + #[rstest] + #[case::file("litellm_settings: {ssl_certificate: /etc/client.pem}", &[], "/etc/client.pem")] + #[case::environment_over_file("litellm_settings: {ssl_certificate: /etc/client.pem}", &[("SSL_CERTIFICATE", "/env/client.pem")], "/env/client.pem")] + fn client_certificate_takes_environment_then_file( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: &str, + ) { + assert_eq!( + lowered(yaml, variables).ssl_certificate, + Some(PathBuf::from(expected)) + ); + } + + #[rstest] + #[case::file("litellm_settings: {http2: true}", &[], true)] + #[case::environment("{}", &[("LITELLM_HTTP2", "TRUE")], true)] + #[case::environment_false_does_not_override_file("litellm_settings: {http2: true}", &[("LITELLM_HTTP2", "false")], true)] + fn flags_are_only_turned_on_by_the_environment( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: bool, + ) { + assert_eq!(lowered(yaml, variables).http2, expected); + } + + #[test] + fn force_ipv4_comes_from_the_file_only() { + assert!(lowered("litellm_settings: {force_ipv4: true}", &[]).force_ipv4); + assert!(!lowered("{}", &[("FORCE_IPV4", "true")]).force_ipv4); + } + + #[rstest] + #[case::file("litellm_settings: {ssl_security_level: '1'}", &[], Some("1"))] + #[case::environment_over_file("litellm_settings: {ssl_security_level: '1'}", &[("SSL_SECURITY_LEVEL", "2")], Some("2"))] + #[case::empty_is_unset("{}", &[("SSL_SECURITY_LEVEL", "")], None)] + fn text_settings_skip_empty_values( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: Option<&str>, + ) { + assert_eq!( + lowered(yaml, variables).ssl_security_level.as_deref(), + expected + ); + } + + #[test] + fn aiohttp_flags_from_the_file_decide_proxy_trust() { + assert!( + !lowered("litellm_settings: {disable_aiohttp_trust_env: true}", &[]).trust_proxy_env + ); + assert!( + lowered( + "litellm_settings: {disable_aiohttp_trust_env: true, aiohttp_trust_env: true}", + &[] + ) + .trust_proxy_env + ); + } + + #[test] + fn later_overlay_wins_and_unrelated_variables_are_ignored() { + let environment = Environment::from_iter([("SSL_VERIFY".to_owned(), "true".to_owned())]) + .overlaid([ + ("SSL_VERIFY".to_owned(), "false".to_owned()), + ("HTTP2".to_owned(), "true".to_owned()), + ]); + let settings = http_settings( + &Config::from_yaml("{}").unwrap().litellm_settings, + &environment, + ); + assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); + assert!(!settings.http2); + } +} diff --git a/litellm-rust/crates/config-load/src/lib.rs b/litellm-rust/crates/config-load/src/lib.rs index 62f1ed876b5..0cd4f3b58a4 100644 --- a/litellm-rust/crates/config-load/src/lib.rs +++ b/litellm-rust/crates/config-load/src/lib.rs @@ -1,16 +1,13 @@ mod error; +mod http; pub mod include; -mod transport; use std::{collections::BTreeMap, ffi::OsString, path::Path}; use litellm_config::Config; use litellm_core_utils::settings::Lookup; -pub use self::{ - error::Error, - transport::{SslVerify, Transport}, -}; +pub use self::{error::Error, http::http_settings}; pub fn load(path: impl AsRef) -> Result { let path = path.as_ref(); diff --git a/litellm-rust/crates/config-load/src/transport.rs b/litellm-rust/crates/config-load/src/transport.rs deleted file mode 100644 index ba1f95b8296..00000000000 --- a/litellm-rust/crates/config-load/src/transport.rs +++ /dev/null @@ -1,166 +0,0 @@ -use std::path::PathBuf; - -use litellm_core_utils::{serde_compat::parse_str_bool, settings::Lookup}; - -use litellm_config::{LiteLlmSettings, Spelled}; - -use crate::Environment; - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub enum SslVerify { - #[default] - Enabled, - Disabled, - CaBundle(PathBuf), -} - -impl SslVerify { - fn parse(value: &str) -> Self { - match parse_str_bool(value) { - Some(true) => Self::Enabled, - Some(false) => Self::Disabled, - None => Self::CaBundle(PathBuf::from(value)), - } - } - - fn written(value: &Spelled) -> Self { - match value { - Spelled::Value(true) => Self::Enabled, - Spelled::Value(false) => Self::Disabled, - Spelled::Text(text) => Self::parse(text), - } - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct Transport { - pub ssl_verify: SslVerify, - pub ssl_certificate: Option, - pub ssl_security_level: Option, - pub ssl_ecdh_curve: Option, - pub force_ipv4: bool, - pub http2: bool, - pub aiohttp_trust_env: bool, - pub disable_aiohttp_trust_env: bool, - pub disable_aiohttp_transport: bool, -} - -impl Transport { - pub fn lower(settings: &LiteLlmSettings, environment: &Environment) -> Self { - let text = |name: &str, written: &Option| { - environment - .get(name) - .or_else(|| written.clone()) - .filter(|value| !value.is_empty()) - }; - let flag = |name: &str, written: Option| { - environment.enabled(name).or(written).unwrap_or(false) - }; - Self { - ssl_verify: environment - .get("SSL_VERIFY") - .map(|value| SslVerify::parse(&value)) - .or_else(|| settings.tls.ssl_verify.as_ref().map(SslVerify::written)) - .unwrap_or_default(), - ssl_certificate: environment - .get("SSL_CERTIFICATE") - .or_else(|| settings.client_certificate.ssl_certificate.clone()) - .map(PathBuf::from), - ssl_security_level: text("SSL_SECURITY_LEVEL", &settings.tls.ssl_security_level), - ssl_ecdh_curve: text("SSL_ECDH_CURVE", &settings.tls.ssl_ecdh_curve), - force_ipv4: settings.socket.force_ipv4.unwrap_or(false), - http2: flag("LITELLM_HTTP2", settings.http2.http2), - aiohttp_trust_env: flag("AIOHTTP_TRUST_ENV", settings.aiohttp.aiohttp_trust_env), - disable_aiohttp_trust_env: flag( - "DISABLE_AIOHTTP_TRUST_ENV", - settings.aiohttp.disable_aiohttp_trust_env, - ), - disable_aiohttp_transport: flag( - "DISABLE_AIOHTTP_TRANSPORT", - settings.aiohttp.disable_aiohttp_transport, - ), - } - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use rstest::rstest; - - use super::{SslVerify, Transport}; - use litellm_config::Config; - - use crate::Environment; - - fn lowered(yaml: &str, variables: &[(&str, &str)]) -> Transport { - let config = Config::from_yaml(yaml).unwrap(); - let environment = variables - .iter() - .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) - .collect(); - Transport::lower(&config.litellm_settings, &environment) - } - - #[rstest] - fn absent_everywhere_is_the_default() { - assert_eq!(lowered("{}", &[]), Transport::default()); - } - - #[rstest] - #[case::default("{}", &[], SslVerify::Enabled)] - #[case::written("litellm_settings: {ssl_verify: false}", &[], SslVerify::Disabled)] - #[case::written_bundle("litellm_settings: {ssl_verify: /etc/roots.pem}", &[], SslVerify::CaBundle(PathBuf::from("/etc/roots.pem")))] - #[case::environment("{}", &[("SSL_VERIFY", "False")], SslVerify::Disabled)] - #[case::environment_over_file("litellm_settings: {ssl_verify: false}", &[("SSL_VERIFY", "true")], SslVerify::Enabled)] - fn ssl_verify_takes_environment_then_file_then_default( - #[case] yaml: &str, - #[case] variables: &[(&str, &str)], - #[case] expected: SslVerify, - ) { - assert_eq!(lowered(yaml, variables).ssl_verify, expected); - } - - #[rstest] - #[case::file("litellm_settings: {http2: true}", &[], true)] - #[case::environment("{}", &[("LITELLM_HTTP2", "TRUE")], true)] - #[case::environment_false_does_not_override_file("litellm_settings: {http2: true}", &[("LITELLM_HTTP2", "false")], true)] - fn flags_are_only_turned_on_by_the_environment( - #[case] yaml: &str, - #[case] variables: &[(&str, &str)], - #[case] expected: bool, - ) { - assert_eq!(lowered(yaml, variables).http2, expected); - } - - #[rstest] - #[case::file("litellm_settings: {ssl_security_level: '1'}", &[], Some("1"))] - #[case::environment_over_file("litellm_settings: {ssl_security_level: '1'}", &[("SSL_SECURITY_LEVEL", "2")], Some("2"))] - #[case::empty_is_unset("{}", &[("SSL_SECURITY_LEVEL", "")], None)] - fn text_settings_skip_empty_values( - #[case] yaml: &str, - #[case] variables: &[(&str, &str)], - #[case] expected: Option<&str>, - ) { - assert_eq!( - lowered(yaml, variables).ssl_security_level.as_deref(), - expected - ); - } - - #[rstest] - fn later_overlay_wins_and_unrelated_variables_are_ignored() { - let environment = Environment::from_iter([("SSL_VERIFY".to_owned(), "true".to_owned())]) - .overlaid([ - ("SSL_VERIFY".to_owned(), "false".to_owned()), - ("HTTP2".to_owned(), "true".to_owned()), - ]); - let transport = Transport::lower( - &Config::from_yaml("{}").unwrap().litellm_settings, - &environment, - ); - assert_eq!(transport.ssl_verify, SslVerify::Disabled); - assert!(!transport.http2); - } -} diff --git a/litellm-rust/crates/gateway-auth/src/lib.rs b/litellm-rust/crates/gateway-auth/src/lib.rs index 1b400a94746..6d5621c8407 100644 --- a/litellm-rust/crates/gateway-auth/src/lib.rs +++ b/litellm-rust/crates/gateway-auth/src/lib.rs @@ -22,7 +22,7 @@ pub use identity::{ AuthenticatedCaller, Authentication, AuthenticationMethod, Principal, PrincipalKind, ResolvedIdentity, SharedCaller, VerifiedIdentity, }; -pub use ui::{UI_CSRF_KEY, UiAuthSession, UiBackend, UiCredentials, UiSession, UiUser}; +pub use ui::{UI_CSRF_KEY, UiAuthSession, UiBackend, UiSession, UiUser}; pub fn hash_token(token: &str) -> String { format!("{:x}", Sha256::digest(token.as_bytes())) diff --git a/litellm-rust/crates/gateway-auth/src/ui.rs b/litellm-rust/crates/gateway-auth/src/ui.rs index 9e6b77a17d0..e1ede25d786 100644 --- a/litellm-rust/crates/gateway-auth/src/ui.rs +++ b/litellm-rust/crates/gateway-auth/src/ui.rs @@ -3,8 +3,7 @@ use std::{convert::Infallible, sync::Arc}; use axum::extract::FromRequestParts; use axum::http::{header::AUTHORIZATION, request::Parts}; use axum_login::{AuthUser, AuthnBackend, UserId}; -use litellm_auth_types::SecretValue; -use serde::Deserialize; +use litellm_auth_types::{SecretValue, UiCredentials}; use sha2::{Digest, Sha256}; use subtle::ConstantTimeEq; use tower_sessions::Session; @@ -38,12 +37,6 @@ impl AuthUser for UiUser { } } -#[derive(Deserialize)] -pub struct UiCredentials { - pub username: String, - pub password: SecretValue, -} - #[derive(Clone)] pub struct UiBackend { user: UiUser, diff --git a/litellm-rust/crates/gateway-auth/tests/ui.rs b/litellm-rust/crates/gateway-auth/tests/ui.rs index 2e5f72d605d..cb8ef97d325 100644 --- a/litellm-rust/crates/gateway-auth/tests/ui.rs +++ b/litellm-rust/crates/gateway-auth/tests/ui.rs @@ -1,6 +1,6 @@ use axum_login::{AuthUser, AuthnBackend}; -use litellm_auth_types::SecretValue; -use litellm_gateway_auth::{UiBackend, UiCredentials}; +use litellm_auth_types::{SecretValue, UiCredentials}; +use litellm_gateway_auth::UiBackend; use rstest::{fixture, rstest}; #[fixture] diff --git a/litellm-rust/crates/gateway-mcp/Cargo.toml b/litellm-rust/crates/gateway-mcp/Cargo.toml index 07706dcb6a4..9b7db77d7ce 100644 --- a/litellm-rust/crates/gateway-mcp/Cargo.toml +++ b/litellm-rust/crates/gateway-mcp/Cargo.toml @@ -7,11 +7,13 @@ license.workspace = true repository.workspace = true [dependencies] -axum = { workspace = true, features = ["json", "query"] } -futures-util.workspace = true litellm-config.workspace = true +litellm-gateway-auth.workspace = true litellm-secrets.workspace = true litellm-auth-types.workspace = true + +axum = { workspace = true, features = ["json", "query"] } +futures-util.workspace = true base64.workspace = true http.workspace = true uuid.workspace = true diff --git a/litellm-rust/crates/gateway/src/auth.rs b/litellm-rust/crates/gateway-mcp/src/auth.rs similarity index 72% rename from litellm-rust/crates/gateway/src/auth.rs rename to litellm-rust/crates/gateway-mcp/src/auth.rs index 72892be783c..9a599372bd1 100644 --- a/litellm-rust/crates/gateway/src/auth.rs +++ b/litellm-rust/crates/gateway-mcp/src/auth.rs @@ -1,13 +1,13 @@ use std::sync::Arc; use axum::{extract::Request, middleware::Next, response::Response}; -use litellm_gateway_auth::{AccessRequest, AuthenticatedRequest, Error, McpAction}; -use litellm_gateway_mcp::{ - Authorization, GatewayFuture, OperationAuthorizer, ServerInfo, SessionOwner, - rmcp::model::ClientRequest, -}; +use rmcp::model::ClientRequest; -pub(crate) async fn bind_session_owner( +use litellm_gateway_auth::{AccessRequest, AuthenticatedRequest, McpAction}; + +use crate::{Authorization, GatewayFuture, OperationAuthorizer, ServerInfo, SessionOwner}; + +pub async fn bind_caller( identity: AuthenticatedRequest, mut request: Request, next: Next, @@ -47,7 +47,7 @@ impl OperationAuthorizer for McpPolicy { Some(ClientRequest::ReadResourceRequest(request)) => { McpAction::ReadResource(request.params.uri.clone()) } - Some(_) => return Err(litellm_gateway_mcp::Error::Forbidden), + Some(_) => return Err(crate::Error::Forbidden), }; let access = AccessRequest::Mcp { server: server.server_id.clone(), @@ -55,19 +55,9 @@ impl OperationAuthorizer for McpPolicy { }; self.0 .authorize(access.clone()) - .await - .map_err(mcp_error)? - .consume(self.0.caller(), &access) - .map_err(mcp_error) + .await? + .consume(self.0.caller(), &access)?; + Ok(()) }) } } - -fn mcp_error(error: Error) -> litellm_gateway_mcp::Error { - match error { - Error::InvalidToken | Error::Expired | Error::Forbidden => { - litellm_gateway_mcp::Error::Forbidden - } - _ => litellm_gateway_mcp::Error::Configuration("MCP authorization unavailable".into()), - } -} diff --git a/litellm-rust/crates/gateway-mcp/src/error.rs b/litellm-rust/crates/gateway-mcp/src/error.rs index 1aa72e34703..cee785a183b 100644 --- a/litellm-rust/crates/gateway-mcp/src/error.rs +++ b/litellm-rust/crates/gateway-mcp/src/error.rs @@ -3,6 +3,7 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; +use litellm_gateway_auth::Error as AuthError; use rmcp::{ model::{ErrorCode, ErrorData}, service::ServiceError, @@ -23,6 +24,8 @@ pub enum Error { UnexpectedResult, #[error("MCP request cancelled")] Cancelled, + #[error(transparent)] + Auth(#[from] AuthError), } impl Error { @@ -30,7 +33,10 @@ impl Error { match self { Self::Upstream(ServiceError::McpError(error)) => error, Self::InvalidRequest(message) => ErrorData::invalid_params(message, None), - Self::Forbidden => ErrorData::new(ErrorCode(-32003), self.to_string(), None), + Self::Forbidden + | Self::Auth(AuthError::Forbidden | AuthError::InvalidToken | AuthError::Expired) => { + ErrorData::new(ErrorCode(-32003), Self::Forbidden.to_string(), None) + } _ => ErrorData::internal_error(self.to_string(), None), } } @@ -44,6 +50,7 @@ impl IntoResponse for Error { Self::Upstream(_) => StatusCode::BAD_GATEWAY, Self::Cancelled => StatusCode::REQUEST_TIMEOUT, Self::Configuration(_) | Self::UnexpectedResult => StatusCode::INTERNAL_SERVER_ERROR, + Self::Auth(error) => error.status(), }; (status, Json(json!({"detail": self.to_string()}))).into_response() } diff --git a/litellm-rust/crates/gateway-mcp/src/lib.rs b/litellm-rust/crates/gateway-mcp/src/lib.rs index dd72940e24b..639af9fc708 100644 --- a/litellm-rust/crates/gateway-mcp/src/lib.rs +++ b/litellm-rust/crates/gateway-mcp/src/lib.rs @@ -1,3 +1,4 @@ +mod auth; mod configured; mod error; mod legacy; @@ -12,6 +13,7 @@ use std::{collections::BTreeMap, future::Future, pin::Pin, sync::Arc}; use http::request::Parts; use rmcp::{RoleServer, model::*, service::RequestContext}; +pub use auth::bind_caller; pub use configured::ConfiguredGateway; pub use error::{ConnectError, Error}; pub use native::{Limits, NativeGateway, Registry, Server, ServerResolver}; diff --git a/litellm-rust/crates/gateway-ui/Cargo.toml b/litellm-rust/crates/gateway-ui/Cargo.toml index 94546ef9118..b69de1886d7 100644 --- a/litellm-rust/crates/gateway-ui/Cargo.toml +++ b/litellm-rust/crates/gateway-ui/Cargo.toml @@ -11,6 +11,7 @@ axum-login.workspace = true base64.workspace = true governor = { version = "0.10.4", default-features = false, features = ["std"] } jsonwebtoken = { workspace = true, features = ["rust_crypto"] } +litellm-auth-types.workspace = true litellm-gateway-auth.workspace = true rand.workspace = true serde.workspace = true @@ -24,7 +25,6 @@ tower-sessions.workspace = true [dev-dependencies] serde_json.workspace = true tower = { version = "0.5", features = ["util"] } -litellm-auth-types.workspace = true rstest.workspace = true tempfile.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/gateway-ui/src/session.rs b/litellm-rust/crates/gateway-ui/src/session.rs index b80f40bb376..a98080abe95 100644 --- a/litellm-rust/crates/gateway-ui/src/session.rs +++ b/litellm-rust/crates/gateway-ui/src/session.rs @@ -10,8 +10,9 @@ use tower_cookies::{Cookie, Cookies}; use tower_sessions::{Expiry, Session, cookie::SameSite}; use crate::Error; +use litellm_auth_types::UiCredentials; use litellm_gateway_auth::{ - AccessRequest, UI_CSRF_KEY, UiAction, UiAuthError, UiAuthSession, UiCredentials, UiSession, + AccessRequest, UI_CSRF_KEY, UiAction, UiAuthError, UiAuthSession, UiSession, }; pub struct State { diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index a495c95238b..001a5a0aa1a 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,4 +1,3 @@ -mod auth; mod error; mod secrets; pub use error::Error; @@ -15,13 +14,10 @@ use axum::{ use http_body_util::BodyExt; use litellm_config::Config; -use litellm_config_load::{Environment, Transport}; +use litellm_config_load::Environment; use litellm_gateway_auth::Auth; use litellm_gateway_inference::{Gateway, ModelRouter}; -use litellm_http::{ - ClientVariant, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, SslVerify, - media::PublicDnsResolver, -}; +use litellm_http::{ClientVariant, HttpClientPool, Resolution, media::PublicDnsResolver}; use litellm_inference::resources::CoreResources; use litellm_secrets::source::EnvironmentSecrets; use litellm_tracing::ByteChunk; @@ -35,11 +31,7 @@ pub fn build_inference(config: &Config, process: &Environment) -> Result Result HttpSettingsLayer { - HttpSettingsLayer { - ssl_verify: Some(match &transport.ssl_verify { - litellm_config_load::SslVerify::Enabled => SslVerify::Enabled, - litellm_config_load::SslVerify::Disabled => SslVerify::Disabled, - litellm_config_load::SslVerify::CaBundle(path) => SslVerify::CaBundle(path.clone()), - }), - ssl_certificate: transport.ssl_certificate.clone(), - ssl_security_level: transport.ssl_security_level.clone(), - ssl_ecdh_curve: transport.ssl_ecdh_curve.clone(), - force_ipv4: Some(transport.force_ipv4), - http2: Some(transport.http2), - aiohttp_trust_env: Some(transport.aiohttp_trust_env), - disable_aiohttp_trust_env: Some(transport.disable_aiohttp_trust_env), - disable_aiohttp_transport: Some(transport.disable_aiohttp_transport), - ..Default::default() - } -} - pub async fn build_mcp( config: &Config, secrets: Arc, @@ -107,8 +80,12 @@ pub fn router( ) -> Router { let auth = Auth::from_config(config, inference.secrets.clone()); let inference = litellm_gateway_inference::router(inference) - .merge(mcp.unwrap_or_default()) - .route_layer(axum::middleware::from_fn(auth::bind_session_owner)) + .merge( + mcp.map(|mcp| { + mcp.route_layer(axum::middleware::from_fn(litellm_gateway_mcp::bind_caller)) + }) + .unwrap_or_default(), + ) .route_layer(axum::middleware::from_fn_with_state( auth, litellm_gateway_auth::authenticate, diff --git a/litellm-rust/crates/http-types/AGENTS.md b/litellm-rust/crates/http-types/AGENTS.md new file mode 100644 index 00000000000..ea60faa77b0 --- /dev/null +++ b/litellm-rust/crates/http-types/AGENTS.md @@ -0,0 +1,21 @@ +## Scope + +- This crate is the HTTP client settings as the user writes them, before anything decides what a client can do with them. `HttpSettingsLayer` is one source's values, where `None` means that source did not set the key. `HttpSettings::from_layers` merges layers highest precedence first and fills defaults. `HttpSettings` is the merged result +- It knows the key names the user writes (`SSL_VERIFY`, `AIOHTTP_TRUST_ENV`, `HTTP_PROXY`, ...) and how Python reads them (`SslVerify::parse` is bool-or-path, lowercase proxy names win, `HTTP_PROXY` is ignored under CGI). It knows nothing about reqwest, rustls or which cipher a security level selects. That is `../http`, through `Resolution::from(&HttpSettings)` +- No I/O. Every environment read goes through an injected `Lookup`, and `without_missing_files` takes the existence check as a closure +- Depends on `litellm-core-utils` (`Lookup`, `Layer`, `merge`) and `veil` only. It must stay buildable without reqwest, so a settings producer can emit a layer without linking a TLS stack + +## Producers and the consumer + +- A host produces one `HttpSettingsLayer` per source it trusts and lets `from_layers` decide. It never pre-merges two sources into one layer +- `config-load` produces the gateway's layers from config.yaml and the environment, `python-bridge` produces the SDK's from the Python settings snapshot. `../http` is the only consumer. Anything that wants a client goes through `HttpClientConfig`, not these types +- Defaults live in `HttpSettings::default` and are applied only in `from_layers`. A layer never carries a default +- Python flags that only exist to be combined (`aiohttp_trust_env`, `disable_aiohttp_trust_env`, `disable_aiohttp_transport`) stay as the user's flags on the layer. `from_layers` collapses them into `trust_proxy_env` once, so no consumer re-derives it + +## Redaction + +- Proxy URLs can embed credentials, so `EnvironmentProxies` redacts `all`, `http` and `https` and exposes them only through accessors. Keep `Debug` derived through `veil`, never hand-written + +## Tests + +- Assert the merged `HttpSettings` for a given stack of layers, and the parsed layer for a given injected environment, never the shape of the helpers diff --git a/litellm-rust/crates/http-types/Cargo.toml b/litellm-rust/crates/http-types/Cargo.toml new file mode 100644 index 00000000000..71d0d86aa1c --- /dev/null +++ b/litellm-rust/crates/http-types/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "litellm-http-types" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-core-utils.workspace = true +veil.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/litellm-rust/crates/http-types/src/lib.rs b/litellm-rust/crates/http-types/src/lib.rs new file mode 100644 index 00000000000..e62d76f049d --- /dev/null +++ b/litellm-rust/crates/http-types/src/lib.rs @@ -0,0 +1,7 @@ +#![forbid(unsafe_code)] + +mod proxy; +mod settings; + +pub use proxy::EnvironmentProxies; +pub use settings::{HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive}; diff --git a/litellm-rust/crates/http-types/src/proxy.rs b/litellm-rust/crates/http-types/src/proxy.rs new file mode 100644 index 00000000000..51c2f658a1a --- /dev/null +++ b/litellm-rust/crates/http-types/src/proxy.rs @@ -0,0 +1,126 @@ +use litellm_core_utils::settings::Lookup; +use veil::Redact; + +#[derive(Clone, Redact, Default, PartialEq, Eq, Hash)] +pub struct EnvironmentProxies { + #[redact] + all: String, + #[redact] + http: String, + #[redact] + https: String, + no: String, +} + +impl EnvironmentProxies { + pub fn from_environment(env: &impl Lookup) -> Self { + Self::resolve(env, cfg!(windows)) + } + + fn resolve(env: &impl Lookup, names_ignore_case: bool) -> Self { + let lowercase_first = |upper: Option<&str>, lower: &str| { + env.get(lower) + .or_else(|| upper.and_then(|name| env.truthy(name))) + .unwrap_or_default() + }; + let is_cgi = env.get("REQUEST_METHOD").is_some(); + Self { + all: lowercase_first(Some("ALL_PROXY"), "all_proxy"), + http: if is_cgi && names_ignore_case { + String::new() + } else { + lowercase_first((!is_cgi).then_some("HTTP_PROXY"), "http_proxy") + }, + https: lowercase_first(Some("HTTPS_PROXY"), "https_proxy"), + no: lowercase_first(Some("NO_PROXY"), "no_proxy"), + } + } + + pub fn all(&self) -> &str { + &self.all + } + + pub fn http(&self) -> &str { + &self.http + } + + pub fn https(&self) -> &str { + &self.https + } + + pub fn no(&self) -> &str { + &self.no + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { + move |name| { + values + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + } + } + + #[rstest] + #[case::lowercase_wins(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])] + #[case::empty_uppercase_falls_through_to_lowercase(&[("HTTPS_PROXY", ""), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])] + #[case::empty_uppercase_alone_is_unset(&[("HTTPS_PROXY", "")], &[])] + #[case::empty_lowercase_clears_the_uppercase_value(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "")], &[])] + #[case::lowercase_no_proxy_wins(&[("NO_PROXY", "upper.test"), ("no_proxy", "lower.test")], &[("no_proxy", "lower.test")])] + #[case::cgi_forgets_the_client_settable_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128")], &[])] + #[case::cgi_keeps_lowercase_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128"), ("http_proxy", "http://lower:3128")], &[("http_proxy", "http://lower:3128")])] + #[case::cgi_keeps_every_other_variable(&[("REQUEST_METHOD", "GET"), ("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")], &[("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")])] + fn variables_resolve_like_urllib_getproxies_environment( + #[case] env: &'static [(&'static str, &'static str)], + #[case] equivalent: &'static [(&'static str, &'static str)], + ) { + assert_eq!( + EnvironmentProxies::from_environment(&env_of(env)), + EnvironmentProxies::from_environment(&env_of(equivalent)) + ); + } + + #[test] + fn cgi_drops_http_proxy_entirely_where_variable_names_ignore_case() { + let windows_env = |name: &str| match name.to_ascii_uppercase().as_str() { + "REQUEST_METHOD" => Some("GET".to_string()), + "HTTP_PROXY" => Some("http://attacker:3128".to_string()), + "HTTPS_PROXY" => Some("http://proxy:3128".to_string()), + _ => None, + }; + let ignoring_case = EnvironmentProxies::resolve(&windows_env, true); + assert_eq!(ignoring_case.http(), ""); + assert_eq!(ignoring_case.https(), "http://proxy:3128"); + assert_eq!( + EnvironmentProxies::resolve(&windows_env, false).http(), + "http://attacker:3128" + ); + } + + #[test] + fn debug_output_hides_proxy_credentials_but_shows_which_variables_are_set() { + let proxies = EnvironmentProxies::from_environment(&env_of(&[ + ("HTTPS_PROXY", "http://operator:hunter2@proxy.corp:3128"), + ("NO_PROXY", "internal.test"), + ])); + let debug = format!("{proxies:?}"); + assert!(!debug.contains("hunter2") && !debug.contains("operator")); + assert!(debug.contains("internal.test")); + assert_ne!(debug, format!("{:?}", EnvironmentProxies::default())); + } + + #[test] + fn an_empty_environment_proxies_nothing() { + assert_eq!( + EnvironmentProxies::from_environment(&env_of(&[])), + EnvironmentProxies::default() + ); + } +} diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http-types/src/settings.rs similarity index 99% rename from litellm-rust/crates/http/src/settings.rs rename to litellm-rust/crates/http-types/src/settings.rs index 1f437df225c..d7092ac97d5 100644 --- a/litellm-rust/crates/http/src/settings.rs +++ b/litellm-rust/crates/http-types/src/settings.rs @@ -8,7 +8,7 @@ use litellm_core_utils::{ settings::{Layer, Lookup, merge}, }; -use crate::proxy::EnvironmentProxies; +use crate::EnvironmentProxies; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum SslVerify { diff --git a/litellm-rust/crates/http/AGENTS.md b/litellm-rust/crates/http/AGENTS.md index db29b59f80e..612287c7792 100644 --- a/litellm-rust/crates/http/AGENTS.md +++ b/litellm-rust/crates/http/AGENTS.md @@ -1,7 +1,38 @@ -- https://github.com/BerriAI/litellm-docs/blob/main/docs/guides/security_settings.md +## Scope -- Own shared HTTP mechanics, including case-insensitive header lookup, defaults, replacement, and transport -- Do not choose provider credentials, OAuth policy, beta requirements, or model behavior -- Apply provider-supplied header decisions without importing provider implementations -- Preserve existing header precedence, duplicate handling, and unrelated forwarded headers -- Verify observable request behavior rather than the structure of helper functions +- This crate is the one place a `reqwest` client is built. `clippy.toml` disallows `reqwest::Client::new`, `Client::builder` and the TLS builder methods everywhere else, so every outbound connection honors the host's TLS, proxy, keepalive and timeout settings by construction +- Input is `HttpClientConfig`, output is a pooled `Client`. `Resolution::from(&HttpSettings)` is the only way to turn user settings into an `HttpClientConfig`, and it is where names the user wrote (`ssl_ecdh_curve`, `ssl_security_level`) meet what rustls supports. A value rustls cannot honor keeps the default and is reported in `Resolution::unsupported`, never silently dropped and never an error +- Settings as the user writes them (`HttpSettings`, `HttpSettingsLayer`, `SslVerify`, `TcpKeepalive`, `EnvironmentProxies`) live in `../http-types`, which links neither reqwest nor rustls. Crates that produce settings depend on that crate, not on this one +- Route-neutral and Rust-only. Nothing here names a route, a provider, an API format, a credential or a config.yaml key. Provider credentials, OAuth, beta headers and model behavior are decided above and arrive as plain headers +- `gateway` and `python-bridge` are both hosts of this crate. Each lowers its own settings source into an `HttpSettingsLayer` and takes clients from one process-wide `HttpClientPool` + +## What lives here + +- `pool.rs`: `HttpClientPool`, keyed by `(HttpClientConfig, ClientVariant)` with a TTL. `ClientVariant` is the full list of ways a client may differ from the configured one (redirect policy, pinned DNS, dropped client certificate and proxies). Add a variant only for a difference that must be fixed when the client is built, otherwise set it on the request +- `config.rs` and `tls.rs`: `HttpClientConfig` to `reqwest::ClientBuilder` and to `rustls::ClientConfig` +- `proxy.rs`: `EnvironmentProxies` to reqwest proxies, and the URL matcher the media fetcher uses to decide whether a URL leaves through the proxy +- `media.rs`: the SSRF-safe `MediaFetcher`. It is here because address pinning is a build-time property of the client (`dns_resolver2`) and because it needs the proxy matcher. `UrlPolicy` and `DownloadPolicy` are the host's inputs, the host never decides a URL is safe on its own +- `request.rs`: case-insensitive header helpers over `Vec<(String, String)>` and `HeaderPolicy` for forwarding. They keep existing precedence, duplicate handling and unrelated forwarded headers. `string_headers` and `HeaderError` validate the shape of a caller's `extra_headers` and are the only place a request field is named +- `outbound.rs`: `OutboundRequest` serializes the body once and a `RequestSigner` sees those exact bytes. A route builds it after its hooks ran and cannot change it afterwards +- `transport.rs`: `Error::{Http, Network, Connect}` is the one transport error every crate above nests. `Connect` means the request never reached the provider, so a caller may reroute. Messages never carry URLs +- `websocket.rs` and `mcp.rs`: the two connections the reqwest 0.12 builder cannot make. `mcp.rs` repeats the builder chain for the reqwest 0.13 that rmcp needs, delete it when the two versions converge + +## Boundary with consumers + +- Consumers take `litellm_http::Client`, which derefs to `reqwest::Client`, and use reqwest's request and response types directly. This crate owns how a client is built and configured, it is not a wrapper over reqwest's request API +- A consumer builds its request from the `Client` it was given. It never reads the environment, a config file or a settings snapshot to decide TLS, proxy or timeouts +- Tests take `Client::for_test`, `plain_for_test` or `no_redirect_for_test` behind the `test-support` feature. Production code never calls them + +## Known gaps + +- `websocket.rs` builds its own rustls config from native roots and ignores `HttpClientConfig` (`ssl_verify`, CA bundle, client certificate, proxies). Make it take an `HttpClientConfig` like the pool does before adding another WebSocket caller +- `HttpSettings::connect_timeout` has no layer field, so it is always the default + +## Tests + +- Verify observable request behavior (which headers were sent, which proxy was picked, whether a client was reused), never helper structure +- TLS tests generate certificates with `rcgen` and run a local listener. They never depend on a public host + +# references + +- https://github.com/BerriAI/litellm-docs/blob/main/docs/guides/security_settings.md owns the user-facing meaning of `ssl_verify`, `ssl_certificate`, `ssl_security_level` and `ssl_ecdh_curve` diff --git a/litellm-rust/crates/http/Cargo.toml b/litellm-rust/crates/http/Cargo.toml index 7d64467f2f7..d88aa742287 100644 --- a/litellm-rust/crates/http/Cargo.toml +++ b/litellm-rust/crates/http/Cargo.toml @@ -14,6 +14,7 @@ rmcp = { version = "=3.4.1", default-features = false, features = ["client", "tr reqwest-mcp = { package = "reqwest", version = "0.13.2", default-features = false, features = ["rustls-no-provider", "json", "stream", "http2"], optional = true } http.workspace = true litellm-core-utils.workspace = true +litellm-http-types.workspace = true hyper-util.workspace = true reqwest.workspace = true rustls.workspace = true diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index 2f36784bc70..62854cb4b29 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -4,10 +4,10 @@ use std::{ time::Duration, }; +use litellm_http_types::{EnvironmentProxies, HttpSettings, SslVerify, TcpKeepalive}; + use crate::{ error::Error, - proxy::EnvironmentProxies, - settings::{HttpSettings, SslVerify, TcpKeepalive}, tls::{CipherSelection, KeyExchangeGroup, Tls12CipherSuite, Unsupported}, }; @@ -122,9 +122,7 @@ impl TryFrom<&HttpClientConfig> for reqwest::ClientBuilder { Some(agent) => with_protocol.user_agent(agent), None => with_protocol, }; - Ok(config - .proxies - .reqwest_proxies() + Ok(crate::proxy::reqwest_proxies(&config.proxies) .into_iter() .fold(with_agent.no_proxy(), reqwest::ClientBuilder::proxy)) } diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs index d53da607d39..2f4002ac74e 100644 --- a/litellm-rust/crates/http/src/lib.rs +++ b/litellm-rust/crates/http/src/lib.rs @@ -14,7 +14,6 @@ pub mod outbound; mod pool; mod proxy; pub mod request; -mod settings; mod tls; pub mod transport; pub mod websocket; @@ -22,7 +21,8 @@ pub mod websocket; pub use client::Client; pub use config::{ClientIdentity, HttpClientConfig, Resolution, Verify}; pub use error::{Error, TlsSource}; +pub use litellm_http_types::{ + EnvironmentProxies, HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive, +}; pub use pool::{ClientVariant, HttpClientPool}; -pub use proxy::EnvironmentProxies; -pub use settings::{HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive}; pub use tls::{KeyExchangeGroup, Tls12CipherSuite, Unsupported}; diff --git a/litellm-rust/crates/http/src/mcp.rs b/litellm-rust/crates/http/src/mcp.rs index 2b101fb9c98..faabbe2648c 100644 --- a/litellm-rust/crates/http/src/mcp.rs +++ b/litellm-rust/crates/http/src/mcp.rs @@ -66,9 +66,7 @@ fn build(config: &HttpClientConfig) -> Result { Some(value) => protocol.user_agent(value), None => protocol, }; - config - .proxies - .mcp_proxies() + crate::proxy::mcp_proxies(&config.proxies) .into_iter() .fold(agent.no_proxy(), reqwest_mcp::ClientBuilder::proxy) .build() diff --git a/litellm-rust/crates/http/src/media.rs b/litellm-rust/crates/http/src/media.rs index 22a881831cf..88f9173d21e 100644 --- a/litellm-rust/crates/http/src/media.rs +++ b/litellm-rust/crates/http/src/media.rs @@ -126,7 +126,7 @@ impl MediaFetcher { config: &HttpClientConfig, url_policy: UrlPolicy, ) -> Result { - let uses_proxy: ProxyMatch = Arc::new(config.proxies.matcher()); + let uses_proxy: ProxyMatch = Arc::new(crate::proxy::matcher(&config.proxies)); Self::with_resolution( pool, config, @@ -375,7 +375,9 @@ mod tests { }; use super::*; - use crate::{HttpSettings, Resolution}; + use litellm_http_types::HttpSettings; + + use crate::Resolution; async fn serve(response: &'static [u8]) -> (Url, tokio::task::JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0") diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index 78932f9e29f..163acfada05 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -6,7 +6,9 @@ use std::{ use reqwest::dns::Resolve; -use crate::{client::Client, config::HttpClientConfig, error::Error, proxy::EnvironmentProxies}; +use litellm_http_types::EnvironmentProxies; + +use crate::{client::Client, config::HttpClientConfig, error::Error}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum ClientVariant { @@ -128,7 +130,9 @@ mod tests { }; use super::*; - use crate::{ClientIdentity, HttpSettings, Resolution, Verify}; + use litellm_http_types::HttpSettings; + + use crate::{ClientIdentity, Resolution, Verify}; struct FixedResolver(SocketAddr); diff --git a/litellm-rust/crates/http/src/proxy.rs b/litellm-rust/crates/http/src/proxy.rs index e7e54ed6ae2..e1c17b0955e 100644 --- a/litellm-rust/crates/http/src/proxy.rs +++ b/litellm-rust/crates/http/src/proxy.rs @@ -1,81 +1,47 @@ use hyper_util::client::proxy::matcher::Matcher; -use litellm_core_utils::settings::Lookup; -use veil::Redact; +use litellm_http_types::EnvironmentProxies; -#[derive(Clone, Redact, Default, PartialEq, Eq, Hash)] -pub struct EnvironmentProxies { - #[redact] - all: String, - #[redact] - http: String, - #[redact] - https: String, - no: String, +pub(crate) fn matcher( + proxies: &EnvironmentProxies, +) -> impl Fn(&reqwest::Url) -> bool + Send + Sync + use<> { + let matcher = Matcher::builder() + .all(proxies.all()) + .http(proxies.http()) + .https(proxies.https()) + .no(proxies.no()) + .build(); + move |url| { + url.as_str() + .parse::() + .is_ok_and(|uri| matcher.intercept(&uri).is_some()) + } } -impl EnvironmentProxies { - pub fn from_environment(env: &impl Lookup) -> Self { - Self::resolve(env, cfg!(windows)) - } +pub(crate) fn reqwest_proxies(proxies: &EnvironmentProxies) -> Vec { + let no_proxy = reqwest::NoProxy::from_string(proxies.no()); + [ + reqwest::Proxy::http(proxies.http()), + reqwest::Proxy::https(proxies.https()), + reqwest::Proxy::all(proxies.all()), + ] + .into_iter() + .filter_map(Result::ok) + .map(|proxy| proxy.no_proxy(no_proxy.clone())) + .collect() +} - fn resolve(env: &impl Lookup, names_ignore_case: bool) -> Self { - let lowercase_first = |upper: Option<&str>, lower: &str| { - env.get(lower) - .or_else(|| upper.and_then(|name| env.truthy(name))) - .unwrap_or_default() - }; - let is_cgi = env.get("REQUEST_METHOD").is_some(); - Self { - all: lowercase_first(Some("ALL_PROXY"), "all_proxy"), - http: if is_cgi && names_ignore_case { - String::new() - } else { - lowercase_first((!is_cgi).then_some("HTTP_PROXY"), "http_proxy") - }, - https: lowercase_first(Some("HTTPS_PROXY"), "https_proxy"), - no: lowercase_first(Some("NO_PROXY"), "no_proxy"), - } - } - - pub(crate) fn matcher(&self) -> impl Fn(&reqwest::Url) -> bool + Send + Sync + use<> { - let matcher = Matcher::builder() - .all(self.all.clone()) - .http(self.http.clone()) - .https(self.https.clone()) - .no(self.no.clone()) - .build(); - move |url| { - url.as_str() - .parse::() - .is_ok_and(|uri| matcher.intercept(&uri).is_some()) - } - } - - pub(crate) fn reqwest_proxies(&self) -> Vec { - let no_proxy = reqwest::NoProxy::from_string(&self.no); - [ - reqwest::Proxy::http(self.http.as_str()), - reqwest::Proxy::https(self.https.as_str()), - reqwest::Proxy::all(self.all.as_str()), - ] - .into_iter() - .filter_map(Result::ok) - .map(|proxy| proxy.no_proxy(no_proxy.clone())) - .collect() - } - #[cfg(feature = "mcp")] - pub(crate) fn mcp_proxies(&self) -> Vec { - let no_proxy = reqwest_mcp::NoProxy::from_string(&self.no); - [ - reqwest_mcp::Proxy::http(self.http.as_str()), - reqwest_mcp::Proxy::https(self.https.as_str()), - reqwest_mcp::Proxy::all(self.all.as_str()), - ] - .into_iter() - .filter_map(Result::ok) - .map(|proxy| proxy.no_proxy(no_proxy.clone())) - .collect() - } +#[cfg(feature = "mcp")] +pub(crate) fn mcp_proxies(proxies: &EnvironmentProxies) -> Vec { + let no_proxy = reqwest_mcp::NoProxy::from_string(proxies.no()); + [ + reqwest_mcp::Proxy::http(proxies.http()), + reqwest_mcp::Proxy::https(proxies.https()), + reqwest_mcp::Proxy::all(proxies.all()), + ] + .into_iter() + .filter_map(Result::ok) + .map(|proxy| proxy.no_proxy(no_proxy.clone())) + .collect() } #[cfg(test)] @@ -103,75 +69,18 @@ mod tests { #[case::all_covers_https(&[("ALL_PROXY", "http://proxy:3128")], "https://api.test/", true)] #[case::lowercase(&[("https_proxy", "http://proxy:3128")], "https://api.test/", true)] #[case::no_proxy_bypass(&[("HTTPS_PROXY", "http://proxy:3128"), ("NO_PROXY", "api.test")], "https://api.test/", false)] + #[case::cgi_still_proxies_https(&[("REQUEST_METHOD", "GET"), ("HTTPS_PROXY", "http://proxy:3128")], "https://api.test/", true)] fn proxies_follow_the_injected_environment( #[case] env: &'static [(&'static str, &'static str)], #[case] target: &str, #[case] expected: bool, ) { let proxies = EnvironmentProxies::from_environment(&env_of(env)); - assert_eq!(proxies.matcher()(&url(target)), expected); - } - - #[rstest] - #[case::lowercase_wins(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])] - #[case::empty_uppercase_falls_through_to_lowercase(&[("HTTPS_PROXY", ""), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])] - #[case::empty_uppercase_alone_is_unset(&[("HTTPS_PROXY", "")], &[])] - #[case::empty_lowercase_clears_the_uppercase_value(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "")], &[])] - #[case::lowercase_no_proxy_wins(&[("NO_PROXY", "upper.test"), ("no_proxy", "lower.test")], &[("no_proxy", "lower.test")])] - #[case::cgi_forgets_the_client_settable_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128")], &[])] - #[case::cgi_keeps_lowercase_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128"), ("http_proxy", "http://lower:3128")], &[("http_proxy", "http://lower:3128")])] - #[case::cgi_keeps_every_other_variable(&[("REQUEST_METHOD", "GET"), ("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")], &[("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")])] - fn variables_resolve_like_urllib_getproxies_environment( - #[case] env: &'static [(&'static str, &'static str)], - #[case] equivalent: &'static [(&'static str, &'static str)], - ) { - assert_eq!( - EnvironmentProxies::from_environment(&env_of(env)), - EnvironmentProxies::from_environment(&env_of(equivalent)) - ); - } - - #[test] - fn cgi_drops_http_proxy_entirely_where_variable_names_ignore_case() { - let windows_env = |name: &str| match name.to_ascii_uppercase().as_str() { - "REQUEST_METHOD" => Some("GET".to_string()), - "HTTP_PROXY" => Some("http://attacker:3128".to_string()), - "HTTPS_PROXY" => Some("http://proxy:3128".to_string()), - _ => None, - }; - let proxies = EnvironmentProxies::resolve(&windows_env, true); - assert!(!proxies.matcher()(&url("http://api.test/"))); - assert!(proxies.matcher()(&url("https://api.test/"))); - assert!(EnvironmentProxies::resolve(&windows_env, false).matcher()( - &url("http://api.test/") - )); - } - - #[test] - fn a_cgi_request_still_proxies_https_through_the_configured_proxy() { - let proxies = EnvironmentProxies::from_environment(&env_of(&[ - ("REQUEST_METHOD", "GET"), - ("HTTPS_PROXY", "http://proxy:3128"), - ])); - assert!(proxies.matcher()(&url("https://api.test/"))); - } - - #[test] - fn debug_output_hides_proxy_credentials_but_shows_which_variables_are_set() { - let proxies = EnvironmentProxies::from_environment(&env_of(&[ - ("HTTPS_PROXY", "http://operator:hunter2@proxy.corp:3128"), - ("NO_PROXY", "internal.test"), - ])); - let debug = format!("{proxies:?}"); - assert!(!debug.contains("hunter2") && !debug.contains("operator")); - assert!(debug.contains("internal.test")); - assert_ne!(debug, format!("{:?}", EnvironmentProxies::default())); + assert_eq!(matcher(&proxies)(&url(target)), expected); } #[test] fn an_empty_environment_proxies_nothing() { - let proxies = EnvironmentProxies::from_environment(&env_of(&[])); - assert_eq!(proxies, EnvironmentProxies::default()); - assert!(proxies.reqwest_proxies().is_empty()); + assert!(reqwest_proxies(&EnvironmentProxies::default()).is_empty()); } } diff --git a/litellm-rust/crates/http/src/tls.rs b/litellm-rust/crates/http/src/tls.rs index 0b1c44bf6fe..391babe643b 100644 --- a/litellm-rust/crates/http/src/tls.rs +++ b/litellm-rust/crates/http/src/tls.rs @@ -315,7 +315,9 @@ mod tests { use rustls::NamedGroup; use super::*; - use crate::{HttpSettings, Resolution}; + use litellm_http_types::HttpSettings; + + use crate::Resolution; fn config(settings: HttpSettings) -> HttpClientConfig { Resolution::from(&settings).config From 8c4d08f1ca8ab3562b14ba2a67e841514cd6dd79 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Thu, 22 Oct 2026 13:19:00 -0700 Subject: [PATCH 4/5] fix(http): open WebSockets through the pooled reqwest client The Responses WebSocket built its own rustls config from native roots, so it ignored ssl_verify, the CA bundle, the client certificate and HTTPS_PROXY/NO_PROXY. The upgrade GET now goes through a ClientVariant::WebSocket client (HTTP/1.1 only, no redirects) and tungstenite only frames the upgraded stream, since tungstenite has no proxy support (snapview/tungstenite-rs#177) tests/websocket.rs covers a private CA, ssl_verify off, mTLS, a CONNECT tunnel proxy, an h2-offering server, a refused upgrade and a wrong accept key Co-Authored-By: Claude Opus 5.5 --- litellm-rust/Cargo.lock | 8 +- litellm-rust/Cargo.toml | 3 +- litellm-rust/crates/http/AGENTS.md | 13 +- litellm-rust/crates/http/Cargo.toml | 2 +- litellm-rust/crates/http/src/pool.rs | 7 +- litellm-rust/crates/http/src/websocket.rs | 130 +++--- litellm-rust/crates/http/tests/websocket.rs | 396 ++++++++++++++++-- .../crates/inference-responses/Cargo.toml | 1 + .../inference-responses/src/websocket.rs | 49 +-- .../inference-responses/tests/responses.rs | 15 +- .../src/routes/responses/websocket.rs | 12 +- 11 files changed, 486 insertions(+), 150 deletions(-) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ff2232ed0e7..958fbbea7a0 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3967,13 +3967,13 @@ dependencies = [ "rmcp", "rstest", "rustls 0.23.42", - "rustls-native-certs", "serde", "serde_json", "strum", "tempfile", "thiserror 2.0.19", "tokio", + "tokio-rustls 0.26.4", "tokio-tungstenite", "tracing", "veil", @@ -7430,11 +7430,7 @@ checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" dependencies = [ "futures-util", "log", - "rustls 0.23.42", - "rustls-native-certs", - "rustls-pki-types", "tokio", - "tokio-rustls 0.26.4", "tungstenite", ] @@ -7779,8 +7775,6 @@ dependencies = [ "httparse", "log", "rand 0.8.7", - "rustls 0.23.42", - "rustls-pki-types", "sha1 0.10.7", "thiserror 1.0.69", "utf-8", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index d290ca4ce7d..ba3e83082d3 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -99,7 +99,6 @@ rustls = { version = "0.23", default-features = false, features = ["ring", "std" rustify = "=0.7.0" rustify_derive = "=0.5.5" vaultrs = { version = "=0.8.0", default-features = false, features = ["rustls"] } -rustls-native-certs = "0.8" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["float_roundtrip"] } serde_with = { version = "=3.21.0", default-features = false, features = ["std", "macros"] } @@ -113,7 +112,7 @@ thiserror = "2.0" tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] } tiktoken-rs = "0.12.0" tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] } -tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } +tokio-tungstenite = { version = "0.24", default-features = false, features = ["handshake"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } base64 = "0.22" flate2 = "1" diff --git a/litellm-rust/crates/http/AGENTS.md b/litellm-rust/crates/http/AGENTS.md index 612287c7792..aae0869280b 100644 --- a/litellm-rust/crates/http/AGENTS.md +++ b/litellm-rust/crates/http/AGENTS.md @@ -8,14 +8,17 @@ ## What lives here -- `pool.rs`: `HttpClientPool`, keyed by `(HttpClientConfig, ClientVariant)` with a TTL. `ClientVariant` is the full list of ways a client may differ from the configured one (redirect policy, pinned DNS, dropped client certificate and proxies). Add a variant only for a difference that must be fixed when the client is built, otherwise set it on the request +- `pool.rs`: `HttpClientPool`, keyed by `(HttpClientConfig, ClientVariant)` with a TTL. `ClientVariant` is the full list of ways a client may differ from the configured one (redirect policy, pinned DNS, dropped client certificate and proxies, HTTP/1.1 only for WebSocket). Add a variant only for a difference that must be fixed when the client is built, otherwise set it on the request - `config.rs` and `tls.rs`: `HttpClientConfig` to `reqwest::ClientBuilder` and to `rustls::ClientConfig` - `proxy.rs`: `EnvironmentProxies` to reqwest proxies, and the URL matcher the media fetcher uses to decide whether a URL leaves through the proxy - `media.rs`: the SSRF-safe `MediaFetcher`. It is here because address pinning is a build-time property of the client (`dns_resolver2`) and because it needs the proxy matcher. `UrlPolicy` and `DownloadPolicy` are the host's inputs, the host never decides a URL is safe on its own - `request.rs`: case-insensitive header helpers over `Vec<(String, String)>` and `HeaderPolicy` for forwarding. They keep existing precedence, duplicate handling and unrelated forwarded headers. `string_headers` and `HeaderError` validate the shape of a caller's `extra_headers` and are the only place a request field is named - `outbound.rs`: `OutboundRequest` serializes the body once and a `RequestSigner` sees those exact bytes. A route builds it after its hooks ran and cannot change it afterwards - `transport.rs`: `Error::{Http, Network, Connect}` is the one transport error every crate above nests. `Connect` means the request never reached the provider, so a caller may reroute. Messages never carry URLs -- `websocket.rs` and `mcp.rs`: the two connections the reqwest 0.12 builder cannot make. `mcp.rs` repeats the builder chain for the reqwest 0.13 that rmcp needs, delete it when the two versions converge +- `websocket.rs`: the WebSocket client handshake over a pooled `ClientVariant::WebSocket` client. reqwest sends the upgrade GET and owns TCP, TLS and proxies, so a WebSocket honors `ssl_verify`, the CA bundle, the client certificate and `HTTPS_PROXY`/`NO_PROXY` exactly like an HTTP call. tungstenite only frames the upgraded stream (`from_raw_socket`) and supplies the key and accept-key math. Never give tungstenite its own TCP or TLS connector (`connect_async*`, a rustls config, native roots): tungstenite has no proxy support (snapview/tungstenite-rs#177) and a second TLS stack drifts from `tls.rs` +- The WebSocket variant forces `http2: false` because an upgrade only exists in HTTP/1.1, and with `h2` in ALPN a server may pick h2 and the upgrade fails. Redirects are off, a WebSocket handshake never follows one +- A refused upgrade is `transport::Error::Http` with the status and body, a wrong `Sec-WebSocket-Accept` is `Network`, and anything before the request left (a bad URL or scheme included) is `Connect` +- `mcp.rs`: the connection the reqwest 0.12 builder cannot make. It repeats the builder chain for the reqwest 0.13 that rmcp needs, delete it when the two versions converge ## Boundary with consumers @@ -23,15 +26,11 @@ - A consumer builds its request from the `Client` it was given. It never reads the environment, a config file or a settings snapshot to decide TLS, proxy or timeouts - Tests take `Client::for_test`, `plain_for_test` or `no_redirect_for_test` behind the `test-support` feature. Production code never calls them -## Known gaps - -- `websocket.rs` builds its own rustls config from native roots and ignores `HttpClientConfig` (`ssl_verify`, CA bundle, client certificate, proxies). Make it take an `HttpClientConfig` like the pool does before adding another WebSocket caller -- `HttpSettings::connect_timeout` has no layer field, so it is always the default - ## Tests - Verify observable request behavior (which headers were sent, which proxy was picked, whether a client was reused), never helper structure - TLS tests generate certificates with `rcgen` and run a local listener. They never depend on a public host +- `tests/websocket.rs` is the transport contract for WebSockets. It runs a private CA with a `tokio-rustls` server that offers `h2` in ALPN (so a client that offers h2 breaks the upgrade), an mTLS server, and a CONNECT tunnel proxy that records the tunnel target. Any change to `tls.rs`, `proxy.rs` or the pool variants must keep it green, and a new TLS or proxy setting gets a case there too # references diff --git a/litellm-rust/crates/http/Cargo.toml b/litellm-rust/crates/http/Cargo.toml index d88aa742287..e81a44cdf5c 100644 --- a/litellm-rust/crates/http/Cargo.toml +++ b/litellm-rust/crates/http/Cargo.toml @@ -18,7 +18,6 @@ litellm-http-types.workspace = true hyper-util.workspace = true reqwest.workspace = true rustls.workspace = true -rustls-native-certs.workspace = true tokio-tungstenite.workspace = true serde.workspace = true serde_json.workspace = true @@ -33,5 +32,6 @@ webpki-roots.workspace = true futures-util.workspace = true rcgen = "0.14.10" tempfile.workspace = true +tokio-rustls = { version = "0.26", default-features = false, features = ["ring"] } rstest.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index 163acfada05..dfefb8e644d 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -16,6 +16,7 @@ pub enum ClientVariant { NoRedirect, Media, UnpinnedMedia, + WebSocket, } const CLIENT_TTL: Duration = Duration::from_secs(3600); @@ -73,6 +74,10 @@ impl HttpClientPool { client_certificate: None, ..config.clone() }, + ClientVariant::WebSocket => HttpClientConfig { + http2: false, + ..config.clone() + }, ClientVariant::Provider | ClientVariant::NoRedirect => config.clone(), }; let key = (effective, variant); @@ -105,7 +110,7 @@ impl HttpClientPool { ) -> reqwest::ClientBuilder { match variant { ClientVariant::Provider => builder, - ClientVariant::NoRedirect | ClientVariant::UnpinnedMedia => { + ClientVariant::NoRedirect | ClientVariant::UnpinnedMedia | ClientVariant::WebSocket => { builder.redirect(reqwest::redirect::Policy::none()) } ClientVariant::Media => builder diff --git a/litellm-rust/crates/http/src/websocket.rs b/litellm-rust/crates/http/src/websocket.rs index e904392152a..e9fc3b6c7fa 100644 --- a/litellm-rust/crates/http/src/websocket.rs +++ b/litellm-rust/crates/http/src/websocket.rs @@ -1,62 +1,86 @@ -use std::{ - io, - sync::{Arc, OnceLock}, +use http::{ + HeaderMap, HeaderValue, StatusCode, + header::{CONNECTION, SEC_WEBSOCKET_ACCEPT, SEC_WEBSOCKET_KEY, SEC_WEBSOCKET_VERSION, UPGRADE}, }; - -use rustls::{ClientConfig, RootCertStore}; -use tokio::net::TcpStream; +use reqwest::Url; use tokio_tungstenite::{ - Connector, MaybeTlsStream, WebSocketStream, connect_async_tls_with_config, - tungstenite::{client::IntoClientRequest, error::TlsError, handshake::client::Response}, + WebSocketStream, + tungstenite::{ + handshake::{client::generate_key, derive_accept_key}, + protocol::Role, + }, }; -pub type UpstreamWebSocket = WebSocketStream>; +use crate::{client::Client, transport::Error}; -static TLS_CONFIG: OnceLock> = OnceLock::new(); - -fn build_tls_config() -> Result> { - let native = rustls_native_certs::load_native_certs(); - let mut store = RootCertStore::empty(); - let (added, _ignored) = store.add_parsable_certificates(native.certs); - if added == 0 { - return Err(Box::new(tokio_tungstenite::tungstenite::Error::Io( - io::Error::other(format!( - "no usable native root certificates: {:?}", - native.errors - )), - ))); - } - ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider())) - .with_safe_default_protocol_versions() - .map(|builder| builder.with_root_certificates(store).with_no_client_auth()) - .map_err(|error| { - Box::new(tokio_tungstenite::tungstenite::Error::Tls( - TlsError::Rustls(error), - )) - }) -} - -fn tls_config() -> Result, Box> { - if let Some(config) = TLS_CONFIG.get() { - return Ok(Arc::clone(config)); - } - let built = Arc::new(build_tls_config()?); - Ok(Arc::clone(TLS_CONFIG.get_or_init(|| built))) -} +pub type UpstreamWebSocket = WebSocketStream; #[tracing::instrument(name = "litellm.websocket.handshake", level = "debug", skip_all)] -pub async fn connect_upstream( - request: R, -) -> Result<(UpstreamWebSocket, Response), Box> -where - R: IntoClientRequest + Unpin, -{ - let request = request.into_client_request().map_err(Box::new)?; - let connector = match request.uri().scheme_str() { - Some("wss") => Some(Connector::Rustls(tls_config()?)), - _ => None, - }; - connect_async_tls_with_config(request, None, false, connector) +pub async fn connect( + client: &Client, + url: &str, + headers: HeaderMap, +) -> Result { + let key = generate_key(); + let response = client + .get(http_url(url)?) + .headers(handshake_headers(headers, &key)?) + .send() .await - .map_err(Box::new) + .map_err(Error::from_reqwest_before_dispatch)?; + let status = response.status(); + if status != StatusCode::SWITCHING_PROTOCOLS { + return Err(Error::Http { + status: status.as_u16(), + body: response.text().await.unwrap_or_default(), + }); + } + let expected = derive_accept_key(key.as_bytes()); + if response + .headers() + .get(SEC_WEBSOCKET_ACCEPT) + .map(HeaderValue::as_bytes) + != Some(expected.as_bytes()) + { + return Err(Error::Network( + "WebSocket handshake returned a wrong Sec-WebSocket-Accept".into(), + )); + } + let upgraded = response.upgrade().await?; + Ok(WebSocketStream::from_raw_socket(upgraded, Role::Client, None).await) +} + +fn http_url(url: &str) -> Result { + let mut url = Url::parse(url).map_err(not_dispatched)?; + let scheme = match url.scheme() { + "ws" | "http" => "http", + "wss" | "https" => "https", + other => { + return Err(not_dispatched(format!( + "unsupported WebSocket URL scheme {other:?}" + ))); + } + }; + url.set_scheme(scheme) + .map_err(|()| not_dispatched("WebSocket URL cannot be sent over HTTP"))?; + Ok(url) +} + +fn handshake_headers(headers: HeaderMap, key: &str) -> Result { + let key = HeaderValue::from_str(key).map_err(not_dispatched)?; + Ok([ + (CONNECTION, HeaderValue::from_static("Upgrade")), + (UPGRADE, HeaderValue::from_static("websocket")), + (SEC_WEBSOCKET_VERSION, HeaderValue::from_static("13")), + (SEC_WEBSOCKET_KEY, key), + ] + .into_iter() + .fold(headers, |mut headers, (name, value)| { + headers.insert(name, value); + headers + })) +} + +fn not_dispatched(error: impl std::fmt::Display) -> Error { + Error::Connect(error.to_string()) } diff --git a/litellm-rust/crates/http/tests/websocket.rs b/litellm-rust/crates/http/tests/websocket.rs index 2fbaa3994e8..515ac618cca 100644 --- a/litellm-rust/crates/http/tests/websocket.rs +++ b/litellm-rust/crates/http/tests/websocket.rs @@ -1,59 +1,375 @@ +use std::{ + net::SocketAddr, + path::PathBuf, + sync::{Arc, Mutex}, +}; + use futures_util::{SinkExt, StreamExt}; -use litellm_http::websocket::connect_upstream; +use http::{HeaderMap, HeaderValue}; +use litellm_http::{ + ClientIdentity, ClientVariant, EnvironmentProxies, HttpClientConfig, HttpClientPool, + HttpSettings, Resolution, Verify, media::PublicDnsResolver, transport::Error, + websocket::connect, +}; +use rcgen::{BasicConstraints, CertificateParams, CertifiedIssuer, IsCa, KeyPair}; use rstest::rstest; -use tokio::net::TcpListener; +use rustls::{ + RootCertStore, ServerConfig, + crypto::ring, + pki_types::{CertificateDer, PrivateKeyDer}, + server::WebPkiClientVerifier, +}; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, + net::{TcpListener, TcpStream}, +}; +use tokio_rustls::TlsAcceptor; use tokio_tungstenite::{ accept_hdr_async, tungstenite::{ Message, - client::IntoClientRequest, handshake::server::{Request, Response}, }, }; +struct Pki { + directory: tempfile::TempDir, + ca: CertifiedIssuer<'static, KeyPair>, + server_chain: Vec>, + server_key: Vec, +} + +impl Pki { + fn new() -> Self { + let mut ca_params = CertificateParams::new(Vec::::new()).unwrap(); + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let ca = CertifiedIssuer::self_signed(ca_params, KeyPair::generate().unwrap()).unwrap(); + let server_key = KeyPair::generate().unwrap(); + let server = CertificateParams::new(vec!["localhost".into()]) + .unwrap() + .signed_by(&server_key, &ca) + .unwrap(); + let client_key = KeyPair::generate().unwrap(); + let client = CertificateParams::new(vec!["client".into()]) + .unwrap() + .signed_by(&client_key, &ca) + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + std::fs::write(directory.path().join("ca.pem"), ca.pem()).unwrap(); + std::fs::write( + directory.path().join("client.pem"), + format!("{}{}", client.pem(), client_key.serialize_pem()), + ) + .unwrap(); + Self { + directory, + server_chain: vec![server.der().clone()], + server_key: server_key.serialize_der(), + ca, + } + } + + fn path(&self, name: &str) -> PathBuf { + self.directory.path().join(name) + } + + fn acceptor(&self, require_client_certificate: bool) -> TlsAcceptor { + let provider = Arc::new(ring::default_provider()); + let builder = ServerConfig::builder_with_provider(Arc::clone(&provider)) + .with_safe_default_protocol_versions() + .unwrap(); + let builder = if require_client_certificate { + let mut roots = RootCertStore::empty(); + roots.add(self.ca.der().clone()).unwrap(); + builder.with_client_cert_verifier( + WebPkiClientVerifier::builder_with_provider(Arc::new(roots), provider) + .build() + .unwrap(), + ) + } else { + builder.with_no_client_auth() + }; + let mut config = builder + .with_single_cert( + self.server_chain.clone(), + PrivateKeyDer::try_from(self.server_key.clone()).unwrap(), + ) + .unwrap(); + config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; + TlsAcceptor::from(Arc::new(config)) + } +} + +fn config(verify: Verify) -> HttpClientConfig { + HttpClientConfig { + verify, + http2: true, + ..Resolution::from(&HttpSettings::default()).config + } +} + +fn proxied_through(proxy: SocketAddr) -> EnvironmentProxies { + EnvironmentProxies::from_environment(&move |name: &str| { + (name == "HTTPS_PROXY").then(|| format!("http://{proxy}")) + }) +} + +async fn open( + config: &HttpClientConfig, + url: &str, + headers: HeaderMap, +) -> Result { + let client = HttpClientPool::new(Arc::new(PublicDnsResolver)) + .client(config, ClientVariant::WebSocket) + .unwrap(); + connect(&client, url, headers).await +} + +async fn echo_once( + stream: S, + seen: Arc>>, +) { + #[expect( + clippy::result_large_err, + reason = "tungstenite requires an unboxed handshake error response" + )] + let Ok(mut socket) = accept_hdr_async(stream, |request: &Request, response: Response| { + *seen.lock().unwrap() = Some(request.clone()); + Ok(response) + }) + .await + else { + return; + }; + if let Some(Ok(message)) = socket.next().await { + let _ = socket.send(message).await; + } +} + +async fn serve_tls( + pki: &Pki, + require_client_certificate: bool, +) -> (SocketAddr, Arc>>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let acceptor = pki.acceptor(require_client_certificate); + let seen = Arc::new(Mutex::new(None)); + let recorded = Arc::clone(&seen); + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + if let Ok(stream) = acceptor.accept(stream).await { + echo_once(stream, Arc::clone(&recorded)).await; + } + } + }); + (address, seen) +} + +async fn read_head(stream: &mut TcpStream) -> String { + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + let mut byte = [0u8; 1]; + if stream.read(&mut byte).await.unwrap() == 0 { + break; + } + head.push(byte[0]); + } + String::from_utf8(head).unwrap() +} + +async fn serve_tunnel_proxy(target: SocketAddr) -> (SocketAddr, Arc>>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = Arc::clone(&requests); + tokio::spawn(async move { + while let Ok((mut client, _)) = listener.accept().await { + let head = read_head(&mut client).await; + seen.lock() + .unwrap() + .push(head.lines().next().unwrap_or_default().to_owned()); + let mut upstream = TcpStream::connect(target).await.unwrap(); + client + .write_all(b"HTTP/1.1 200 Connection established\r\n\r\n") + .await + .unwrap(); + let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await; + } + }); + (address, requests) +} + +async fn serve_raw(response: &'static str) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + read_head(&mut stream).await; + stream.write_all(response.as_bytes()).await.unwrap(); + let _ = stream.read(&mut [0u8; 1]).await; + }); + address +} + +async fn round_trip(socket: &mut litellm_http::websocket::UpstreamWebSocket) { + socket.send(Message::Text("ping".into())).await.unwrap(); + assert_eq!( + socket.next().await.unwrap().unwrap(), + Message::Text("ping".into()) + ); +} + #[rstest] #[case::without_query("/responses")] #[case::with_query("/responses?model=test-model")] #[tokio::test] -async fn connects_with_caller_headers_and_exchanges_frames(#[case] path: &'static str) { +async fn plain_websocket_sends_caller_headers_and_path(#[case] path: &'static str) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { + let seen = Arc::new(Mutex::new(None)); + let recorded = Arc::clone(&seen); + tokio::spawn(async move { let (stream, _) = listener.accept().await.unwrap(); - #[expect( - clippy::result_large_err, - reason = "tungstenite requires an unboxed handshake error response" - )] - let mut socket = accept_hdr_async(stream, move |request: &Request, response: Response| { - assert_eq!(request.uri().path_and_query().unwrap().as_str(), path); - assert_eq!(request.headers()["authorization"], "Bearer test-key"); - Ok(response) - }) - .await - .unwrap(); - assert_eq!( - socket.next().await.unwrap().unwrap(), - Message::Text("request".into()) - ); - socket.send(Message::Text("response".into())).await.unwrap(); - socket.close(None).await.unwrap(); + echo_once(stream, recorded).await; }); - let mut request = format!("ws://{address}{path}") - .into_client_request() - .unwrap(); - request - .headers_mut() - .insert("authorization", "Bearer test-key".parse().unwrap()); - let (mut socket, response) = connect_upstream(request).await.unwrap(); - assert_eq!(response.status(), 101); - socket.send(Message::Text("request".into())).await.unwrap(); - assert_eq!( - socket.next().await.unwrap().unwrap(), - Message::Text("response".into()) - ); - assert!(matches!( - socket.next().await.unwrap().unwrap(), - Message::Close(_) - )); - server.await.unwrap(); + let headers = HeaderMap::from_iter([( + http::header::AUTHORIZATION, + HeaderValue::from_static("Bearer test-key"), + )]); + + let mut socket = open( + &config(Verify::BuiltInRoots), + &format!("ws://{address}{path}"), + headers, + ) + .await + .unwrap(); + + round_trip(&mut socket).await; + let request = seen.lock().unwrap().take().unwrap(); + assert_eq!(request.uri().path_and_query().unwrap().as_str(), path); + assert_eq!(request.headers()["authorization"], "Bearer test-key"); +} + +#[rstest] +#[case::ca_bundle_trusts_the_private_ca(|pki: &Pki| Verify::CaBundle(pki.path("ca.pem")), true)] +#[case::ssl_verify_off_skips_verification(|_: &Pki| Verify::Disabled, true)] +#[case::built_in_roots_reject_the_private_ca(|_: &Pki| Verify::BuiltInRoots, false)] +#[tokio::test] +async fn wss_honors_ssl_verify(#[case] verify: fn(&Pki) -> Verify, #[case] trusted: bool) { + let pki = Pki::new(); + let (address, _) = serve_tls(&pki, false).await; + + let result = open( + &config(verify(&pki)), + &format!("wss://localhost:{}/responses", address.port()), + HeaderMap::new(), + ) + .await; + + match result { + Ok(mut socket) if trusted => round_trip(&mut socket).await, + Err(Error::Connect(_)) if !trusted => {} + other => panic!("unexpected handshake outcome: {:?}", other.map(drop)), + } +} + +#[rstest] +#[case::with_certificate(true)] +#[case::without_certificate(false)] +#[tokio::test] +async fn wss_presents_the_configured_client_certificate(#[case] configured: bool) { + let pki = Pki::new(); + let (address, _) = serve_tls(&pki, true).await; + let config = HttpClientConfig { + client_certificate: configured.then(|| ClientIdentity::Pem(pki.path("client.pem"))), + ..config(Verify::CaBundle(pki.path("ca.pem"))) + }; + + let result = open( + &config, + &format!("wss://localhost:{}/responses", address.port()), + HeaderMap::new(), + ) + .await; + + match result { + Ok(mut socket) if configured => round_trip(&mut socket).await, + Err(_) if !configured => {} + other => panic!("unexpected handshake outcome: {:?}", other.map(drop)), + } +} + +#[tokio::test] +async fn wss_tunnels_through_the_https_proxy() { + let pki = Pki::new(); + let (target, _) = serve_tls(&pki, false).await; + let (proxy, requests) = serve_tunnel_proxy(target).await; + let config = HttpClientConfig { + proxies: proxied_through(proxy), + ..config(Verify::CaBundle(pki.path("ca.pem"))) + }; + + let mut socket = open( + &config, + &format!("wss://localhost:{}/responses", target.port()), + HeaderMap::new(), + ) + .await + .unwrap(); + + round_trip(&mut socket).await; + assert_eq!( + *requests.lock().unwrap(), + [format!("CONNECT localhost:{} HTTP/1.1", target.port())] + ); +} + +#[tokio::test] +async fn refused_upgrade_reports_the_status_and_body() { + let address = + serve_raw("HTTP/1.1 403 Forbidden\r\nContent-Length: 6\r\nConnection: close\r\n\r\ndenied") + .await; + + let result = open( + &config(Verify::BuiltInRoots), + &format!("ws://{address}/responses"), + HeaderMap::new(), + ) + .await; + + assert_eq!( + result.map(drop), + Err(Error::Http { + status: 403, + body: "denied".into() + }) + ); +} + +#[tokio::test] +async fn upgrade_with_a_wrong_accept_key_is_rejected() { + let address = serve_raw( + "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Accept: wrong\r\n\r\n", + ) + .await; + + let result = open( + &config(Verify::BuiltInRoots), + &format!("ws://{address}/responses"), + HeaderMap::new(), + ) + .await; + + assert!(matches!(result, Err(Error::Network(_)))); +} + +#[rstest] +#[case::non_websocket_scheme("ftp://localhost/responses")] +#[case::unparseable("not a url")] +#[tokio::test] +async fn invalid_url_is_rejected_before_dispatch(#[case] url: &str) { + let result = open(&config(Verify::BuiltInRoots), url, HeaderMap::new()).await; + + assert!(matches!(result, Err(Error::Connect(_)))); } diff --git a/litellm-rust/crates/inference-responses/Cargo.toml b/litellm-rust/crates/inference-responses/Cargo.toml index 793483ee76d..3dcd9976e0a 100644 --- a/litellm-rust/crates/inference-responses/Cargo.toml +++ b/litellm-rust/crates/inference-responses/Cargo.toml @@ -25,6 +25,7 @@ tokio-tungstenite.workspace = true tracing.workspace = true [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } litellm-cache.workspace = true litellm-cache-memory.workspace = true litellm-host-native.workspace = true diff --git a/litellm-rust/crates/inference-responses/src/websocket.rs b/litellm-rust/crates/inference-responses/src/websocket.rs index 213a9a95831..c2ca89ecbaf 100644 --- a/litellm-rust/crates/inference-responses/src/websocket.rs +++ b/litellm-rust/crates/inference-responses/src/websocket.rs @@ -1,12 +1,14 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use futures_util::{SinkExt, StreamExt}; -use litellm_http::websocket::{UpstreamWebSocket, connect_upstream}; +use litellm_http::{ + Client, + websocket::{self, UpstreamWebSocket}, +}; use tokio::sync::Mutex; use tokio_tungstenite::tungstenite::{ Message, - client::IntoClientRequest, - http::{HeaderName, HeaderValue}, + http::{HeaderMap, HeaderName, HeaderValue}, }; use super::Error; @@ -24,42 +26,33 @@ impl ResponsesWebSocketConnection { fields(outcome) )] pub async fn connect_url( + client: &Client, url: &str, headers: &HashMap, timeout: Option, ) -> Result { litellm_inference::diagnostic::operation("litellm.websocket.connect_url", async { - let mut request = url.into_client_request().map_err(|error| { - Error::Transport(litellm_http::transport::Error::Network(error.to_string())) - })?; - for (name, value) in headers { - let header_name = name - .parse::() - .map_err(|error| Error::InvalidRequest(error.to_string().into()))?; - let header_value = HeaderValue::from_str(value) - .map_err(|error| Error::InvalidRequest(error.to_string().into()))?; - request.headers_mut().insert(header_name, header_value); - } - let connect = connect_upstream(request); - let result = match timeout { + let headers = headers + .iter() + .map(|(name, value)| { + Ok(( + name.parse::() + .map_err(|error| Error::InvalidRequest(error.to_string().into()))?, + HeaderValue::from_str(value) + .map_err(|error| Error::InvalidRequest(error.to_string().into()))?, + )) + }) + .collect::>()?; + let connect = websocket::connect(client, url, headers); + let socket = match timeout { Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| { Error::Transport(litellm_http::transport::Error::Network( "Responses WebSocket connection timed out".into(), )) })?, None => connect.await, - }; - let (socket, _) = result.map_err(|error| match *error { - tokio_tungstenite::tungstenite::Error::Http(response) => { - Error::Transport(litellm_http::transport::Error::Http { - status: response.status().as_u16(), - body: String::new(), - }) - } - other => { - Error::Transport(litellm_http::transport::Error::Network(other.to_string())) - } - })?; + } + .map_err(Error::Transport)?; Ok(Self { socket: Arc::new(Mutex::new(Some(socket))), }) diff --git a/litellm-rust/crates/inference-responses/tests/responses.rs b/litellm-rust/crates/inference-responses/tests/responses.rs index ae07a71dabd..66eff404360 100644 --- a/litellm-rust/crates/inference-responses/tests/responses.rs +++ b/litellm-rust/crates/inference-responses/tests/responses.rs @@ -1,20 +1,20 @@ +use futures_util::TryStreamExt; +use rstest::{fixture, rstest}; +use serde_json::json; +use std::sync::Arc; +use wiremock::ResponseTemplate; + +use litellm_host::{call::HostedCompletion, lifecycle::CallEvent}; use litellm_host::{ interceptors::{ExecutionFacts, ResultSource}, lifecycle::ExecutionEvent, }; -use std::sync::Arc; - -use futures_util::TryStreamExt; -use litellm_host::{call::HostedCompletion, lifecycle::CallEvent}; use litellm_inference_responses::{ Error, route::Responses, types::{ResponsesCall, ResponsesOutput}, }; use litellm_inference_testing::{RecordingSecrets, no_secrets}; -use rstest::{fixture, rstest}; -use serde_json::json; -use wiremock::ResponseTemplate; mod support; use support::*; @@ -538,6 +538,7 @@ async fn websocket_operations_trace_outcomes_without_capturing_frames_or_credent .logger() .instrument(async { let connection = ResponsesWebSocketConnection::connect_url( + &litellm_http::Client::plain_for_test(), &format!("ws://{address}/responses"), &std::collections::HashMap::from([( "authorization".into(), diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/websocket.rs b/litellm-rust/crates/python-bridge/src/routes/responses/websocket.rs index 615a838a597..c89185dc21f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/websocket.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/websocket.rs @@ -1,7 +1,9 @@ -use litellm_inference_responses::websocket::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; use pyo3::prelude::*; use serde_json::Value; +use litellm_http::ClientVariant; +use litellm_inference_responses::websocket::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; + use crate::{ errors::route_error_to_pyerr, marshal::{marshal_headers, optional_timeout}, @@ -25,10 +27,12 @@ impl ResponsesWebSocketConnection { ) -> PyResult> { let headers = marshal_headers(headers)?; let timeout = optional_timeout(timeout_seconds); + let client = crate::http::host_client(py, ClientVariant::WebSocket)?; crate::execution::run_async_value(py, async move { - let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout) - .await - .map_err(route_error_to_pyerr)?; + let inner = + RustResponsesWebSocketConnection::connect_url(&client, &url, &headers, timeout) + .await + .map_err(route_error_to_pyerr)?; Ok(ResponsesWebSocketConnection { inner }) }) } From 1afe420d046c1fd1cb13eb3f51d85363bde1f194 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Thu, 22 Oct 2026 13:24:03 -0700 Subject: [PATCH 5/5] wip --- litellm-rust/crates/config/AGENTS.md | 1 + litellm-rust/crates/config/src/lib.rs | 12 +- .../crates/config/src/litellm/budget.rs | 104 +++++++++++++++++- litellm-rust/crates/config/src/litellm/mod.rs | 14 ++- litellm-rust/crates/gateway/src/lib.rs | 11 +- litellm-rust/crates/gateway/tests/boot.rs | 62 +++++++++++ litellm-rust/crates/gateway/tests/mcp.rs | 19 ---- litellm-rust/crates/http-types/AGENTS.md | 5 +- .../crates/http-types/src/settings.rs | 80 +++++++++++--- litellm-rust/crates/http/src/config.rs | 7 +- 10 files changed, 260 insertions(+), 55 deletions(-) create mode 100644 litellm-rust/crates/gateway/tests/boot.rs diff --git a/litellm-rust/crates/config/AGENTS.md b/litellm-rust/crates/config/AGENTS.md index 310c97bb566..b5dcd8c1768 100644 --- a/litellm-rust/crates/config/AGENTS.md +++ b/litellm-rust/crates/config/AGENTS.md @@ -42,6 +42,7 @@ Sections split one YAML mapping into several structs with nested `#[serde(flatte Which gives these rules - Each mapping has exactly one `#[serde(flatten)] additional_fields: Object`, declared last on the outermost struct of that mapping. Flattened sub-structs stay plain, with no flatten field inside +- The exception is a mapping where Python gives every unnamed key one meaning, like the callback vars in a `default_team_settings` entry. Its catch-all is a typed map named for that meaning (`inline_callback_vars`), so a value of the wrong shape fails to parse - A key belongs to exactly one struct. Which one is decided by who reads it, see `src/AGENTS.md` - Never combine `deny_unknown_fields` with flatten, on either the outer or the flattened struct. It is fine on a struct that is never flattened and has no flatten fields diff --git a/litellm-rust/crates/config/src/lib.rs b/litellm-rust/crates/config/src/lib.rs index f831dbd710a..e9d0f158c7b 100644 --- a/litellm-rust/crates/config/src/lib.rs +++ b/litellm-rust/crates/config/src/lib.rs @@ -40,12 +40,12 @@ pub use general::{ pub use guardrail::{Guardrail, GuardrailParams}; pub use litellm::{ AiohttpSettings, AzureBlobCacheParams, BackgroundMode, BudgetSettings, CacheParams, - CachingSettings, CallerIdentity, ClientCertificateSettings, CustomProvider, DiskCacheParams, - FallbackSettings, GcsCacheParams, Http2Settings, LiteLlmSettings, LoggingSettings, - McpToolSettings, ModelGroupSettings, PriorityReservationSettings, ProxyCacheTtlParams, - QdrantCacheParams, RedisCacheParams, RedisServerCacheParams, ResponsesSettings, S3CacheParams, - SemanticCacheParams, SocketSettings, TeamDefaultSettings, TlsSettings, - UpperboundKeyGenerateParams, ValkeyCacheParams, + CachingSettings, CallbackVar, CallerIdentity, ClientCertificateSettings, CustomProvider, + DiskCacheParams, FallbackSettings, GcsCacheParams, Http2Settings, LiteLlmSettings, + LoggingSettings, McpToolSettings, ModelGroupSettings, PriorityReservationSettings, + ProxyCacheTtlParams, QdrantCacheParams, RedisCacheParams, RedisServerCacheParams, + ResponsesSettings, S3CacheParams, SemanticCacheParams, SocketSettings, TeamDefaultSettings, + TlsSettings, UpperboundKeyGenerateParams, ValkeyCacheParams, }; pub use litellm_python_compat::read::Spelled; pub use litellm_router_types::LitellmParams; diff --git a/litellm-rust/crates/config/src/litellm/budget.rs b/litellm-rust/crates/config/src/litellm/budget.rs index 12acac2c1c2..de326161815 100644 --- a/litellm-rust/crates/config/src/litellm/budget.rs +++ b/litellm-rust/crates/config/src/litellm/budget.rs @@ -1,4 +1,9 @@ -use crate::{Object, Raw, Spelled}; +use serde::Deserialize; +use std::collections::BTreeMap; + +use litellm_auth_types::SecretValue; + +use crate::{Raw, Spelled}; #[macro_rules_attribute::apply(crate::config_section)] pub struct BudgetSettings { @@ -14,11 +19,25 @@ pub struct BudgetSettings { pub disable_end_user_cost_tracking: Option>, } -#[macro_rules_attribute::apply(crate::config_section)] +#[derive(Clone, Debug, Deserialize)] pub struct TeamDefaultSettings { - pub team_id: Option, + pub team_id: String, + #[serde(default)] + pub success_callback: Option>, + #[serde(default)] + pub failure_callback: Option>, + #[serde(default)] + pub callback_vars: Option>, #[serde(flatten)] - pub additional_fields: Object, + pub inline_callback_vars: BTreeMap, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum CallbackVar { + Bool(bool), + Number(serde_yaml_ng::Number), + Text(SecretValue), } #[macro_rules_attribute::apply(crate::config_section)] @@ -30,3 +49,80 @@ pub struct UpperboundKeyGenerateParams { pub tpm_limit: Option>, pub rpm_limit: Option>, } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use litellm_auth_types::SecretValue; + use rstest::rstest; + + use super::{CallbackVar, TeamDefaultSettings}; + use crate::Config; + + fn teams(yaml: &str) -> Result, crate::Error> { + Config::from_yaml(&format!( + "litellm_settings:\n default_team_settings: {yaml}" + )) + .map(|config| { + config + .litellm_settings + .budget + .default_team_settings + .unwrap() + }) + } + + fn team(yaml: &str) -> TeamDefaultSettings { + teams(yaml).unwrap()[0].clone() + } + + #[rstest] + fn named_keys_stay_out_of_inline_callback_vars() { + let team = team( + "[{team_id: t, success_callback: [langfuse], failure_callback: [otel], callback_vars: {k: v}, pk: x}]", + ); + assert_eq!(team.team_id, "t"); + assert_eq!( + team.success_callback.as_deref(), + Some(&["langfuse".to_owned()][..]) + ); + assert_eq!( + team.failure_callback.as_deref(), + Some(&["otel".to_owned()][..]) + ); + assert_eq!( + team.callback_vars, + Some(BTreeMap::from([( + "k".to_owned(), + CallbackVar::Text(SecretValue::new("v")) + )])) + ); + assert_eq!(team.inline_callback_vars.keys().collect::>(), ["pk"]); + } + + #[rstest] + #[case::text("pk", CallbackVar::Text(SecretValue::new("pk")))] + #[case::reference("os.environ/PK", CallbackVar::Text(SecretValue::new("os.environ/PK")))] + #[case::quoted_number("'5'", CallbackVar::Text(SecretValue::new("5")))] + #[case::integer("5", CallbackVar::Number(5.into()))] + #[case::float("0.5", CallbackVar::Number(0.5.into()))] + #[case::bool("true", CallbackVar::Bool(true))] + fn callback_vars_accept_scalars(#[case] yaml: &str, #[case] expected: CallbackVar) { + let inline = team(&format!("[{{team_id: t, var: {yaml}}}]")); + let nested = team(&format!("[{{team_id: t, callback_vars: {{var: {yaml}}}}}]")); + assert_eq!(inline.inline_callback_vars.get("var"), Some(&expected)); + assert_eq!(nested.callback_vars.unwrap().get("var"), Some(&expected)); + } + + #[rstest] + #[case::missing_team_id("[{success_callback: [langfuse]}]")] + #[case::inline_list("[{team_id: t, var: [a]}]")] + #[case::inline_mapping("[{team_id: t, var: {a: 1}}]")] + #[case::inline_null("[{team_id: t, var: null}]")] + #[case::nested_mapping("[{team_id: t, callback_vars: {var: {a: 1}}}]")] + #[case::single_callback("[{team_id: t, success_callback: langfuse}]")] + fn invalid_team_settings_fail_to_parse(#[case] yaml: &str) { + assert!(teams(yaml).is_err()); + } +} diff --git a/litellm-rust/crates/config/src/litellm/mod.rs b/litellm-rust/crates/config/src/litellm/mod.rs index a5d3ee3ea1a..41635107836 100644 --- a/litellm-rust/crates/config/src/litellm/mod.rs +++ b/litellm-rust/crates/config/src/litellm/mod.rs @@ -8,7 +8,7 @@ mod transport; use serde::Deserialize; pub use self::{ - budget::{BudgetSettings, TeamDefaultSettings, UpperboundKeyGenerateParams}, + budget::{BudgetSettings, CallbackVar, TeamDefaultSettings, UpperboundKeyGenerateParams}, caching::{ AzureBlobCacheParams, CacheParams, CachingSettings, DiskCacheParams, GcsCacheParams, ProxyCacheTtlParams, QdrantCacheParams, RedisCacheParams, RedisServerCacheParams, @@ -297,8 +297,11 @@ mod tests { ); assert_eq!(background.ttl, Some(Spelled::Value(60))); let team = &settings.budget.default_team_settings.unwrap()[0]; - assert_eq!(team.team_id.as_deref(), Some("team-1")); - assert!(team.additional_fields.contains_key("success_callback")); + assert_eq!(team.team_id, "team-1"); + assert_eq!( + team.success_callback.as_deref(), + Some(&["langfuse".to_owned()][..]) + ); let upperbound = settings.budget.upperbound_key_generate_params.unwrap(); assert_eq!(upperbound.max_budget, Some(Spelled::Value(100.0))); assert_eq!(upperbound.duration.as_deref(), Some("30d")); @@ -319,7 +322,10 @@ mod tests { #[case::cache_raw(" cache_params: {s3_config: {secret: SENTINEL}}")] #[case::gitlab_object(" global_gitlab_config: {access_token: SENTINEL}")] #[case::s3_callback_object(" s3_callback_params: {s3_aws_secret_access_key: SENTINEL}")] - #[case::team_object(" default_team_settings: [{team_id: t, langfuse_secret: SENTINEL}]")] + #[case::team_inline_var(" default_team_settings: [{team_id: t, langfuse_secret: SENTINEL}]")] + #[case::team_callback_vars( + " default_team_settings: [{team_id: t, callback_vars: {langfuse_secret: SENTINEL}}]" + )] #[case::fallbacks_raw(" default_fallbacks: [SENTINEL]")] #[case::user_params_raw(" default_internal_user_params: {models: [SENTINEL]}")] #[case::callback_entry(" callbacks: [SENTINEL]")] diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 001a5a0aa1a..801f3272bfe 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,6 +1,5 @@ mod error; mod secrets; -pub use error::Error; use std::{sync::Arc, time::Instant}; @@ -12,6 +11,7 @@ use axum::{ response::Response, }; use http_body_util::BodyExt; +use uuid::Uuid; use litellm_config::Config; use litellm_config_load::Environment; @@ -21,7 +21,8 @@ use litellm_http::{ClientVariant, HttpClientPool, Resolution, media::PublicDnsRe use litellm_inference::resources::CoreResources; use litellm_secrets::source::EnvironmentSecrets; use litellm_tracing::ByteChunk; -use uuid::Uuid; + +pub use error::Error; pub fn build_inference(config: &Config, process: &Environment) -> Result, Error> { let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))); @@ -32,7 +33,11 @@ pub fn build_inference(config: &Config, process: &Environment) -> Result); + +impl Sink for LogSink { + fn enabled(&self, _: &Metadata<'_>) -> bool { + true + } + + fn emit(&self, record: &Record) { + self.0.send(record.message.clone()).unwrap(); + } +} + +#[rstest] +fn warns_about_tls_settings_the_native_client_cannot_honor() { + let config = Config::from_yaml( + "litellm_settings: {ssl_security_level: 'DEFAULT@SECLEVEL=1', ssl_ecdh_curve: secp521r1}", + ) + .unwrap(); + let (sender, receiver) = mpsc::channel(); + let logger = Logger::new(LogSink(sender)); + logger + .scope(|| litellm_gateway::build_inference(&config, &Environment::default())) + .unwrap(); + let messages: Vec = receiver.try_iter().collect(); + assert!( + messages + .iter() + .any(|message| message.contains("ssl_ecdh_curve \"secp521r1\"")) + ); + assert!( + messages + .iter() + .any(|message| message.contains("ssl_security_level \"@SECLEVEL=1\"")) + ); +} diff --git a/litellm-rust/crates/gateway/tests/mcp.rs b/litellm-rust/crates/gateway/tests/mcp.rs index 9e52a73f191..841d4412114 100644 --- a/litellm-rust/crates/gateway/tests/mcp.rs +++ b/litellm-rust/crates/gateway/tests/mcp.rs @@ -303,22 +303,3 @@ impl litellm_secrets::source::SecretSource for NoSecrets { Box::pin(async { Ok(None) }) } } - -#[rstest] -fn applies_configured_http_environment_before_building_clients() { - let directory = tempfile::tempdir().unwrap(); - let missing = directory.path().join("missing-ca.pem"); - let config = Config::from_yaml(&format!( - "environment_variables: {{SSL_VERIFY: '{}'}}\nlitellm_settings: {{ssl_verify: false}}", - missing.display(), - )) - .unwrap(); - let error = litellm_gateway::build_inference(&config, &Environment::default()) - .err() - .unwrap(); - assert!(matches!( - error, - litellm_gateway::Error::Http(litellm_http::Error::Read { path, .. }) - if path == missing - )); -} diff --git a/litellm-rust/crates/http-types/AGENTS.md b/litellm-rust/crates/http-types/AGENTS.md index ea60faa77b0..e03716be122 100644 --- a/litellm-rust/crates/http-types/AGENTS.md +++ b/litellm-rust/crates/http-types/AGENTS.md @@ -9,7 +9,10 @@ - A host produces one `HttpSettingsLayer` per source it trusts and lets `from_layers` decide. It never pre-merges two sources into one layer - `config-load` produces the gateway's layers from config.yaml and the environment, `python-bridge` produces the SDK's from the Python settings snapshot. `../http` is the only consumer. Anything that wants a client goes through `HttpClientConfig`, not these types -- Defaults live in `HttpSettings::default` and are applied only in `from_layers`. A layer never carries a default +- The environment layer sits above the file and the module global for every key here, because that is Python's order for each of them (`os.getenv("SSL_VERIFY", litellm.ssl_verify)` in `litellm/llms/custom_httpx/http_handler.py`). The on-only flags (`LITELLM_HTTP2`, `DISABLE_AIOHTTP_*`) are `global or env` in Python, which the env layer reproduces by only ever contributing `Some(true)` +- Defaults live in `HttpSettings::default` and `TcpKeepalive::default` and are applied only in `from_layers`. A layer never carries a default, so `from_environment` reads each `AIOHTTP_TCP_*` variable into its own field and `from_layers` composes `TcpKeepalive` +- The connect timeout is not a setting. Python hard-codes it (`HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS`), so it is a constant in `../http`, not a field here +- `without_missing_files` is Python's `get_ssl_configuration` fallback, where a CA path that does not exist yields to `SSL_CERT_FILE` and then the built-in roots. Only `python-bridge` applies it. The gateway skips it on purpose, so a missing bundle in config.yaml fails at boot with `Error::Read` instead of silently verifying against other roots (config compat doc C13) - Python flags that only exist to be combined (`aiohttp_trust_env`, `disable_aiohttp_trust_env`, `disable_aiohttp_transport`) stay as the user's flags on the layer. `from_layers` collapses them into `trust_proxy_env` once, so no consumer re-derives it ## Redaction diff --git a/litellm-rust/crates/http-types/src/settings.rs b/litellm-rust/crates/http-types/src/settings.rs index d7092ac97d5..36a9663034e 100644 --- a/litellm-rust/crates/http-types/src/settings.rs +++ b/litellm-rust/crates/http-types/src/settings.rs @@ -34,6 +34,16 @@ pub struct TcpKeepalive { pub retries: u32, } +impl Default for TcpKeepalive { + fn default() -> Self { + Self { + idle: Duration::from_secs(60), + interval: Duration::from_secs(30), + retries: 5, + } + } +} + #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct HttpSettingsLayer { pub ssl_verify: Option, @@ -47,15 +57,19 @@ pub struct HttpSettingsLayer { pub disable_aiohttp_trust_env: Option, pub disable_aiohttp_transport: Option, pub user_agent: Option, - pub tcp_keepalive: Option, + pub tcp_keepalive: Option, + pub tcp_keepalive_idle: Option, + pub tcp_keepalive_interval: Option, + pub tcp_keepalive_retries: Option, pub pool_idle_timeout: Option, pub proxies: Option, } impl HttpSettingsLayer { pub fn from_environment(env: &impl Lookup) -> Self { - let seconds = |name: &str, default: u32| { - Duration::from_secs(u64::from(env.parsed::(name).unwrap_or(default))) + let seconds = |name: &str| { + env.parsed::(name) + .map(|value| Duration::from_secs(u64::from(value))) }; Self { ssl_verify: env.get("SSL_VERIFY").map(|value| SslVerify::parse(&value)), @@ -63,22 +77,19 @@ impl HttpSettingsLayer { ssl_certificate: env.get("SSL_CERTIFICATE").map(PathBuf::from), ssl_security_level: env.get("SSL_SECURITY_LEVEL"), ssl_ecdh_curve: env.get("SSL_ECDH_CURVE"), - force_ipv4: None, http2: env.enabled("LITELLM_HTTP2"), aiohttp_trust_env: env.enabled("AIOHTTP_TRUST_ENV"), disable_aiohttp_trust_env: env.enabled("DISABLE_AIOHTTP_TRUST_ENV"), disable_aiohttp_transport: env.enabled("DISABLE_AIOHTTP_TRANSPORT"), user_agent: env.get("LITELLM_USER_AGENT"), - tcp_keepalive: env.enabled("AIOHTTP_SO_KEEPALIVE").map(|_| TcpKeepalive { - idle: seconds("AIOHTTP_TCP_KEEPIDLE", 60), - interval: seconds("AIOHTTP_TCP_KEEPINTVL", 30), - retries: env.parsed("AIOHTTP_TCP_KEEPCNT").unwrap_or(5), - }), - pool_idle_timeout: env - .parsed::("AIOHTTP_KEEPALIVE_TIMEOUT") - .map(|timeout| Duration::from_secs(u64::from(timeout))), + tcp_keepalive: env.enabled("AIOHTTP_SO_KEEPALIVE"), + tcp_keepalive_idle: seconds("AIOHTTP_TCP_KEEPIDLE"), + tcp_keepalive_interval: seconds("AIOHTTP_TCP_KEEPINTVL"), + tcp_keepalive_retries: env.parsed("AIOHTTP_TCP_KEEPCNT"), + pool_idle_timeout: seconds("AIOHTTP_KEEPALIVE_TIMEOUT"), proxies: Some(EnvironmentProxies::from_environment(env)) .filter(|proxies| *proxies != EnvironmentProxies::default()), + ..Self::default() } } } @@ -102,6 +113,9 @@ impl Layer for HttpSettingsLayer { .or(lower.disable_aiohttp_transport), user_agent: self.user_agent.or(lower.user_agent), tcp_keepalive: self.tcp_keepalive.or(lower.tcp_keepalive), + tcp_keepalive_idle: self.tcp_keepalive_idle.or(lower.tcp_keepalive_idle), + tcp_keepalive_interval: self.tcp_keepalive_interval.or(lower.tcp_keepalive_interval), + tcp_keepalive_retries: self.tcp_keepalive_retries.or(lower.tcp_keepalive_retries), pool_idle_timeout: self.pool_idle_timeout.or(lower.pool_idle_timeout), proxies: self.proxies.or(lower.proxies), } @@ -120,7 +134,6 @@ pub struct HttpSettings { pub user_agent: Option, pub trust_proxy_env: bool, pub proxies: EnvironmentProxies, - pub connect_timeout: Duration, pub tcp_keepalive: Option, pub pool_idle_timeout: Duration, } @@ -138,7 +151,6 @@ impl Default for HttpSettings { user_agent: None, trust_proxy_env: true, proxies: EnvironmentProxies::default(), - connect_timeout: Duration::from_secs(10), tcp_keepalive: None, pool_idle_timeout: Duration::from_secs(15), } @@ -152,6 +164,7 @@ impl HttpSettings { let merged = merge(highest_precedence_first); let defaults = Self::default(); let http2 = merged.http2.unwrap_or(defaults.http2); + let keepalive = TcpKeepalive::default(); Self { ssl_verify: merged.ssl_verify, ssl_cert_file: merged.ssl_cert_file, @@ -165,7 +178,14 @@ impl HttpSettings { || merged.aiohttp_trust_env.unwrap_or(false) || merged.disable_aiohttp_transport.unwrap_or(false) || http2, - tcp_keepalive: merged.tcp_keepalive, + tcp_keepalive: merged + .tcp_keepalive + .filter(|enabled| *enabled) + .map(|_| TcpKeepalive { + idle: merged.tcp_keepalive_idle.unwrap_or(keepalive.idle), + interval: merged.tcp_keepalive_interval.unwrap_or(keepalive.interval), + retries: merged.tcp_keepalive_retries.unwrap_or(keepalive.retries), + }), pool_idle_timeout: merged .pool_idle_timeout .unwrap_or(defaults.pool_idle_timeout), @@ -324,6 +344,36 @@ mod tests { assert_eq!(tuned.pool_idle_timeout, Duration::from_secs(30)); } + #[test] + fn keepalive_tuning_composes_across_layers_and_an_explicit_off_wins() { + let enabling = HttpSettingsLayer { + tcp_keepalive: Some(true), + ..HttpSettingsLayer::default() + }; + let tuned = HttpSettingsLayer { + tcp_keepalive: Some(true), + tcp_keepalive_idle: Some(Duration::from_secs(45)), + tcp_keepalive_retries: Some(9), + ..HttpSettingsLayer::default() + }; + assert_eq!( + HttpSettings::from_layers([enabling, tuned.clone()]).tcp_keepalive, + Some(TcpKeepalive { + idle: Duration::from_secs(45), + interval: Duration::from_secs(30), + retries: 9, + }) + ); + let disabling = HttpSettingsLayer { + tcp_keepalive: Some(false), + ..HttpSettingsLayer::default() + }; + assert_eq!( + HttpSettings::from_layers([disabling, tuned]).tcp_keepalive, + None + ); + } + #[test] fn socket_keepalive_stays_off_unless_enabled() { let settings = HttpSettings::from_layers([HttpSettingsLayer::from_environment(&env_of( diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index 62854cb4b29..e830782d2e2 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -11,6 +11,8 @@ use crate::{ tls::{CipherSelection, KeyExchangeGroup, Tls12CipherSuite, Unsupported}, }; +const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum Verify { Disabled, @@ -84,7 +86,7 @@ impl From<&HttpSettings> for Resolution { } else { EnvironmentProxies::default() }, - connect_timeout: settings.connect_timeout, + connect_timeout: CONNECT_TIMEOUT, tcp_keepalive: settings.tcp_keepalive, pool_idle_timeout: settings.pool_idle_timeout, }, @@ -270,7 +272,6 @@ mod tests { user_agent: Some("litellm/1.0".into()), trust_proxy_env: true, proxies: proxies(), - connect_timeout: Duration::from_secs(7), tcp_keepalive: Some(keepalive), pool_idle_timeout: Duration::from_secs(45), ..HttpSettings::default() @@ -287,7 +288,7 @@ mod tests { http2: true, user_agent: Some("litellm/1.0".into()), proxies: proxies(), - connect_timeout: Duration::from_secs(7), + connect_timeout: Duration::from_secs(5), tcp_keepalive: Some(keepalive), pool_idle_timeout: Duration::from_secs(45), }