mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-19 09:53:33 +00:00
Merge 1afe420d04 into 4710c3b30e
This commit is contained in:
commit
56d89a4bae
138 changed files with 6212 additions and 1566 deletions
35
litellm-rust/Cargo.lock
generated
35
litellm-rust/Cargo.lock
generated
|
|
@ -3647,12 +3647,26 @@ name = "litellm-config"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-auth-types",
|
||||
"litellm-python-compat",
|
||||
"litellm-router-types",
|
||||
"macro_rules_attribute",
|
||||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml_ng",
|
||||
"strum",
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-config-load"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-config",
|
||||
"litellm-core-utils",
|
||||
"litellm-http-types",
|
||||
"rstest",
|
||||
"serde_yaml_ng",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
|
@ -3737,6 +3751,7 @@ dependencies = [
|
|||
"http-body-util",
|
||||
"litellm-auth-types",
|
||||
"litellm-config",
|
||||
"litellm-config-load",
|
||||
"litellm-gateway-auth",
|
||||
"litellm-gateway-inference",
|
||||
"litellm-gateway-mcp",
|
||||
|
|
@ -3837,6 +3852,7 @@ dependencies = [
|
|||
"http 1.4.2",
|
||||
"litellm-auth-types",
|
||||
"litellm-config",
|
||||
"litellm-gateway-auth",
|
||||
"litellm-secrets",
|
||||
"moka",
|
||||
"rmcp",
|
||||
|
|
@ -3944,25 +3960,35 @@ dependencies = [
|
|||
"http 1.4.2",
|
||||
"hyper-util",
|
||||
"litellm-core-utils",
|
||||
"litellm-http-types",
|
||||
"rcgen",
|
||||
"reqwest 0.12.28",
|
||||
"reqwest 0.13.5",
|
||||
"rmcp",
|
||||
"rstest",
|
||||
"rustls 0.23.42",
|
||||
"rustls-native-certs",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.4",
|
||||
"tokio-tungstenite",
|
||||
"tracing",
|
||||
"veil",
|
||||
"webpki-roots",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-http-types"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-core-utils",
|
||||
"rstest",
|
||||
"veil",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-inference"
|
||||
version = "0.1.0"
|
||||
|
|
@ -4291,6 +4317,7 @@ name = "litellm-router-types"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-auth-types",
|
||||
"litellm-python-compat",
|
||||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -7403,11 +7430,7 @@ checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"
|
|||
dependencies = [
|
||||
"futures-util",
|
||||
"log",
|
||||
"rustls 0.23.42",
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.4",
|
||||
"tungstenite",
|
||||
]
|
||||
|
||||
|
|
@ -7752,8 +7775,6 @@ dependencies = [
|
|||
"httparse",
|
||||
"log",
|
||||
"rand 0.8.7",
|
||||
"rustls 0.23.42",
|
||||
"rustls-pki-types",
|
||||
"sha1 0.10.7",
|
||||
"thiserror 1.0.69",
|
||||
"utf-8",
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ repository = "https://github.com/BerriAI/litellm"
|
|||
|
||||
[workspace.dependencies]
|
||||
litellm-config = { path = "crates/config" }
|
||||
litellm-config-load = { path = "crates/config-load" }
|
||||
litellm-router = { path = "crates/router" }
|
||||
litellm-router-types = { path = "crates/router-types" }
|
||||
litellm-tracing = { path = "crates/tracing" }
|
||||
|
|
@ -47,6 +48,7 @@ litellm-secrets-hashicorp = { path = "crates/secrets-hashicorp" }
|
|||
litellm-secrets-azure = { path = "crates/secrets-azure" }
|
||||
litellm-secrets-cyberark = { path = "crates/secrets-cyberark" }
|
||||
litellm-http = { path = "crates/http" }
|
||||
litellm-http-types = { path = "crates/http-types" }
|
||||
litellm-llms = { path = "crates/llms" }
|
||||
litellm-llms-types = { path = "crates/llms-types" }
|
||||
litellm-core-utils = { path = "crates/core-utils" }
|
||||
|
|
@ -97,7 +99,6 @@ rustls = { version = "0.23", default-features = false, features = ["ring", "std"
|
|||
rustify = "=0.7.0"
|
||||
rustify_derive = "=0.5.5"
|
||||
vaultrs = { version = "=0.8.0", default-features = false, features = ["rustls"] }
|
||||
rustls-native-certs = "0.8"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = { version = "1.0", features = ["float_roundtrip"] }
|
||||
serde_with = { version = "=3.21.0", default-features = false, features = ["std", "macros"] }
|
||||
|
|
@ -111,7 +112,7 @@ thiserror = "2.0"
|
|||
tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
|
||||
tiktoken-rs = "0.12.0"
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] }
|
||||
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
|
||||
tokio-tungstenite = { version = "0.24", default-features = false, features = ["handshake"] }
|
||||
futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
|
||||
base64 = "0.22"
|
||||
flate2 = "1"
|
||||
|
|
|
|||
35
litellm-rust/crates/auth-types/AGENTS.md
Normal file
35
litellm-rust/crates/auth-types/AGENTS.md
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
## Scope
|
||||
|
||||
- This crate is the vocabulary for authenticating LiteLLM to a provider: how a credential is named before it is known (`CredentialRef`, `CredentialPlan`, `ParamSpec`), how it is obtained (`CredentialResolver`, `TokenProvider`, `ResolvedCredential`), how it is carried (`SecretValue`, `Sourced`), how it lands on a request (`CredentialPlacement`, `ProviderAuthPolicy`) and how that fails (`Error`)
|
||||
- Outbound only. Who is calling the gateway is `gateway-auth`. The one exception is `UiCredentials`, which is here because `gateway-auth` and `gateway-ui` both read it and neither may depend on the other
|
||||
- No I/O and no clock. Resolving a reference, fetching a token or reading a file is a `CredentialResolver` or `TokenProvider` implementation in `auth-aws`, `auth-azure`, `auth-gcp` or a host. This crate only defines the traits and runs the plan over an injected handle
|
||||
- Depends on `serde`, `serde_json`, `subtle`, `thiserror` and `veil` only. Nothing here may pull in an HTTP client, a cloud SDK or a provider crate
|
||||
|
||||
## Layering
|
||||
|
||||
- `auth-types` (contracts) is below `auth-aws`, `auth-azure` and `auth-gcp` (one cloud's acquisition each), which are below `auth` (the `AuthServices` facade that bundles them behind features). `llms`, `inference-*`, `secrets-*` and `cache-*` take the types from here and the services from `auth`
|
||||
- Import every type from `litellm_auth_types`. `auth` still glob re-exports this crate and `secrets-types` re-exports `SecretValue`, both against the workspace rule in `../inference/AGENTS.md`. Do not add a third path, and move consumers off those two when touching them
|
||||
- A provider's own credential shape (`AwsParams`, `VertexParams`) lives in `src/params/` only because routing, config and the provider all read it. A field or env name used by one provider stays in that provider's crate
|
||||
|
||||
## Contracts
|
||||
|
||||
- `SecretValue` is the only way a secret moves between crates. `Debug` is redacted through `veil`, equality is constant time, and `expose()` is the single read. Never add `Display`, `Serialize` or an `AsRef<str>` to it
|
||||
- `Sourced<T>` records whether a value came from the request, the deployment or the environment. Precedence between those is decided by whoever builds the `Sourced`, never by this crate
|
||||
- `CredentialPlan::resolve` has three outcomes and they are not interchangeable. `Resolved` carries a credential, `Unavailable` means the caller may fall back to the next plan, and `Err` is terminal. A resolver that cannot answer returns `Declined` or `Missing`, it never errors to mean "not mine"
|
||||
- A `CredentialRef::Explicit` resolves without consulting the resolver. Everything else, including `Host`, is resolved at acquisition time so rotated secrets are read fresh
|
||||
- `apply_credential` prepends the credential header and refuses an empty credential or a header already present. Whether an existing header is preserved or rejected is `ProviderAuthPolicy::existing_header_behavior`, chosen per provider, never a default here
|
||||
- `ParamSpec` lists every place Python reads one connection param, in order: wire names, the `litellm.<name>` module global, then environment names. `resolve` walks that order and treats blank as absent. The static tables in `src/params/` are the one copy of those names
|
||||
|
||||
## Errors
|
||||
|
||||
- `Error` is the single credential error every provider nests. `auth-aws` maps its SDK failures into it with `From`, `auth-azure` and `auth-gcp` build the variants directly. Either way a consumer never sees a cloud SDK error type
|
||||
- `MissingApiKey`, `MissingApiBase` and `MissingParam` are what the end user reads when a setting is absent. The message names what to pass or set, and `ParamSpec::guidance` derives that from the spec so the two never drift
|
||||
- `ErrorDetail::Failed` wraps a source error by `Arc`, so equality is pointer identity. Tests compare the `Message`, `Empty`, `InvalidType` and `DuplicateHeader` variants and only match on `Failed`
|
||||
|
||||
## Known gaps
|
||||
|
||||
- `ProviderAuthPolicy` and `CredentialPlan` have no production consumer yet, `auth/tests/facade.rs` is the only user. Wire them through `llms/src/base_llm/auth.rs` or delete them, do not grow them unused
|
||||
|
||||
## Tests
|
||||
|
||||
- Test what crosses the boundary: redaction in `Debug` output, which header a placement writes, which plan outcome a resolver answer produces. Never test that a type derives something
|
||||
|
|
@ -7,6 +7,7 @@ mod params;
|
|||
mod policy;
|
||||
mod secret;
|
||||
mod token;
|
||||
mod ui;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
|
|
@ -57,3 +58,4 @@ pub use params::{AwsParams, ParamSpec, VertexParams};
|
|||
pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
|
||||
pub use secret::SecretValue;
|
||||
pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};
|
||||
pub use ui::UiCredentials;
|
||||
|
|
|
|||
9
litellm-rust/crates/auth-types/src/ui.rs
Normal file
9
litellm-rust/crates/auth-types/src/ui.rs
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
use serde::Deserialize;
|
||||
|
||||
use crate::SecretValue;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
|
||||
pub struct UiCredentials {
|
||||
pub username: String,
|
||||
pub password: SecretValue,
|
||||
}
|
||||
12
litellm-rust/crates/config-load/AGENTS.md
Normal file
12
litellm-rust/crates/config-load/AGENTS.md
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
## Scope
|
||||
|
||||
- This crate is the gateway's config loader. It takes the types `litellm-config` parses and turns them into the values the gateway runs on, so env lookups, the DB overlay, reference substitution, defaults and the domain types live here, never in `litellm-config`
|
||||
- It never reads process state. The caller reads the process environment once and passes it in as an `Environment`, so tests pass their own variables
|
||||
- Python never uses this crate. It shares only the `litellm-config` types, so behavior here may differ from Python where Python is wrong, and each difference is listed in the config compat doc
|
||||
|
||||
- Fixture and corpus rules live in `tests/AGENTS.md`
|
||||
|
||||
## Precedence
|
||||
|
||||
- Each lowered value takes the env, the file and then the default in the order Python uses for that key today, until that order is normalized
|
||||
- HTTP client settings are lowered into `litellm-http-types` layers (`http_settings`), one layer per source, so this crate never defines its own transport types
|
||||
17
litellm-rust/crates/config-load/Cargo.toml
Normal file
17
litellm-rust/crates/config-load/Cargo.toml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
[package]
|
||||
name = "litellm-config-load"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-config.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-http-types.workspace = true
|
||||
serde_yaml_ng = "0.10.0"
|
||||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
tempfile.workspace = true
|
||||
3
litellm-rust/crates/config-load/build.rs
Normal file
3
litellm-rust/crates/config-load/build.rs
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
fn main() {
|
||||
println!("cargo::rerun-if-changed=tests/fixtures");
|
||||
}
|
||||
26
litellm-rust/crates/config-load/src/error.rs
Normal file
26
litellm-rust/crates/config-load/src/error.rs
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
use std::fmt;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("could not read config")]
|
||||
Read(#[from] std::io::Error),
|
||||
#[error("invalid YAML config")]
|
||||
Parse(#[from] serde_yaml_ng::Error),
|
||||
#[error("invalid config")]
|
||||
Config(#[from] litellm_config::Error),
|
||||
#[error("could not load config {key}")]
|
||||
Document {
|
||||
key: String,
|
||||
#[source]
|
||||
source: Box<Error>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Error {
|
||||
pub(crate) fn document(key: &impl fmt::Debug, source: Error) -> Self {
|
||||
Self::Document {
|
||||
key: format!("{key:?}"),
|
||||
source: Box::new(source),
|
||||
}
|
||||
}
|
||||
}
|
||||
156
litellm-rust/crates/config-load/src/http.rs
Normal file
156
litellm-rust/crates/config-load/src/http.rs
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
use litellm_config::{LiteLlmSettings, Spelled};
|
||||
use litellm_http_types::{HttpSettings, HttpSettingsLayer, SslVerify};
|
||||
|
||||
use crate::Environment;
|
||||
|
||||
pub fn http_settings(settings: &LiteLlmSettings, environment: &Environment) -> HttpSettings {
|
||||
HttpSettings::from_layers([
|
||||
HttpSettingsLayer::from_environment(environment),
|
||||
written(settings),
|
||||
])
|
||||
}
|
||||
|
||||
fn written(settings: &LiteLlmSettings) -> HttpSettingsLayer {
|
||||
HttpSettingsLayer {
|
||||
ssl_verify: settings.tls.ssl_verify.as_ref().map(ssl_verify),
|
||||
ssl_certificate: settings
|
||||
.client_certificate
|
||||
.ssl_certificate
|
||||
.clone()
|
||||
.map(Into::into),
|
||||
ssl_security_level: settings.tls.ssl_security_level.clone(),
|
||||
ssl_ecdh_curve: settings.tls.ssl_ecdh_curve.clone(),
|
||||
force_ipv4: settings.socket.force_ipv4,
|
||||
http2: settings.http2.http2,
|
||||
aiohttp_trust_env: settings.aiohttp.aiohttp_trust_env,
|
||||
disable_aiohttp_trust_env: settings.aiohttp.disable_aiohttp_trust_env,
|
||||
disable_aiohttp_transport: settings.aiohttp.disable_aiohttp_transport,
|
||||
..HttpSettingsLayer::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn ssl_verify(value: &Spelled<bool>) -> SslVerify {
|
||||
match value {
|
||||
Spelled::Value(true) => SslVerify::Enabled,
|
||||
Spelled::Value(false) => SslVerify::Disabled,
|
||||
Spelled::Text(text) => SslVerify::parse(text),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::PathBuf;
|
||||
|
||||
use rstest::rstest;
|
||||
|
||||
use litellm_config::Config;
|
||||
use litellm_http_types::{HttpSettings, SslVerify};
|
||||
|
||||
use super::http_settings;
|
||||
use crate::Environment;
|
||||
|
||||
fn lowered(yaml: &str, variables: &[(&str, &str)]) -> HttpSettings {
|
||||
let config = Config::from_yaml(yaml).unwrap();
|
||||
let environment = variables
|
||||
.iter()
|
||||
.map(|(name, value)| ((*name).to_owned(), (*value).to_owned()))
|
||||
.collect();
|
||||
http_settings(&config.litellm_settings, &environment)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_everywhere_is_the_default() {
|
||||
assert_eq!(lowered("{}", &[]), HttpSettings::default());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::default("{}", &[], None)]
|
||||
#[case::written("litellm_settings: {ssl_verify: false}", &[], Some(SslVerify::Disabled))]
|
||||
#[case::written_text("litellm_settings: {ssl_verify: 'False'}", &[], Some(SslVerify::Disabled))]
|
||||
#[case::written_bundle("litellm_settings: {ssl_verify: /etc/roots.pem}", &[], Some(SslVerify::CaBundle(PathBuf::from("/etc/roots.pem"))))]
|
||||
#[case::environment("{}", &[("SSL_VERIFY", "False")], Some(SslVerify::Disabled))]
|
||||
#[case::environment_over_file("litellm_settings: {ssl_verify: false}", &[("SSL_VERIFY", "true")], Some(SslVerify::Enabled))]
|
||||
fn ssl_verify_takes_environment_then_file(
|
||||
#[case] yaml: &str,
|
||||
#[case] variables: &[(&str, &str)],
|
||||
#[case] expected: Option<SslVerify>,
|
||||
) {
|
||||
assert_eq!(lowered(yaml, variables).ssl_verify, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::file("litellm_settings: {ssl_certificate: /etc/client.pem}", &[], "/etc/client.pem")]
|
||||
#[case::environment_over_file("litellm_settings: {ssl_certificate: /etc/client.pem}", &[("SSL_CERTIFICATE", "/env/client.pem")], "/env/client.pem")]
|
||||
fn client_certificate_takes_environment_then_file(
|
||||
#[case] yaml: &str,
|
||||
#[case] variables: &[(&str, &str)],
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
lowered(yaml, variables).ssl_certificate,
|
||||
Some(PathBuf::from(expected))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::file("litellm_settings: {http2: true}", &[], true)]
|
||||
#[case::environment("{}", &[("LITELLM_HTTP2", "TRUE")], true)]
|
||||
#[case::environment_false_does_not_override_file("litellm_settings: {http2: true}", &[("LITELLM_HTTP2", "false")], true)]
|
||||
fn flags_are_only_turned_on_by_the_environment(
|
||||
#[case] yaml: &str,
|
||||
#[case] variables: &[(&str, &str)],
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
assert_eq!(lowered(yaml, variables).http2, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn force_ipv4_comes_from_the_file_only() {
|
||||
assert!(lowered("litellm_settings: {force_ipv4: true}", &[]).force_ipv4);
|
||||
assert!(!lowered("{}", &[("FORCE_IPV4", "true")]).force_ipv4);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::file("litellm_settings: {ssl_security_level: '1'}", &[], Some("1"))]
|
||||
#[case::environment_over_file("litellm_settings: {ssl_security_level: '1'}", &[("SSL_SECURITY_LEVEL", "2")], Some("2"))]
|
||||
#[case::empty_is_unset("{}", &[("SSL_SECURITY_LEVEL", "")], None)]
|
||||
fn text_settings_skip_empty_values(
|
||||
#[case] yaml: &str,
|
||||
#[case] variables: &[(&str, &str)],
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
assert_eq!(
|
||||
lowered(yaml, variables).ssl_security_level.as_deref(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aiohttp_flags_from_the_file_decide_proxy_trust() {
|
||||
assert!(
|
||||
!lowered("litellm_settings: {disable_aiohttp_trust_env: true}", &[]).trust_proxy_env
|
||||
);
|
||||
assert!(
|
||||
lowered(
|
||||
"litellm_settings: {disable_aiohttp_trust_env: true, aiohttp_trust_env: true}",
|
||||
&[]
|
||||
)
|
||||
.trust_proxy_env
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn later_overlay_wins_and_unrelated_variables_are_ignored() {
|
||||
let environment = Environment::from_iter([("SSL_VERIFY".to_owned(), "true".to_owned())])
|
||||
.overlaid([
|
||||
("SSL_VERIFY".to_owned(), "false".to_owned()),
|
||||
("HTTP2".to_owned(), "true".to_owned()),
|
||||
]);
|
||||
let settings = http_settings(
|
||||
&Config::from_yaml("{}").unwrap().litellm_settings,
|
||||
&environment,
|
||||
);
|
||||
assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled));
|
||||
assert!(!settings.http2);
|
||||
}
|
||||
}
|
||||
39
litellm-rust/crates/config-load/src/include/documents.rs
Normal file
39
litellm-rust/crates/config-load/src/include/documents.rs
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
use std::collections::{BTreeSet, VecDeque};
|
||||
|
||||
use serde_yaml_ng::{Mapping, Value};
|
||||
|
||||
use super::Store;
|
||||
use crate::Error;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Document<K> {
|
||||
pub key: K,
|
||||
pub body: Mapping,
|
||||
}
|
||||
|
||||
pub fn documents<S: Store>(store: &S, root: S::Key) -> Result<Vec<Document<S::Key>>, Error> {
|
||||
let mut pending = VecDeque::from([root.clone()]);
|
||||
let mut loaded = BTreeSet::new();
|
||||
let mut documents = Vec::new();
|
||||
while let Some(key) = pending.pop_front() {
|
||||
if !loaded.insert(key.clone()) {
|
||||
continue;
|
||||
}
|
||||
let (body, includes) = read(store, &key).map_err(|error| Error::document(&key, error))?;
|
||||
for entry in includes {
|
||||
pending.push_back(store.locate(&entry, &key, &root)?);
|
||||
}
|
||||
documents.push(Document { key, body });
|
||||
}
|
||||
Ok(documents)
|
||||
}
|
||||
|
||||
fn read<S: Store>(store: &S, key: &S::Key) -> Result<(Mapping, Vec<String>), Error> {
|
||||
let mut body = litellm_config::yaml::mapping(&store.read(key)?)?;
|
||||
let includes = body
|
||||
.remove(Value::from("include"))
|
||||
.map(serde_yaml_ng::from_value)
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
Ok((body, includes))
|
||||
}
|
||||
90
litellm-rust/crates/config-load/src/include/merge.rs
Normal file
90
litellm-rust/crates/config-load/src/include/merge.rs
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
use std::collections::BTreeMap;
|
||||
|
||||
use serde_yaml_ng::{Mapping, Value};
|
||||
|
||||
use super::Document;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Merged<K> {
|
||||
pub document: Mapping,
|
||||
origins: BTreeMap<String, Vec<(K, usize)>>,
|
||||
}
|
||||
|
||||
impl<K> Merged<K> {
|
||||
/// Maps item `index` of the merged top-level `field` back to its document and its index there.
|
||||
/// A value that is not a list is item 0.
|
||||
pub fn locate(&self, field: &str, index: usize) -> Option<(&K, usize)> {
|
||||
self.origins
|
||||
.get(field)?
|
||||
.iter()
|
||||
.scan(0, |start, (key, len)| {
|
||||
let run = (key, *start, *len);
|
||||
*start += len;
|
||||
Some(run)
|
||||
})
|
||||
.find(|(_, start, len)| (*start..start + len).contains(&index))
|
||||
.map(|(key, start, _)| (key, index - start))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn merge<K: Clone>(documents: &[Document<K>]) -> Merged<K> {
|
||||
let fields = documents
|
||||
.iter()
|
||||
.flat_map(|document| document.body.keys())
|
||||
.fold(Vec::new(), |mut fields, field| {
|
||||
if !fields.contains(&field) {
|
||||
fields.push(field);
|
||||
}
|
||||
fields
|
||||
});
|
||||
let (document, origins) = fields
|
||||
.into_iter()
|
||||
.map(|field| {
|
||||
let layers: Vec<_> = documents
|
||||
.iter()
|
||||
.filter_map(|document| Some((&document.key, document.body.get(field)?)))
|
||||
.collect();
|
||||
(field, combine(&layers))
|
||||
})
|
||||
.map(|(field, (value, runs))| {
|
||||
(
|
||||
(field.clone(), value),
|
||||
(field.as_str().map(str::to_owned), runs),
|
||||
)
|
||||
})
|
||||
.unzip::<_, _, Mapping, Vec<_>>();
|
||||
Merged {
|
||||
document,
|
||||
origins: origins
|
||||
.into_iter()
|
||||
.filter_map(|(field, runs)| Some((field?, runs)))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Python's rule (config L16): a list extends a list, anything else replaces. So the merged value is the
|
||||
/// lists after the last non-list concatenated, or that non-list alone when nothing follows it.
|
||||
fn combine<K: Clone>(layers: &[(&K, &Value)]) -> (Value, Vec<(K, usize)>) {
|
||||
let tail = layers
|
||||
.iter()
|
||||
.rposition(|(_, value)| !value.is_sequence())
|
||||
.map_or(0, |last| last + 1);
|
||||
match &layers[tail..] {
|
||||
[] => {
|
||||
let (key, value) = layers[layers.len() - 1];
|
||||
(value.clone(), vec![(key.clone(), 1)])
|
||||
}
|
||||
lists => (
|
||||
Value::Sequence(
|
||||
lists
|
||||
.iter()
|
||||
.flat_map(|(_, value)| value.as_sequence().into_iter().flatten().cloned())
|
||||
.collect(),
|
||||
),
|
||||
lists
|
||||
.iter()
|
||||
.map(|(key, value)| ((*key).clone(), value.as_sequence().map_or(0, Vec::len)))
|
||||
.collect(),
|
||||
),
|
||||
}
|
||||
}
|
||||
169
litellm-rust/crates/config-load/src/include/mod.rs
Normal file
169
litellm-rust/crates/config-load/src/include/mod.rs
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
mod documents;
|
||||
mod merge;
|
||||
mod store;
|
||||
|
||||
pub use documents::{Document, documents};
|
||||
pub use merge::{Merged, merge};
|
||||
pub use store::{Files, Store};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use rstest::rstest;
|
||||
use serde_yaml_ng::Value;
|
||||
|
||||
use super::*;
|
||||
use crate::Error;
|
||||
|
||||
struct Memory(BTreeMap<&'static str, &'static str>);
|
||||
|
||||
impl<const N: usize> From<[(&'static str, &'static str); N]> for Memory {
|
||||
fn from(files: [(&'static str, &'static str); N]) -> Self {
|
||||
Self(files.into())
|
||||
}
|
||||
}
|
||||
|
||||
impl Store for Memory {
|
||||
type Key = &'static str;
|
||||
|
||||
fn locate(
|
||||
&self,
|
||||
entry: &str,
|
||||
_: &&'static str,
|
||||
_: &&'static str,
|
||||
) -> Result<&'static str, Error> {
|
||||
self.0
|
||||
.get_key_value(entry)
|
||||
.map(|(key, _)| *key)
|
||||
.ok_or_else(|| {
|
||||
Error::document(
|
||||
&entry,
|
||||
std::io::Error::from(std::io::ErrorKind::NotFound).into(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn read(&self, key: &&'static str) -> Result<String, Error> {
|
||||
Ok(self.0[key].to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
fn load(files: impl Into<Memory>) -> Result<Merged<&'static str>, Error> {
|
||||
documents(&files.into(), "root").map(|documents| merge(&documents))
|
||||
}
|
||||
|
||||
fn yaml(text: &str) -> Value {
|
||||
serde_yaml_ng::from_str(text).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn reads_each_document_once_in_breadth_first_order() {
|
||||
let order = documents(
|
||||
&Memory::from([
|
||||
("root", "include: [a, b]"),
|
||||
("a", "include: [c, root]"),
|
||||
("b", "include: [a]"),
|
||||
("c", "{}"),
|
||||
]),
|
||||
"root",
|
||||
)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|document| document.key)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(order, ["root", "a", "b", "c"]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::lists_concatenate("x: [1]", "x: [2]", "[1, 2]")]
|
||||
#[case::mappings_replace_whole("x: {a: 1, b: 2}", "x: {c: 3}", "{c: 3}")]
|
||||
#[case::scalar_replaces_list("x: [1]", "x: 2", "2")]
|
||||
#[case::list_replaces_scalar("x: 1", "x: [2]", "[2]")]
|
||||
fn merges_top_level_keys_like_python(
|
||||
#[case] root: &'static str,
|
||||
#[case] other: &'static str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let root = Box::leak(format!("include: [other]\n{root}").into_boxed_str());
|
||||
let merged = load([("root", &*root), ("other", other)]).unwrap();
|
||||
assert_eq!(merged.document.get("x"), Some(&yaml(expected)));
|
||||
assert!(!merged.document.contains_key("include"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn keeps_concatenating_after_a_replace() {
|
||||
let merged = load([
|
||||
("root", "include: [a, b, c]\nx: [1]"),
|
||||
("a", "x: 2"),
|
||||
("b", "x: [3]"),
|
||||
("c", "x: [4, 5]"),
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(merged.document.get("x"), Some(&yaml("[3, 4, 5]")));
|
||||
assert_eq!(merged.locate("x", 0), Some((&"b", 0)));
|
||||
assert_eq!(merged.locate("x", 2), Some((&"c", 1)));
|
||||
assert_eq!(merged.locate("x", 3), None);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn locates_list_items_and_replaced_values_in_their_documents() {
|
||||
let merged = load([
|
||||
(
|
||||
"root",
|
||||
"include: [a]\nmodel_list: [r0, r1]\ngeneral_settings: {k: 1}\nonly_root: 1",
|
||||
),
|
||||
("a", "model_list: [a0]\ngeneral_settings: {k: 2}"),
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(merged.locate("model_list", 1), Some((&"root", 1)));
|
||||
assert_eq!(merged.locate("model_list", 2), Some((&"a", 0)));
|
||||
assert_eq!(merged.locate("general_settings", 0), Some((&"a", 0)));
|
||||
assert_eq!(merged.locate("only_root", 0), Some((&"root", 0)));
|
||||
assert_eq!(merged.locate("missing", 0), None);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn names_the_missing_include() {
|
||||
let error = load([("root", "include: [gone.yaml]")]).unwrap_err();
|
||||
assert!(
|
||||
matches!(error, Error::Document { ref key, .. } if key.contains("gone.yaml")),
|
||||
"{error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::bare_string("include: a.yaml")]
|
||||
#[case::number_item("include: [1]")]
|
||||
fn rejects_include_that_is_not_a_list_of_paths(#[case] root: &'static str) {
|
||||
assert!(load([("root", root)]).is_err());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::declaring_directory_wins(&["nested/x.yaml", "x.yaml"], "nested/x.yaml")]
|
||||
#[case::falls_back_to_root_directory(&["x.yaml"], "x.yaml")]
|
||||
#[case::reports_declaring_path_when_neither_exists(&[], "nested/x.yaml")]
|
||||
fn files_resolve_against_declaring_then_root_directory(
|
||||
#[case] existing: &[&str],
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let directory = tempfile::TempDir::new().unwrap();
|
||||
let root = directory.path().canonicalize().unwrap();
|
||||
std::fs::create_dir(root.join("nested")).unwrap();
|
||||
for file in ["config.yaml", "nested/first.yaml"].iter().chain(existing) {
|
||||
std::fs::write(root.join(file), "{}").unwrap();
|
||||
}
|
||||
let located = Files.locate(
|
||||
"x.yaml",
|
||||
&root.join("nested/first.yaml"),
|
||||
&root.join("config.yaml"),
|
||||
);
|
||||
match located {
|
||||
Ok(path) => assert_eq!(path, root.join(expected)),
|
||||
Err(Error::Document { key, .. }) if !existing.contains(&expected) => {
|
||||
assert!(key.contains(&*root.join(expected).to_string_lossy()))
|
||||
}
|
||||
Err(error) => panic!("{error:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
46
litellm-rust/crates/config-load/src/include/store.rs
Normal file
46
litellm-rust/crates/config-load/src/include/store.rs
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
use std::{
|
||||
fmt,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use crate::Error;
|
||||
|
||||
pub trait Store {
|
||||
type Key: Ord + Clone + fmt::Debug;
|
||||
|
||||
fn locate(
|
||||
&self,
|
||||
entry: &str,
|
||||
declaring: &Self::Key,
|
||||
root: &Self::Key,
|
||||
) -> Result<Self::Key, Error>;
|
||||
|
||||
fn read(&self, key: &Self::Key) -> Result<String, Error>;
|
||||
}
|
||||
|
||||
pub struct Files;
|
||||
|
||||
impl Store for Files {
|
||||
type Key = PathBuf;
|
||||
|
||||
fn locate(&self, entry: &str, declaring: &PathBuf, root: &PathBuf) -> Result<PathBuf, Error> {
|
||||
let declared = sibling(declaring, entry);
|
||||
let fallback = sibling(root, entry);
|
||||
let location = if declared.exists() || !fallback.exists() {
|
||||
declared
|
||||
} else {
|
||||
fallback
|
||||
};
|
||||
location
|
||||
.canonicalize()
|
||||
.map_err(|error| Error::document(&location, error.into()))
|
||||
}
|
||||
|
||||
fn read(&self, key: &PathBuf) -> Result<String, Error> {
|
||||
Ok(std::fs::read_to_string(key)?)
|
||||
}
|
||||
}
|
||||
|
||||
fn sibling(file: &Path, entry: &str) -> PathBuf {
|
||||
file.parent().unwrap_or(Path::new(".")).join(entry)
|
||||
}
|
||||
62
litellm-rust/crates/config-load/src/lib.rs
Normal file
62
litellm-rust/crates/config-load/src/lib.rs
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
mod error;
|
||||
mod http;
|
||||
pub mod include;
|
||||
|
||||
use std::{collections::BTreeMap, ffi::OsString, path::Path};
|
||||
|
||||
use litellm_config::Config;
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
|
||||
pub use self::{error::Error, http::http_settings};
|
||||
|
||||
pub fn load(path: impl AsRef<Path>) -> Result<Config, Error> {
|
||||
let path = path.as_ref();
|
||||
let root = path
|
||||
.canonicalize()
|
||||
.map_err(|error| Error::document(&path, error.into()))?;
|
||||
load_from(&include::Files, root)
|
||||
}
|
||||
|
||||
pub fn load_from<S: include::Store>(store: &S, root: S::Key) -> Result<Config, Error> {
|
||||
Ok(Config::from_mapping(
|
||||
include::merge(&include::documents(store, root)?).document,
|
||||
)?)
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct Environment(BTreeMap<String, String>);
|
||||
|
||||
impl Environment {
|
||||
pub fn from_os(variables: impl IntoIterator<Item = (OsString, OsString)>) -> Self {
|
||||
variables
|
||||
.into_iter()
|
||||
.filter_map(|(name, value)| Some((name.into_string().ok()?, value.into_string().ok()?)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn overlaid(self, higher: impl IntoIterator<Item = (String, String)>) -> Self {
|
||||
Self(self.0.into_iter().chain(higher).collect())
|
||||
}
|
||||
|
||||
pub fn var(&self, name: &str) -> Option<&str> {
|
||||
self.0.get(name).map(String::as_str)
|
||||
}
|
||||
|
||||
pub fn variables(&self) -> impl Iterator<Item = (String, String)> + '_ {
|
||||
self.0
|
||||
.iter()
|
||||
.map(|(name, value)| (name.clone(), value.clone()))
|
||||
}
|
||||
}
|
||||
|
||||
impl FromIterator<(String, String)> for Environment {
|
||||
fn from_iter<I: IntoIterator<Item = (String, String)>>(variables: I) -> Self {
|
||||
Self(variables.into_iter().collect())
|
||||
}
|
||||
}
|
||||
|
||||
impl Lookup for Environment {
|
||||
fn get(&self, name: &str) -> Option<String> {
|
||||
self.0.get(name).cloned()
|
||||
}
|
||||
}
|
||||
5
litellm-rust/crates/config-load/tests/AGENTS.md
Normal file
5
litellm-rust/crates/config-load/tests/AGENTS.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
## Fixtures
|
||||
|
||||
- `corpus.rs` requires every config in `fixtures/valid/` to load and every config in `fixtures/invalid/` to fail. There is no known-gap list, so a valid config that fails to load is fixed in `src/`
|
||||
- A config goes in `invalid/` only when Python rejects it too, either by raising on read or by crashing when the proxy loads it
|
||||
- `#[files]` globs at compile time, so `build.rs` reruns the build when `fixtures/` changes
|
||||
16
litellm-rust/crates/config-load/tests/corpus.rs
Normal file
16
litellm-rust/crates/config-load/tests/corpus.rs
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
use std::path::PathBuf;
|
||||
|
||||
use litellm_config_load::load;
|
||||
use rstest::rstest;
|
||||
|
||||
#[rstest]
|
||||
fn loads_every_valid_fixture(#[files("tests/fixtures/valid/*.yaml")] path: PathBuf) {
|
||||
if let Err(error) = load(&path) {
|
||||
panic!("{}: {error:?}", path.display());
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn rejects_every_invalid_fixture(#[files("tests/fixtures/invalid/*.yaml")] path: PathBuf) {
|
||||
assert!(load(&path).is_err(), "{} loads", path.display());
|
||||
}
|
||||
3
litellm-rust/crates/config-load/tests/fixtures/AGENTS.md
vendored
Normal file
3
litellm-rust/crates/config-load/tests/fixtures/AGENTS.md
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
These configs are copied as-is from `tests/unit/proxy/example_config_yaml` and `tests/unit/proxy/test_configs` for now. Keep each of `valid/` and `invalid/` flat so relative `include` paths still resolve
|
||||
|
||||
`invalid/config_with_missing_include.yaml` is in Python's `INVALID_FILES`. `invalid/langfuse_config.yaml` passes Python's read-only `get_config` test, but its model has no `litellm_params`, so Python's `load_config` raises `KeyError` on it
|
||||
5
litellm-rust/crates/config-load/tests/fixtures/invalid/config_with_missing_include.yaml
vendored
Normal file
5
litellm-rust/crates/config-load/tests/fixtures/invalid/config_with_missing_include.yaml
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
include:
|
||||
- non-existent-file.yaml
|
||||
|
||||
litellm_settings:
|
||||
callbacks: ["prometheus"]
|
||||
7
litellm-rust/crates/config-load/tests/fixtures/invalid/langfuse_config.yaml
vendored
Normal file
7
litellm-rust/crates/config-load/tests/fixtures/invalid/langfuse_config.yaml
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
model_list:
|
||||
- model_name: gpt-3.5-turbo
|
||||
|
||||
litellm_settings:
|
||||
drop_params: True
|
||||
success_callback: ["langfuse"] # https://docs.litellm.ai/docs/observability/langfuse_integration
|
||||
|
||||
30
litellm-rust/crates/config-load/tests/fixtures/valid/aliases_config.yaml
vendored
Normal file
30
litellm-rust/crates/config-load/tests/fixtures/valid/aliases_config.yaml
vendored
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
model_list:
|
||||
- model_name: gpt-3.5-turbo-instruct
|
||||
litellm_params:
|
||||
model: ollama/zephyr
|
||||
- model_name: gpt-4
|
||||
litellm_params:
|
||||
model: ollama/llama2
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: ollama/llama2
|
||||
temperature: 0.1
|
||||
max_tokens: 20
|
||||
|
||||
|
||||
# request to gpt-4, response from ollama/llama2
|
||||
# curl --location 'http://0.0.0.0:8000/chat/completions' \
|
||||
# --header 'Content-Type: application/json' \
|
||||
# --data ' {
|
||||
# "model": "gpt-4",
|
||||
# "messages": [
|
||||
# {
|
||||
# "role": "user",
|
||||
# "content": "what llm are you"
|
||||
# }
|
||||
# ],
|
||||
# }
|
||||
# '
|
||||
#
|
||||
|
||||
# {"id":"chatcmpl-27c85cf0-ab09-4bcf-8cb1-0ee950520743","choices":[{"finish_reason":"stop","index":0,"message":{"content":" Hello! I'm just an AI, I don't have personal experiences or emotions like humans do. However, I can help you with any questions or tasks you may have! Is there something specific you'd like to know or discuss?","role":"assistant","_logprobs":null}}],"created":1700094955.373751,"model":"ollama/llama2","object":"chat.completion","system_fingerprint":null,"usage":{"prompt_tokens":12,"completion_tokens":47,"total_tokens":59},"_response_ms":8028.017999999999}%
|
||||
15
litellm-rust/crates/config-load/tests/fixtures/valid/azure_config.yaml
vendored
Normal file
15
litellm-rust/crates/config-load/tests/fixtures/valid/azure_config.yaml
vendored
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
model_list:
|
||||
- model_name: gpt-4-team1
|
||||
litellm_params:
|
||||
model: azure/gpt-4.1-mini
|
||||
api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
|
||||
api_version: "2023-05-15"
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
tpm: 20_000
|
||||
- model_name: gpt-4-team2
|
||||
litellm_params:
|
||||
model: azure/gpt-4
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
api_base: https://openai-gpt-4-test-v-2.openai.azure.com/
|
||||
tpm: 100_000
|
||||
|
||||
7
litellm-rust/crates/config-load/tests/fixtures/valid/cache_no_params.yaml
vendored
Normal file
7
litellm-rust/crates/config-load/tests/fixtures/valid/cache_no_params.yaml
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
model_list:
|
||||
- model_name: "openai-model"
|
||||
litellm_params:
|
||||
model: "gpt-3.5-turbo"
|
||||
|
||||
litellm_settings:
|
||||
cache: True
|
||||
11
litellm-rust/crates/config-load/tests/fixtures/valid/cache_with_params.yaml
vendored
Normal file
11
litellm-rust/crates/config-load/tests/fixtures/valid/cache_with_params.yaml
vendored
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
model_list:
|
||||
- model_name: "openai-model"
|
||||
litellm_params:
|
||||
model: "gpt-3.5-turbo"
|
||||
|
||||
litellm_settings:
|
||||
cache: True
|
||||
cache_params:
|
||||
type: "redis"
|
||||
supported_call_types: ["embedding", "aembedding"]
|
||||
host: "os.environ/REDIS_HOST"
|
||||
48
litellm-rust/crates/config-load/tests/fixtures/valid/config_with_env_vars.yaml
vendored
Normal file
48
litellm-rust/crates/config-load/tests/fixtures/valid/config_with_env_vars.yaml
vendored
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
model_list:
|
||||
################################################################################
|
||||
# Azure
|
||||
- model_name: gpt-4o-mini
|
||||
litellm_params:
|
||||
model: azure/gpt-4o-mini
|
||||
api_base: https://amazin-prod.openai.azure.com
|
||||
api_key: "os.environ/AZURE_GPT_4O"
|
||||
deployment_id: gpt-4o-mini
|
||||
- model_name: gpt-4o
|
||||
litellm_params:
|
||||
model: azure/gpt-4o
|
||||
api_base: https://very-cool-prod.openai.azure.com
|
||||
api_key: "os.environ/AZURE_GPT_4O"
|
||||
deployment_id: gpt-4o
|
||||
|
||||
################################################################################
|
||||
# Fireworks
|
||||
- model_name: fireworks-llama-v3p1-405b-instruct
|
||||
litellm_params:
|
||||
model: fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct
|
||||
api_key: "os.environ/FIREWORKS"
|
||||
- model_name: fireworks-llama-v3p1-70b-instruct
|
||||
litellm_params:
|
||||
model: fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct
|
||||
api_key: "os.environ/FIREWORKS"
|
||||
|
||||
general_settings:
|
||||
alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+
|
||||
litellm_settings: # module level litellm settings - https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py
|
||||
success_callback: ["prometheus"]
|
||||
service_callback: ["prometheus_system"]
|
||||
drop_params: False # Raise an exception if the openai param being passed in isn't supported.
|
||||
cache: false
|
||||
default_internal_user_params:
|
||||
user_role: os.environ/DEFAULT_USER_ROLE
|
||||
|
||||
success_callback: ["s3"]
|
||||
s3_callback_params:
|
||||
s3_bucket_name: logs-bucket-litellm # AWS Bucket Name for S3
|
||||
s3_region_name: us-west-2 # AWS Region Name for S3
|
||||
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
|
||||
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
|
||||
s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to
|
||||
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets
|
||||
|
||||
router_settings:
|
||||
routing_strategy: simple-shuffle # "simple-shuffle" shown to result in highest throughput. https://docs.litellm.ai/docs/proxy/configs#load-balancing
|
||||
5
litellm-rust/crates/config-load/tests/fixtures/valid/config_with_include.yaml
vendored
Normal file
5
litellm-rust/crates/config-load/tests/fixtures/valid/config_with_include.yaml
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
include:
|
||||
- included_models.yaml
|
||||
|
||||
litellm_settings:
|
||||
callbacks: ["prometheus"]
|
||||
6
litellm-rust/crates/config-load/tests/fixtures/valid/config_with_multiple_includes.yaml
vendored
Normal file
6
litellm-rust/crates/config-load/tests/fixtures/valid/config_with_multiple_includes.yaml
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
include:
|
||||
- models_file_1.yaml
|
||||
- models_file_2.yaml
|
||||
|
||||
litellm_settings:
|
||||
callbacks: ["prometheus"]
|
||||
4
litellm-rust/crates/config-load/tests/fixtures/valid/included_models.yaml
vendored
Normal file
4
litellm-rust/crates/config-load/tests/fixtures/valid/included_models.yaml
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
model_list:
|
||||
- model_name: included-model
|
||||
litellm_params:
|
||||
model: gpt-4
|
||||
28
litellm-rust/crates/config-load/tests/fixtures/valid/load_balancer.yaml
vendored
Normal file
28
litellm-rust/crates/config-load/tests/fixtures/valid/load_balancer.yaml
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
litellm_settings:
|
||||
drop_params: True
|
||||
|
||||
# Model-specific settings
|
||||
model_list: # use the same model_name for using the litellm router. LiteLLM will use the router between gpt-3.5-turbo
|
||||
- model_name: gpt-3.5-turbo # litellm will
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
api_key: sk-uj6F
|
||||
tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm
|
||||
rpm: 3 # [OPTIONAL] REPLACE with your openai rpm
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
api_key: sk-Imn
|
||||
tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm
|
||||
rpm: 3 # [OPTIONAL] REPLACE with your openai rpm
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: openrouter/gpt-3.5-turbo
|
||||
- model_name: mistral-7b-instruct
|
||||
litellm_params:
|
||||
model: mistralai/mistral-7b-instruct
|
||||
|
||||
environment_variables:
|
||||
REDIS_HOST: localhost
|
||||
REDIS_PASSWORD:
|
||||
REDIS_PORT:
|
||||
4
litellm-rust/crates/config-load/tests/fixtures/valid/models_file_1.yaml
vendored
Normal file
4
litellm-rust/crates/config-load/tests/fixtures/valid/models_file_1.yaml
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
model_list:
|
||||
- model_name: included-model-1
|
||||
litellm_params:
|
||||
model: gpt-4
|
||||
4
litellm-rust/crates/config-load/tests/fixtures/valid/models_file_2.yaml
vendored
Normal file
4
litellm-rust/crates/config-load/tests/fixtures/valid/models_file_2.yaml
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
model_list:
|
||||
- model_name: included-model-2
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
7
litellm-rust/crates/config-load/tests/fixtures/valid/opentelemetry_config.yaml
vendored
Normal file
7
litellm-rust/crates/config-load/tests/fixtures/valid/opentelemetry_config.yaml
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
model_list:
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
|
||||
general_settings:
|
||||
otel: True # OpenTelemetry Logger this logs OTEL data to your collector
|
||||
4
litellm-rust/crates/config-load/tests/fixtures/valid/simple_config.yaml
vendored
Normal file
4
litellm-rust/crates/config-load/tests/fixtures/valid/simple_config.yaml
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
model_list:
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
20
litellm-rust/crates/config-load/tests/fixtures/valid/test_bad_config.yaml
vendored
Normal file
20
litellm-rust/crates/config-load/tests/fixtures/valid/test_bad_config.yaml
vendored
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
model_list:
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
api_key: bad-key
|
||||
model: gpt-3.5-turbo
|
||||
- model_name: working-azure-gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: azure/gpt-4.1-mini
|
||||
api_base: https://azure-unit.example.com
|
||||
api_key: bad-key
|
||||
- model_name: azure-gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: azure/gpt-4.1-mini
|
||||
api_base: https://azure-unit.example.com
|
||||
api_key: bad-key
|
||||
- model_name: azure-embedding
|
||||
litellm_params:
|
||||
model: azure/text-embedding-ada-002
|
||||
api_base: https://azure-unit.example.com
|
||||
api_key: bad-key
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
model_list:
|
||||
- model_name: azure-cloudflare
|
||||
litellm_params:
|
||||
model: azure/gpt-4.1-mini
|
||||
api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
api_version: 2023-07-01-preview
|
||||
|
||||
litellm_settings:
|
||||
set_verbose: True
|
||||
cache: True # set cache responses to True
|
||||
cache_params: # set cache params for s3
|
||||
type: s3
|
||||
s3_bucket_name: litellm-my-test-bucket-2 # AWS Bucket Name for S3
|
||||
s3_region_name: us-east-1 # AWS Region Name for S3
|
||||
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # AWS Access Key ID for S3
|
||||
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
|
||||
28
litellm-rust/crates/config-load/tests/fixtures/valid/test_config.yaml
vendored
Normal file
28
litellm-rust/crates/config-load/tests/fixtures/valid/test_config.yaml
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
general_settings:
|
||||
database_url: os.environ/DATABASE_URL
|
||||
master_key: os.environ/PROXY_MASTER_KEY
|
||||
litellm_settings:
|
||||
drop_params: true
|
||||
success_callback: ["langfuse"]
|
||||
|
||||
model_list:
|
||||
- litellm_params:
|
||||
api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
|
||||
api_key: os.environ/AZURE_EUROPE_API_KEY
|
||||
model: azure/gpt-35-turbo
|
||||
model_name: azure-model
|
||||
- litellm_params:
|
||||
api_base: https://my-endpoint-canada-berri992.openai.azure.com
|
||||
api_key: os.environ/AZURE_CANADA_API_KEY
|
||||
model: azure/gpt-35-turbo
|
||||
model_name: azure-model
|
||||
- litellm_params:
|
||||
api_base: https://openai-france-1234.openai.azure.com
|
||||
api_key: os.environ/AZURE_FRANCE_API_KEY
|
||||
model: azure/gpt-turbo
|
||||
model_name: azure-model
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
model_name: test_openai_models
|
||||
11
litellm-rust/crates/config-load/tests/fixtures/valid/test_config_custom_auth.yaml
vendored
Normal file
11
litellm-rust/crates/config-load/tests/fixtures/valid/test_config_custom_auth.yaml
vendored
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
model_list:
|
||||
- model_name: "openai-model"
|
||||
litellm_params:
|
||||
model: "gpt-3.5-turbo"
|
||||
|
||||
litellm_settings:
|
||||
drop_params: True
|
||||
set_verbose: True
|
||||
|
||||
general_settings:
|
||||
custom_auth: custom_auth.user_api_key_auth
|
||||
6
litellm-rust/crates/config-load/tests/fixtures/valid/test_config_hosted_vllm_embedding.yaml
vendored
Normal file
6
litellm-rust/crates/config-load/tests/fixtures/valid/test_config_hosted_vllm_embedding.yaml
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
model_list:
|
||||
- litellm_params:
|
||||
model: hosted_vllm/embed_model
|
||||
model_info:
|
||||
description: this is a test embedding hosted_vllm model
|
||||
model_name: vllm_embed_model
|
||||
127
litellm-rust/crates/config-load/tests/fixtures/valid/test_config_no_auth.yaml
vendored
Normal file
127
litellm-rust/crates/config-load/tests/fixtures/valid/test_config_no_auth.yaml
vendored
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
model_list:
|
||||
- litellm_params:
|
||||
api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
|
||||
api_key: os.environ/AZURE_EUROPE_API_KEY
|
||||
model: azure/gpt-35-turbo
|
||||
model_name: azure-model
|
||||
- litellm_params:
|
||||
api_base: https://my-endpoint-canada-berri992.openai.azure.com
|
||||
api_key: os.environ/AZURE_CANADA_API_KEY
|
||||
model: azure/gpt-35-turbo
|
||||
model_name: azure-model
|
||||
- litellm_params:
|
||||
api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
model: azure/gpt-4.1-mini
|
||||
model_name: azure-cloudflare-model
|
||||
- litellm_params:
|
||||
api_base: https://openai-france-1234.openai.azure.com
|
||||
api_key: os.environ/AZURE_FRANCE_API_KEY
|
||||
model: azure/gpt-turbo
|
||||
model_name: azure-model
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
model_name: test_openai_models
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
id: 56f1bd94-3b54-4b67-9ea2-7c70e9a3a709
|
||||
model_name: test_openai_models
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
id: 4d1ee26c-abca-450c-8744-8e87fd6755e9
|
||||
model_name: test_openai_models
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
id: 00e19c0f-b63d-42bb-88e9-016fb0c60764
|
||||
model_name: test_openai_models
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
id: 79fc75bf-8e1b-47d5-8d24-9365a854af03
|
||||
model_name: test_openai_models
|
||||
- litellm_params:
|
||||
api_base: os.environ/AZURE_AI_API_BASE
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
api_version: 2023-07-01-preview
|
||||
model: azure/text-embedding-ada-002
|
||||
model_info:
|
||||
mode: embedding
|
||||
model_name: azure-embedding-model
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
id: 55848c55-4162-40f9-a6e2-9a722b9ef404
|
||||
model_name: test_openai_models
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
id: 34339b1e-e030-4bcc-a531-c48559f10ce4
|
||||
model_name: test_openai_models
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
id: f6f74e14-ac64-4403-9365-319e584dcdc5
|
||||
model_name: test_openai_models
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
id: 9b1ef341-322c-410a-8992-903987fef439
|
||||
model_name: test_openai_models
|
||||
- litellm_params:
|
||||
model: dall-e-3
|
||||
model_info:
|
||||
mode: image_generation
|
||||
model_name: dall-e-3
|
||||
- litellm_params:
|
||||
api_base: os.environ/AZURE_SWEDEN_API_BASE
|
||||
api_key: os.environ/AZURE_SWEDEN_API_KEY
|
||||
api_version: 2023-12-01-preview
|
||||
model: azure/dall-e-3-test
|
||||
model_info:
|
||||
mode: image_generation
|
||||
model_name: dall-e-3
|
||||
- litellm_params:
|
||||
api_base: os.environ/AZURE_AI_API_BASE
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
api_version: 2023-06-01-preview
|
||||
model: azure/
|
||||
model_info:
|
||||
mode: image_generation
|
||||
model_name: dall-e-2
|
||||
- litellm_params:
|
||||
api_base: os.environ/AZURE_AI_API_BASE
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
api_version: 2023-07-01-preview
|
||||
model: azure/text-embedding-ada-002
|
||||
model_info:
|
||||
base_model: text-embedding-ada-002
|
||||
mode: embedding
|
||||
model_name: text-embedding-ada-002
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
id: 34cb2419-7c63-44ae-a189-53f1d1ce5953
|
||||
model_name: test_openai_models
|
||||
- litellm_params:
|
||||
model: amazon.titan-embed-text-v1
|
||||
model_name: amazon-embeddings
|
||||
- litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
description: this is a test openai model
|
||||
id: 753dca9a-898d-4ff7-9961-5acf7cdf38cf
|
||||
model_name: test_openai_models
|
||||
21
litellm-rust/crates/config-load/tests/fixtures/valid/test_custom_logger.yaml
vendored
Normal file
21
litellm-rust/crates/config-load/tests/fixtures/valid/test_custom_logger.yaml
vendored
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
model_list:
|
||||
- model_name: Azure OpenAI GPT-4 Canada
|
||||
litellm_params:
|
||||
model: gpt-4.1-nano
|
||||
api_key: sk-unit-fake
|
||||
model_info:
|
||||
mode: chat
|
||||
input_cost_per_token: 0.0002
|
||||
id: gm
|
||||
- model_name: azure-embedding-model
|
||||
litellm_params:
|
||||
model: text-embedding-ada-002
|
||||
api_key: sk-unit-fake
|
||||
model_info:
|
||||
mode: embedding
|
||||
input_cost_per_token: 0.002
|
||||
id: hello
|
||||
|
||||
litellm_settings:
|
||||
drop_params: True
|
||||
callbacks: custom_callbacks.my_custom_logger
|
||||
32
litellm-rust/crates/config-load/tests/fixtures/valid/test_guardrails_config.yaml
vendored
Normal file
32
litellm-rust/crates/config-load/tests/fixtures/valid/test_guardrails_config.yaml
vendored
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
|
||||
|
||||
model_list:
|
||||
- litellm_params:
|
||||
api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
|
||||
api_key: os.environ/AZURE_EUROPE_API_KEY
|
||||
model: azure/gpt-35-turbo
|
||||
model_name: azure-model
|
||||
- litellm_params:
|
||||
api_base: https://my-endpoint-canada-berri992.openai.azure.com
|
||||
api_key: os.environ/AZURE_CANADA_API_KEY
|
||||
model: azure/gpt-35-turbo
|
||||
model_name: azure-model
|
||||
- litellm_params:
|
||||
api_base: https://openai-france-1234.openai.azure.com
|
||||
api_key: os.environ/AZURE_FRANCE_API_KEY
|
||||
model: azure/gpt-turbo
|
||||
model_name: azure-model
|
||||
|
||||
|
||||
|
||||
litellm_settings:
|
||||
guardrails:
|
||||
- prompt_injection:
|
||||
callbacks: [lakera_prompt_injection, detect_prompt_injection]
|
||||
default_on: true
|
||||
- hide_secrets:
|
||||
callbacks: [hide_secrets]
|
||||
default_on: true
|
||||
- moderations:
|
||||
callbacks: [openai_moderations]
|
||||
default_on: false
|
||||
111
litellm-rust/crates/config-load/tests/load.rs
Normal file
111
litellm-rust/crates/config-load/tests/load.rs
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
use std::fs;
|
||||
|
||||
use litellm_config_load::{Error, load};
|
||||
use rstest::{fixture, rstest};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[fixture]
|
||||
fn directory() -> TempDir {
|
||||
tempfile::tempdir().unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn loads_model_list_from_file(directory: TempDir) {
|
||||
let path = directory.path().join("config.yaml");
|
||||
fs::write(
|
||||
&path,
|
||||
r#"
|
||||
model_list:
|
||||
- model_name: assistant
|
||||
litellm_params:
|
||||
model: anthropic/test-model
|
||||
- model_name: local
|
||||
litellm_params:
|
||||
model: test-model
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config = load(path).unwrap();
|
||||
assert_eq!(
|
||||
config
|
||||
.model_list
|
||||
.iter()
|
||||
.map(|model| model.model_name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["assistant", "local"]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing_file(None, |error: &Error| matches!(error, Error::Read(error) if error.kind() == std::io::ErrorKind::NotFound))]
|
||||
#[case::malformed_yaml(Some("model_list: ["), |error: &Error| matches!(error, Error::Parse(_)))]
|
||||
fn distinguishes_read_errors_from_parse_errors(
|
||||
directory: TempDir,
|
||||
#[case] contents: Option<&str>,
|
||||
#[case] expected: fn(&Error) -> bool,
|
||||
) {
|
||||
let path = directory.path().join("config.yaml");
|
||||
if let Some(contents) = contents {
|
||||
fs::write(&path, contents).unwrap();
|
||||
}
|
||||
|
||||
let error = load(&path).unwrap_err();
|
||||
let Error::Document { source, .. } = &error else {
|
||||
panic!("{error:?}");
|
||||
};
|
||||
assert!(expected(source), "{error:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn resolves_nested_includes_once_in_breadth_first_order(directory: TempDir) {
|
||||
let root = directory.path();
|
||||
fs::create_dir(root.join("nested")).unwrap();
|
||||
fs::write(
|
||||
root.join("config.yaml"),
|
||||
r#"
|
||||
include: [nested/first.yaml, second.yaml]
|
||||
model_list: [{model_name: root, litellm_params: {model: root}}]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
root.join("nested/first.yaml"),
|
||||
r#"
|
||||
include: [third.yaml]
|
||||
model_list: [{model_name: first, litellm_params: {model: first}}]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
root.join("second.yaml"),
|
||||
r#"
|
||||
general_settings: {master_key: second}
|
||||
model_list: [{model_name: second, litellm_params: {model: second}}]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
root.join("nested/third.yaml"),
|
||||
r#"
|
||||
include: [../config.yaml]
|
||||
general_settings: {master_key: third}
|
||||
model_list: [{model_name: third, litellm_params: {model: third}}]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config = load(root.join("config.yaml")).unwrap();
|
||||
assert_eq!(
|
||||
config
|
||||
.model_list
|
||||
.iter()
|
||||
.map(|model| model.model_name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["root", "first", "second", "third"]
|
||||
);
|
||||
assert_eq!(
|
||||
config.general_settings.master_key.unwrap().expose(),
|
||||
"third"
|
||||
);
|
||||
}
|
||||
61
litellm-rust/crates/config/AGENTS.md
Normal file
61
litellm-rust/crates/config/AGENTS.md
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
## Scope
|
||||
|
||||
- The crate is the type definition of `config.yaml`, what a user writes and nothing more. Parsing one YAML text into those types (`Config::from_yaml`, `Config::from_mapping`, the PyYAML reading rules in `yaml.rs`) is all it does
|
||||
- Everything that turns those types into what the gateway runs on lives in `crates/config-load`: resolving `include`, overlaying DB rows, reading the environment, substituting references, applying defaults and building the domain types. So this crate never reads files, the environment or a secret manager
|
||||
- It knows nothing about which settings a consumer implements or rolls out, so checks like "refuse setting X because it is not supported yet" live in the consuming crate (e.g. `gateway`), never here
|
||||
|
||||
## Defaults
|
||||
|
||||
- The value the existing proxy uses when a key is absent (`health_check_interval` is 300, `database_connection_pool_limit` is 10) is part of what `config.yaml` means, but it is applied in `crates/config-load`, never here. Parsing never fills a default in, so every key here keeps whether the file set it. The proxy treats a key set in the file as owned by the file, so the DB value and Admin UI edits for it are ignored
|
||||
- Other sections have no DB overlay, so a key there whose default is empty (an empty list or map) can be the plain collection, filled in by `#[serde(default)]`
|
||||
|
||||
## Module layout
|
||||
|
||||
Each top-level `config.yaml` key belongs to exactly one module under `src/`. A module is a single `<name>.rs` file until it has flattened sub-groups, then it becomes a `<name>/` folder with one file per sub-group. This only applies one level down, so groups flattened inside a sub-group (like the cache backends in `litellm/caching.rs`) stay in that sub-group's file. Keys typed as a plain `Object` have no module and stay on `Config` in `lib.rs`. Keep this list in sync when adding a key
|
||||
|
||||
- `model.rs`: `model_list`
|
||||
- `general/`: `general_settings`
|
||||
- `router.rs`: `router_settings`
|
||||
- `litellm/`: `litellm_settings`
|
||||
- `callback.rs`: `callback_settings`
|
||||
- `provider.rs`: `assistant_settings`, `default_vertex_config`, `files_settings`, `finetune_settings`, `worker_registry`
|
||||
- `mcp.rs`: `mcp_servers`
|
||||
- `credential.rs`: `credential_list`
|
||||
- `guardrail.rs`: `guardrails`
|
||||
- `prompt.rs`: `prompts`
|
||||
- `tool.rs`: `sandbox_tools`, `search_tools`
|
||||
- `vector_store.rs`: `vector_store_registry`
|
||||
- `agent.rs`: `agents`, `agent_list`
|
||||
- `policy.rs`: `policies`, `policy_attachments`
|
||||
- `lib.rs`: `environment_variables`, `mcp_tools`, `include`
|
||||
|
||||
`error.rs`, `value.rs` and `yaml.rs` are crate infrastructure, not config keys
|
||||
|
||||
## Serde flatten
|
||||
|
||||
Sections split one YAML mapping into several structs with nested `#[serde(flatten)]` (`LitellmSettings` flattens `CachingSettings`, `GeneralSettings` flattens `AuthSettings`, and so on). Serde hands keys to flattened fields like this
|
||||
|
||||
- A flattened struct without a flatten field of its own takes only the keys it declares, and later siblings and the catch-all never see them
|
||||
- A flattened struct or map that has its own flatten field is read as a map and claims nothing. It sees every key still unclaimed, including keys a later sibling declares, and those keys also reach the outer catch-all
|
||||
- Fields claim keys in declaration order, so when two flattened structs declare the same key only the first one gets it
|
||||
|
||||
Which gives these rules
|
||||
|
||||
- Each mapping has exactly one `#[serde(flatten)] additional_fields: Object`, declared last on the outermost struct of that mapping. Flattened sub-structs stay plain, with no flatten field inside
|
||||
- The exception is a mapping where Python gives every unnamed key one meaning, like the callback vars in a `default_team_settings` entry. Its catch-all is a typed map named for that meaning (`inline_callback_vars`), so a value of the wrong shape fails to parse
|
||||
- A key belongs to exactly one struct. Which one is decided by who reads it, see `src/AGENTS.md`
|
||||
- Never combine `deny_unknown_fields` with flatten, on either the outer or the flattened struct. It is fine on a struct that is never flattened and has no flatten fields
|
||||
|
||||
## Inline YAML
|
||||
|
||||
- Write inline YAML as a multi-line raw string (`r#"..."#`), not as a one-line string with `\n` escapes like `"key:\n value"`
|
||||
- This includes `#[case(...)]` arguments. Open the raw string on the attribute line and start the YAML at column 0, e.g. `#[case::name(r#"` then the YAML lines then `"#)]`
|
||||
- YAML that fits on one line, like flow-style `key: {a: 1}`, stays a plain string with no trailing `\n`
|
||||
|
||||
## Debug and redaction
|
||||
|
||||
Every struct derives `Debug`, and redaction lives on the field types, so no struct has a hand-written `Debug`
|
||||
|
||||
- Anything that is or can carry a secret (keys, tokens, passwords, URLs that may embed credentials, MCP `command` and `args`) is `SecretValue`, which prints `[REDACTED]`
|
||||
- YAML we don't model is `Object` (prints its keys) or `Raw` (prints its kind). They are the only hand-written `Debug` impls. Never put a bare `Value` in a field
|
||||
- Redaction tests put a unique `SENTINEL` under the slot and assert it never shows up in `format!("{config:?}")`. Words like `password` also match field names such as `redis_password`
|
||||
|
|
@ -7,7 +7,9 @@ repository.workspace = true
|
|||
|
||||
[dependencies]
|
||||
litellm-auth-types.workspace = true
|
||||
litellm-python-compat.workspace = true
|
||||
litellm-router-types.workspace = true
|
||||
macro_rules_attribute.workspace = true
|
||||
serde.workspace = true
|
||||
serde_yaml_ng = "0.10.0"
|
||||
strum.workspace = true
|
||||
|
|
@ -16,4 +18,3 @@ thiserror.workspace = true
|
|||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
serde_json.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
|
|
|||
32
litellm-rust/crates/config/src/AGENTS.md
Normal file
32
litellm-rust/crates/config/src/AGENTS.md
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
## Grouping lives in the types
|
||||
|
||||
The YAML shape is Python's `config.yaml` and we never change it, so a mapping stays flat even when its keys serve different consumers. The grouping is encoded in the types instead, with flattened structs (see the "Serde flatten" rules in `../AGENTS.md`)
|
||||
|
||||
### Rules
|
||||
|
||||
- Every distinct set of readers gets its own flattened struct, even when only one key has that set. `namespace` in `cache_params` is read only by `redis`, so it gets its own struct and does not sit with `host` and `port`, which `redis-semantic` and `valkey-semantic` also read
|
||||
- Only keys that every reader of the mapping uses stay on the parent struct. A key that some readers ignore never stays on the parent just because no group fits yet. Make the group
|
||||
- Once a mapping has keys with different readers, all of them move into groups, never only some. `router_settings` typed as one flat struct, or a parent that keeps keys an existing group already covers, breaks this
|
||||
|
||||
### Finding the readers
|
||||
|
||||
- A reader is the code that receives or branches on the value, found in the Python that consumes the key. The key's name does not tell you. `default_in_redis_ttl` looks Redis-only but Valkey reads it too, and `default_redis_ttl` sits under `cache_params` yet configures the proxy's internal caches, not the response cache
|
||||
- Follow the value to where it is used, not just to where it is passed. A constructor argument read in one `if type == ...` arm has that arm as its reader. A value applied behind an `isinstance` guard is read by that class and its subclasses only (`namespace` reaches `RedisCache` and `RedisClusterCache`, not `RedisSemanticCache`)
|
||||
- A key read by the shared code path for every variant (e.g. `ttl` and `mode`, read by `Cache` itself) is read by all of them and stays on the parent
|
||||
|
||||
### Naming
|
||||
|
||||
- Name each struct for its reader set (`RedisServerCacheParams`, `SemanticCacheParams`), and name the parent's flattened field for the same set (`redis_server`, `semantic`)
|
||||
- When the name alone does not say who reads the keys, add a one-line doc comment on the struct that lists the readers, like the one on `RedisServerCacheParams`
|
||||
|
||||
### Tests
|
||||
|
||||
- Each mapping has one rstest with a case per group, putting one key of that group in the YAML and asserting it lands in that group's field and not in `additional_fields`
|
||||
- When a key's readers were easy to get wrong (the examples above), add a case for that key too, so moving it back into a broader group fails a test
|
||||
|
||||
## Lists are boxed slices
|
||||
|
||||
Config is parsed once and only read after that, so every list field on a config type is `Box<[T]>` (or `Option<Box<[T]>>`), never `Vec<T>`. The type then says the list can't grow, and serde deserializes into it directly
|
||||
|
||||
- `Vec<T>` is only for values that get built up and thrown away, like the layers and origins in `include/`. Those never become config fields
|
||||
- Code that builds a field by hand collects into a `Vec` and converts at the end (`Box::from`, `.into_boxed_slice()`), and never keeps the `Vec` around
|
||||
75
litellm-rust/crates/config/src/agent.rs
Normal file
75
litellm-rust/crates/config/src/agent.rs
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
use crate::{Object, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct Agent {
|
||||
pub agent_name: Option<String>,
|
||||
pub agent_card_params: Option<Object>,
|
||||
pub litellm_params: Option<Object>,
|
||||
pub identity: Option<Object>,
|
||||
pub enabled: Option<Spelled<bool>>,
|
||||
pub execution_mode: Option<String>,
|
||||
pub object_permission: Option<Object>,
|
||||
pub tpm_limit: Option<Spelled<i64>>,
|
||||
pub rpm_limit: Option<Spelled<i64>>,
|
||||
pub session_tpm_limit: Option<Spelled<i64>>,
|
||||
pub session_rpm_limit: Option<Spelled<i64>>,
|
||||
pub static_headers: Option<Object>,
|
||||
pub extra_headers: Option<Box<[String]>>,
|
||||
pub access_group_ids: Option<Box<[String]>>,
|
||||
pub kill_switch: Option<Object>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::{Config, Spelled};
|
||||
|
||||
fn parse(yaml: &str) -> Config {
|
||||
Config::from_yaml(yaml).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::agents("agents")]
|
||||
#[case::agent_list("agent_list")]
|
||||
fn agent_entries_keep_typed_limits_and_flags(#[case] section: &str) {
|
||||
let config = parse(&format!(
|
||||
"{section}:\n - {{agent_name: reviewer, agent_card_params: {{name: x}}, tpm_limit: 100, enabled: false}}\n"
|
||||
));
|
||||
let agent = config.agents.first().or(config.agent_list.first()).unwrap();
|
||||
|
||||
assert_eq!(agent.agent_name.as_deref(), Some("reviewer"));
|
||||
assert!(
|
||||
agent
|
||||
.agent_card_params
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.contains_key("name")
|
||||
);
|
||||
assert_eq!(agent.tpm_limit, Some(Spelled::Value(100)));
|
||||
assert_eq!(agent.enabled, Some(Spelled::Value(false)));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::agent_headers(
|
||||
r#"
|
||||
agents:
|
||||
- static_headers: {Authorization: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::agent_params(
|
||||
r#"
|
||||
agents:
|
||||
- litellm_params: {api_key: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::agent_identity(
|
||||
r#"
|
||||
agent_list:
|
||||
- identity: {token: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
fn debug_never_prints_agent_values(#[case] yaml: &str) {
|
||||
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
|
||||
}
|
||||
}
|
||||
77
litellm-rust/crates/config/src/callback.rs
Normal file
77
litellm-rust/crates/config/src/callback.rs
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
use litellm_auth_types::SecretValue;
|
||||
|
||||
use crate::{Object, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct CallbackSettings {
|
||||
pub callback_type: Option<String>,
|
||||
pub endpoint: Option<SecretValue>,
|
||||
pub headers: Option<Object>,
|
||||
pub event_types: Option<Box<[String]>>,
|
||||
pub log_format: Option<String>,
|
||||
pub max_retries: Option<Spelled<i64>>,
|
||||
pub retry_delay: Option<Spelled<f64>>,
|
||||
pub timeout: Option<Spelled<f64>>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::{Config, Spelled};
|
||||
|
||||
fn parse(yaml: &str) -> Config {
|
||||
Config::from_yaml(yaml).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn callback_settings_keep_typed_fields_and_unknown_keys() {
|
||||
let config = parse(
|
||||
"callback_settings:\n hook:\n callback_type: generic_api\n retry_delay: 0.5\n batch_size: 10\n",
|
||||
);
|
||||
let hook = &config.callback_settings["hook"];
|
||||
|
||||
assert_eq!(hook.callback_type.as_deref(), Some("generic_api"));
|
||||
assert_eq!(hook.retry_delay, Some(Spelled::Value(0.5)));
|
||||
assert!(hook.additional_fields.contains_key("batch_size"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::env_reference("os.environ/RETRIES", Spelled::Text("os.environ/RETRIES".to_owned()))]
|
||||
#[case::number("4", Spelled::Value(4))]
|
||||
fn spelled_int_fields_keep_references_as_text(
|
||||
#[case] spelled: &str,
|
||||
#[case] expected: Spelled<i64>,
|
||||
) {
|
||||
let config = parse(&format!(
|
||||
"callback_settings:\n hook:\n max_retries: {spelled}\n"
|
||||
));
|
||||
|
||||
assert_eq!(config.callback_settings["hook"].max_retries, Some(expected));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::callback_endpoint(
|
||||
r#"
|
||||
callback_settings:
|
||||
hook: {endpoint: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::callback_headers(
|
||||
r#"
|
||||
callback_settings:
|
||||
hook: {headers: {Authorization: SENTINEL}}
|
||||
"#
|
||||
)]
|
||||
#[case::callback_extra(
|
||||
r#"
|
||||
callback_settings:
|
||||
hook: {vendor_secret: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
fn debug_never_prints_callback_values(#[case] yaml: &str) {
|
||||
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
|
||||
}
|
||||
}
|
||||
65
litellm-rust/crates/config/src/credential.rs
Normal file
65
litellm-rust/crates/config/src/credential.rs
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
use crate::Object;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct Credential {
|
||||
pub credential_name: Option<String>,
|
||||
pub credential_info: Option<Object>,
|
||||
pub credential_values: Option<Object>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::Config;
|
||||
|
||||
fn parse(yaml: &str) -> Config {
|
||||
Config::from_yaml(yaml).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn credential_entries_keep_name_info_and_values() {
|
||||
let config = parse(
|
||||
r#"
|
||||
credential_list:
|
||||
- credential_name: azure-prod
|
||||
credential_info: {description: prod}
|
||||
credential_values: {api_key: os.environ/AZURE_API_KEY}
|
||||
"#,
|
||||
);
|
||||
let credential = &config.credential_list[0];
|
||||
|
||||
assert_eq!(credential.credential_name.as_deref(), Some("azure-prod"));
|
||||
assert!(
|
||||
credential
|
||||
.credential_info
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.contains_key("description")
|
||||
);
|
||||
assert!(
|
||||
credential
|
||||
.credential_values
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.contains_key("api_key")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::credential_values(
|
||||
r#"
|
||||
credential_list:
|
||||
- credential_values: {api_key: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::credential_info(
|
||||
r#"
|
||||
credential_list:
|
||||
- credential_info: {owner: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
fn debug_never_prints_credential_values(#[case] yaml: &str) {
|
||||
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,5 @@
|
|||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("could not read config")]
|
||||
Read(#[from] std::io::Error),
|
||||
#[error("invalid YAML config")]
|
||||
Parse(#[from] serde_yaml_ng::Error),
|
||||
}
|
||||
|
|
|
|||
14
litellm-rust/crates/config/src/general/AGENTS.md
Normal file
14
litellm-rust/crates/config/src/general/AGENTS.md
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
## Docs
|
||||
|
||||
- https://github.com/BerriAI/litellm-docs/blob/main/docs/proxy/config_settings.md
|
||||
|
||||
## Python
|
||||
|
||||
- `ConfigGeneralSettings` in `litellm/proxy/_types.py` is the Python model but is incomplete. What the proxy reads is `general_settings.get(...)`, mostly in `proxy_server.py`
|
||||
- `general_settings` is one of the sections the proxy persists to the DB (`_CONFIG_PERSISTED_SECTIONS`) and the Admin UI edits, so at runtime the YAML is only one source of the effective values. Merging the DB overlay is the consumer's job, not this crate's
|
||||
|
||||
## Fields that need care
|
||||
|
||||
- `master_key`, `database_url` and `drain_endpoint_token` fall back to `LITELLM_MASTER_KEY`, `DATABASE_URL` and `DRAIN_ENDPOINT_TOKEN`. That fallback belongs to the consumer, so the fields stay optional here
|
||||
- `custom_auth`, `custom_key_generate`, `custom_key_update`, `custom_key_policy`, `custom_team_metadata_validate`, `custom_sso`, `custom_ui_sso_sign_in_handler`, `litellm_jwtauth.custom_validate` and `pass_through_endpoints[].target` are Python module paths that the proxy imports. Type them as plain strings and never resolve or validate them here
|
||||
- `key_management_system` and `key_management_settings` configure the secret manager itself, so they can only use `os.environ/` references, not secret manager ones
|
||||
39
litellm-rust/crates/config/src/general/alerting.rs
Normal file
39
litellm-rust/crates/config/src/general/alerting.rs
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
use serde::Deserialize;
|
||||
|
||||
use crate::{Object, Spelled};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum Alerting {
|
||||
List(Box<[String]>),
|
||||
One(String),
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct AlertingSettings {
|
||||
pub alerting: Option<Alerting>,
|
||||
pub alerting_threshold: Option<Spelled<f64>>,
|
||||
pub alert_types: Option<Box<[String]>>,
|
||||
pub alert_to_webhook_url: Option<Object>,
|
||||
pub alerting_args: Option<AlertingArgs>,
|
||||
pub alert_type_config: Option<Object>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct AlertingArgs {
|
||||
pub daily_report_frequency: Option<Spelled<i64>>,
|
||||
pub report_check_interval: Option<Spelled<i64>>,
|
||||
pub budget_alert_ttl: Option<Spelled<i64>>,
|
||||
pub outage_alert_ttl: Option<Spelled<i64>>,
|
||||
pub region_outage_alert_ttl: Option<Spelled<i64>>,
|
||||
pub minor_outage_alert_threshold: Option<Spelled<i64>>,
|
||||
pub major_outage_alert_threshold: Option<Spelled<i64>>,
|
||||
pub max_outage_alert_list_size: Option<Spelled<i64>>,
|
||||
pub log_to_console: Option<Spelled<bool>>,
|
||||
pub daily_spend_per_user_threshold: Option<Spelled<f64>>,
|
||||
pub monthly_spend_per_user_threshold: Option<Spelled<f64>>,
|
||||
pub spend_anomaly_multiplier: Option<Spelled<f64>>,
|
||||
pub spend_anomaly_baseline_days: Option<Spelled<i64>>,
|
||||
pub spend_anomaly_min_spend: Option<Spelled<f64>>,
|
||||
pub user_spend_check_interval: Option<Spelled<i64>>,
|
||||
}
|
||||
77
litellm-rust/crates/config/src/general/auth.rs
Normal file
77
litellm-rust/crates/config/src/general/auth.rs
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
use litellm_auth_types::SecretValue;
|
||||
|
||||
use crate::{Object, Raw, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct AuthSettings {
|
||||
pub litellm_key_header_name: Option<String>,
|
||||
pub enable_jwt_auth: Option<Spelled<bool>>,
|
||||
pub litellm_jwtauth: Option<Object>,
|
||||
pub enable_oauth2_auth: Option<Spelled<bool>>,
|
||||
pub enable_oauth2_proxy_auth: Option<Spelled<bool>>,
|
||||
pub oauth2_config_mappings: Option<Object>,
|
||||
pub custom_auth: Option<String>,
|
||||
pub custom_auth_run_common_checks: Option<Spelled<bool>>,
|
||||
pub allow_user_auth: Option<Spelled<bool>>,
|
||||
pub allowed_routes: Option<Box<[String]>>,
|
||||
pub admin_only_routes: Option<Box<[String]>>,
|
||||
pub public_routes: Option<Box<[String]>>,
|
||||
pub allowed_ips: Option<Box<[String]>>,
|
||||
pub use_x_forwarded_for: Option<Spelled<bool>>,
|
||||
pub trusted_proxy_ranges: Option<Box<[String]>>,
|
||||
pub user_header_name: Option<String>,
|
||||
pub user_header_mappings: Option<Box<[UserHeaderMapping]>>,
|
||||
pub role_permissions: Option<Box<[RoleBasedPermissions]>>,
|
||||
pub enforce_rbac: Option<Spelled<bool>>,
|
||||
pub allow_client_side_credentials: Option<Spelled<bool>>,
|
||||
pub enforce_user_param: Option<Spelled<bool>>,
|
||||
pub reject_clientside_metadata_tags: Option<Spelled<bool>>,
|
||||
pub enforced_params: Option<Box<[String]>>,
|
||||
pub service_account_settings: Option<ServiceAccountSettings>,
|
||||
pub litellm_license: Option<SecretValue>,
|
||||
pub control_plane_url: Option<String>,
|
||||
pub key_management_system: Option<String>,
|
||||
pub key_management_settings: Option<KeyManagementSettings>,
|
||||
pub use_google_kms: Option<Spelled<bool>>,
|
||||
pub use_azure_key_vault: Option<Spelled<bool>>,
|
||||
pub encryption_algorithm: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct UserHeaderMapping {
|
||||
pub header_name: Option<String>,
|
||||
pub litellm_user_role: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct RoleBasedPermissions {
|
||||
pub role: Option<String>,
|
||||
pub models: Option<Box<[String]>>,
|
||||
pub routes: Option<Box<[String]>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct ServiceAccountSettings {
|
||||
pub enforced_params: Option<Box<[String]>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct KeyManagementSettings {
|
||||
pub hosted_keys: Option<Box<[Raw]>>,
|
||||
pub store_virtual_keys: Option<Spelled<bool>>,
|
||||
pub prefix_for_stored_virtual_keys: Option<String>,
|
||||
pub access_mode: Option<String>,
|
||||
pub primary_secret_name: Option<String>,
|
||||
pub description: Option<String>,
|
||||
pub tags: Option<Object>,
|
||||
pub kms_key_id: Option<String>,
|
||||
pub custom_secret_manager: Option<String>,
|
||||
pub aws_region_name: Option<String>,
|
||||
pub aws_role_name: Option<String>,
|
||||
pub aws_session_name: Option<String>,
|
||||
pub aws_external_id: Option<SecretValue>,
|
||||
pub aws_profile_name: Option<String>,
|
||||
pub aws_web_identity_token: Option<SecretValue>,
|
||||
pub aws_sts_endpoint: Option<String>,
|
||||
pub replica_regions: Option<Box<[String]>>,
|
||||
}
|
||||
15
litellm-rust/crates/config/src/general/database.rs
Normal file
15
litellm-rust/crates/config/src/general/database.rs
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
use crate::{Object, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct DatabaseSettings {
|
||||
pub database_connection_pool_timeout: Option<Spelled<f64>>,
|
||||
pub database_extra_connection_params: Option<Object>,
|
||||
pub database_disable_prepared_statements: Option<Spelled<bool>>,
|
||||
pub database_statement_timeout: Option<Spelled<f64>>,
|
||||
pub database_lock_timeout: Option<Spelled<f64>>,
|
||||
pub database_idle_in_transaction_session_timeout: Option<Spelled<f64>>,
|
||||
pub disable_prisma_schema_update: Option<Spelled<bool>>,
|
||||
pub allow_requests_on_db_unavailable: Option<Spelled<bool>>,
|
||||
pub supported_db_objects: Option<Box<[String]>>,
|
||||
pub use_redis_transaction_buffer: Option<Spelled<bool>>,
|
||||
}
|
||||
15
litellm-rust/crates/config/src/general/files.rs
Normal file
15
litellm-rust/crates/config/src/general/files.rs
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
use crate::Spelled;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct FilesSettings {
|
||||
pub max_file_size_mb: Option<Spelled<i64>>,
|
||||
pub max_batch_file_size_mb: Option<Spelled<i64>>,
|
||||
pub max_batch_file_records: Option<Spelled<i64>>,
|
||||
pub max_batch_file_uploads_per_day: Option<Spelled<i64>>,
|
||||
pub max_file_downloads_per_minute: Option<Spelled<i64>>,
|
||||
pub allowed_file_extensions: Option<Box<[String]>>,
|
||||
pub blocked_file_extensions: Option<Box<[String]>>,
|
||||
pub disable_batch_input_file_rate_limiting: Option<Spelled<bool>>,
|
||||
pub skip_batch_input_file_rate_limiting_for_providers: Option<Box<[String]>>,
|
||||
pub skip_batch_input_file_rate_limiting_for_models: Option<Box<[String]>>,
|
||||
}
|
||||
22
litellm-rust/crates/config/src/general/health.rs
Normal file
22
litellm-rust/crates/config/src/general/health.rs
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
use litellm_auth_types::SecretValue;
|
||||
|
||||
use crate::Spelled;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct HealthSettings {
|
||||
pub use_shared_health_check: Option<Spelled<bool>>,
|
||||
pub health_check_details: Option<Spelled<bool>>,
|
||||
pub enable_health_check_routing: Option<Spelled<bool>>,
|
||||
pub health_check_staleness_threshold: Option<Spelled<f64>>,
|
||||
pub health_check_ignore_transient_errors: Option<Spelled<bool>>,
|
||||
pub health_check_skip_disabled_background_models: Option<Spelled<bool>>,
|
||||
pub background_health_check_model_groups: Option<Box<[String]>>,
|
||||
pub model_list_healthy_only: Option<Spelled<bool>>,
|
||||
pub allow_public_health_readiness_details: Option<Spelled<bool>>,
|
||||
pub enable_drain_endpoint: Option<Spelled<bool>>,
|
||||
pub drain_endpoint_token: Option<SecretValue>,
|
||||
pub disable_model_info_refresh: Option<Spelled<bool>>,
|
||||
pub use_team_public_model_name: Option<Spelled<bool>>,
|
||||
pub image_generation_model: Option<String>,
|
||||
pub moderation_model: Option<String>,
|
||||
}
|
||||
11
litellm-rust/crates/config/src/general/limits.rs
Normal file
11
litellm-rust/crates/config/src/general/limits.rs
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
use crate::Spelled;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct LimitsSettings {
|
||||
pub fail_closed_rate_limit_enforcement: Option<Spelled<bool>>,
|
||||
pub token_rate_limit_type: Option<String>,
|
||||
pub disable_retry_on_max_parallel_request_limit_error: Option<Spelled<bool>>,
|
||||
pub disable_fallbacks_on_per_model_rate_limits: Option<Spelled<bool>>,
|
||||
pub user_api_key_cache_ttl: Option<Spelled<f64>>,
|
||||
pub user_api_key_cache_max_size: Option<Spelled<i64>>,
|
||||
}
|
||||
20
litellm-rust/crates/config/src/general/mcp.rs
Normal file
20
litellm-rust/crates/config/src/general/mcp.rs
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
use crate::{Raw, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct McpSettings {
|
||||
pub mcp_allowed_hosts: Option<Box<[String]>>,
|
||||
pub mcp_allowed_origins: Box<[String]>,
|
||||
pub mcp_advertised_versions: Option<Raw>,
|
||||
pub mcp_disabled_capabilities: Option<Raw>,
|
||||
pub mcp_allowed_clients: Option<Raw>,
|
||||
pub mcp_require_explicit_server_scope: Option<Spelled<bool>>,
|
||||
pub mcp_client_id_header: Option<String>,
|
||||
pub mcp_client_side_auth_header_name: Option<String>,
|
||||
pub mcp_allow_all_keys_respects_mcp_scope: Option<Spelled<bool>>,
|
||||
pub mcp_prefer_client_id_metadata_document: Option<Spelled<bool>>,
|
||||
pub mcp_internal_ip_ranges: Option<Raw>,
|
||||
pub mcp_trusted_proxy_ranges: Option<Raw>,
|
||||
pub mcp_xff_num_trusted_hops: Option<Spelled<i64>>,
|
||||
pub mcp_required_fields: Option<Raw>,
|
||||
pub mcp_session_token_signing: Option<Raw>,
|
||||
}
|
||||
535
litellm-rust/crates/config/src/general/mod.rs
Normal file
535
litellm-rust/crates/config/src/general/mod.rs
Normal file
|
|
@ -0,0 +1,535 @@
|
|||
mod alerting;
|
||||
mod auth;
|
||||
mod database;
|
||||
mod files;
|
||||
mod health;
|
||||
mod limits;
|
||||
mod mcp;
|
||||
mod objects;
|
||||
mod requests;
|
||||
mod spend;
|
||||
mod tracing;
|
||||
mod ui;
|
||||
|
||||
use litellm_auth_types::SecretValue;
|
||||
use serde::Deserialize;
|
||||
|
||||
pub use self::{
|
||||
alerting::{Alerting, AlertingArgs, AlertingSettings},
|
||||
auth::{
|
||||
AuthSettings, KeyManagementSettings, RoleBasedPermissions, ServiceAccountSettings,
|
||||
UserHeaderMapping,
|
||||
},
|
||||
database::DatabaseSettings,
|
||||
files::FilesSettings,
|
||||
health::HealthSettings,
|
||||
limits::LimitsSettings,
|
||||
mcp::McpSettings,
|
||||
objects::{ObjectsSettings, TeamMetadataField},
|
||||
requests::{PassThroughEndpoint, RequestTagHeader, RequestTagMapping, RequestsSettings},
|
||||
spend::{
|
||||
ScheduledJobStaggerSettings, SpendCaptureRateCheckSettings, SpendLogsMetadataFields,
|
||||
SpendSettings,
|
||||
},
|
||||
tracing::{ClickHouseStoreSettings, TracingSettings, TracingStoreSettings},
|
||||
ui::{UiAccessMode, UiAccessSsoGroup, UiSettings},
|
||||
};
|
||||
use crate::Object;
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct GeneralSettings {
|
||||
pub completion_model: Option<String>,
|
||||
pub max_in_flight_requests_per_worker: Option<u64>,
|
||||
pub max_queued_requests_per_worker: Option<u64>,
|
||||
pub admission_queue_timeout_seconds: f64,
|
||||
pub master_key: Option<SecretValue>,
|
||||
pub database_url: Option<SecretValue>,
|
||||
pub tracing: Option<TracingSettings>,
|
||||
pub database_connection_pool_limit: Option<u64>,
|
||||
pub database_connection_timeout: Option<f64>,
|
||||
pub database_connect_timeout: Option<f64>,
|
||||
pub database_socket_timeout: Option<f64>,
|
||||
pub database_max_idle_connection_lifetime: Option<f64>,
|
||||
pub max_parallel_requests: Option<u64>,
|
||||
pub global_max_parallel_requests: Option<u64>,
|
||||
pub max_request_size_mb: Option<u64>,
|
||||
pub max_response_size_mb: Option<u64>,
|
||||
pub proxy_config_reload_interval_seconds: u64,
|
||||
pub background_health_checks: Option<bool>,
|
||||
pub health_check_interval: u64,
|
||||
pub health_check_concurrency: Option<u64>,
|
||||
pub store_model_in_db: Option<bool>,
|
||||
pub forward_client_headers_to_llm_api: Option<bool>,
|
||||
pub cancel_on_disconnect: Option<bool>,
|
||||
pub infer_model_from_keys: Option<bool>,
|
||||
pub enable_public_model_hub: bool,
|
||||
pub dangerously_permit_weak_or_unset_master_key: Option<bool>,
|
||||
pub plugins: Option<Box<[Object]>>,
|
||||
pub coordination_redis: Option<Object>,
|
||||
#[serde(flatten)]
|
||||
pub auth: AuthSettings,
|
||||
#[serde(flatten)]
|
||||
pub database: DatabaseSettings,
|
||||
#[serde(flatten)]
|
||||
pub spend: SpendSettings,
|
||||
#[serde(flatten)]
|
||||
pub limits: LimitsSettings,
|
||||
#[serde(flatten)]
|
||||
pub health: HealthSettings,
|
||||
#[serde(flatten)]
|
||||
pub alerting: AlertingSettings,
|
||||
#[serde(flatten)]
|
||||
pub ui: UiSettings,
|
||||
#[serde(flatten)]
|
||||
pub objects: ObjectsSettings,
|
||||
#[serde(flatten)]
|
||||
pub requests: RequestsSettings,
|
||||
#[serde(flatten)]
|
||||
pub files: FilesSettings,
|
||||
#[serde(flatten)]
|
||||
pub mcp: McpSettings,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
impl Default for GeneralSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
completion_model: None,
|
||||
max_in_flight_requests_per_worker: None,
|
||||
max_queued_requests_per_worker: None,
|
||||
admission_queue_timeout_seconds: 1.0,
|
||||
master_key: None,
|
||||
database_url: None,
|
||||
tracing: None,
|
||||
database_connection_pool_limit: Some(10),
|
||||
database_connection_timeout: Some(60.0),
|
||||
database_connect_timeout: None,
|
||||
database_socket_timeout: None,
|
||||
database_max_idle_connection_lifetime: Some(60.0),
|
||||
max_parallel_requests: None,
|
||||
global_max_parallel_requests: None,
|
||||
max_request_size_mb: None,
|
||||
max_response_size_mb: None,
|
||||
proxy_config_reload_interval_seconds: 30,
|
||||
background_health_checks: None,
|
||||
health_check_interval: 300,
|
||||
health_check_concurrency: None,
|
||||
store_model_in_db: None,
|
||||
forward_client_headers_to_llm_api: None,
|
||||
cancel_on_disconnect: None,
|
||||
infer_model_from_keys: None,
|
||||
enable_public_model_hub: false,
|
||||
dangerously_permit_weak_or_unset_master_key: None,
|
||||
plugins: None,
|
||||
coordination_redis: None,
|
||||
auth: AuthSettings::default(),
|
||||
database: DatabaseSettings::default(),
|
||||
spend: SpendSettings::default(),
|
||||
limits: LimitsSettings::default(),
|
||||
health: HealthSettings::default(),
|
||||
alerting: AlertingSettings::default(),
|
||||
ui: UiSettings::default(),
|
||||
objects: ObjectsSettings::default(),
|
||||
requests: RequestsSettings::default(),
|
||||
files: FilesSettings::default(),
|
||||
mcp: McpSettings::default(),
|
||||
additional_fields: Object::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::{Alerting, Config, GeneralSettings, Spelled, UiAccessMode};
|
||||
|
||||
fn general(yaml: &str) -> GeneralSettings {
|
||||
Config::from_yaml(&format!("general_settings:\n{yaml}"))
|
||||
.unwrap()
|
||||
.general_settings
|
||||
}
|
||||
|
||||
fn text<T>(value: &str) -> Option<Spelled<T>> {
|
||||
Some(Spelled::Text(value.to_string()))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::yaml_bool("true", Some(Spelled::Value(true)))]
|
||||
#[case::yaml_1_1_word("yes", text("yes"))]
|
||||
#[case::quoted("'false'", text("false"))]
|
||||
#[case::environment_reference("os.environ/ENABLE_JWT", text("os.environ/ENABLE_JWT"))]
|
||||
fn bool_settings_keep_every_python_spelling(
|
||||
#[case] spelling: &str,
|
||||
#[case] expected: Option<Spelled<bool>>,
|
||||
) {
|
||||
let settings = general(&format!(
|
||||
" enable_jwt_auth: {spelling}\n disable_spend_logs: {spelling}\n use_redis_transaction_buffer: {spelling}\n model_list_healthy_only: {spelling}\n"
|
||||
));
|
||||
assert_eq!(settings.auth.enable_jwt_auth, expected);
|
||||
assert_eq!(settings.spend.disable_spend_logs, expected);
|
||||
assert_eq!(settings.database.use_redis_transaction_buffer, expected);
|
||||
assert_eq!(settings.health.model_list_healthy_only, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::bare_days("7", Some(Spelled::Value(7)))]
|
||||
#[case::duration("'60d'", text("60d"))]
|
||||
#[case::environment_reference("os.environ/RETENTION", text("os.environ/RETENTION"))]
|
||||
fn retention_periods_accept_days_or_a_duration(
|
||||
#[case] spelling: &str,
|
||||
#[case] expected: Option<Spelled<i64>>,
|
||||
) {
|
||||
let settings = general(&format!(
|
||||
" maximum_spend_logs_retention_period: {spelling}\n"
|
||||
));
|
||||
assert_eq!(settings.spend.maximum_spend_logs_retention_period, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::integer("5", Some(Spelled::Value(5.0)))]
|
||||
#[case::fraction("0.5", Some(Spelled::Value(0.5)))]
|
||||
#[case::environment_reference("os.environ/TTL", text("os.environ/TTL"))]
|
||||
fn number_settings_accept_integers_fractions_and_references(
|
||||
#[case] spelling: &str,
|
||||
#[case] expected: Option<Spelled<f64>>,
|
||||
) {
|
||||
let settings = general(&format!(
|
||||
" user_api_key_cache_ttl: {spelling}\n database_statement_timeout: {spelling}\n health_check_staleness_threshold: {spelling}\n"
|
||||
));
|
||||
assert_eq!(settings.limits.user_api_key_cache_ttl, expected);
|
||||
assert_eq!(settings.database.database_statement_timeout, expected);
|
||||
assert_eq!(settings.health.health_check_staleness_threshold, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_settings_are_typed_and_keep_unknown_keys() {
|
||||
let settings = general(
|
||||
r#"
|
||||
allowed_routes: [/chat/completions, /models]
|
||||
custom_auth: custom_auth.user_api_key_auth
|
||||
user_header_mappings:
|
||||
- header_name: X-OpenWebUI-User-Id
|
||||
litellm_user_role: internal_user
|
||||
role_permissions:
|
||||
- role: team
|
||||
models: [gpt]
|
||||
routes: [/chat/completions]
|
||||
service_account_settings:
|
||||
enforced_params: [user]
|
||||
key_management_settings:
|
||||
access_mode: read_and_write
|
||||
store_virtual_keys: true
|
||||
future_option: 1
|
||||
spend_logs_metadata_fields:
|
||||
include: [user_api_key_alias]
|
||||
spend_capture_rate_check:
|
||||
threshold: 0.8
|
||||
lookback_days: 3
|
||||
scheduled_job_stagger:
|
||||
window_seconds: 120
|
||||
offsets: {reset_budget_job: 0}
|
||||
"#,
|
||||
);
|
||||
assert_eq!(
|
||||
settings.auth.allowed_routes.as_deref(),
|
||||
Some(["/chat/completions".to_string(), "/models".to_string()].as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
settings.auth.custom_auth.as_deref(),
|
||||
Some("custom_auth.user_api_key_auth")
|
||||
);
|
||||
let mapping = &settings.auth.user_header_mappings.as_deref().unwrap()[0];
|
||||
assert_eq!(mapping.header_name.as_deref(), Some("X-OpenWebUI-User-Id"));
|
||||
assert_eq!(mapping.litellm_user_role.as_deref(), Some("internal_user"));
|
||||
let permission = &settings.auth.role_permissions.as_deref().unwrap()[0];
|
||||
assert_eq!(permission.role.as_deref(), Some("team"));
|
||||
assert_eq!(
|
||||
permission.routes.as_deref(),
|
||||
Some(["/chat/completions".to_string()].as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
settings
|
||||
.auth
|
||||
.service_account_settings
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.enforced_params
|
||||
.as_deref(),
|
||||
Some(["user".to_string()].as_slice())
|
||||
);
|
||||
let key_management = settings.auth.key_management_settings.as_ref().unwrap();
|
||||
assert_eq!(
|
||||
key_management.access_mode.as_deref(),
|
||||
Some("read_and_write")
|
||||
);
|
||||
assert_eq!(
|
||||
key_management.store_virtual_keys,
|
||||
Some(Spelled::Value(true))
|
||||
);
|
||||
assert_eq!(
|
||||
settings
|
||||
.spend
|
||||
.spend_logs_metadata_fields
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.include
|
||||
.as_deref(),
|
||||
Some(["user_api_key_alias".to_string()].as_slice())
|
||||
);
|
||||
let capture = settings.spend.spend_capture_rate_check.as_ref().unwrap();
|
||||
assert_eq!(capture.threshold, Some(Spelled::Value(0.8)));
|
||||
assert_eq!(capture.lookback_days, Some(Spelled::Value(3)));
|
||||
let stagger = settings.spend.scheduled_job_stagger.as_ref().unwrap();
|
||||
assert_eq!(stagger.window_seconds, Some(Spelled::Value(120)));
|
||||
assert!(
|
||||
stagger
|
||||
.offsets
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.contains_key("reset_budget_job")
|
||||
);
|
||||
assert!(settings.additional_fields.is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::license("litellm_license: SENTINEL")]
|
||||
#[case::drain_token("drain_endpoint_token: SENTINEL")]
|
||||
#[case::jwt_auth_object("litellm_jwtauth: {team_id_jwt_field: SENTINEL}")]
|
||||
#[case::oauth2_mappings("oauth2_config_mappings: {X-Header: SENTINEL}")]
|
||||
#[case::database_params("database_extra_connection_params: {sslpassword: SENTINEL}")]
|
||||
#[case::key_management_token("key_management_settings: {aws_web_identity_token: SENTINEL}")]
|
||||
#[case::key_management_external_id("key_management_settings: {aws_external_id: SENTINEL}")]
|
||||
#[case::key_management_hosted_keys("key_management_settings: {hosted_keys: [SENTINEL]}")]
|
||||
#[case::key_management_tags("key_management_settings: {tags: {owner: SENTINEL}}")]
|
||||
#[case::mcp_session_signing("mcp_session_token_signing: {private_key: SENTINEL}")]
|
||||
#[case::master_key("master_key: SENTINEL")]
|
||||
#[case::database_url("database_url: postgres://user:SENTINEL@db/litellm")]
|
||||
#[case::coordination_redis("coordination_redis: {password: SENTINEL}")]
|
||||
#[case::plugins("plugins: [{token: SENTINEL}]")]
|
||||
#[case::unknown_setting("future_setting: SENTINEL")]
|
||||
fn debug_output_does_not_expose_setting_values(#[case] setting: &str) {
|
||||
let config = Config::from_yaml(&format!("general_settings:\n {setting}\n")).unwrap();
|
||||
let printed = format!("{config:?}");
|
||||
assert!(!printed.contains("SENTINEL"), "{printed}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_keys_still_land_in_additional_fields() {
|
||||
let settings = general(" future_setting: 1\n custom_auth: auth.py\n");
|
||||
assert!(settings.additional_fields.contains_key("future_setting"));
|
||||
assert!(!settings.additional_fields.contains_key("custom_auth"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("true", Spelled::Value(true))]
|
||||
#[case("\"false\"", Spelled::Text("false".to_string()))]
|
||||
#[case("yes", Spelled::Text("yes".to_string()))]
|
||||
#[case("os.environ/GATEWAY_FLAG", Spelled::Text("os.environ/GATEWAY_FLAG".to_string()))]
|
||||
fn bool_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled<bool>) {
|
||||
let settings = general(&format!(" enable_claude_code_gateway: {value}\n"));
|
||||
assert_eq!(settings.requests.enable_claude_code_gateway, Some(expected));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("100", Spelled::Value(100))]
|
||||
#[case("os.environ/MAX_FILE_MB", Spelled::Text("os.environ/MAX_FILE_MB".to_string()))]
|
||||
fn int_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled<i64>) {
|
||||
let settings = general(&format!(" max_file_size_mb: {value}\n"));
|
||||
assert_eq!(settings.files.max_file_size_mb, Some(expected));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("600", Spelled::Value(600.0))]
|
||||
#[case("2.5", Spelled::Value(2.5))]
|
||||
#[case("os.environ/ALERT_THRESHOLD", Spelled::Text("os.environ/ALERT_THRESHOLD".to_string()))]
|
||||
fn float_spellings_land_in_typed_field(#[case] value: &str, #[case] expected: Spelled<f64>) {
|
||||
let settings = general(&format!(" alerting_threshold: {value}\n"));
|
||||
assert_eq!(settings.alerting.alerting_threshold, Some(expected));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn nested_mappings_land_in_typed_structs() {
|
||||
let settings = general(
|
||||
r#"
|
||||
alerting: [slack, email]
|
||||
alerting_args:
|
||||
daily_report_frequency: 3600
|
||||
log_to_console: true
|
||||
spend_anomaly_multiplier: 2.5
|
||||
future_arg: 1
|
||||
team_metadata_schema:
|
||||
- key: cost_center
|
||||
label: Cost center
|
||||
request_tag_mappings:
|
||||
- header: baggage
|
||||
key: tenant
|
||||
tag_prefix: tenant-
|
||||
required: true
|
||||
pass_through_endpoints:
|
||||
- path: /bria
|
||||
target: https://engine.prod.bria-api.com
|
||||
include_subpath: true
|
||||
cost_per_request: 0.5
|
||||
methods: [GET, POST]
|
||||
custom_handler: handler.fn
|
||||
allowed_file_extensions: [jsonl, csv]
|
||||
"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
settings.alerting.alerting,
|
||||
Some(Alerting::List(
|
||||
["slack".to_string(), "email".to_string()].into()
|
||||
))
|
||||
);
|
||||
let args = settings.alerting.alerting_args.unwrap();
|
||||
assert_eq!(args.daily_report_frequency, Some(Spelled::Value(3600)));
|
||||
assert_eq!(args.log_to_console, Some(Spelled::Value(true)));
|
||||
assert_eq!(args.spend_anomaly_multiplier, Some(Spelled::Value(2.5)));
|
||||
|
||||
let schema = settings.objects.team_metadata_schema.unwrap();
|
||||
assert_eq!(schema[0].key.as_deref(), Some("cost_center"));
|
||||
assert_eq!(schema[0].label.as_deref(), Some("Cost center"));
|
||||
|
||||
let mapping = &settings.requests.request_tag_mappings.unwrap()[0];
|
||||
let header: &'static str = mapping.header.unwrap().into();
|
||||
assert_eq!(header, "baggage");
|
||||
assert_eq!(mapping.key.as_deref(), Some("tenant"));
|
||||
assert_eq!(mapping.tag_prefix.as_deref(), Some("tenant-"));
|
||||
assert_eq!(mapping.required, Some(Spelled::Value(true)));
|
||||
|
||||
let endpoint = &settings.requests.pass_through_endpoints.unwrap()[0];
|
||||
assert_eq!(endpoint.path.as_deref(), Some("/bria"));
|
||||
assert_eq!(
|
||||
endpoint.target.as_ref().unwrap().expose(),
|
||||
"https://engine.prod.bria-api.com"
|
||||
);
|
||||
assert_eq!(endpoint.include_subpath, Some(Spelled::Value(true)));
|
||||
assert_eq!(endpoint.cost_per_request, Some(Spelled::Value(0.5)));
|
||||
assert_eq!(
|
||||
endpoint.methods.as_deref(),
|
||||
Some(&["GET".to_string(), "POST".to_string()][..])
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
settings.files.allowed_file_extensions.as_deref(),
|
||||
Some(&["jsonl".to_string(), "csv".to_string()][..])
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn ui_access_mode_accepts_a_mode_string() {
|
||||
let settings = general(" ui_access_mode: admin_only\n");
|
||||
assert!(
|
||||
matches!(settings.ui.ui_access_mode, Some(UiAccessMode::Mode(mode)) if mode == "admin_only")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn ui_access_mode_accepts_an_sso_group_mapping() {
|
||||
let settings = general(
|
||||
" ui_access_mode: {type: restricted_sso_group, restricted_sso_group: ai-admins, sso_group_jwt_field: groups}\n",
|
||||
);
|
||||
let Some(UiAccessMode::SsoGroup(group)) = settings.ui.ui_access_mode else {
|
||||
panic!("expected an SSO group mapping")
|
||||
};
|
||||
assert_eq!(group.kind.as_deref(), Some("restricted_sso_group"));
|
||||
assert_eq!(group.restricted_sso_group.as_deref(), Some("ai-admins"));
|
||||
assert_eq!(group.sso_group_jwt_field.as_deref(), Some("groups"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn alerting_accepts_a_single_name() {
|
||||
let settings = general(" alerting: slack\n");
|
||||
assert_eq!(
|
||||
settings.alerting.alerting,
|
||||
Some(Alerting::One("slack".to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn rejects_request_tag_header_python_rejects_at_boot() {
|
||||
assert!(
|
||||
Config::from_yaml(
|
||||
"general_settings:\n request_tag_mappings: [{header: x-tenant, key: tenant, tag: t}]\n"
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("alert_to_webhook_url: {budget_alerts: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))]
|
||||
#[case("alert_type_config: {budget_alerts: {webhook: SENTINEL}}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))]
|
||||
#[case("alerting_args: {webhook: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.alerting))]
|
||||
#[case("pass_through_endpoints: [{path: /x, target: SENTINEL}]", |settings: &GeneralSettings| format!("{:?}", settings.requests))]
|
||||
#[case("pass_through_endpoints: [{path: /x, target: t, headers: {Authorization: SENTINEL}}]", |settings: &GeneralSettings| format!("{:?}", settings.requests))]
|
||||
#[case("claude_code_gateway_managed_settings: {token: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.requests))]
|
||||
#[case("search_tools: [{litellm_params: {api_key: SENTINEL}}]", |settings: &GeneralSettings| format!("{:?}", settings.objects))]
|
||||
#[case("max_failed_login_attempts_per_source_overrides: {10.0.0.1: SENTINEL}", |settings: &GeneralSettings| format!("{:?}", settings.ui))]
|
||||
fn debug_hides_values_that_can_carry_secrets(
|
||||
#[case] entry: &str,
|
||||
#[case] render: fn(&GeneralSettings) -> String,
|
||||
) {
|
||||
let settings = general(&format!(" {entry}\n"));
|
||||
assert!(!render(&settings).contains("SENTINEL"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn unknown_key_still_lands_in_additional_fields() {
|
||||
let settings = general(" future_setting: 1\n forward_openai_org_id: true\n");
|
||||
assert!(settings.additional_fields.contains_key("future_setting"));
|
||||
assert!(
|
||||
!settings
|
||||
.additional_fields
|
||||
.contains_key("forward_openai_org_id")
|
||||
);
|
||||
assert_eq!(
|
||||
settings.requests.forward_openai_org_id,
|
||||
Some(Spelled::Value(true))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn mcp_allowed_hosts_and_origins_land_typed() {
|
||||
let settings = general(
|
||||
" mcp_allowed_hosts: [localhost]\n mcp_allowed_origins: ['https://app.test']\n",
|
||||
);
|
||||
assert_eq!(
|
||||
settings.mcp.mcp_allowed_hosts.as_deref(),
|
||||
Some(&["localhost".to_string()][..])
|
||||
);
|
||||
assert_eq!(
|
||||
settings.mcp.mcp_allowed_origins.as_ref(),
|
||||
["https://app.test".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::literal("secret-master-key")]
|
||||
#[case::reference("os.environ/LITELLM_MASTER_KEY")]
|
||||
fn loads_and_redacts_the_master_key(#[case] key: &str) {
|
||||
let config = Config::from_yaml(&format!(
|
||||
"model_list: []\ngeneral_settings:\n master_key: {key}\n"
|
||||
))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
config
|
||||
.general_settings
|
||||
.master_key
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.expose(),
|
||||
key
|
||||
);
|
||||
assert!(!format!("{config:?}").contains(key));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn missing_general_settings_has_no_master_key() {
|
||||
let config = Config::from_yaml("model_list: []").unwrap();
|
||||
assert!(config.general_settings.master_key.is_none());
|
||||
}
|
||||
}
|
||||
26
litellm-rust/crates/config/src/general/objects.rs
Normal file
26
litellm-rust/crates/config/src/general/objects.rs
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
use crate::{Object, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct ObjectsSettings {
|
||||
pub custom_key_generate: Option<String>,
|
||||
pub custom_key_update: Option<String>,
|
||||
pub custom_key_policy: Option<String>,
|
||||
pub custom_team_metadata_validate: Option<String>,
|
||||
pub team_metadata_schema: Option<Box<[TeamMetadataField]>>,
|
||||
pub team_metadata_validation_timeout: Option<Spelled<f64>>,
|
||||
pub team_metadata_validation_error_message: Option<String>,
|
||||
pub disable_auto_add_proxy_admin_to_teams: Option<Spelled<bool>>,
|
||||
pub self_serve_budget_policy: Option<String>,
|
||||
pub enforce_rpm_tpm_on_model_add: Option<Spelled<bool>>,
|
||||
pub agent_access_default_deny: Option<Spelled<bool>>,
|
||||
pub vector_store_deny_by_default: Option<Spelled<bool>>,
|
||||
pub search_tool_deny_by_default: Option<Spelled<bool>>,
|
||||
pub search_tools: Option<Box<[Object]>>,
|
||||
pub enforce_fallback_model_access: Option<Spelled<bool>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct TeamMetadataField {
|
||||
pub key: Option<String>,
|
||||
pub label: Option<String>,
|
||||
}
|
||||
66
litellm-rust/crates/config/src/general/requests.rs
Normal file
66
litellm-rust/crates/config/src/general/requests.rs
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
use litellm_auth_types::SecretValue;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{Object, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct RequestsSettings {
|
||||
pub forward_llm_provider_auth_headers: Option<Spelled<bool>>,
|
||||
pub forward_openai_org_id: Option<Spelled<bool>>,
|
||||
pub always_include_stream_usage: Option<Spelled<bool>>,
|
||||
pub expose_fallback_errors_to_caller: Option<Spelled<bool>>,
|
||||
pub include_call_id_in_error_body: Option<Spelled<bool>>,
|
||||
pub request_tag_mappings: Option<Box<[RequestTagMapping]>>,
|
||||
pub pass_through_endpoints: Option<Box<[PassThroughEndpoint]>>,
|
||||
pub pass_through_request_timeout: Option<Spelled<f64>>,
|
||||
pub passthrough_managed_object_ids: Option<Spelled<bool>>,
|
||||
pub disable_bedrock_agent_runtime_passthrough: Option<Spelled<bool>>,
|
||||
pub enable_openai_websocket_passthrough: Option<Spelled<bool>>,
|
||||
pub vertex_ai_pass_through_drop_anthropic_betas: Option<Box<[String]>>,
|
||||
pub transcribe_media_buckets: Option<Box<[String]>>,
|
||||
pub responses_websocket_session_limit_seconds: Option<Spelled<f64>>,
|
||||
pub disable_responses_id_security: Option<Spelled<bool>>,
|
||||
pub allow_unmanaged_response_ids: Option<Spelled<bool>>,
|
||||
pub dangerously_allow_mock_testing_request_params: Option<Spelled<bool>>,
|
||||
pub enable_claude_code_gateway: Option<Spelled<bool>>,
|
||||
pub claude_code_gateway_managed_settings: Option<Object>,
|
||||
pub context_management_summary_model: Option<String>,
|
||||
pub context_management_summary_max_tokens: Option<Spelled<i64>>,
|
||||
pub safeguards_classifier_model: Option<String>,
|
||||
pub user_url_validation: Option<Spelled<bool>>,
|
||||
pub user_url_allowed_hosts: Option<Box<[String]>>,
|
||||
pub provider_url_destination_allowed_hosts: Option<Box<[String]>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, strum::IntoStaticStr)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum RequestTagHeader {
|
||||
#[strum(serialize = "baggage")]
|
||||
Baggage,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct RequestTagMapping {
|
||||
pub header: Option<RequestTagHeader>,
|
||||
pub key: Option<String>,
|
||||
pub value: Option<String>,
|
||||
pub tag: Option<String>,
|
||||
pub tag_prefix: Option<String>,
|
||||
pub required: Option<Spelled<bool>>,
|
||||
pub require_budgeted_tag: Option<Spelled<bool>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct PassThroughEndpoint {
|
||||
pub id: Option<String>,
|
||||
pub path: Option<String>,
|
||||
pub target: Option<SecretValue>,
|
||||
pub headers: Option<Object>,
|
||||
pub default_query_params: Option<Object>,
|
||||
pub include_subpath: Option<Spelled<bool>>,
|
||||
pub cost_per_request: Option<Spelled<f64>>,
|
||||
pub timeout: Option<Spelled<f64>>,
|
||||
pub auth: Option<Spelled<bool>>,
|
||||
pub guardrails: Option<Object>,
|
||||
pub methods: Option<Box<[String]>>,
|
||||
}
|
||||
61
litellm-rust/crates/config/src/general/spend.rs
Normal file
61
litellm-rust/crates/config/src/general/spend.rs
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
use crate::{Object, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct SpendSettings {
|
||||
pub disable_spend_logs: Option<Spelled<bool>>,
|
||||
pub disable_spend_updates: Option<Spelled<bool>>,
|
||||
pub disable_reset_budget: Option<Spelled<bool>>,
|
||||
pub disable_error_logs: Option<Spelled<bool>>,
|
||||
pub store_prompts_in_spend_logs: Option<Spelled<bool>>,
|
||||
pub disable_requester_ip_address_logging: Option<Spelled<bool>>,
|
||||
pub spend_logs_metadata_fields: Option<SpendLogsMetadataFields>,
|
||||
pub proxy_batch_write_at: Option<Spelled<f64>>,
|
||||
pub proxy_batch_polling_interval: Option<Spelled<f64>>,
|
||||
pub proxy_budget_rescheduler_min_time: Option<Spelled<f64>>,
|
||||
pub proxy_budget_rescheduler_max_time: Option<Spelled<f64>>,
|
||||
pub disable_budget_reservation: Option<Spelled<bool>>,
|
||||
pub fail_closed_budget_enforcement: Option<Spelled<bool>>,
|
||||
pub apply_user_budget_to_team_keys: Option<Spelled<bool>>,
|
||||
pub enforce_fallback_budget: Option<Spelled<bool>>,
|
||||
pub track_unmanaged_batch_cost: Option<Spelled<bool>>,
|
||||
pub spend_report_frequency: Option<String>,
|
||||
pub spend_capture_rate_check: Option<SpendCaptureRateCheckSettings>,
|
||||
pub maximum_spend_logs_retention_period: Option<Spelled<i64>>,
|
||||
pub maximum_autorouter_session_retention_period: Option<Spelled<i64>>,
|
||||
pub maximum_health_check_retention_period: Option<Spelled<i64>>,
|
||||
pub maximum_daily_tag_spend_retention_period: Option<Spelled<i64>>,
|
||||
pub maximum_spend_logs_size: Option<Spelled<i64>>,
|
||||
pub maximum_spend_logs_cleanup_target_size: Option<Spelled<i64>>,
|
||||
pub maximum_spend_logs_size_check_interval: Option<String>,
|
||||
pub maximum_spend_logs_retention_interval: Option<String>,
|
||||
pub maximum_spend_logs_cleanup_cron: Option<String>,
|
||||
pub maximum_spend_logs_cleanup_batch_size: Option<Spelled<i64>>,
|
||||
pub maximum_spend_logs_cleanup_max_batches: Option<Spelled<i64>>,
|
||||
pub maximum_spend_logs_cleanup_run_budget: Option<Spelled<f64>>,
|
||||
pub maximum_spend_logs_cleanup_batch_timeout: Option<Spelled<f64>>,
|
||||
pub use_spend_logs_partitioning: Option<Spelled<bool>>,
|
||||
pub scheduled_job_stagger: Option<ScheduledJobStaggerSettings>,
|
||||
pub missing_session_id: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct SpendLogsMetadataFields {
|
||||
pub include: Option<Box<[String]>>,
|
||||
pub exclude: Option<Box<[String]>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct SpendCaptureRateCheckSettings {
|
||||
pub providers: Option<Box<[String]>>,
|
||||
pub threshold: Option<Spelled<f64>>,
|
||||
pub lookback_days: Option<Spelled<i64>>,
|
||||
pub openai_project_ids: Option<Box<[String]>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct ScheduledJobStaggerSettings {
|
||||
pub enabled: Option<Spelled<bool>>,
|
||||
pub window_seconds: Option<Spelled<i64>>,
|
||||
pub identity: Option<String>,
|
||||
pub offsets: Option<Object>,
|
||||
}
|
||||
84
litellm-rust/crates/config/src/general/tracing.rs
Normal file
84
litellm-rust/crates/config/src/general/tracing.rs
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
use litellm_auth_types::SecretValue;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::Spelled;
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum TracingStoreKind {
|
||||
Clickhouse,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ClickHouseStoreSettings {
|
||||
#[serde(rename = "type")]
|
||||
pub kind: TracingStoreKind,
|
||||
pub url: Option<SecretValue>,
|
||||
pub database: Option<String>,
|
||||
pub retention_days: Option<Spelled<f64>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TracingStoreSettings {
|
||||
ClickHouse(ClickHouseStoreSettings),
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct TracingSettings {
|
||||
pub store: Option<TracingStoreSettings>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::{Config, Spelled, TracingStoreSettings};
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_are_typed_and_redact_the_url() {
|
||||
let config = Config::from_yaml(
|
||||
"general_settings:\n tracing:\n store:\n type: clickhouse\n url: https://writer:SENTINEL@example.com\n database: analytics\n retention_days: 7\n",
|
||||
)
|
||||
.unwrap();
|
||||
let tracing = config.general_settings.tracing.as_ref().unwrap();
|
||||
let Some(TracingStoreSettings::ClickHouse(store)) = tracing.store.as_ref() else {
|
||||
panic!("expected ClickHouse tracing store")
|
||||
};
|
||||
assert_eq!(
|
||||
store.url.as_ref().unwrap().expose(),
|
||||
"https://writer:SENTINEL@example.com"
|
||||
);
|
||||
assert_eq!(store.database.as_deref(), Some("analytics"));
|
||||
assert_eq!(store.retention_days, Some(Spelled::Value(7.0)));
|
||||
assert!(!format!("{config:?}").contains("SENTINEL"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_accept_environment_references() {
|
||||
let config = Config::from_yaml(
|
||||
"general_settings:\n tracing:\n store:\n type: clickhouse\n url: os.environ/CLICKHOUSE_URL\n retention_days: os.environ/RETENTION_DAYS\n",
|
||||
)
|
||||
.unwrap();
|
||||
let Some(TracingStoreSettings::ClickHouse(store)) =
|
||||
config.general_settings.tracing.unwrap().store
|
||||
else {
|
||||
panic!("expected ClickHouse tracing store")
|
||||
};
|
||||
assert_eq!(
|
||||
store.retention_days,
|
||||
Some(Spelled::Text("os.environ/RETENTION_DAYS".to_owned()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_reject_string_store() {
|
||||
assert!(
|
||||
Config::from_yaml("general_settings:\n tracing:\n store: clickhouse\n").is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_reject_removed_reader_configuration() {
|
||||
assert!(Config::from_yaml("general_settings:\n tracing:\n store:\n type: clickhouse\n reader_url: http://localhost:8123\n").is_err());
|
||||
}
|
||||
}
|
||||
50
litellm-rust/crates/config/src/general/ui.rs
Normal file
50
litellm-rust/crates/config/src/general/ui.rs
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
use serde::Deserialize;
|
||||
|
||||
use crate::{Object, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct UiSettings {
|
||||
pub ui_access_mode: Option<UiAccessMode>,
|
||||
pub custom_sso: Option<String>,
|
||||
pub custom_ui_sso_sign_in_handler: Option<String>,
|
||||
pub auto_redirect_ui_login_to_sso: Option<Spelled<bool>>,
|
||||
pub default_team_disabled: Option<Spelled<bool>>,
|
||||
pub allow_cli_sso_verification_uri_complete: Option<Spelled<bool>>,
|
||||
pub disable_env_credential_login: Option<Spelled<bool>>,
|
||||
pub disable_password_login_when_sso_enabled: Option<Spelled<bool>>,
|
||||
pub password_policy_min_length: Option<Spelled<f64>>,
|
||||
pub password_policy_require_uppercase: Option<Spelled<bool>>,
|
||||
pub password_policy_require_lowercase: Option<Spelled<bool>>,
|
||||
pub password_policy_require_numbers: Option<Spelled<bool>>,
|
||||
pub password_policy_require_special_characters: Option<Spelled<bool>>,
|
||||
pub password_policy_check_breached_passwords: Option<Spelled<bool>>,
|
||||
pub max_failed_login_attempts_per_source: Option<Spelled<i64>>,
|
||||
pub failed_login_window_seconds: Option<Spelled<i64>>,
|
||||
pub failed_login_block_seconds: Option<Spelled<i64>>,
|
||||
pub max_failed_login_attempts_per_source_overrides: Option<Object>,
|
||||
pub hide_default_credentials_hint: Option<Spelled<bool>>,
|
||||
pub disable_master_key_return: Option<Spelled<bool>>,
|
||||
pub disable_custom_api_keys: Option<Spelled<bool>>,
|
||||
pub disable_key_generate_for_org_admin: Option<Spelled<bool>>,
|
||||
pub team_admin_editable_team_fields: Option<Box<[String]>>,
|
||||
pub disable_model_add_for_internal_users: Option<Spelled<bool>>,
|
||||
pub disable_agents_for_internal_users: Option<Spelled<bool>>,
|
||||
pub allow_agents_for_team_admins: Option<Spelled<bool>>,
|
||||
pub disable_vector_stores_for_internal_users: Option<Spelled<bool>>,
|
||||
pub allow_vector_stores_for_team_admins: Option<Spelled<bool>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum UiAccessMode {
|
||||
Mode(String),
|
||||
SsoGroup(UiAccessSsoGroup),
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct UiAccessSsoGroup {
|
||||
#[serde(rename = "type")]
|
||||
pub kind: Option<String>,
|
||||
pub restricted_sso_group: Option<String>,
|
||||
pub sso_group_jwt_field: Option<String>,
|
||||
}
|
||||
102
litellm-rust/crates/config/src/guardrail.rs
Normal file
102
litellm-rust/crates/config/src/guardrail.rs
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
use litellm_auth_types::SecretValue;
|
||||
|
||||
use crate::{Object, Raw, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct Guardrail {
|
||||
pub guardrail_id: Option<String>,
|
||||
pub guardrail_name: Option<String>,
|
||||
pub litellm_params: Option<GuardrailParams>,
|
||||
pub guardrail_info: Option<Object>,
|
||||
pub policy_template: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct GuardrailParams {
|
||||
pub guardrail: Option<String>,
|
||||
pub mode: Option<Raw>,
|
||||
pub default_on: Option<Spelled<bool>>,
|
||||
pub api_key: Option<SecretValue>,
|
||||
pub api_base: Option<String>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::{Config, Spelled, Value};
|
||||
|
||||
fn parse(yaml: &str) -> Config {
|
||||
Config::from_yaml(yaml).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn guardrail_entries_keep_typed_params_and_unknown_keys() {
|
||||
let config = parse(
|
||||
r#"
|
||||
guardrails:
|
||||
- guardrail_name: pii
|
||||
litellm_params:
|
||||
guardrail: presidio
|
||||
mode: [pre_call, post_call]
|
||||
default_on: "true"
|
||||
api_key: os.environ/PRESIDIO_KEY
|
||||
presidio_language: en
|
||||
guardrail_info:
|
||||
owner: security
|
||||
"#,
|
||||
);
|
||||
let guardrail = &config.guardrails[0];
|
||||
let params = guardrail.litellm_params.as_ref().unwrap();
|
||||
|
||||
assert_eq!(guardrail.guardrail_name.as_deref(), Some("pii"));
|
||||
assert_eq!(params.guardrail.as_deref(), Some("presidio"));
|
||||
assert!(
|
||||
matches!(params.mode.as_ref().unwrap().get(), Value::Sequence(modes) if modes.len() == 2)
|
||||
);
|
||||
assert_eq!(params.default_on, Some(Spelled::Text("true".to_owned())));
|
||||
assert_eq!(
|
||||
params.api_key.as_ref().unwrap().expose(),
|
||||
"os.environ/PRESIDIO_KEY"
|
||||
);
|
||||
assert!(params.additional_fields.contains_key("presidio_language"));
|
||||
assert!(
|
||||
guardrail
|
||||
.guardrail_info
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.contains_key("owner")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::guardrail_key(
|
||||
r#"
|
||||
guardrails:
|
||||
- litellm_params: {api_key: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::guardrail_extra(
|
||||
r#"
|
||||
guardrails:
|
||||
- litellm_params: {vendor_secret: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::guardrail_info(
|
||||
r#"
|
||||
guardrails:
|
||||
- guardrail_info: {owner: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::guardrail_mode(
|
||||
r#"
|
||||
guardrails:
|
||||
- litellm_params: {mode: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
fn debug_never_prints_guardrail_values(#[case] yaml: &str) {
|
||||
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
use std::{
|
||||
collections::{BTreeSet, VecDeque},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use serde::Deserialize;
|
||||
use serde_yaml_ng::{Mapping, Value};
|
||||
|
||||
use crate::Error;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Includes {
|
||||
#[serde(default)]
|
||||
include: Vec<String>,
|
||||
}
|
||||
|
||||
fn read(path: &Path) -> Result<Mapping, Error> {
|
||||
Ok(serde_yaml_ng::from_str(&std::fs::read_to_string(path)?)?)
|
||||
}
|
||||
|
||||
fn entries(config: &Mapping, path: &Path) -> Result<Vec<(String, PathBuf)>, Error> {
|
||||
let includes: Includes = serde_yaml_ng::from_value(Value::Mapping(config.clone()))?;
|
||||
Ok(includes
|
||||
.include
|
||||
.into_iter()
|
||||
.map(|entry| (entry, path.to_owned()))
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub(super) fn load(path: &Path) -> Result<Value, Error> {
|
||||
let root = path.canonicalize()?;
|
||||
let mut merged = read(&root)?;
|
||||
let mut pending: VecDeque<_> = entries(&merged, &root)?.into();
|
||||
let mut loaded = BTreeSet::from([root.clone()]);
|
||||
merged.remove(Value::String("include".into()));
|
||||
while let Some((entry, declaring)) = pending.pop_front() {
|
||||
let declared = declaring.parent().unwrap_or(Path::new(".")).join(&entry);
|
||||
let fallback = root.parent().unwrap_or(Path::new(".")).join(&entry);
|
||||
let location = if declared.exists() {
|
||||
declared
|
||||
} else {
|
||||
fallback
|
||||
}
|
||||
.canonicalize()?;
|
||||
if !loaded.insert(location.clone()) {
|
||||
continue;
|
||||
}
|
||||
let mut included = read(&location)?;
|
||||
pending.extend(entries(&included, &location)?);
|
||||
included.remove(Value::String("include".into()));
|
||||
for (key, value) in included {
|
||||
match (merged.get_mut(&key), value) {
|
||||
(Some(Value::Sequence(base)), Value::Sequence(extra)) => base.extend(extra),
|
||||
(_, value) => {
|
||||
merged.insert(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Value::Mapping(merged))
|
||||
}
|
||||
|
|
@ -1,94 +1,349 @@
|
|||
macro_rules_attribute::attribute_alias! {
|
||||
#[apply(config_section)] =
|
||||
#[derive(Clone, Debug, Default, ::serde::Deserialize)]
|
||||
#[serde(default)];
|
||||
}
|
||||
|
||||
mod agent;
|
||||
mod callback;
|
||||
mod credential;
|
||||
mod error;
|
||||
mod includes;
|
||||
mod general;
|
||||
mod guardrail;
|
||||
mod litellm;
|
||||
mod mcp;
|
||||
mod model;
|
||||
mod settings;
|
||||
mod policy;
|
||||
mod prompt;
|
||||
mod provider;
|
||||
mod router;
|
||||
mod tool;
|
||||
mod value;
|
||||
mod vector_store;
|
||||
pub mod yaml;
|
||||
|
||||
use std::{fmt, path::Path};
|
||||
|
||||
use serde::Deserialize;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
pub use agent::Agent;
|
||||
pub use callback::CallbackSettings;
|
||||
pub use credential::Credential;
|
||||
pub use error::Error;
|
||||
pub use litellm_router_types::{LitellmParams, Spelled};
|
||||
pub use mcp::{McpAuth, McpServer, McpTransport};
|
||||
pub use model::Model;
|
||||
pub use settings::{
|
||||
ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings,
|
||||
TracingStoreSettings,
|
||||
pub use general::{
|
||||
Alerting, AlertingArgs, AlertingSettings, AuthSettings, ClickHouseStoreSettings,
|
||||
DatabaseSettings, FilesSettings, GeneralSettings, HealthSettings, KeyManagementSettings,
|
||||
LimitsSettings, McpSettings, ObjectsSettings, PassThroughEndpoint, RequestTagHeader,
|
||||
RequestTagMapping, RequestsSettings, RoleBasedPermissions, ScheduledJobStaggerSettings,
|
||||
ServiceAccountSettings, SpendCaptureRateCheckSettings, SpendLogsMetadataFields, SpendSettings,
|
||||
TeamMetadataField, TracingSettings, TracingStoreSettings, UiAccessMode, UiAccessSsoGroup,
|
||||
UiSettings, UserHeaderMapping,
|
||||
};
|
||||
pub use value::{AdditionalFields, Object, Value};
|
||||
pub use guardrail::{Guardrail, GuardrailParams};
|
||||
pub use litellm::{
|
||||
AiohttpSettings, AzureBlobCacheParams, BackgroundMode, BudgetSettings, CacheParams,
|
||||
CachingSettings, CallbackVar, CallerIdentity, ClientCertificateSettings, CustomProvider,
|
||||
DiskCacheParams, FallbackSettings, GcsCacheParams, Http2Settings, LiteLlmSettings,
|
||||
LoggingSettings, McpToolSettings, ModelGroupSettings, PriorityReservationSettings,
|
||||
ProxyCacheTtlParams, QdrantCacheParams, RedisCacheParams, RedisServerCacheParams,
|
||||
ResponsesSettings, S3CacheParams, SemanticCacheParams, SocketSettings, TeamDefaultSettings,
|
||||
TlsSettings, UpperboundKeyGenerateParams, ValkeyCacheParams,
|
||||
};
|
||||
pub use litellm_python_compat::read::Spelled;
|
||||
pub use litellm_router_types::LitellmParams;
|
||||
pub use mcp::{McpAuth, McpServer, McpTransport};
|
||||
pub use model::{AccessWindow, Model, ModelId, ModelInfo};
|
||||
pub use policy::{
|
||||
Policy, PolicyAttachment, PolicyCondition, PolicyGuardrailChanges, PolicyGuardrails,
|
||||
PolicyPipeline, PolicyPipelineStep,
|
||||
};
|
||||
pub use prompt::{Prompt, PromptInfo, PromptParams};
|
||||
pub use provider::{AssistantSettings, ProviderSettings, VertexConfig, Worker};
|
||||
pub use router::{AlertingConfig, AllowedFailsPolicy, ProviderBudget, RouterSettings};
|
||||
pub use tool::{SandboxTool, SandboxToolParams, SearchTool, SearchToolInfo, SearchToolParams};
|
||||
pub use value::{Object, Raw, Value};
|
||||
pub use vector_store::{VectorStore, VectorStoreParams};
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
#[macro_rules_attribute::apply(config_section)]
|
||||
pub struct Config {
|
||||
pub model_list: Box<[Model]>,
|
||||
pub general_settings: GeneralSettings,
|
||||
pub router_settings: RouterSettings,
|
||||
pub litellm_settings: LiteLlmSettings,
|
||||
pub environment_variables: Object,
|
||||
pub callback_settings: Object,
|
||||
pub assistant_settings: Object,
|
||||
pub default_vertex_config: Object,
|
||||
pub mcp_servers: std::collections::BTreeMap<String, McpServer>,
|
||||
pub credential_list: Box<[Object]>,
|
||||
pub guardrails: Box<[Object]>,
|
||||
pub prompts: Box<[Object]>,
|
||||
pub sandbox_tools: Box<[Object]>,
|
||||
pub search_tools: Box<[Object]>,
|
||||
pub files_settings: Box<[Object]>,
|
||||
pub finetune_settings: Box<[Object]>,
|
||||
pub callback_settings: BTreeMap<String, CallbackSettings>,
|
||||
pub assistant_settings: Option<AssistantSettings>,
|
||||
pub default_vertex_config: Option<VertexConfig>,
|
||||
pub mcp_servers: BTreeMap<String, McpServer>,
|
||||
pub credential_list: Box<[Credential]>,
|
||||
pub guardrails: Box<[Guardrail]>,
|
||||
pub prompts: Box<[Prompt]>,
|
||||
pub sandbox_tools: Box<[SandboxTool]>,
|
||||
pub search_tools: Box<[SearchTool]>,
|
||||
pub files_settings: Box<[ProviderSettings]>,
|
||||
pub finetune_settings: Box<[ProviderSettings]>,
|
||||
pub mcp_tools: Box<[Object]>,
|
||||
pub vector_store_registry: Box<[Object]>,
|
||||
pub worker_registry: Box<[Object]>,
|
||||
pub agents: Box<[Object]>,
|
||||
pub agent_list: Box<[Object]>,
|
||||
pub policies: Object,
|
||||
pub policy_attachments: Box<[Object]>,
|
||||
pub vector_store_registry: Box<[VectorStore]>,
|
||||
pub worker_registry: Box<[Worker]>,
|
||||
pub agents: Box<[Agent]>,
|
||||
pub agent_list: Box<[Agent]>,
|
||||
pub policies: BTreeMap<String, Policy>,
|
||||
pub policy_attachments: Box<[PolicyAttachment]>,
|
||||
pub include: Box<[String]>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: AdditionalFields,
|
||||
}
|
||||
|
||||
impl fmt::Debug for Config {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("Config")
|
||||
.field("model_list", &self.model_list)
|
||||
.field("general_settings", &self.general_settings)
|
||||
.field("router_settings", &self.router_settings)
|
||||
.field("litellm_settings", &self.litellm_settings)
|
||||
.field("environment_variables", &self.environment_variables)
|
||||
.field("callback_settings", &self.callback_settings)
|
||||
.field("assistant_settings", &self.assistant_settings)
|
||||
.field("default_vertex_config", &self.default_vertex_config)
|
||||
.field("mcp_servers", &self.mcp_servers)
|
||||
.field("credential_list", &self.credential_list)
|
||||
.field("guardrails", &self.guardrails)
|
||||
.field("prompts", &self.prompts)
|
||||
.field("sandbox_tools", &self.sandbox_tools)
|
||||
.field("search_tools", &self.search_tools)
|
||||
.field("files_settings", &self.files_settings)
|
||||
.field("finetune_settings", &self.finetune_settings)
|
||||
.field("mcp_tools", &self.mcp_tools)
|
||||
.field("vector_store_registry", &self.vector_store_registry)
|
||||
.field("worker_registry", &self.worker_registry)
|
||||
.field("agents", &self.agents)
|
||||
.field("agent_list", &self.agent_list)
|
||||
.field("policies", &self.policies)
|
||||
.field("policy_attachments", &self.policy_attachments)
|
||||
.field("include", &self.include)
|
||||
.field("additional_fields", &self.additional_fields.keys())
|
||||
.finish()
|
||||
}
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn from_yaml(yaml: &str) -> Result<Self, Error> {
|
||||
Ok(serde_yaml_ng::from_str(yaml)?)
|
||||
Self::from_mapping(yaml::mapping(yaml)?)
|
||||
}
|
||||
|
||||
pub fn load(path: impl AsRef<Path>) -> Result<Self, Error> {
|
||||
Ok(serde_yaml_ng::from_value(includes::load(path.as_ref())?)?)
|
||||
pub fn from_mapping(document: serde_yaml_ng::Mapping) -> Result<Self, Error> {
|
||||
Ok(serde_yaml_ng::from_value(serde_yaml_ng::Value::Mapping(
|
||||
document,
|
||||
))?)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::{Config, Error, Spelled};
|
||||
|
||||
#[rstest]
|
||||
fn config_debug_redacts_api_keys() {
|
||||
let config = Config::from_yaml(
|
||||
"model_list: [{model_name: assistant, litellm_params: {model: anthropic/test-model, api_key: secret-value}}]",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
config.model_list[0]
|
||||
.litellm_params
|
||||
.api_key
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.expose(),
|
||||
"secret-value"
|
||||
);
|
||||
assert!(!format!("{config:?}").contains("secret-value"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::malformed_yaml("model_list: [")]
|
||||
#[case::missing_params("model_list: [{model_name: assistant}]")]
|
||||
#[case::missing_model("model_list: [{model_name: assistant, litellm_params: {api_key: key}}]")]
|
||||
#[case::provider_param_of_the_wrong_type(
|
||||
"model_list: [{model_name: assistant, litellm_params: {model: test, aws_region_name: [eu-central-1]}}]"
|
||||
)]
|
||||
#[case::negative_model_rpm(
|
||||
"model_list: [{model_name: assistant, rpm: -1, litellm_params: {model: test}}]"
|
||||
)]
|
||||
fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) {
|
||||
let error = Config::from_yaml(yaml).unwrap_err();
|
||||
assert!(matches!(error, Error::Parse(_)), "{error:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn empty_config_matches_python_defaults() {
|
||||
let config = Config::from_yaml("{}").unwrap();
|
||||
assert!(config.model_list.is_empty());
|
||||
assert_eq!(config.general_settings.admission_queue_timeout_seconds, 1.0);
|
||||
assert_eq!(
|
||||
config.general_settings.database_connection_pool_limit,
|
||||
Some(10)
|
||||
);
|
||||
assert_eq!(
|
||||
config.general_settings.proxy_config_reload_interval_seconds,
|
||||
30
|
||||
);
|
||||
assert_eq!(config.general_settings.health_check_interval, 300);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn parses_typed_settings_and_preserves_extension_fields() {
|
||||
let config = Config::from_yaml(
|
||||
r#"
|
||||
model_list:
|
||||
- model_name: assistant
|
||||
litellm_params:
|
||||
model: vertex_ai/test-model
|
||||
timeout: os.environ/REQUEST_TIMEOUT
|
||||
tpm: os.environ/TPM_LIMIT
|
||||
rpm: 5
|
||||
drop_params: "true"
|
||||
vertex_project: test-project
|
||||
vertex_credentials:
|
||||
type: service_account
|
||||
future_param: 1
|
||||
model_info:
|
||||
mode: chat
|
||||
access_groups: [internal]
|
||||
general_settings:
|
||||
master_key: secret-master-key
|
||||
store_model_in_db: true
|
||||
custom_auth: auth.py
|
||||
router_settings:
|
||||
routing_strategy: simple-shuffle
|
||||
allowed_fails: 2
|
||||
future_router_setting: cache.internal
|
||||
litellm_settings:
|
||||
drop_params: true
|
||||
cache: true
|
||||
custom_callback_name: audit
|
||||
future_section:
|
||||
enabled: true
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let model = &config.model_list[0];
|
||||
assert_eq!(
|
||||
model.litellm_params.timeout,
|
||||
Some(Spelled::Text("os.environ/REQUEST_TIMEOUT".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
model.litellm_params.tpm,
|
||||
Some(Spelled::Text("os.environ/TPM_LIMIT".to_string()))
|
||||
);
|
||||
assert_eq!(model.litellm_params.rpm, Some(Spelled::Value(5)));
|
||||
assert_eq!(
|
||||
model.litellm_params.drop_params,
|
||||
Some(Spelled::Text("true".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
model.litellm_params.vertex.vertex_project.as_deref(),
|
||||
Some("test-project")
|
||||
);
|
||||
assert_eq!(
|
||||
model.litellm_params.vertex.vertex_credentials.as_deref(),
|
||||
Some(r#"{"type":"service_account"}"#)
|
||||
);
|
||||
assert!(model.litellm_params.extra.contains_key("future_param"));
|
||||
assert!(model.additional_fields.contains_key("access_groups"));
|
||||
assert_eq!(config.router_settings.allowed_fails, Some(2));
|
||||
assert!(
|
||||
config
|
||||
.router_settings
|
||||
.additional_fields
|
||||
.contains_key("future_router_setting")
|
||||
);
|
||||
assert!(
|
||||
config
|
||||
.litellm_settings
|
||||
.additional_fields
|
||||
.contains_key("custom_callback_name")
|
||||
);
|
||||
assert!(config.additional_fields.contains_key("future_section"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn parses_python_config_sections() {
|
||||
let config = Config::from_yaml(
|
||||
r#"
|
||||
environment_variables:
|
||||
REDIS_PORT: 6379
|
||||
callback_settings:
|
||||
otel:
|
||||
message_logging: false
|
||||
assistant_settings:
|
||||
custom_llm_provider: openai
|
||||
credential_list:
|
||||
- credential_name: bedrock
|
||||
credential_values:
|
||||
aws_region_name: us-east-1
|
||||
guardrails:
|
||||
- guardrail_name: pii
|
||||
litellm_params:
|
||||
guardrail: presidio
|
||||
prompts:
|
||||
- prompt_id: support
|
||||
sandbox_tools:
|
||||
- sandbox_tool_name: e2b
|
||||
search_tools:
|
||||
- search_tool_name: web
|
||||
files_settings:
|
||||
- custom_llm_provider: openai
|
||||
finetune_settings:
|
||||
- custom_llm_provider: openai
|
||||
mcp_tools:
|
||||
- name: lookup
|
||||
mcp_servers:
|
||||
docs:
|
||||
url: https://example.test/mcp
|
||||
vector_store_registry:
|
||||
- vector_store_name: docs
|
||||
worker_registry:
|
||||
- worker_id: regional
|
||||
agents:
|
||||
- agent_name: reviewer
|
||||
agent_list:
|
||||
- agent_name: legacy-reviewer
|
||||
policies:
|
||||
safe:
|
||||
guardrails:
|
||||
add: [pii]
|
||||
policy_attachments:
|
||||
- policy_id: safe
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(config.environment_variables.len(), 1);
|
||||
assert_eq!(config.credential_list.len(), 1);
|
||||
assert_eq!(config.guardrails.len(), 1);
|
||||
assert_eq!(config.prompts.len(), 1);
|
||||
assert_eq!(config.sandbox_tools.len(), 1);
|
||||
assert_eq!(config.search_tools.len(), 1);
|
||||
assert_eq!(config.files_settings.len(), 1);
|
||||
assert_eq!(config.finetune_settings.len(), 1);
|
||||
assert_eq!(config.mcp_tools.len(), 1);
|
||||
assert_eq!(config.mcp_servers.len(), 1);
|
||||
assert_eq!(config.vector_store_registry.len(), 1);
|
||||
assert_eq!(config.worker_registry.len(), 1);
|
||||
assert_eq!(config.agents.len(), 1);
|
||||
assert_eq!(config.agent_list.len(), 1);
|
||||
assert!(config.policies.contains_key("safe"));
|
||||
assert_eq!(config.policy_attachments.len(), 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::api_key("api_key", "provider-secret")]
|
||||
#[case::aws_secret("aws_secret_access_key", "aws-secret")]
|
||||
#[case::vertex_credentials("vertex_credentials", "vertex-secret")]
|
||||
#[case::provider_extension("azure_ad_token", "azure-secret")]
|
||||
#[case::general_extension("custom_auth_secret", "auth-secret")]
|
||||
#[case::router_extension("future_router_token", "router-secret")]
|
||||
#[case::litellm_extension("callback_token", "callback-secret")]
|
||||
#[case::root_extension("private_token", "root-secret")]
|
||||
fn debug_output_does_not_expose_config_values(#[case] field: &str, #[case] secret: &str) {
|
||||
let yaml = match field {
|
||||
"api_key" => format!(
|
||||
"model_list: [{{model_name: assistant, litellm_params: {{model: test, api_key: {secret}}}}}]"
|
||||
),
|
||||
"aws_secret_access_key" => format!(
|
||||
"model_list: [{{model_name: assistant, litellm_params: {{model: test, aws_secret_access_key: {secret}}}}}]"
|
||||
),
|
||||
"vertex_credentials" => format!(
|
||||
"model_list: [{{model_name: assistant, litellm_params: {{model: test, vertex_credentials: {secret}}}}}]"
|
||||
),
|
||||
"azure_ad_token" => format!(
|
||||
"model_list: [{{model_name: assistant, litellm_params: {{model: test, azure_ad_token: {secret}}}}}]"
|
||||
),
|
||||
"custom_auth_secret" => format!("general_settings: {{{field}: {secret}}}"),
|
||||
"future_router_token" => format!("router_settings: {{{field}: {secret}}}"),
|
||||
"callback_token" => format!("litellm_settings: {{{field}: {secret}}}"),
|
||||
"private_token" => format!("{field}: {secret}"),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let config = Config::from_yaml(&yaml).unwrap();
|
||||
assert!(!format!("{config:?}").contains(secret));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn unknown_top_level_key_lands_in_additional_fields() {
|
||||
let config = Config::from_yaml("future_section:\n a: 1\nenable_jwt_auth: true\n").unwrap();
|
||||
|
||||
assert!(config.additional_fields.contains_key("future_section"));
|
||||
assert!(config.additional_fields.contains_key("enable_jwt_auth"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
9
litellm-rust/crates/config/src/litellm/AGENTS.md
Normal file
9
litellm-rust/crates/config/src/litellm/AGENTS.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
## Docs
|
||||
|
||||
- https://github.com/BerriAI/litellm-docs/blob/main/docs/proxy/config_settings.md
|
||||
|
||||
## Python
|
||||
|
||||
- There is no Python model. `ConfigYAML.litellm_settings` in `litellm/proxy/_types.py` is a plain `dict`
|
||||
- `ProxyConfig.load_config` in `proxy_server.py` handles a few keys specially (`cache`, `cache_params`, `callbacks`, `success_callback`, `guardrails`, `drop_params` and others) and does `setattr(litellm, key, value)` for everything else. So any module-level attribute in `litellm/__init__.py` is a valid key, and that is the real key set
|
||||
- `litellm_settings` is one of the sections the proxy persists to the DB (`_CONFIG_PERSISTED_SECTIONS`) and the Admin UI edits, so at runtime the YAML is only one source of the effective values. Merging the DB overlay is the consumer's job, not this crate's
|
||||
128
litellm-rust/crates/config/src/litellm/budget.rs
Normal file
128
litellm-rust/crates/config/src/litellm/budget.rs
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
use serde::Deserialize;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_auth_types::SecretValue;
|
||||
|
||||
use crate::{Raw, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct BudgetSettings {
|
||||
pub max_budget: Option<Spelled<f64>>,
|
||||
pub max_internal_user_budget: Option<Spelled<f64>>,
|
||||
pub default_max_internal_user_budget: Option<Spelled<f64>>,
|
||||
pub default_internal_user_params: Option<Raw>,
|
||||
pub max_ui_session_budget: Option<Spelled<f64>>,
|
||||
pub default_team_settings: Option<Box<[TeamDefaultSettings]>>,
|
||||
pub upperbound_key_generate_params: Option<UpperboundKeyGenerateParams>,
|
||||
pub key_alias_pattern: Option<String>,
|
||||
pub budget_reset_time: Option<String>,
|
||||
pub disable_end_user_cost_tracking: Option<Spelled<bool>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct TeamDefaultSettings {
|
||||
pub team_id: String,
|
||||
#[serde(default)]
|
||||
pub success_callback: Option<Box<[String]>>,
|
||||
#[serde(default)]
|
||||
pub failure_callback: Option<Box<[String]>>,
|
||||
#[serde(default)]
|
||||
pub callback_vars: Option<BTreeMap<String, CallbackVar>>,
|
||||
#[serde(flatten)]
|
||||
pub inline_callback_vars: BTreeMap<String, CallbackVar>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
#[serde(untagged)]
|
||||
pub enum CallbackVar {
|
||||
Bool(bool),
|
||||
Number(serde_yaml_ng::Number),
|
||||
Text(SecretValue),
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct UpperboundKeyGenerateParams {
|
||||
pub max_budget: Option<Spelled<f64>>,
|
||||
pub budget_duration: Option<String>,
|
||||
pub duration: Option<String>,
|
||||
pub max_parallel_requests: Option<Spelled<i64>>,
|
||||
pub tpm_limit: Option<Spelled<i64>>,
|
||||
pub rpm_limit: Option<Spelled<i64>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_auth_types::SecretValue;
|
||||
use rstest::rstest;
|
||||
|
||||
use super::{CallbackVar, TeamDefaultSettings};
|
||||
use crate::Config;
|
||||
|
||||
fn teams(yaml: &str) -> Result<Box<[TeamDefaultSettings]>, crate::Error> {
|
||||
Config::from_yaml(&format!(
|
||||
"litellm_settings:\n default_team_settings: {yaml}"
|
||||
))
|
||||
.map(|config| {
|
||||
config
|
||||
.litellm_settings
|
||||
.budget
|
||||
.default_team_settings
|
||||
.unwrap()
|
||||
})
|
||||
}
|
||||
|
||||
fn team(yaml: &str) -> TeamDefaultSettings {
|
||||
teams(yaml).unwrap()[0].clone()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn named_keys_stay_out_of_inline_callback_vars() {
|
||||
let team = team(
|
||||
"[{team_id: t, success_callback: [langfuse], failure_callback: [otel], callback_vars: {k: v}, pk: x}]",
|
||||
);
|
||||
assert_eq!(team.team_id, "t");
|
||||
assert_eq!(
|
||||
team.success_callback.as_deref(),
|
||||
Some(&["langfuse".to_owned()][..])
|
||||
);
|
||||
assert_eq!(
|
||||
team.failure_callback.as_deref(),
|
||||
Some(&["otel".to_owned()][..])
|
||||
);
|
||||
assert_eq!(
|
||||
team.callback_vars,
|
||||
Some(BTreeMap::from([(
|
||||
"k".to_owned(),
|
||||
CallbackVar::Text(SecretValue::new("v"))
|
||||
)]))
|
||||
);
|
||||
assert_eq!(team.inline_callback_vars.keys().collect::<Vec<_>>(), ["pk"]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::text("pk", CallbackVar::Text(SecretValue::new("pk")))]
|
||||
#[case::reference("os.environ/PK", CallbackVar::Text(SecretValue::new("os.environ/PK")))]
|
||||
#[case::quoted_number("'5'", CallbackVar::Text(SecretValue::new("5")))]
|
||||
#[case::integer("5", CallbackVar::Number(5.into()))]
|
||||
#[case::float("0.5", CallbackVar::Number(0.5.into()))]
|
||||
#[case::bool("true", CallbackVar::Bool(true))]
|
||||
fn callback_vars_accept_scalars(#[case] yaml: &str, #[case] expected: CallbackVar) {
|
||||
let inline = team(&format!("[{{team_id: t, var: {yaml}}}]"));
|
||||
let nested = team(&format!("[{{team_id: t, callback_vars: {{var: {yaml}}}}}]"));
|
||||
assert_eq!(inline.inline_callback_vars.get("var"), Some(&expected));
|
||||
assert_eq!(nested.callback_vars.unwrap().get("var"), Some(&expected));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing_team_id("[{success_callback: [langfuse]}]")]
|
||||
#[case::inline_list("[{team_id: t, var: [a]}]")]
|
||||
#[case::inline_mapping("[{team_id: t, var: {a: 1}}]")]
|
||||
#[case::inline_null("[{team_id: t, var: null}]")]
|
||||
#[case::nested_mapping("[{team_id: t, callback_vars: {var: {a: 1}}}]")]
|
||||
#[case::single_callback("[{team_id: t, success_callback: langfuse}]")]
|
||||
fn invalid_team_settings_fail_to_parse(#[case] yaml: &str) {
|
||||
assert!(teams(yaml).is_err());
|
||||
}
|
||||
}
|
||||
137
litellm-rust/crates/config/src/litellm/caching.rs
Normal file
137
litellm-rust/crates/config/src/litellm/caching.rs
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
use serde::{Deserialize, Deserializer};
|
||||
|
||||
use litellm_auth_types::SecretValue;
|
||||
|
||||
use crate::{Object, Raw, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct CachingSettings {
|
||||
pub cache: Option<bool>,
|
||||
pub cache_params: Option<CacheParams>,
|
||||
pub enable_redis_auth_cache: Option<Spelled<bool>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct CacheParams {
|
||||
#[serde(rename = "type")]
|
||||
pub cache_type: Option<String>,
|
||||
pub mode: Option<String>,
|
||||
pub namespace: Option<String>,
|
||||
pub ttl: Option<Spelled<f64>>,
|
||||
pub supported_call_types: Option<Box<[String]>>,
|
||||
#[serde(deserialize_with = "present")]
|
||||
pub max_messages: Option<Option<Spelled<i64>>>,
|
||||
#[serde(flatten)]
|
||||
pub redis_server: RedisServerCacheParams,
|
||||
#[serde(flatten)]
|
||||
pub semantic: SemanticCacheParams,
|
||||
#[serde(flatten)]
|
||||
pub proxy_ttl: ProxyCacheTtlParams,
|
||||
#[serde(flatten)]
|
||||
pub redis: RedisCacheParams,
|
||||
#[serde(flatten)]
|
||||
pub valkey: ValkeyCacheParams,
|
||||
#[serde(flatten)]
|
||||
pub s3: S3CacheParams,
|
||||
#[serde(flatten)]
|
||||
pub gcs: GcsCacheParams,
|
||||
#[serde(flatten)]
|
||||
pub azure_blob: AzureBlobCacheParams,
|
||||
#[serde(flatten)]
|
||||
pub qdrant: QdrantCacheParams,
|
||||
#[serde(flatten)]
|
||||
pub disk: DiskCacheParams,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
/// Read by every backend that talks to a Redis-protocol server: `redis`, `redis-semantic` and `valkey-semantic`.
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct RedisServerCacheParams {
|
||||
pub host: Option<String>,
|
||||
pub port: Option<Spelled<i64>>,
|
||||
pub password: Option<SecretValue>,
|
||||
pub redis_startup_nodes: Option<Raw>,
|
||||
pub default_in_redis_ttl: Option<Spelled<f64>>,
|
||||
}
|
||||
|
||||
/// Read by the semantic backends: `redis-semantic`, `valkey-semantic` and `qdrant-semantic`.
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct SemanticCacheParams {
|
||||
pub similarity_threshold: Option<Spelled<f64>>,
|
||||
pub semantic_cache_embedding_max_input_tokens: Option<Spelled<i64>>,
|
||||
pub semantic_cache_embedding_timeout: Option<Spelled<f64>>,
|
||||
pub semantic_cache_scope: Option<String>,
|
||||
}
|
||||
|
||||
/// Not response-cache settings. The proxy copies them onto `litellm.default_*_ttl` for its own internal caches.
|
||||
/// `default_in_memory_ttl` also replaces `ttl` when `type` is `local`.
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct ProxyCacheTtlParams {
|
||||
pub default_in_memory_ttl: Option<Spelled<f64>>,
|
||||
pub default_redis_ttl: Option<Spelled<f64>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct RedisCacheParams {
|
||||
pub redis_semantic_cache_embedding_model: Option<String>,
|
||||
pub redis_semantic_cache_index_name: Option<String>,
|
||||
pub redis_flush_size: Option<Spelled<i64>>,
|
||||
pub gcp_service_account: Option<String>,
|
||||
pub gcp_ssl_ca_certs: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct ValkeyCacheParams {
|
||||
pub valkey_semantic_cache_embedding_model: Option<String>,
|
||||
pub valkey_semantic_cache_index_name: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct S3CacheParams {
|
||||
pub s3_bucket_name: Option<String>,
|
||||
pub s3_region_name: Option<String>,
|
||||
pub s3_api_version: Option<String>,
|
||||
pub s3_use_ssl: Option<Spelled<bool>>,
|
||||
pub s3_verify: Option<Spelled<bool>>,
|
||||
pub s3_endpoint_url: Option<SecretValue>,
|
||||
pub s3_aws_access_key_id: Option<SecretValue>,
|
||||
pub s3_aws_secret_access_key: Option<SecretValue>,
|
||||
pub s3_aws_session_token: Option<SecretValue>,
|
||||
pub s3_config: Option<Raw>,
|
||||
pub s3_path: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct GcsCacheParams {
|
||||
pub gcs_bucket_name: Option<String>,
|
||||
pub gcs_path_service_account: Option<String>,
|
||||
pub gcs_path: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct AzureBlobCacheParams {
|
||||
pub azure_account_url: Option<SecretValue>,
|
||||
pub azure_blob_container: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct QdrantCacheParams {
|
||||
pub qdrant_api_base: Option<SecretValue>,
|
||||
pub qdrant_api_key: Option<SecretValue>,
|
||||
pub qdrant_collection_name: Option<String>,
|
||||
pub qdrant_quantization_config: Option<String>,
|
||||
pub qdrant_semantic_cache_embedding_model: Option<String>,
|
||||
pub qdrant_semantic_cache_vector_size: Option<Spelled<i64>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct DiskCacheParams {
|
||||
pub disk_cache_dir: Option<String>,
|
||||
}
|
||||
|
||||
fn present<'de, D: Deserializer<'de>, T: Deserialize<'de>>(
|
||||
deserializer: D,
|
||||
) -> Result<Option<Option<T>>, D::Error> {
|
||||
Ok(Some(Option::deserialize(deserializer)?))
|
||||
}
|
||||
7
litellm-rust/crates/config/src/litellm/fallbacks.rs
Normal file
7
litellm-rust/crates/config/src/litellm/fallbacks.rs
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
use crate::Raw;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct FallbackSettings {
|
||||
pub default_fallbacks: Option<Raw>,
|
||||
pub context_window_fallbacks: Option<Raw>,
|
||||
}
|
||||
24
litellm-rust/crates/config/src/litellm/logging.rs
Normal file
24
litellm-rust/crates/config/src/litellm/logging.rs
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
use litellm_python_compat::read::one_or_many;
|
||||
|
||||
use crate::{Object, Raw, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct LoggingSettings {
|
||||
#[serde(deserialize_with = "one_or_many")]
|
||||
pub callbacks: Option<Box<[Raw]>>,
|
||||
#[serde(deserialize_with = "one_or_many")]
|
||||
pub success_callback: Option<Box<[Raw]>>,
|
||||
#[serde(deserialize_with = "one_or_many")]
|
||||
pub failure_callback: Option<Box<[Raw]>>,
|
||||
pub service_callback: Option<Box<[String]>>,
|
||||
pub audit_log_callbacks: Option<Box<[String]>>,
|
||||
pub s3_callback_params: Option<Object>,
|
||||
pub s3_audit_callback_params: Option<Object>,
|
||||
pub store_audit_logs: Option<Spelled<bool>>,
|
||||
pub json_logs: Option<bool>,
|
||||
pub set_verbose: Option<bool>,
|
||||
pub turn_off_message_logging: Option<Spelled<bool>>,
|
||||
pub redact_messages_in_exceptions: Option<Spelled<bool>>,
|
||||
pub log_raw_request_response: Option<Spelled<bool>>,
|
||||
pub presidio_logging_only: Option<Spelled<bool>>,
|
||||
}
|
||||
8
litellm-rust/crates/config/src/litellm/mcp.rs
Normal file
8
litellm-rust/crates/config/src/litellm/mcp.rs
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
use crate::Raw;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct McpToolSettings {
|
||||
pub mcp_aliases: Option<Raw>,
|
||||
pub mcp_semantic_tool_filter: Option<Raw>,
|
||||
pub mcp_tool_search: Option<Raw>,
|
||||
}
|
||||
406
litellm-rust/crates/config/src/litellm/mod.rs
Normal file
406
litellm-rust/crates/config/src/litellm/mod.rs
Normal file
|
|
@ -0,0 +1,406 @@
|
|||
mod budget;
|
||||
mod caching;
|
||||
mod fallbacks;
|
||||
mod logging;
|
||||
mod mcp;
|
||||
mod transport;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
pub use self::{
|
||||
budget::{BudgetSettings, CallbackVar, TeamDefaultSettings, UpperboundKeyGenerateParams},
|
||||
caching::{
|
||||
AzureBlobCacheParams, CacheParams, CachingSettings, DiskCacheParams, GcsCacheParams,
|
||||
ProxyCacheTtlParams, QdrantCacheParams, RedisCacheParams, RedisServerCacheParams,
|
||||
S3CacheParams, SemanticCacheParams, ValkeyCacheParams,
|
||||
},
|
||||
fallbacks::FallbackSettings,
|
||||
logging::LoggingSettings,
|
||||
mcp::McpToolSettings,
|
||||
transport::{
|
||||
AiohttpSettings, ClientCertificateSettings, Http2Settings, SocketSettings, TlsSettings,
|
||||
},
|
||||
};
|
||||
use crate::{Object, Raw, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct LiteLlmSettings {
|
||||
pub drop_params: Option<Spelled<bool>>,
|
||||
pub request_timeout: Option<Spelled<f64>>,
|
||||
pub num_retries: Option<u64>,
|
||||
pub prometheus_deployment_and_latency_caller_identity: Option<CallerIdentity>,
|
||||
pub guardrails: Option<Box<[Object]>>,
|
||||
pub prompt_injection_params: Option<Object>,
|
||||
pub azure_content_safety_params: Option<Object>,
|
||||
pub global_prompt_directory: Option<String>,
|
||||
pub global_bitbucket_config: Option<Object>,
|
||||
pub global_gitlab_config: Option<Object>,
|
||||
pub priority_reservation_settings: Option<PriorityReservationSettings>,
|
||||
pub model_group_settings: Option<ModelGroupSettings>,
|
||||
pub post_call_rules: Option<String>,
|
||||
pub custom_provider_map: Option<Box<[CustomProvider]>>,
|
||||
pub responses: Option<ResponsesSettings>,
|
||||
pub enable_preview_features: Option<Spelled<bool>>,
|
||||
pub public_model_groups: Option<Box<[String]>>,
|
||||
pub timezone: Option<String>,
|
||||
#[serde(flatten)]
|
||||
pub tls: TlsSettings,
|
||||
#[serde(flatten)]
|
||||
pub client_certificate: ClientCertificateSettings,
|
||||
#[serde(flatten)]
|
||||
pub socket: SocketSettings,
|
||||
#[serde(flatten)]
|
||||
pub http2: Http2Settings,
|
||||
#[serde(flatten)]
|
||||
pub aiohttp: AiohttpSettings,
|
||||
#[serde(flatten)]
|
||||
pub caching: CachingSettings,
|
||||
#[serde(flatten)]
|
||||
pub budget: BudgetSettings,
|
||||
#[serde(flatten)]
|
||||
pub logging: LoggingSettings,
|
||||
#[serde(flatten)]
|
||||
pub fallbacks: FallbackSettings,
|
||||
#[serde(flatten)]
|
||||
pub mcp: McpToolSettings,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, strum::IntoStaticStr)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CallerIdentity {
|
||||
#[strum(serialize = "api_key_alias")]
|
||||
ApiKeyAlias,
|
||||
#[strum(serialize = "user_email")]
|
||||
UserEmail,
|
||||
#[strum(serialize = "both")]
|
||||
Both,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct PriorityReservationSettings {
|
||||
pub default_priority: Option<Spelled<f64>>,
|
||||
pub saturation_threshold: Option<Spelled<f64>>,
|
||||
pub saturation_check_cache_ttl: Option<Spelled<i64>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct ModelGroupSettings {
|
||||
pub forward_client_headers_to_llm_api: Option<Box<[String]>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct CustomProvider {
|
||||
pub provider: Option<String>,
|
||||
pub custom_handler: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct ResponsesSettings {
|
||||
pub background_mode: Option<BackgroundMode>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct BackgroundMode {
|
||||
pub polling_via_cache: Option<Raw>,
|
||||
pub native_background_mode: Option<Box<[String]>>,
|
||||
pub ttl: Option<Spelled<i64>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::{CallerIdentity, Config, LiteLlmSettings, Spelled};
|
||||
|
||||
fn settings(yaml: &str) -> LiteLlmSettings {
|
||||
Config::from_yaml(&format!("litellm_settings:\n{yaml}"))
|
||||
.unwrap()
|
||||
.litellm_settings
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::bool_word(" turn_off_message_logging: 'false'", Spelled::Text("false".into()))]
|
||||
#[case::bool_value(" turn_off_message_logging: true", Spelled::Value(true))]
|
||||
#[case::env_reference(" turn_off_message_logging: os.environ/REDACT", Spelled::Text("os.environ/REDACT".into()))]
|
||||
fn bool_settings_keep_python_spellings(#[case] yaml: &str, #[case] expected: Spelled<bool>) {
|
||||
assert_eq!(
|
||||
settings(yaml).logging.turn_off_message_logging,
|
||||
Some(expected)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::number(" max_budget: 10", Spelled::Value(10.0))]
|
||||
#[case::string(" max_budget: '10'", Spelled::Text("10".into()))]
|
||||
#[case::env_reference(" max_budget: os.environ/MAX_BUDGET", Spelled::Text("os.environ/MAX_BUDGET".into()))]
|
||||
fn budget_accepts_number_or_string(#[case] yaml: &str, #[case] expected: Spelled<f64>) {
|
||||
assert_eq!(settings(yaml).budget.max_budget, Some(expected));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::api_key_alias("api_key_alias", CallerIdentity::ApiKeyAlias)]
|
||||
#[case::user_email("user_email", CallerIdentity::UserEmail)]
|
||||
#[case::both("both", CallerIdentity::Both)]
|
||||
fn caller_identity_accepts_python_values(
|
||||
#[case] value: &str,
|
||||
#[case] expected: CallerIdentity,
|
||||
) {
|
||||
assert_eq!(
|
||||
settings(&format!(
|
||||
" prometheus_deployment_and_latency_caller_identity: {value}"
|
||||
))
|
||||
.prometheus_deployment_and_latency_caller_identity,
|
||||
Some(expected)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn caller_identity_rejects_values_python_refuses() {
|
||||
assert!(
|
||||
Config::from_yaml(
|
||||
"litellm_settings:\n prometheus_deployment_and_latency_caller_identity: team"
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn cache_params_reads_cache_keywords() {
|
||||
let cache = settings(
|
||||
" cache_params:\n type: redis\n port: 6379\n password: os.environ/REDIS_PASSWORD\n ttl: 600\n supported_call_types: [acompletion]\n namespace_extra: x",
|
||||
)
|
||||
.caching
|
||||
.cache_params
|
||||
.unwrap();
|
||||
assert_eq!(cache.cache_type.as_deref(), Some("redis"));
|
||||
assert_eq!(cache.redis_server.port, Some(Spelled::Value(6379)));
|
||||
assert_eq!(
|
||||
cache.redis_server.password.unwrap().expose(),
|
||||
"os.environ/REDIS_PASSWORD"
|
||||
);
|
||||
assert_eq!(cache.ttl, Some(Spelled::Value(600.0)));
|
||||
assert_eq!(
|
||||
cache.supported_call_types.as_deref(),
|
||||
Some(&["acompletion".to_owned()][..])
|
||||
);
|
||||
assert!(cache.additional_fields.contains_key("namespace_extra"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn cache_params_route_each_key_to_its_backend_group() {
|
||||
let cache = settings(
|
||||
r#" cache_params:
|
||||
type: s3
|
||||
namespace: shared
|
||||
port: 6379
|
||||
similarity_threshold: 0.8
|
||||
default_redis_ttl: 30
|
||||
redis_flush_size: 10
|
||||
valkey_semantic_cache_index_name: valkey-index
|
||||
s3_bucket_name: bucket
|
||||
gcs_path: gcs/path
|
||||
azure_blob_container: container
|
||||
qdrant_collection_name: collection
|
||||
disk_cache_dir: /tmp/cache
|
||||
namespace_extra: x"#,
|
||||
)
|
||||
.caching
|
||||
.cache_params
|
||||
.unwrap();
|
||||
assert_eq!(cache.namespace.as_deref(), Some("shared"));
|
||||
assert_eq!(cache.redis_server.port, Some(Spelled::Value(6379)));
|
||||
assert_eq!(
|
||||
cache.semantic.similarity_threshold,
|
||||
Some(Spelled::Value(0.8))
|
||||
);
|
||||
assert_eq!(
|
||||
cache.proxy_ttl.default_redis_ttl,
|
||||
Some(Spelled::Value(30.0))
|
||||
);
|
||||
assert_eq!(cache.redis.redis_flush_size, Some(Spelled::Value(10)));
|
||||
assert_eq!(
|
||||
cache.valkey.valkey_semantic_cache_index_name.as_deref(),
|
||||
Some("valkey-index")
|
||||
);
|
||||
assert_eq!(cache.s3.s3_bucket_name.as_deref(), Some("bucket"));
|
||||
assert_eq!(cache.gcs.gcs_path.as_deref(), Some("gcs/path"));
|
||||
assert_eq!(
|
||||
cache.azure_blob.azure_blob_container.as_deref(),
|
||||
Some("container")
|
||||
);
|
||||
assert_eq!(
|
||||
cache.qdrant.qdrant_collection_name.as_deref(),
|
||||
Some("collection")
|
||||
);
|
||||
assert_eq!(cache.disk.disk_cache_dir.as_deref(), Some("/tmp/cache"));
|
||||
assert_eq!(
|
||||
cache.additional_fields.keys().collect::<Vec<_>>(),
|
||||
["namespace_extra"]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::absent(" ttl: 1", None)]
|
||||
#[case::explicit_null(" max_messages: null", Some(None))]
|
||||
#[case::limit(" max_messages: 8", Some(Some(Spelled::Value(8))))]
|
||||
fn cache_max_messages_tells_null_from_absent(
|
||||
#[case] yaml: &str,
|
||||
#[case] expected: Option<Option<Spelled<i64>>>,
|
||||
) {
|
||||
assert_eq!(
|
||||
settings(&format!(" cache_params:\n{yaml}"))
|
||||
.caching
|
||||
.cache_params
|
||||
.unwrap()
|
||||
.max_messages,
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn nested_settings_land_in_typed_fields() {
|
||||
let settings = settings(
|
||||
" priority_reservation_settings: {default_priority: 0.5, saturation_check_cache_ttl: 30}
|
||||
model_group_settings: {forward_client_headers_to_llm_api: [gpt]}
|
||||
custom_provider_map: [{provider: my-llm, custom_handler: handler.instance}]
|
||||
responses: {background_mode: {polling_via_cache: all, native_background_mode: [o3], ttl: 60}}
|
||||
default_team_settings: [{team_id: team-1, success_callback: [langfuse]}]
|
||||
upperbound_key_generate_params: {max_budget: 100, duration: 30d, rpm_limit: os.environ/RPM}",
|
||||
);
|
||||
let priority = settings.priority_reservation_settings.unwrap();
|
||||
assert_eq!(priority.default_priority, Some(Spelled::Value(0.5)));
|
||||
assert_eq!(
|
||||
priority.saturation_check_cache_ttl,
|
||||
Some(Spelled::Value(30))
|
||||
);
|
||||
assert_eq!(
|
||||
settings
|
||||
.model_group_settings
|
||||
.unwrap()
|
||||
.forward_client_headers_to_llm_api
|
||||
.as_deref(),
|
||||
Some(&["gpt".to_owned()][..])
|
||||
);
|
||||
let provider = &settings.custom_provider_map.unwrap()[0];
|
||||
assert_eq!(provider.provider.as_deref(), Some("my-llm"));
|
||||
assert_eq!(provider.custom_handler.as_deref(), Some("handler.instance"));
|
||||
let background = settings.responses.unwrap().background_mode.unwrap();
|
||||
assert_eq!(
|
||||
background.polling_via_cache.unwrap().get().as_str(),
|
||||
Some("all")
|
||||
);
|
||||
assert_eq!(
|
||||
background.native_background_mode.as_deref(),
|
||||
Some(&["o3".to_owned()][..])
|
||||
);
|
||||
assert_eq!(background.ttl, Some(Spelled::Value(60)));
|
||||
let team = &settings.budget.default_team_settings.unwrap()[0];
|
||||
assert_eq!(team.team_id, "team-1");
|
||||
assert_eq!(
|
||||
team.success_callback.as_deref(),
|
||||
Some(&["langfuse".to_owned()][..])
|
||||
);
|
||||
let upperbound = settings.budget.upperbound_key_generate_params.unwrap();
|
||||
assert_eq!(upperbound.max_budget, Some(Spelled::Value(100.0)));
|
||||
assert_eq!(upperbound.duration.as_deref(), Some("30d"));
|
||||
assert_eq!(
|
||||
upperbound.rpm_limit,
|
||||
Some(Spelled::Text("os.environ/RPM".into()))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::cache_password(
|
||||
r#"
|
||||
cache: true
|
||||
cache_params: {password: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::cache_endpoint(" cache_params: {s3_endpoint_url: 'https://user:SENTINEL@s3.local'}")]
|
||||
#[case::cache_raw(" cache_params: {s3_config: {secret: SENTINEL}}")]
|
||||
#[case::gitlab_object(" global_gitlab_config: {access_token: SENTINEL}")]
|
||||
#[case::s3_callback_object(" s3_callback_params: {s3_aws_secret_access_key: SENTINEL}")]
|
||||
#[case::team_inline_var(" default_team_settings: [{team_id: t, langfuse_secret: SENTINEL}]")]
|
||||
#[case::team_callback_vars(
|
||||
" default_team_settings: [{team_id: t, callback_vars: {langfuse_secret: SENTINEL}}]"
|
||||
)]
|
||||
#[case::fallbacks_raw(" default_fallbacks: [SENTINEL]")]
|
||||
#[case::user_params_raw(" default_internal_user_params: {models: [SENTINEL]}")]
|
||||
#[case::callback_entry(" callbacks: [SENTINEL]")]
|
||||
#[case::success_callback_map(" success_callback: {name: SENTINEL}")]
|
||||
#[case::failure_callback_one(" failure_callback: SENTINEL")]
|
||||
#[case::audit_object(" s3_audit_callback_params: {secret: SENTINEL}")]
|
||||
#[case::context_window_raw(" context_window_fallbacks: [{gpt: [SENTINEL]}]")]
|
||||
#[case::mcp_raw(" mcp_aliases: {docs: SENTINEL}")]
|
||||
#[case::injection_object(" prompt_injection_params: {key: SENTINEL}")]
|
||||
#[case::unknown_setting(" future_setting: SENTINEL")]
|
||||
fn debug_output_does_not_expose_settings_values(#[case] yaml: &str) {
|
||||
let config = Config::from_yaml(&format!("litellm_settings:\n{yaml}")).unwrap();
|
||||
assert!(!format!("{config:?}").contains("SENTINEL"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn unknown_setting_stays_in_additional_fields() {
|
||||
assert!(
|
||||
settings(" future_setting: 1")
|
||||
.additional_fields
|
||||
.contains_key("future_setting")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn transport_keys_land_in_their_reader_groups() {
|
||||
let settings = settings(
|
||||
r#" ssl_verify: false
|
||||
ssl_security_level: '1'
|
||||
ssl_certificate: /etc/client.pem
|
||||
force_ipv4: true
|
||||
http2: true
|
||||
aiohttp_trust_env: true"#,
|
||||
);
|
||||
assert_eq!(settings.tls.ssl_verify, Some(Spelled::Value(false)));
|
||||
assert_eq!(settings.tls.ssl_security_level.as_deref(), Some("1"));
|
||||
assert_eq!(
|
||||
settings.client_certificate.ssl_certificate.as_deref(),
|
||||
Some("/etc/client.pem")
|
||||
);
|
||||
assert_eq!(settings.socket.force_ipv4, Some(true));
|
||||
assert_eq!(settings.http2.http2, Some(true));
|
||||
assert_eq!(settings.aiohttp.aiohttp_trust_env, Some(true));
|
||||
assert!(settings.additional_fields.is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::tls(" ssl_verify: false", "ssl_verify")]
|
||||
#[case::client_certificate(" ssl_certificate: /etc/client.pem", "ssl_certificate")]
|
||||
#[case::socket(" force_ipv4: true", "force_ipv4")]
|
||||
#[case::http2(" http2: true", "http2")]
|
||||
#[case::aiohttp(" disable_aiohttp_transport: true", "disable_aiohttp_transport")]
|
||||
#[case::caching(" cache: true", "cache")]
|
||||
#[case::budget(" max_budget: 10", "max_budget")]
|
||||
#[case::logging(" json_logs: true", "json_logs")]
|
||||
#[case::fallbacks(" default_fallbacks: [gpt]", "default_fallbacks")]
|
||||
#[case::mcp(" mcp_aliases: {docs: docs_server}", "mcp_aliases")]
|
||||
fn grouped_settings_stay_out_of_additional_fields(#[case] yaml: &str, #[case] key: &str) {
|
||||
assert!(!settings(yaml).additional_fields.contains_key(key));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::one("callbacks: custom_callbacks.logger", 1)]
|
||||
#[case::many("callbacks: [prometheus, otel]", 2)]
|
||||
fn accepts_python_callback_shorthand(#[case] setting: &str, #[case] expected_len: usize) {
|
||||
let config = Config::from_yaml(&format!("litellm_settings:\n {setting}")).unwrap();
|
||||
assert_eq!(
|
||||
config
|
||||
.litellm_settings
|
||||
.logging
|
||||
.callbacks
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.len(),
|
||||
expected_len
|
||||
);
|
||||
}
|
||||
}
|
||||
34
litellm-rust/crates/config/src/litellm/transport.rs
Normal file
34
litellm-rust/crates/config/src/litellm/transport.rs
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
use crate::Spelled;
|
||||
|
||||
/// Read by the SSL context builder, `get_ssl_configuration`.
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct TlsSettings {
|
||||
pub ssl_verify: Option<Spelled<bool>>,
|
||||
pub ssl_security_level: Option<String>,
|
||||
pub ssl_ecdh_curve: Option<String>,
|
||||
}
|
||||
|
||||
/// Read by the sync and async HTTP clients, which send it as the client certificate.
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct ClientCertificateSettings {
|
||||
pub ssl_certificate: Option<String>,
|
||||
}
|
||||
|
||||
/// Read by the httpx and aiohttp transports when they bind the local socket.
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct SocketSettings {
|
||||
pub force_ipv4: Option<bool>,
|
||||
}
|
||||
|
||||
/// Read by the httpx client.
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct Http2Settings {
|
||||
pub http2: Option<bool>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct AiohttpSettings {
|
||||
pub aiohttp_trust_env: Option<bool>,
|
||||
pub disable_aiohttp_trust_env: Option<bool>,
|
||||
pub disable_aiohttp_transport: Option<bool>,
|
||||
}
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
use std::{collections::BTreeMap, fmt};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_auth_types::SecretValue;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::Object;
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct McpServer {
|
||||
pub server_id: Option<String>,
|
||||
|
|
@ -14,8 +14,8 @@ pub struct McpServer {
|
|||
pub mcp_info: Object,
|
||||
pub transport: McpTransport,
|
||||
pub url: Option<SecretValue>,
|
||||
pub command: Option<String>,
|
||||
pub args: Box<[String]>,
|
||||
pub command: Option<SecretValue>,
|
||||
pub args: Box<[SecretValue]>,
|
||||
pub env: BTreeMap<String, SecretValue>,
|
||||
pub auth_type: Option<McpAuth>,
|
||||
#[serde(alias = "auth_value")]
|
||||
|
|
@ -29,18 +29,6 @@ pub struct McpServer {
|
|||
pub unsupported: Object,
|
||||
}
|
||||
|
||||
impl fmt::Debug for McpServer {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("McpServer")
|
||||
.field("transport", &self.transport)
|
||||
.field("auth_type", &self.auth_type)
|
||||
.field("timeout", &self.timeout)
|
||||
.field("max_concurrent_requests", &self.max_concurrent_requests)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, strum::IntoStaticStr)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum McpTransport {
|
||||
|
|
@ -81,3 +69,75 @@ pub enum McpAuth {
|
|||
#[strum(serialize = "oauth_delegate")]
|
||||
OauthDelegate,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use crate::{Config, McpAuth, McpTransport};
|
||||
|
||||
#[rstest]
|
||||
#[case::http(McpTransport::Http, "http")]
|
||||
#[case::sse(McpTransport::Sse, "sse")]
|
||||
#[case::stdio(McpTransport::Stdio, "stdio")]
|
||||
fn mcp_transport_as_str_matches_the_serde_spelling(
|
||||
#[case] transport: McpTransport,
|
||||
#[case] name: &str,
|
||||
) {
|
||||
assert_eq!(<&'static str>::from(transport), name);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<McpTransport>(json!(name)).unwrap(),
|
||||
transport
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::none(McpAuth::None, "none")]
|
||||
#[case::api_key(McpAuth::ApiKey, "api_key")]
|
||||
#[case::bearer_token(McpAuth::BearerToken, "bearer_token")]
|
||||
#[case::basic(McpAuth::Basic, "basic")]
|
||||
#[case::authorization(McpAuth::Authorization, "authorization")]
|
||||
#[case::token(McpAuth::Token, "token")]
|
||||
#[case::oauth2(McpAuth::Oauth2, "oauth2")]
|
||||
#[case::aws_sigv4(McpAuth::AwsSigv4, "aws_sigv4")]
|
||||
#[case::oauth2_token_exchange(McpAuth::Oauth2TokenExchange, "oauth2_token_exchange")]
|
||||
#[case::oauth2_id_jag(McpAuth::Oauth2IdJag, "oauth2_id_jag")]
|
||||
#[case::true_passthrough(McpAuth::TruePassthrough, "true_passthrough")]
|
||||
#[case::oauth_delegate(McpAuth::OauthDelegate, "oauth_delegate")]
|
||||
fn mcp_auth_as_str_matches_the_serde_spelling(#[case] auth: McpAuth, #[case] name: &str) {
|
||||
assert_eq!(<&'static str>::from(auth), name);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<McpAuth>(json!(name)).unwrap(),
|
||||
auth
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() {
|
||||
let config = Config::from_yaml("mcp_servers:\n docs:\n url: https://example.test/private-secret/mcp\n authentication_token: upstream-secret\n static_headers: {x-token: header-secret}\n env: {TOKEN: env-secret}\n args: [argument-secret]\n client_secret: oauth-secret\n allowed_tools: [search]\n").unwrap();
|
||||
let server = &config.mcp_servers["docs"];
|
||||
assert_eq!(server.allowed_tools.as_deref().unwrap(), ["search"]);
|
||||
assert!(server.unsupported.contains_key("client_secret"));
|
||||
let debug = format!("{config:?}");
|
||||
for secret in [
|
||||
"private-secret",
|
||||
"upstream-secret",
|
||||
"header-secret",
|
||||
"env-secret",
|
||||
"oauth-secret",
|
||||
"argument-secret",
|
||||
] {
|
||||
assert!(!debug.contains(secret));
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::transport("transport: invalid")]
|
||||
#[case::auth("auth_type: invalid")]
|
||||
#[case::concurrency("max_concurrent_requests: -1")]
|
||||
#[case::headers("static_headers: {x-token: [not, a, string]}")]
|
||||
fn rejects_invalid_typed_mcp_settings(#[case] setting: &str) {
|
||||
assert!(Config::from_yaml(&format!("mcp_servers:\n docs:\n {setting}\n")).is_err());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,30 +1,187 @@
|
|||
use std::fmt;
|
||||
|
||||
use litellm_router_types::LitellmParams;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{AdditionalFields, Object};
|
||||
use crate::{Object, Raw, Spelled};
|
||||
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct Model {
|
||||
pub model_name: String,
|
||||
pub litellm_params: LitellmParams,
|
||||
#[serde(default)]
|
||||
pub model_info: Object,
|
||||
pub model_info: ModelInfo,
|
||||
pub blocked: Option<bool>,
|
||||
pub rpm: Option<Spelled<u64>>,
|
||||
pub tpm: Option<Spelled<u64>>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: AdditionalFields,
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
impl fmt::Debug for Model {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("Model")
|
||||
.field("model_name", &self.model_name)
|
||||
.field("litellm_params", &self.litellm_params)
|
||||
.field("model_info", &self.model_info)
|
||||
.field("blocked", &self.blocked)
|
||||
.field("additional_fields", &self.additional_fields.keys())
|
||||
.finish()
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct ModelInfo {
|
||||
pub id: Option<ModelId>,
|
||||
pub db_model: Option<Spelled<bool>>,
|
||||
pub updated_at: Option<Raw>,
|
||||
pub created_at: Option<Raw>,
|
||||
pub updated_by: Option<String>,
|
||||
pub created_by: Option<String>,
|
||||
pub base_model: Option<String>,
|
||||
pub pricing_provider: Option<String>,
|
||||
pub tier: Option<String>,
|
||||
pub team_id: Option<String>,
|
||||
pub team_public_model_name: Option<String>,
|
||||
pub member_auto_router: Option<Spelled<bool>>,
|
||||
pub blocked: Option<Spelled<bool>>,
|
||||
pub discoverable: Option<Spelled<bool>>,
|
||||
pub access_windows: Option<Box<[AccessWindow]>>,
|
||||
pub ptu_count: Option<Spelled<i64>>,
|
||||
pub cost_per_ptu_per_hour: Option<Spelled<f64>>,
|
||||
pub ptu_effective_from: Option<Raw>,
|
||||
pub ptu_effective_to: Option<Raw>,
|
||||
pub ptu_shares: Option<Object>,
|
||||
pub allow_fail_open: Option<Spelled<bool>>,
|
||||
pub enable_tag_filtering: Option<Spelled<bool>>,
|
||||
pub internal_router_model: Option<Spelled<bool>>,
|
||||
pub max_embedding_batch_size: Option<Spelled<i64>>,
|
||||
pub input_cost_per_token: Option<Spelled<f64>>,
|
||||
pub output_cost_per_token: Option<Spelled<f64>>,
|
||||
pub input_cost_per_character: Option<Spelled<f64>>,
|
||||
pub output_cost_per_character: Option<Spelled<f64>>,
|
||||
pub cache_read_input_token_cost: Option<Spelled<f64>>,
|
||||
pub explicit_cache_read_input_token_cost: Option<Spelled<f64>>,
|
||||
pub cache_creation_input_token_cost: Option<Spelled<f64>>,
|
||||
pub cache_creation_input_token_cost_above_1hr: Option<Spelled<f64>>,
|
||||
pub tiered_pricing: Option<Box<[Object]>>,
|
||||
pub mode: Option<String>,
|
||||
pub access_groups: Option<Box<[String]>>,
|
||||
pub supported_environments: Option<Box<[String]>>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ModelId {
|
||||
Text(String),
|
||||
Number(i64),
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct AccessWindow {
|
||||
pub start: Option<Raw>,
|
||||
pub end: Option<Raw>,
|
||||
pub timezone: Option<String>,
|
||||
pub team_ids: Option<Box<[String]>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::{Config, Model, ModelId, Spelled};
|
||||
|
||||
fn model(entry: &str) -> Model {
|
||||
let config = Config::from_yaml(&format!(
|
||||
"model_list:\n - model_name: assistant\n litellm_params: {{model: openai/test}}\n{entry}"
|
||||
))
|
||||
.unwrap();
|
||||
config.model_list[0].clone()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::text(" model_info: {id: primary}", Some(ModelId::Text("primary".to_string())))]
|
||||
#[case::number(" model_info: {id: 7}", Some(ModelId::Number(7)))]
|
||||
#[case::null(" model_info: {id: null}", None)]
|
||||
#[case::absent(" model_info: {mode: chat}", None)]
|
||||
fn model_info_id_accepts_what_python_accepts(#[case] entry: &str, #[case] id: Option<ModelId>) {
|
||||
assert_eq!(model(entry).model_info.id, id);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing("")]
|
||||
#[case::null(" model_info:")]
|
||||
fn model_info_may_be_missing_or_null(#[case] entry: &str) {
|
||||
let model = model(entry);
|
||||
assert!(model.model_info.id.is_none());
|
||||
assert!(model.model_info.additional_fields.is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn model_info_fields_land_typed() {
|
||||
let model = model(
|
||||
r#" model_info:
|
||||
mode: embedding
|
||||
base_model: azure/test-base
|
||||
team_id: team-1
|
||||
access_groups: [internal, beta]
|
||||
db_model: "1"
|
||||
blocked: true
|
||||
ptu_count: 4
|
||||
cost_per_ptu_per_hour: 2
|
||||
input_cost_per_token: os.environ/INPUT_COST
|
||||
tiered_pricing: [{range: [0, 1000]}]
|
||||
access_windows:
|
||||
- {start: "09:00", end: "17:00", timezone: UTC, team_ids: [team-1]}
|
||||
supports_vision: true
|
||||
"#,
|
||||
);
|
||||
let info = &model.model_info;
|
||||
assert_eq!(info.mode.as_deref(), Some("embedding"));
|
||||
assert_eq!(info.base_model.as_deref(), Some("azure/test-base"));
|
||||
assert_eq!(info.team_id.as_deref(), Some("team-1"));
|
||||
assert_eq!(
|
||||
info.access_groups.as_deref(),
|
||||
Some(&["internal".to_string(), "beta".to_string()][..])
|
||||
);
|
||||
assert_eq!(info.db_model, Some(Spelled::Text("1".to_string())));
|
||||
assert_eq!(info.blocked, Some(Spelled::Value(true)));
|
||||
assert_eq!(info.ptu_count, Some(Spelled::Value(4)));
|
||||
assert_eq!(info.cost_per_ptu_per_hour, Some(Spelled::Value(2.0)));
|
||||
assert_eq!(
|
||||
info.input_cost_per_token,
|
||||
Some(Spelled::Text("os.environ/INPUT_COST".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
info.tiered_pricing.as_ref().map(|tiers| tiers.len()),
|
||||
Some(1)
|
||||
);
|
||||
let window = &info.access_windows.as_ref().unwrap()[0];
|
||||
assert_eq!(window.timezone.as_deref(), Some("UTC"));
|
||||
assert_eq!(window.start.as_ref().unwrap().get().as_str(), Some("09:00"));
|
||||
assert_eq!(
|
||||
window.team_ids.as_deref(),
|
||||
Some(&["team-1".to_string()][..])
|
||||
);
|
||||
assert!(info.additional_fields.contains_key("supports_vision"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn top_level_limits_and_unknown_keys_land_on_the_model() {
|
||||
let model = model(" rpm: 100\n tpm: os.environ/TPM\n team: research\n");
|
||||
assert_eq!(model.rpm, Some(Spelled::Value(100)));
|
||||
assert_eq!(model.tpm, Some(Spelled::Text("os.environ/TPM".to_string())));
|
||||
assert!(model.additional_fields.contains_key("team"));
|
||||
assert!(!model.additional_fields.contains_key("rpm"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::ptu_shares(
|
||||
" model_info: {ptu_shares: {team-1: sentinel-share-value}}",
|
||||
"sentinel-share-value"
|
||||
)]
|
||||
#[case::created_at(
|
||||
" model_info: {created_at: sentinel-created-at}",
|
||||
"sentinel-created-at"
|
||||
)]
|
||||
#[case::tiered_pricing(
|
||||
" model_info: {tiered_pricing: [{label: sentinel-tier-value}]}",
|
||||
"sentinel-tier-value"
|
||||
)]
|
||||
#[case::window_start(
|
||||
" model_info: {access_windows: [{start: sentinel-window-start, end: '17:00', timezone: UTC, team_ids: [t]}]}",
|
||||
"sentinel-window-start"
|
||||
)]
|
||||
#[case::model_extension(" team_token: sentinel-model-extension", "sentinel-model-extension")]
|
||||
fn opaque_model_info_values_stay_out_of_debug(#[case] entry: &str, #[case] sentinel: &str) {
|
||||
assert!(!format!("{:?}", model(entry)).contains(sentinel));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
144
litellm-rust/crates/config/src/policy.rs
Normal file
144
litellm-rust/crates/config/src/policy.rs
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
use serde::Deserialize;
|
||||
|
||||
use crate::{Raw, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct Policy {
|
||||
pub guardrails: Option<PolicyGuardrails>,
|
||||
pub condition: Option<PolicyCondition>,
|
||||
pub pipeline: Option<PolicyPipeline>,
|
||||
pub inherit: Option<String>,
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum PolicyGuardrails {
|
||||
Added(Box<[String]>),
|
||||
Changes(PolicyGuardrailChanges),
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct PolicyGuardrailChanges {
|
||||
pub add: Option<Box<[String]>>,
|
||||
pub remove: Option<Box<[String]>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct PolicyCondition {
|
||||
pub model: Option<Raw>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct PolicyPipeline {
|
||||
pub mode: Option<String>,
|
||||
pub steps: Option<Box<[PolicyPipelineStep]>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct PolicyPipelineStep {
|
||||
pub guardrail: Option<String>,
|
||||
pub on_fail: Option<String>,
|
||||
pub on_pass: Option<String>,
|
||||
pub on_error: Option<String>,
|
||||
pub pass_data: Option<Spelled<bool>>,
|
||||
pub modify_response_message: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct PolicyAttachment {
|
||||
pub policy: Option<String>,
|
||||
pub scope: Option<String>,
|
||||
pub teams: Option<Box<[String]>>,
|
||||
pub keys: Option<Box<[String]>>,
|
||||
pub models: Option<Box<[String]>>,
|
||||
pub tags: Option<Box<[String]>>,
|
||||
pub priority: Option<Spelled<i64>>,
|
||||
pub default: Option<Spelled<bool>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::{Config, PolicyGuardrails, Spelled, Value};
|
||||
|
||||
fn parse(yaml: &str) -> Config {
|
||||
Config::from_yaml(yaml).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::legacy_list("guardrails: [pii, toxicity]", &["pii", "toxicity"], &[])]
|
||||
#[case::add_remove("guardrails: {add: [pii], remove: [toxicity]}", &["pii"], &["toxicity"])]
|
||||
fn policy_guardrails_accept_both_python_spellings(
|
||||
#[case] guardrails: &str,
|
||||
#[case] added: &[&str],
|
||||
#[case] removed: &[&str],
|
||||
) {
|
||||
let config = parse(&format!("policies:\n safe:\n {guardrails}\n"));
|
||||
let (add, remove) = match config.policies["safe"].guardrails.as_ref().unwrap() {
|
||||
PolicyGuardrails::Added(add) => (add.to_vec(), Vec::new()),
|
||||
PolicyGuardrails::Changes(changes) => (
|
||||
changes.add.as_deref().unwrap_or_default().to_vec(),
|
||||
changes.remove.as_deref().unwrap_or_default().to_vec(),
|
||||
),
|
||||
};
|
||||
|
||||
assert_eq!(add, added);
|
||||
assert_eq!(remove, removed);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn policies_and_attachments_parse_pipeline_condition_and_scope() {
|
||||
let config = parse(
|
||||
r#"
|
||||
policies:
|
||||
safe:
|
||||
inherit: base
|
||||
condition:
|
||||
model: [gpt-a, gpt-b]
|
||||
pipeline:
|
||||
mode: pre_call
|
||||
steps:
|
||||
- guardrail: pii
|
||||
on_fail: next
|
||||
pass_data: true
|
||||
policy_attachments:
|
||||
- policy: safe
|
||||
teams: [research]
|
||||
priority: 10
|
||||
default: true
|
||||
"#,
|
||||
);
|
||||
let policy = &config.policies["safe"];
|
||||
let step = &policy.pipeline.as_ref().unwrap().steps.as_ref().unwrap()[0];
|
||||
let attachment = &config.policy_attachments[0];
|
||||
|
||||
assert_eq!(policy.inherit.as_deref(), Some("base"));
|
||||
assert!(matches!(
|
||||
policy.condition.as_ref().unwrap().model.as_ref().unwrap().get(),
|
||||
Value::Sequence(models) if models.len() == 2
|
||||
));
|
||||
assert_eq!(step.guardrail.as_deref(), Some("pii"));
|
||||
assert_eq!(step.on_fail.as_deref(), Some("next"));
|
||||
assert_eq!(step.pass_data, Some(Spelled::Value(true)));
|
||||
assert_eq!(attachment.policy.as_deref(), Some("safe"));
|
||||
assert_eq!(
|
||||
attachment.teams.as_deref(),
|
||||
Some(&["research".to_owned()][..])
|
||||
);
|
||||
assert_eq!(attachment.priority, Some(Spelled::Value(10)));
|
||||
assert_eq!(attachment.default, Some(Spelled::Value(true)));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::policy_model(
|
||||
r#"
|
||||
policies:
|
||||
safe: {condition: {model: SENTINEL}}
|
||||
"#
|
||||
)]
|
||||
fn debug_never_prints_policy_values(#[case] yaml: &str) {
|
||||
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
|
||||
}
|
||||
}
|
||||
99
litellm-rust/crates/config/src/prompt.rs
Normal file
99
litellm-rust/crates/config/src/prompt.rs
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
use litellm_auth_types::SecretValue;
|
||||
|
||||
use crate::{Object, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct Prompt {
|
||||
pub prompt_id: Option<String>,
|
||||
pub litellm_params: Option<PromptParams>,
|
||||
pub prompt_info: Option<PromptInfo>,
|
||||
pub version: Option<Spelled<i64>>,
|
||||
pub environment: Option<String>,
|
||||
pub created_by: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct PromptParams {
|
||||
pub prompt_id: Option<String>,
|
||||
pub prompt_integration: Option<String>,
|
||||
pub api_base: Option<String>,
|
||||
pub api_key: Option<SecretValue>,
|
||||
pub provider_specific_query_params: Option<Object>,
|
||||
pub ignore_prompt_manager_model: Option<Spelled<bool>>,
|
||||
pub ignore_prompt_manager_optional_params: Option<Spelled<bool>>,
|
||||
pub dotprompt_content: Option<String>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct PromptInfo {
|
||||
pub prompt_type: Option<String>,
|
||||
pub environment: Option<String>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::{Config, Spelled};
|
||||
|
||||
fn parse(yaml: &str) -> Config {
|
||||
Config::from_yaml(yaml).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn prompt_entries_keep_typed_params_and_info() {
|
||||
let config = parse(
|
||||
r#"
|
||||
prompts:
|
||||
- prompt_id: greeting
|
||||
version: 2
|
||||
litellm_params:
|
||||
prompt_integration: dotprompt
|
||||
ignore_prompt_manager_model: true
|
||||
prompt_extension: x
|
||||
prompt_info:
|
||||
prompt_type: config
|
||||
"#,
|
||||
);
|
||||
let prompt = &config.prompts[0];
|
||||
let params = prompt.litellm_params.as_ref().unwrap();
|
||||
|
||||
assert_eq!(prompt.prompt_id.as_deref(), Some("greeting"));
|
||||
assert_eq!(prompt.version, Some(Spelled::Value(2)));
|
||||
assert_eq!(params.prompt_integration.as_deref(), Some("dotprompt"));
|
||||
assert_eq!(
|
||||
params.ignore_prompt_manager_model,
|
||||
Some(Spelled::Value(true))
|
||||
);
|
||||
assert!(params.additional_fields.contains_key("prompt_extension"));
|
||||
assert_eq!(
|
||||
prompt.prompt_info.as_ref().unwrap().prompt_type.as_deref(),
|
||||
Some("config")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::prompt_key(
|
||||
r#"
|
||||
prompts:
|
||||
- litellm_params: {api_key: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::prompt_query_params(
|
||||
r#"
|
||||
prompts:
|
||||
- litellm_params: {provider_specific_query_params: {token: SENTINEL}}
|
||||
"#
|
||||
)]
|
||||
#[case::prompt_extra(
|
||||
r#"
|
||||
prompts:
|
||||
- litellm_params: {vendor_secret: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
fn debug_never_prints_prompt_values(#[case] yaml: &str) {
|
||||
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
|
||||
}
|
||||
}
|
||||
118
litellm-rust/crates/config/src/provider.rs
Normal file
118
litellm-rust/crates/config/src/provider.rs
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
use litellm_auth_types::SecretValue;
|
||||
|
||||
use crate::{Object, Raw};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct AssistantSettings {
|
||||
pub custom_llm_provider: Option<String>,
|
||||
pub litellm_params: Option<Object>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct ProviderSettings {
|
||||
pub custom_llm_provider: Option<String>,
|
||||
pub api_key: Option<SecretValue>,
|
||||
pub api_base: Option<String>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct VertexConfig {
|
||||
pub vertex_project: Option<String>,
|
||||
pub vertex_location: Option<String>,
|
||||
pub vertex_credentials: Option<Raw>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct Worker {
|
||||
pub worker_id: Option<String>,
|
||||
pub name: Option<String>,
|
||||
pub url: Option<String>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::Config;
|
||||
|
||||
fn parse(yaml: &str) -> Config {
|
||||
Config::from_yaml(yaml).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::files("files_settings")]
|
||||
#[case::finetune("finetune_settings")]
|
||||
fn provider_settings_keep_the_provider_and_unknown_keys(#[case] section: &str) {
|
||||
let config = parse(&format!(
|
||||
"{section}:\n - custom_llm_provider: azure\n api_version: '2024-01-01'\n"
|
||||
));
|
||||
let settings = config
|
||||
.files_settings
|
||||
.first()
|
||||
.or(config.finetune_settings.first())
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(settings.custom_llm_provider.as_deref(), Some("azure"));
|
||||
assert!(settings.additional_fields.contains_key("api_version"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn single_provider_sections_land_in_typed_fields() {
|
||||
let config = parse(
|
||||
r#"
|
||||
assistant_settings:
|
||||
custom_llm_provider: openai
|
||||
litellm_params:
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
default_vertex_config:
|
||||
vertex_project: proj
|
||||
vertex_location: us-central1
|
||||
worker_registry:
|
||||
- worker_id: eu
|
||||
name: Europe
|
||||
url: https://eu.example.test
|
||||
"#,
|
||||
);
|
||||
let assistant = config.assistant_settings.as_ref().unwrap();
|
||||
let vertex = config.default_vertex_config.as_ref().unwrap();
|
||||
let worker = &config.worker_registry[0];
|
||||
|
||||
assert_eq!(assistant.custom_llm_provider.as_deref(), Some("openai"));
|
||||
assert!(
|
||||
assistant
|
||||
.litellm_params
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.contains_key("api_key")
|
||||
);
|
||||
assert_eq!(vertex.vertex_project.as_deref(), Some("proj"));
|
||||
assert_eq!(vertex.vertex_location.as_deref(), Some("us-central1"));
|
||||
assert_eq!(worker.worker_id.as_deref(), Some("eu"));
|
||||
assert_eq!(worker.name.as_deref(), Some("Europe"));
|
||||
assert_eq!(worker.url.as_deref(), Some("https://eu.example.test"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::vertex_credentials("default_vertex_config: {vertex_credentials: SENTINEL}")]
|
||||
#[case::vertex_credentials_map(
|
||||
"default_vertex_config: {vertex_credentials: {private_key: SENTINEL}}"
|
||||
)]
|
||||
#[case::assistant_params("assistant_settings: {litellm_params: {api_key: SENTINEL}}")]
|
||||
#[case::files_key(
|
||||
r#"
|
||||
files_settings:
|
||||
- api_key: SENTINEL
|
||||
"#
|
||||
)]
|
||||
#[case::finetune_extra(
|
||||
r#"
|
||||
finetune_settings:
|
||||
- vendor_secret: SENTINEL
|
||||
"#
|
||||
)]
|
||||
fn debug_never_prints_provider_values(#[case] yaml: &str) {
|
||||
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
|
||||
}
|
||||
}
|
||||
191
litellm-rust/crates/config/src/router.rs
Normal file
191
litellm-rust/crates/config/src/router.rs
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_auth_types::SecretValue;
|
||||
|
||||
use crate::{Object, Raw, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct RouterSettings {
|
||||
pub redis_url: Option<SecretValue>,
|
||||
pub redis_host: Option<String>,
|
||||
pub redis_port: Option<Spelled<i64>>,
|
||||
pub redis_password: Option<SecretValue>,
|
||||
pub redis_db: Option<Spelled<i64>>,
|
||||
pub cache_responses: Option<Spelled<bool>>,
|
||||
pub cache_kwargs: Option<Object>,
|
||||
pub caching_groups: Option<Raw>,
|
||||
pub client_ttl: Option<Spelled<i64>>,
|
||||
pub polling_interval: Option<Spelled<f64>>,
|
||||
pub default_priority: Option<Raw>,
|
||||
pub num_retries: Option<u64>,
|
||||
pub max_fallbacks: Option<Spelled<i64>>,
|
||||
pub timeout: Option<f64>,
|
||||
pub stream_timeout: Option<Spelled<f64>>,
|
||||
pub default_litellm_params: Option<Object>,
|
||||
pub default_max_parallel_requests: Option<Spelled<i64>>,
|
||||
pub set_verbose: Option<Spelled<bool>>,
|
||||
pub debug_level: Option<String>,
|
||||
pub default_fallbacks: Option<Box<[String]>>,
|
||||
pub fallbacks: Option<Box<[Object]>>,
|
||||
pub context_window_fallbacks: Option<Box<[Object]>>,
|
||||
pub content_policy_fallbacks: Option<Box<[Object]>>,
|
||||
pub model_group_alias: Option<Object>,
|
||||
pub enable_pre_call_checks: Option<Spelled<bool>>,
|
||||
pub enable_tag_filtering: Option<bool>,
|
||||
pub tag_filtering_match_any: Option<Spelled<bool>>,
|
||||
pub tag_routing_prefix: Option<String>,
|
||||
pub plugins: Option<Box<[String]>>,
|
||||
pub retry_after: Option<f64>,
|
||||
pub retry_policy: Option<Object>,
|
||||
pub model_group_retry_policy: Option<Object>,
|
||||
pub allowed_fails: Option<u64>,
|
||||
pub allowed_fails_policy: Option<AllowedFailsPolicy>,
|
||||
pub cooldown_time: Option<f64>,
|
||||
pub disable_cooldowns: Option<Spelled<bool>>,
|
||||
pub fallback_on_unsupported_params: Option<Spelled<bool>>,
|
||||
pub routing_strategy: Option<String>,
|
||||
pub optional_pre_call_checks: Option<Box<[String]>>,
|
||||
pub routing_strategy_args: Option<Object>,
|
||||
pub routing_groups: Option<Box<[Object]>>,
|
||||
pub provider_budget_config: Option<BTreeMap<String, ProviderBudget>>,
|
||||
pub alerting_config: Option<AlertingConfig>,
|
||||
pub deployment_affinity_ttl_seconds: Option<Spelled<i64>>,
|
||||
pub model_group_affinity_config: Option<Object>,
|
||||
pub enable_health_check_routing: Option<Spelled<bool>>,
|
||||
pub health_check_staleness_threshold: Option<Spelled<f64>>,
|
||||
pub health_check_ignore_transient_errors: Option<Spelled<bool>>,
|
||||
pub background_health_check_model_groups: Option<Box<[String]>>,
|
||||
pub enable_weighted_failover: Option<Spelled<bool>>,
|
||||
pub enable_mid_stream_fallback_continuation: Option<Spelled<bool>>,
|
||||
pub guardrail_list: Option<Box<[Object]>>,
|
||||
pub max_retries: Option<u64>,
|
||||
pub weights: Option<Object>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
#[serde(rename_all = "PascalCase")]
|
||||
pub struct AllowedFailsPolicy {
|
||||
pub bad_request_error_allowed_fails: Option<Spelled<i64>>,
|
||||
pub authentication_error_allowed_fails: Option<Spelled<i64>>,
|
||||
pub timeout_error_allowed_fails: Option<Spelled<i64>>,
|
||||
pub rate_limit_error_allowed_fails: Option<Spelled<i64>>,
|
||||
pub content_policy_violation_error_allowed_fails: Option<Spelled<i64>>,
|
||||
pub internal_server_error_allowed_fails: Option<Spelled<i64>>,
|
||||
pub service_unavailable_error_allowed_fails: Option<Spelled<i64>>,
|
||||
pub bad_gateway_error_allowed_fails: Option<Spelled<i64>>,
|
||||
pub not_found_error_allowed_fails: Option<Spelled<i64>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct ProviderBudget {
|
||||
pub budget_limit: Option<Spelled<f64>>,
|
||||
pub time_period: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct AlertingConfig {
|
||||
pub webhook_url: Option<SecretValue>,
|
||||
pub alerting_threshold: Option<Spelled<f64>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::{Config, RouterSettings, Spelled};
|
||||
|
||||
fn router(yaml: &str) -> RouterSettings {
|
||||
Config::from_yaml(&format!("router_settings:\n{yaml}"))
|
||||
.unwrap()
|
||||
.router_settings
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::int(" redis_port: 6379", |s: &RouterSettings| s.redis_port == Some(Spelled::Value(6379)))]
|
||||
#[case::int_env(
|
||||
" client_ttl: os.environ/CLIENT_TTL",
|
||||
|s: &RouterSettings| s.client_ttl == Some(Spelled::Text("os.environ/CLIENT_TTL".to_string()))
|
||||
)]
|
||||
#[case::float_from_int(" stream_timeout: 30", |s: &RouterSettings| s.stream_timeout == Some(Spelled::Value(30.0)))]
|
||||
#[case::float(
|
||||
" health_check_staleness_threshold: 2.5",
|
||||
|s: &RouterSettings| s.health_check_staleness_threshold == Some(Spelled::Value(2.5))
|
||||
)]
|
||||
#[case::bool(" enable_weighted_failover: true", |s: &RouterSettings| s.enable_weighted_failover == Some(Spelled::Value(true)))]
|
||||
#[case::bool_word(
|
||||
" enable_mid_stream_fallback_continuation: \"true\"",
|
||||
|s: &RouterSettings| s.enable_mid_stream_fallback_continuation == Some(Spelled::Text("true".to_string()))
|
||||
)]
|
||||
#[case::string(" redis_host: cache.internal", |s: &RouterSettings| s.redis_host.as_deref() == Some("cache.internal"))]
|
||||
#[case::string_list(
|
||||
" default_fallbacks: [gpt-small, gpt-large]",
|
||||
|s: &RouterSettings| s.default_fallbacks.as_deref() == Some(&["gpt-small".to_string(), "gpt-large".to_string()][..])
|
||||
)]
|
||||
#[case::mapping_list(
|
||||
" content_policy_fallbacks: [{gpt: [claude]}]",
|
||||
|s: &RouterSettings| s.content_policy_fallbacks.as_ref().is_some_and(|items| items[0].contains_key("gpt"))
|
||||
)]
|
||||
#[case::mapping(
|
||||
" default_litellm_params: {temperature: 0}",
|
||||
|s: &RouterSettings| s.default_litellm_params.as_ref().is_some_and(|params| params.contains_key("temperature"))
|
||||
)]
|
||||
#[case::non_integer_priority(" default_priority: 1.5", |s: &RouterSettings| s.default_priority.is_some())]
|
||||
#[case::bool_priority(" default_priority: true", |s: &RouterSettings| s.default_priority.is_some())]
|
||||
#[case::caching_groups(
|
||||
" caching_groups: [[gpt, [gpt-a, gpt-b]]]",
|
||||
|s: &RouterSettings| s.caching_groups.as_ref().is_some_and(|groups| groups.get().is_sequence())
|
||||
)]
|
||||
#[case::allowed_fails_policy(
|
||||
" allowed_fails_policy: {RateLimitErrorAllowedFails: \"3\", BadGatewayErrorAllowedFails: 2, FutureAllowedFails: 1}",
|
||||
|s: &RouterSettings| s.allowed_fails_policy.as_ref().is_some_and(|policy| {
|
||||
policy.rate_limit_error_allowed_fails == Some(Spelled::Text("3".to_string()))
|
||||
&& policy.bad_gateway_error_allowed_fails == Some(Spelled::Value(2))
|
||||
})
|
||||
)]
|
||||
#[case::provider_budget_config(
|
||||
" provider_budget_config: {openai: {budget_limit: 100, time_period: 1d, rpm_limit: 5}}",
|
||||
|s: &RouterSettings| s.provider_budget_config.as_ref().is_some_and(|budgets| {
|
||||
let openai = &budgets["openai"];
|
||||
openai.budget_limit == Some(Spelled::Value(100.0))
|
||||
&& openai.time_period.as_deref() == Some("1d")
|
||||
})
|
||||
)]
|
||||
#[case::alerting_config(
|
||||
" alerting_config: {webhook_url: https://hooks.example/a, alerting_threshold: 10, channel: ops}",
|
||||
|s: &RouterSettings| s.alerting_config.as_ref().is_some_and(|alerting| {
|
||||
alerting.webhook_url.as_ref().map(|url| url.expose()) == Some("https://hooks.example/a")
|
||||
&& alerting.alerting_threshold == Some(Spelled::Value(10.0))
|
||||
})
|
||||
)]
|
||||
fn parses_python_router_spellings(
|
||||
#[case] yaml: &str,
|
||||
#[case] check: fn(&RouterSettings) -> bool,
|
||||
) {
|
||||
assert!(check(&router(yaml)));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unknown("future_router_setting")]
|
||||
#[case::moved_to_general_settings("health_check_interval")]
|
||||
#[case::managed_outside_config("model_list")]
|
||||
fn keeps_keys_outside_the_router_signature(#[case] key: &str) {
|
||||
let settings = router(&format!(" {key}: 60"));
|
||||
assert!(settings.additional_fields.contains_key(key));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::redis_password(" redis_password: router-sentinel")]
|
||||
#[case::redis_url(" redis_url: redis://user:router-sentinel@cache:6379")]
|
||||
#[case::webhook_url(" alerting_config: {webhook_url: https://hooks.example/router-sentinel}")]
|
||||
#[case::cache_kwargs(" cache_kwargs: {password: router-sentinel}")]
|
||||
#[case::default_priority(" default_priority: router-sentinel")]
|
||||
#[case::caching_groups(" caching_groups: [[router-sentinel, [a]]]")]
|
||||
#[case::guardrail_list(" guardrail_list: [{api_key: router-sentinel}]")]
|
||||
#[case::nested_extension(" allowed_fails_policy: {FutureAllowedFails: router-sentinel}")]
|
||||
fn debug_output_does_not_expose_router_values(#[case] yaml: &str) {
|
||||
let config = Config::from_yaml(&format!("router_settings:\n{yaml}")).unwrap();
|
||||
assert!(!format!("{config:?}").contains("router-sentinel"));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,268 +0,0 @@
|
|||
use std::fmt;
|
||||
|
||||
use litellm_auth_types::SecretValue;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{AdditionalFields, Object, Spelled, Value, value::one_or_many};
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum TracingStoreKind {
|
||||
Clickhouse,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ClickHouseStoreSettings {
|
||||
#[serde(rename = "type")]
|
||||
pub kind: TracingStoreKind,
|
||||
pub url: Option<SecretValue>,
|
||||
pub database: Option<String>,
|
||||
pub retention_days: Option<Spelled<f64>>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ClickHouseStoreSettings {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ClickHouseStoreSettings")
|
||||
.field("kind", &self.kind)
|
||||
.field("database", &self.database)
|
||||
.field("retention_days", &self.retention_days)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TracingStoreSettings {
|
||||
ClickHouse(ClickHouseStoreSettings),
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Debug, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct TracingSettings {
|
||||
pub store: Option<TracingStoreSettings>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: AdditionalFields,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct GeneralSettings {
|
||||
pub completion_model: Option<String>,
|
||||
pub max_in_flight_requests_per_worker: Option<u64>,
|
||||
pub max_queued_requests_per_worker: Option<u64>,
|
||||
pub admission_queue_timeout_seconds: f64,
|
||||
pub master_key: Option<SecretValue>,
|
||||
pub database_url: Option<SecretValue>,
|
||||
pub tracing: Option<TracingSettings>,
|
||||
pub database_connection_pool_limit: Option<u64>,
|
||||
pub database_connection_timeout: Option<f64>,
|
||||
pub database_connect_timeout: Option<f64>,
|
||||
pub database_socket_timeout: Option<f64>,
|
||||
pub database_max_idle_connection_lifetime: Option<f64>,
|
||||
pub max_parallel_requests: Option<u64>,
|
||||
pub global_max_parallel_requests: Option<u64>,
|
||||
pub max_request_size_mb: Option<u64>,
|
||||
pub max_response_size_mb: Option<u64>,
|
||||
pub proxy_config_reload_interval_seconds: u64,
|
||||
pub background_health_checks: Option<bool>,
|
||||
pub health_check_interval: u64,
|
||||
pub health_check_concurrency: Option<u64>,
|
||||
pub store_model_in_db: Option<bool>,
|
||||
pub forward_client_headers_to_llm_api: Option<bool>,
|
||||
pub cancel_on_disconnect: Option<bool>,
|
||||
pub infer_model_from_keys: Option<bool>,
|
||||
pub enable_public_model_hub: bool,
|
||||
pub dangerously_permit_weak_or_unset_master_key: Option<bool>,
|
||||
pub plugins: Option<Box<[Object]>>,
|
||||
pub coordination_redis: Option<Object>,
|
||||
pub mcp_allowed_hosts: Option<Box<[String]>>,
|
||||
pub mcp_allowed_origins: Box<[String]>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: AdditionalFields,
|
||||
}
|
||||
|
||||
impl Default for GeneralSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
completion_model: None,
|
||||
max_in_flight_requests_per_worker: None,
|
||||
max_queued_requests_per_worker: None,
|
||||
admission_queue_timeout_seconds: 1.0,
|
||||
master_key: None,
|
||||
database_url: None,
|
||||
tracing: None,
|
||||
database_connection_pool_limit: Some(10),
|
||||
database_connection_timeout: Some(60.0),
|
||||
database_connect_timeout: None,
|
||||
database_socket_timeout: None,
|
||||
database_max_idle_connection_lifetime: Some(60.0),
|
||||
max_parallel_requests: None,
|
||||
global_max_parallel_requests: None,
|
||||
max_request_size_mb: None,
|
||||
max_response_size_mb: None,
|
||||
proxy_config_reload_interval_seconds: 30,
|
||||
background_health_checks: None,
|
||||
health_check_interval: 300,
|
||||
health_check_concurrency: None,
|
||||
store_model_in_db: None,
|
||||
forward_client_headers_to_llm_api: None,
|
||||
cancel_on_disconnect: None,
|
||||
infer_model_from_keys: None,
|
||||
enable_public_model_hub: false,
|
||||
dangerously_permit_weak_or_unset_master_key: None,
|
||||
plugins: None,
|
||||
coordination_redis: None,
|
||||
mcp_allowed_hosts: None,
|
||||
mcp_allowed_origins: Box::default(),
|
||||
additional_fields: AdditionalFields::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for GeneralSettings {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("GeneralSettings")
|
||||
.field("completion_model", &self.completion_model)
|
||||
.field(
|
||||
"max_in_flight_requests_per_worker",
|
||||
&self.max_in_flight_requests_per_worker,
|
||||
)
|
||||
.field(
|
||||
"max_queued_requests_per_worker",
|
||||
&self.max_queued_requests_per_worker,
|
||||
)
|
||||
.field(
|
||||
"admission_queue_timeout_seconds",
|
||||
&self.admission_queue_timeout_seconds,
|
||||
)
|
||||
.field("master_key", &self.master_key)
|
||||
.field("database_url", &self.database_url)
|
||||
.field("tracing", &self.tracing)
|
||||
.field("store_model_in_db", &self.store_model_in_db)
|
||||
.field("additional_fields", &self.additional_fields.keys())
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct RouterSettings {
|
||||
pub routing_strategy: Option<String>,
|
||||
pub routing_strategy_args: Option<Object>,
|
||||
pub routing_groups: Option<Box<[Object]>>,
|
||||
pub retry_policy: Option<Object>,
|
||||
pub model_group_retry_policy: Option<Object>,
|
||||
pub model_group_affinity_config: Option<Object>,
|
||||
pub allowed_fails: Option<u64>,
|
||||
pub cooldown_time: Option<f64>,
|
||||
pub num_retries: Option<u64>,
|
||||
pub timeout: Option<f64>,
|
||||
pub max_retries: Option<u64>,
|
||||
pub retry_after: Option<f64>,
|
||||
pub fallbacks: Option<Box<[Object]>>,
|
||||
pub context_window_fallbacks: Option<Box<[Object]>>,
|
||||
pub model_group_alias: Option<Object>,
|
||||
pub enable_tag_filtering: Option<bool>,
|
||||
pub weights: Option<Object>,
|
||||
pub tag_routing_prefix: Option<String>,
|
||||
pub optional_pre_call_checks: Option<Box<[String]>>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: AdditionalFields,
|
||||
}
|
||||
|
||||
impl fmt::Debug for RouterSettings {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("RouterSettings")
|
||||
.field("routing_strategy", &self.routing_strategy)
|
||||
.field("routing_strategy_args", &self.routing_strategy_args)
|
||||
.field("routing_groups", &self.routing_groups)
|
||||
.field("retry_policy", &self.retry_policy)
|
||||
.field("model_group_retry_policy", &self.model_group_retry_policy)
|
||||
.field(
|
||||
"model_group_affinity_config",
|
||||
&self.model_group_affinity_config,
|
||||
)
|
||||
.field("allowed_fails", &self.allowed_fails)
|
||||
.field("cooldown_time", &self.cooldown_time)
|
||||
.field("num_retries", &self.num_retries)
|
||||
.field("timeout", &self.timeout)
|
||||
.field("max_retries", &self.max_retries)
|
||||
.field("retry_after", &self.retry_after)
|
||||
.field("fallbacks", &self.fallbacks)
|
||||
.field("context_window_fallbacks", &self.context_window_fallbacks)
|
||||
.field("model_group_alias", &self.model_group_alias)
|
||||
.field("enable_tag_filtering", &self.enable_tag_filtering)
|
||||
.field("weights", &self.weights)
|
||||
.field("tag_routing_prefix", &self.tag_routing_prefix)
|
||||
.field("optional_pre_call_checks", &self.optional_pre_call_checks)
|
||||
.field("additional_fields", &self.additional_fields.keys())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct LiteLlmSettings {
|
||||
pub ssl_verify: Option<Spelled<bool>>,
|
||||
pub ssl_certificate: Option<String>,
|
||||
pub ssl_security_level: Option<String>,
|
||||
pub ssl_ecdh_curve: Option<String>,
|
||||
pub force_ipv4: Option<bool>,
|
||||
pub http2: Option<bool>,
|
||||
pub aiohttp_trust_env: Option<bool>,
|
||||
pub disable_aiohttp_trust_env: Option<bool>,
|
||||
pub disable_aiohttp_transport: Option<bool>,
|
||||
pub drop_params: Option<Spelled<bool>>,
|
||||
pub request_timeout: Option<Spelled<f64>>,
|
||||
pub num_retries: Option<u64>,
|
||||
pub cache: Option<bool>,
|
||||
pub cache_params: Option<Object>,
|
||||
#[serde(deserialize_with = "one_or_many")]
|
||||
pub callbacks: Option<Vec<Value>>,
|
||||
#[serde(deserialize_with = "one_or_many")]
|
||||
pub success_callback: Option<Vec<Value>>,
|
||||
#[serde(deserialize_with = "one_or_many")]
|
||||
pub failure_callback: Option<Vec<Value>>,
|
||||
pub json_logs: Option<bool>,
|
||||
pub set_verbose: Option<bool>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: AdditionalFields,
|
||||
}
|
||||
|
||||
impl fmt::Debug for LiteLlmSettings {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("LiteLlmSettings")
|
||||
.field("drop_params", &self.drop_params)
|
||||
.field("request_timeout", &self.request_timeout)
|
||||
.field("num_retries", &self.num_retries)
|
||||
.field("cache", &self.cache)
|
||||
.field("cache_params", &self.cache_params)
|
||||
.field(
|
||||
"callbacks",
|
||||
&self.callbacks.as_ref().map(|callbacks| callbacks.len()),
|
||||
)
|
||||
.field(
|
||||
"success_callback",
|
||||
&self
|
||||
.success_callback
|
||||
.as_ref()
|
||||
.map(|callbacks| callbacks.len()),
|
||||
)
|
||||
.field(
|
||||
"failure_callback",
|
||||
&self
|
||||
.failure_callback
|
||||
.as_ref()
|
||||
.map(|callbacks| callbacks.len()),
|
||||
)
|
||||
.field("json_logs", &self.json_logs)
|
||||
.field("set_verbose", &self.set_verbose)
|
||||
.field("additional_fields", &self.additional_fields.keys())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
128
litellm-rust/crates/config/src/tool.rs
Normal file
128
litellm-rust/crates/config/src/tool.rs
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
use litellm_auth_types::SecretValue;
|
||||
|
||||
use crate::{Object, Spelled};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct SearchTool {
|
||||
pub search_tool_name: Option<String>,
|
||||
pub litellm_params: Option<SearchToolParams>,
|
||||
pub search_tool_info: Option<SearchToolInfo>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct SearchToolParams {
|
||||
pub search_provider: Option<String>,
|
||||
pub api_key: Option<SecretValue>,
|
||||
pub api_base: Option<String>,
|
||||
pub timeout: Option<Spelled<f64>>,
|
||||
pub max_retries: Option<Spelled<i64>>,
|
||||
pub input_cost_per_query: Option<Spelled<f64>>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct SearchToolInfo {
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct SandboxTool {
|
||||
pub sandbox_tool_name: Option<String>,
|
||||
pub litellm_params: Option<SandboxToolParams>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct SandboxToolParams {
|
||||
pub sandbox_provider: Option<String>,
|
||||
pub api_key: Option<SecretValue>,
|
||||
pub api_base: Option<String>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::{Config, Spelled};
|
||||
|
||||
fn parse(yaml: &str) -> Config {
|
||||
Config::from_yaml(yaml).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn search_tools_keep_typed_params_and_info() {
|
||||
let config = parse(
|
||||
r#"
|
||||
search_tools:
|
||||
- search_tool_name: web
|
||||
litellm_params:
|
||||
search_provider: perplexity
|
||||
max_retries: 3
|
||||
timeout: 2.5
|
||||
region: eu
|
||||
search_tool_info:
|
||||
description: web search
|
||||
"#,
|
||||
);
|
||||
let tool = &config.search_tools[0];
|
||||
let params = tool.litellm_params.as_ref().unwrap();
|
||||
|
||||
assert_eq!(tool.search_tool_name.as_deref(), Some("web"));
|
||||
assert_eq!(params.search_provider.as_deref(), Some("perplexity"));
|
||||
assert_eq!(params.max_retries, Some(Spelled::Value(3)));
|
||||
assert_eq!(params.timeout, Some(Spelled::Value(2.5)));
|
||||
assert!(params.additional_fields.contains_key("region"));
|
||||
assert_eq!(
|
||||
tool.search_tool_info
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.description
|
||||
.as_deref(),
|
||||
Some("web search")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn sandbox_tools_keep_typed_params() {
|
||||
let config = parse(
|
||||
"sandbox_tools:\n - sandbox_tool_name: e2b\n litellm_params: {sandbox_provider: e2b, template: base}\n",
|
||||
);
|
||||
let tool = &config.sandbox_tools[0];
|
||||
let params = tool.litellm_params.as_ref().unwrap();
|
||||
|
||||
assert_eq!(tool.sandbox_tool_name.as_deref(), Some("e2b"));
|
||||
assert_eq!(params.sandbox_provider.as_deref(), Some("e2b"));
|
||||
assert!(params.additional_fields.contains_key("template"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::search_key(
|
||||
r#"
|
||||
search_tools:
|
||||
- litellm_params: {api_key: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::search_extra(
|
||||
r#"
|
||||
search_tools:
|
||||
- litellm_params: {vendor_secret: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::sandbox_key(
|
||||
r#"
|
||||
sandbox_tools:
|
||||
- litellm_params: {api_key: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::sandbox_extra(
|
||||
r#"
|
||||
sandbox_tools:
|
||||
- litellm_params: {vendor_secret: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
fn debug_never_prints_tool_values(#[case] yaml: &str) {
|
||||
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,28 +1,13 @@
|
|||
use std::{collections::BTreeMap, fmt, ops::Deref};
|
||||
|
||||
use serde::{Deserialize, Deserializer};
|
||||
use serde::Deserialize;
|
||||
|
||||
pub type Value = serde_yaml_ng::Value;
|
||||
pub type AdditionalFields = BTreeMap<String, Value>;
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct Object(BTreeMap<String, Value>);
|
||||
|
||||
impl Object {
|
||||
pub fn get(&self, key: &str) -> Option<&Value> {
|
||||
self.0.get(key)
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.is_empty()
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.0.len()
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for Object {
|
||||
type Target = BTreeMap<String, Value>;
|
||||
|
||||
|
|
@ -40,14 +25,27 @@ impl fmt::Debug for Object {
|
|||
}
|
||||
}
|
||||
|
||||
/// Python's callback shorthand: one entry or a list of them, kept as the values written.
|
||||
pub(crate) fn one_or_many<'de, D: Deserializer<'de>>(
|
||||
deserializer: D,
|
||||
) -> Result<Option<Vec<Value>>, D::Error> {
|
||||
Ok(
|
||||
Option::<Value>::deserialize(deserializer)?.map(|value| match value {
|
||||
Value::Sequence(values) => values,
|
||||
one => vec![one],
|
||||
}),
|
||||
)
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct Raw(Value);
|
||||
|
||||
impl Raw {
|
||||
pub fn get(&self) -> &Value {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for Raw {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let kind = match &self.0 {
|
||||
Value::Null => "null",
|
||||
Value::Bool(_) => "bool",
|
||||
Value::Number(_) => "number",
|
||||
Value::String(_) => "string",
|
||||
Value::Sequence(_) => "sequence",
|
||||
Value::Mapping(_) => "mapping",
|
||||
Value::Tagged(_) => "tagged",
|
||||
};
|
||||
formatter.debug_tuple("Raw").field(&kind).finish()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
63
litellm-rust/crates/config/src/vector_store.rs
Normal file
63
litellm-rust/crates/config/src/vector_store.rs
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
use crate::{Object, Raw};
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct VectorStore {
|
||||
pub vector_store_name: Option<String>,
|
||||
pub litellm_params: Option<VectorStoreParams>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::config_section)]
|
||||
pub struct VectorStoreParams {
|
||||
pub vector_store_id: Option<String>,
|
||||
pub custom_llm_provider: Option<String>,
|
||||
pub vector_store_description: Option<String>,
|
||||
pub vector_store_metadata: Option<Raw>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: Object,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::Config;
|
||||
|
||||
fn parse(yaml: &str) -> Config {
|
||||
Config::from_yaml(yaml).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn vector_store_entries_keep_typed_params_and_unknown_keys() {
|
||||
let config = parse(
|
||||
r#"
|
||||
vector_store_registry:
|
||||
- vector_store_name: kb
|
||||
litellm_params: {vector_store_id: vs_123, custom_llm_provider: openai, region: eu}
|
||||
"#,
|
||||
);
|
||||
let store = &config.vector_store_registry[0];
|
||||
let params = store.litellm_params.as_ref().unwrap();
|
||||
|
||||
assert_eq!(store.vector_store_name.as_deref(), Some("kb"));
|
||||
assert_eq!(params.vector_store_id.as_deref(), Some("vs_123"));
|
||||
assert_eq!(params.custom_llm_provider.as_deref(), Some("openai"));
|
||||
assert!(params.additional_fields.contains_key("region"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::vector_store_metadata(
|
||||
r#"
|
||||
vector_store_registry:
|
||||
- litellm_params: {vector_store_metadata: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
#[case::vector_store_extra(
|
||||
r#"
|
||||
vector_store_registry:
|
||||
- litellm_params: {api_key: SENTINEL}
|
||||
"#
|
||||
)]
|
||||
fn debug_never_prints_vector_store_values(#[case] yaml: &str) {
|
||||
assert!(!format!("{:?}", parse(yaml)).contains("SENTINEL"));
|
||||
}
|
||||
}
|
||||
177
litellm-rust/crates/config/src/yaml.rs
Normal file
177
litellm-rust/crates/config/src/yaml.rs
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
use std::fmt;
|
||||
|
||||
use serde::{
|
||||
Deserialize, Deserializer,
|
||||
de::{MapAccess, SeqAccess, Visitor},
|
||||
};
|
||||
use serde_yaml_ng::{Mapping, Value};
|
||||
|
||||
pub fn mapping(text: &str) -> Result<Mapping, serde_yaml_ng::Error> {
|
||||
serde_yaml_ng::from_value(serde_yaml_ng::from_str::<LastWins>(text)?.0)
|
||||
}
|
||||
|
||||
struct LastWins(Value);
|
||||
|
||||
impl<'de> Deserialize<'de> for LastWins {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
deserializer.deserialize_any(LastWinsVisitor).map(Self)
|
||||
}
|
||||
}
|
||||
|
||||
// PyYAML safe_load keeps the later value of a duplicate key, serde_yaml_ng rejects it
|
||||
struct LastWinsVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for LastWinsVisitor {
|
||||
type Value = Value;
|
||||
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
|
||||
formatter.write_str("any YAML value")
|
||||
}
|
||||
|
||||
fn visit_bool<E>(self, value: bool) -> Result<Value, E> {
|
||||
Ok(Value::Bool(value))
|
||||
}
|
||||
|
||||
fn visit_i64<E>(self, value: i64) -> Result<Value, E> {
|
||||
Ok(Value::Number(value.into()))
|
||||
}
|
||||
|
||||
fn visit_u64<E>(self, value: u64) -> Result<Value, E> {
|
||||
Ok(Value::Number(value.into()))
|
||||
}
|
||||
|
||||
fn visit_f64<E>(self, value: f64) -> Result<Value, E> {
|
||||
Ok(Value::Number(value.into()))
|
||||
}
|
||||
|
||||
fn visit_str<E>(self, value: &str) -> Result<Value, E> {
|
||||
Ok(Value::String(value.to_owned()))
|
||||
}
|
||||
|
||||
fn visit_string<E>(self, value: String) -> Result<Value, E> {
|
||||
Ok(Value::String(value))
|
||||
}
|
||||
|
||||
fn visit_unit<E>(self) -> Result<Value, E> {
|
||||
Ok(Value::Null)
|
||||
}
|
||||
|
||||
fn visit_none<E>(self) -> Result<Value, E> {
|
||||
Ok(Value::Null)
|
||||
}
|
||||
|
||||
fn visit_some<D: Deserializer<'de>>(self, deserializer: D) -> Result<Value, D::Error> {
|
||||
LastWins::deserialize(deserializer).map(|value| value.0)
|
||||
}
|
||||
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, mut items: A) -> Result<Value, A::Error> {
|
||||
let mut sequence = Vec::new();
|
||||
while let Some(LastWins(item)) = items.next_element()? {
|
||||
sequence.push(item);
|
||||
}
|
||||
Ok(Value::Sequence(sequence))
|
||||
}
|
||||
|
||||
fn visit_map<A: MapAccess<'de>>(self, mut entries: A) -> Result<Value, A::Error> {
|
||||
let mut mapping = Mapping::new();
|
||||
while let Some((LastWins(key), LastWins(value))) = entries.next_entry()? {
|
||||
mapping.insert(key, value);
|
||||
}
|
||||
Ok(Value::Mapping(mapping))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn yaml(text: &str) -> Mapping {
|
||||
serde_yaml_ng::from_str(text).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::top_level(
|
||||
r#"
|
||||
a: 1
|
||||
b: 2
|
||||
a: 3
|
||||
"#,
|
||||
"{a: 3, b: 2}"
|
||||
)]
|
||||
#[case::sequence_value(
|
||||
r#"
|
||||
cb: [prometheus]
|
||||
cb: [s3]
|
||||
"#,
|
||||
"cb: [s3]"
|
||||
)]
|
||||
#[case::nested("outer: {x: [1], y: 2, x: [3]}", "outer: {x: [3], y: 2}")]
|
||||
#[case::deeply_nested("a: {b: {c: 1, c: 2}}", "a: {b: {c: 2}}")]
|
||||
#[case::inside_sequence("list: [{k: old, k: new}]", "list: [{k: new}]")]
|
||||
#[case::mapping_replaced_by_scalar(
|
||||
r#"
|
||||
a: {b: 1}
|
||||
a: 2
|
||||
"#,
|
||||
"a: 2"
|
||||
)]
|
||||
#[case::value_replaced_by_null(
|
||||
r#"
|
||||
a: 1
|
||||
a:
|
||||
"#,
|
||||
"a: null"
|
||||
)]
|
||||
#[case::three_occurrences(
|
||||
r#"
|
||||
a: 1
|
||||
a: 2
|
||||
a: 3
|
||||
"#,
|
||||
"a: 3"
|
||||
)]
|
||||
#[case::non_string_keys(
|
||||
r#"
|
||||
1: a
|
||||
true: b
|
||||
1: c
|
||||
"#,
|
||||
"{1: c, true: b}"
|
||||
)]
|
||||
#[case::scalars_without_duplicates(
|
||||
r#"
|
||||
i: -1
|
||||
u: 18446744073709551615
|
||||
f: 1.5
|
||||
b: false
|
||||
n: ~
|
||||
s: text
|
||||
"#,
|
||||
"{i: -1, u: 18446744073709551615, f: 1.5, b: false, n: ~, s: text}"
|
||||
)]
|
||||
#[case::empty_document("", "{}")]
|
||||
fn later_duplicate_keys_override_earlier_ones(#[case] text: &str, #[case] expected: &str) {
|
||||
assert_eq!(mapping(text).unwrap(), yaml(expected));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn duplicate_key_keeps_its_first_position() {
|
||||
let keys: Vec<_> = mapping("a: 1\nb: 2\na: 3\n").unwrap().into_keys().collect();
|
||||
assert_eq!(keys, [Value::from("a"), Value::from("b")]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::sequence(
|
||||
r#"
|
||||
- a
|
||||
- b
|
||||
"#
|
||||
)]
|
||||
#[case::scalar("text")]
|
||||
#[case::malformed("a: [")]
|
||||
fn rejects_a_document_that_is_not_a_mapping(#[case] text: &str) {
|
||||
assert!(mapping(text).is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,436 +0,0 @@
|
|||
use litellm_config::{Config, Error, Spelled, TracingStoreSettings};
|
||||
use rstest::{fixture, rstest};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[fixture]
|
||||
fn directory() -> TempDir {
|
||||
tempfile::tempdir().unwrap()
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn model_list_yaml() -> &'static str {
|
||||
r#"
|
||||
model_list:
|
||||
- model_name: assistant
|
||||
litellm_params:
|
||||
model: anthropic/test-model
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
- model_name: local
|
||||
litellm_params:
|
||||
model: test-model
|
||||
api_base: http://localhost:8000/v1
|
||||
custom_llm_provider: openai
|
||||
"#
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn loads_model_list_from_file(directory: TempDir, model_list_yaml: &str) {
|
||||
let path = directory.path().join("config.yaml");
|
||||
std::fs::write(&path, model_list_yaml).unwrap();
|
||||
|
||||
let config = Config::load(path).unwrap();
|
||||
assert_eq!(config.model_list.len(), 2);
|
||||
let anthropic = &config.model_list[0];
|
||||
assert_eq!(anthropic.model_name, "assistant");
|
||||
assert_eq!(anthropic.litellm_params.model, "anthropic/test-model");
|
||||
assert_eq!(
|
||||
anthropic.litellm_params.api_key.as_ref().unwrap().expose(),
|
||||
"os.environ/ANTHROPIC_API_KEY"
|
||||
);
|
||||
assert!(anthropic.litellm_params.api_base.is_none());
|
||||
assert!(anthropic.litellm_params.custom_llm_provider.is_none());
|
||||
let local = &config.model_list[1];
|
||||
assert_eq!(local.model_name, "local");
|
||||
assert_eq!(local.litellm_params.model, "test-model");
|
||||
assert!(local.litellm_params.api_key.is_none());
|
||||
assert_eq!(
|
||||
local.litellm_params.api_base.as_deref(),
|
||||
Some("http://localhost:8000/v1")
|
||||
);
|
||||
assert_eq!(
|
||||
local.litellm_params.custom_llm_provider.as_deref(),
|
||||
Some("openai")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn config_debug_redacts_api_keys() {
|
||||
let config = Config::from_yaml(
|
||||
"model_list: [{model_name: assistant, litellm_params: {model: anthropic/test-model, api_key: secret-value}}]",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
config.model_list[0]
|
||||
.litellm_params
|
||||
.api_key
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.expose(),
|
||||
"secret-value"
|
||||
);
|
||||
assert!(!format!("{config:?}").contains("secret-value"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::malformed_yaml("model_list: [")]
|
||||
#[case::missing_params("model_list: [{model_name: assistant}]")]
|
||||
#[case::missing_model("model_list: [{model_name: assistant, litellm_params: {api_key: key}}]")]
|
||||
#[case::provider_param_of_the_wrong_type(
|
||||
"model_list: [{model_name: assistant, litellm_params: {model: test, aws_region_name: [eu-central-1]}}]"
|
||||
)]
|
||||
fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) {
|
||||
assert!(matches!(Config::from_yaml(yaml), Err(Error::Parse(_))));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn distinguishes_read_errors_from_parse_errors(directory: TempDir) {
|
||||
assert!(matches!(
|
||||
Config::load(directory.path().join("missing.yaml")),
|
||||
Err(Error::Read(error)) if error.kind() == std::io::ErrorKind::NotFound
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::literal("secret-master-key")]
|
||||
#[case::reference("os.environ/LITELLM_MASTER_KEY")]
|
||||
fn loads_and_redacts_the_master_key(#[case] key: &str) {
|
||||
let config = Config::from_yaml(&format!(
|
||||
"model_list: []\ngeneral_settings:\n master_key: {key}\n"
|
||||
))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
config
|
||||
.general_settings
|
||||
.master_key
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.expose(),
|
||||
key
|
||||
);
|
||||
assert!(!format!("{config:?}").contains(key));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn missing_general_settings_has_no_master_key() {
|
||||
let config = Config::from_yaml("model_list: []").unwrap();
|
||||
assert!(config.general_settings.master_key.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_are_typed_and_redact_the_url() {
|
||||
let config = Config::from_yaml(
|
||||
"general_settings:\n tracing:\n store:\n type: clickhouse\n url: https://writer:password@example.com\n database: analytics\n retention_days: 7\n",
|
||||
)
|
||||
.unwrap();
|
||||
let tracing = config.general_settings.tracing.as_ref().unwrap();
|
||||
let Some(TracingStoreSettings::ClickHouse(store)) = tracing.store.as_ref() else {
|
||||
panic!("expected ClickHouse tracing store")
|
||||
};
|
||||
assert_eq!(
|
||||
store.url.as_ref().unwrap().expose(),
|
||||
"https://writer:password@example.com"
|
||||
);
|
||||
assert_eq!(store.database.as_deref(), Some("analytics"));
|
||||
assert_eq!(store.retention_days, Some(Spelled::Value(7.0)));
|
||||
assert!(!format!("{config:?}").contains("password"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_accept_environment_references() {
|
||||
let config = Config::from_yaml(
|
||||
"general_settings:\n tracing:\n store:\n type: clickhouse\n url: os.environ/CLICKHOUSE_URL\n retention_days: os.environ/RETENTION_DAYS\n",
|
||||
)
|
||||
.unwrap();
|
||||
let Some(TracingStoreSettings::ClickHouse(store)) =
|
||||
config.general_settings.tracing.unwrap().store
|
||||
else {
|
||||
panic!("expected ClickHouse tracing store")
|
||||
};
|
||||
assert_eq!(
|
||||
store.retention_days,
|
||||
Some(Spelled::Text("os.environ/RETENTION_DAYS".to_owned()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_reject_string_store() {
|
||||
assert!(Config::from_yaml("general_settings:\n tracing:\n store: clickhouse\n").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_reject_removed_reader_configuration() {
|
||||
assert!(Config::from_yaml("general_settings:\n tracing:\n store:\n type: clickhouse\n reader_url: http://localhost:8123\n").is_err());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn empty_config_matches_python_defaults() {
|
||||
let config = Config::from_yaml("{}").unwrap();
|
||||
assert!(config.model_list.is_empty());
|
||||
assert_eq!(config.general_settings.admission_queue_timeout_seconds, 1.0);
|
||||
assert_eq!(
|
||||
config.general_settings.database_connection_pool_limit,
|
||||
Some(10)
|
||||
);
|
||||
assert_eq!(
|
||||
config.general_settings.proxy_config_reload_interval_seconds,
|
||||
30
|
||||
);
|
||||
assert_eq!(config.general_settings.health_check_interval, 300);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn parses_typed_settings_and_preserves_extension_fields() {
|
||||
let config = Config::from_yaml(
|
||||
r#"
|
||||
model_list:
|
||||
- model_name: assistant
|
||||
litellm_params:
|
||||
model: vertex_ai/test-model
|
||||
timeout: os.environ/REQUEST_TIMEOUT
|
||||
tpm: os.environ/TPM_LIMIT
|
||||
rpm: 5
|
||||
drop_params: "true"
|
||||
vertex_project: test-project
|
||||
vertex_credentials:
|
||||
type: service_account
|
||||
future_param: 1
|
||||
model_info:
|
||||
mode: chat
|
||||
access_groups: [internal]
|
||||
general_settings:
|
||||
master_key: secret-master-key
|
||||
store_model_in_db: true
|
||||
custom_auth: auth.py
|
||||
router_settings:
|
||||
routing_strategy: simple-shuffle
|
||||
allowed_fails: 2
|
||||
redis_host: cache.internal
|
||||
litellm_settings:
|
||||
drop_params: true
|
||||
cache: true
|
||||
custom_callback_name: audit
|
||||
future_section:
|
||||
enabled: true
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let model = &config.model_list[0];
|
||||
assert_eq!(
|
||||
model.litellm_params.timeout,
|
||||
Some(Spelled::Text("os.environ/REQUEST_TIMEOUT".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
model.litellm_params.tpm,
|
||||
Some(Spelled::Text("os.environ/TPM_LIMIT".to_string()))
|
||||
);
|
||||
assert_eq!(model.litellm_params.rpm, Some(Spelled::Value(5.0)));
|
||||
assert_eq!(
|
||||
model.litellm_params.drop_params,
|
||||
Some(Spelled::Text("true".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
model.litellm_params.vertex.vertex_project.as_deref(),
|
||||
Some("test-project")
|
||||
);
|
||||
assert_eq!(
|
||||
model.litellm_params.vertex.vertex_credentials.as_deref(),
|
||||
Some(r#"{"type":"service_account"}"#)
|
||||
);
|
||||
assert!(model.litellm_params.extra.contains_key("future_param"));
|
||||
assert!(model.additional_fields.contains_key("access_groups"));
|
||||
assert_eq!(config.router_settings.allowed_fails, Some(2));
|
||||
assert!(
|
||||
config
|
||||
.router_settings
|
||||
.additional_fields
|
||||
.contains_key("redis_host")
|
||||
);
|
||||
assert!(
|
||||
config
|
||||
.litellm_settings
|
||||
.additional_fields
|
||||
.contains_key("custom_callback_name")
|
||||
);
|
||||
assert!(config.additional_fields.contains_key("future_section"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn parses_python_config_sections() {
|
||||
let config = Config::from_yaml(
|
||||
r#"
|
||||
environment_variables:
|
||||
REDIS_PORT: 6379
|
||||
callback_settings:
|
||||
otel:
|
||||
message_logging: false
|
||||
assistant_settings:
|
||||
custom_llm_provider: openai
|
||||
credential_list:
|
||||
- credential_name: bedrock
|
||||
credential_values:
|
||||
aws_region_name: us-east-1
|
||||
guardrails:
|
||||
- guardrail_name: pii
|
||||
litellm_params:
|
||||
guardrail: presidio
|
||||
prompts:
|
||||
- prompt_id: support
|
||||
sandbox_tools:
|
||||
- sandbox_tool_name: e2b
|
||||
search_tools:
|
||||
- search_tool_name: web
|
||||
files_settings:
|
||||
- custom_llm_provider: openai
|
||||
finetune_settings:
|
||||
- custom_llm_provider: openai
|
||||
mcp_tools:
|
||||
- name: lookup
|
||||
mcp_servers:
|
||||
docs:
|
||||
url: https://example.test/mcp
|
||||
vector_store_registry:
|
||||
- vector_store_name: docs
|
||||
worker_registry:
|
||||
- worker_id: regional
|
||||
agents:
|
||||
- agent_name: reviewer
|
||||
agent_list:
|
||||
- agent_name: legacy-reviewer
|
||||
policies:
|
||||
safe:
|
||||
guardrails:
|
||||
add: [pii]
|
||||
policy_attachments:
|
||||
- policy_id: safe
|
||||
include:
|
||||
- models.yaml
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(config.environment_variables.len(), 1);
|
||||
assert_eq!(config.credential_list.len(), 1);
|
||||
assert_eq!(config.guardrails.len(), 1);
|
||||
assert_eq!(config.prompts.len(), 1);
|
||||
assert_eq!(config.sandbox_tools.len(), 1);
|
||||
assert_eq!(config.search_tools.len(), 1);
|
||||
assert_eq!(config.files_settings.len(), 1);
|
||||
assert_eq!(config.finetune_settings.len(), 1);
|
||||
assert_eq!(config.mcp_tools.len(), 1);
|
||||
assert_eq!(config.mcp_servers.len(), 1);
|
||||
assert_eq!(config.vector_store_registry.len(), 1);
|
||||
assert_eq!(config.worker_registry.len(), 1);
|
||||
assert_eq!(config.agents.len(), 1);
|
||||
assert_eq!(config.agent_list.len(), 1);
|
||||
assert!(config.policies.contains_key("safe"));
|
||||
assert_eq!(config.policy_attachments.len(), 1);
|
||||
assert_eq!(&*config.include, &["models.yaml"]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::policy_pipeline("../../../litellm/proxy/example_config_yaml/test_pipeline_config.yaml")]
|
||||
#[case::gateway("../../../tests/e2e/gateway/stage_mirror_ci_config.yml")]
|
||||
fn parses_representative_python_configs(#[case] relative_path: &str) {
|
||||
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(relative_path);
|
||||
let config = Config::load(path).unwrap();
|
||||
assert!(!config.model_list.is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::one("callbacks: custom_callbacks.logger", 1)]
|
||||
#[case::many("callbacks: [prometheus, otel]", 2)]
|
||||
fn accepts_python_callback_shorthand(#[case] setting: &str, #[case] expected_len: usize) {
|
||||
let config = Config::from_yaml(&format!("litellm_settings:\n {setting}")).unwrap();
|
||||
assert_eq!(
|
||||
config.litellm_settings.callbacks.as_ref().unwrap().len(),
|
||||
expected_len
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::api_key("api_key", "provider-secret")]
|
||||
#[case::aws_secret("aws_secret_access_key", "aws-secret")]
|
||||
#[case::vertex_credentials("vertex_credentials", "vertex-secret")]
|
||||
#[case::provider_extension("azure_ad_token", "azure-secret")]
|
||||
#[case::general_extension("custom_auth_secret", "auth-secret")]
|
||||
#[case::router_extension("redis_password", "redis-secret")]
|
||||
#[case::litellm_extension("callback_token", "callback-secret")]
|
||||
#[case::root_extension("private_token", "root-secret")]
|
||||
fn debug_output_does_not_expose_config_values(#[case] field: &str, #[case] secret: &str) {
|
||||
let yaml = match field {
|
||||
"api_key" => format!(
|
||||
"model_list: [{{model_name: assistant, litellm_params: {{model: test, api_key: {secret}}}}}]"
|
||||
),
|
||||
"aws_secret_access_key" => format!(
|
||||
"model_list: [{{model_name: assistant, litellm_params: {{model: test, aws_secret_access_key: {secret}}}}}]"
|
||||
),
|
||||
"vertex_credentials" => format!(
|
||||
"model_list: [{{model_name: assistant, litellm_params: {{model: test, vertex_credentials: {secret}}}}}]"
|
||||
),
|
||||
"azure_ad_token" => format!(
|
||||
"model_list: [{{model_name: assistant, litellm_params: {{model: test, azure_ad_token: {secret}}}}}]"
|
||||
),
|
||||
"custom_auth_secret" => format!("general_settings: {{{field}: {secret}}}"),
|
||||
"redis_password" => format!("router_settings: {{{field}: {secret}}}"),
|
||||
"callback_token" => format!("litellm_settings: {{{field}: {secret}}}"),
|
||||
"private_token" => format!("{field}: {secret}"),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let config = Config::from_yaml(&yaml).unwrap();
|
||||
assert!(!format!("{config:?}").contains(secret));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn resolves_nested_includes_once_in_breadth_first_order() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let root = directory.path();
|
||||
std::fs::create_dir(root.join("nested")).unwrap();
|
||||
std::fs::write(root.join("config.yaml"), "include: [nested/first.yaml, second.yaml]\nmodel_list: [{model_name: root, litellm_params: {model: root}}]\n").unwrap();
|
||||
std::fs::write(root.join("nested/first.yaml"), "include: [third.yaml]\nmodel_list: [{model_name: first, litellm_params: {model: first}}]\n").unwrap();
|
||||
std::fs::write(root.join("second.yaml"), "general_settings: {master_key: second}\nmodel_list: [{model_name: second, litellm_params: {model: second}}]\n").unwrap();
|
||||
std::fs::write(root.join("nested/third.yaml"), "include: [../config.yaml]\ngeneral_settings: {master_key: third}\nmodel_list: [{model_name: third, litellm_params: {model: third}}]\n").unwrap();
|
||||
|
||||
let config = Config::load(root.join("config.yaml")).unwrap();
|
||||
assert_eq!(
|
||||
config
|
||||
.model_list
|
||||
.iter()
|
||||
.map(|model| model.model_name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["root", "first", "second", "third"]
|
||||
);
|
||||
assert_eq!(
|
||||
config.general_settings.master_key.unwrap().expose(),
|
||||
"third"
|
||||
);
|
||||
assert!(config.include.is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() {
|
||||
let config = Config::from_yaml("mcp_servers:\n docs:\n url: https://example.test/private-secret/mcp\n authentication_token: upstream-secret\n static_headers: {x-token: header-secret}\n env: {TOKEN: env-secret}\n args: [argument-secret]\n client_secret: oauth-secret\n allowed_tools: [search]\n").unwrap();
|
||||
let server = &config.mcp_servers["docs"];
|
||||
assert_eq!(server.allowed_tools.as_deref().unwrap(), ["search"]);
|
||||
assert!(server.unsupported.contains_key("client_secret"));
|
||||
let debug = format!("{config:?}");
|
||||
for secret in [
|
||||
"private-secret",
|
||||
"upstream-secret",
|
||||
"header-secret",
|
||||
"env-secret",
|
||||
"oauth-secret",
|
||||
"argument-secret",
|
||||
] {
|
||||
assert!(!debug.contains(secret));
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::transport("transport: invalid")]
|
||||
#[case::auth("auth_type: invalid")]
|
||||
#[case::concurrency("max_concurrent_requests: -1")]
|
||||
#[case::headers("static_headers: {x-token: [not, a, string]}")]
|
||||
fn rejects_invalid_typed_mcp_settings(#[case] setting: &str) {
|
||||
assert!(Config::from_yaml(&format!("mcp_servers:\n docs:\n {setting}\n")).is_err());
|
||||
}
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
use litellm_config::{McpAuth, McpTransport};
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
#[rstest]
|
||||
#[case::http(McpTransport::Http, "http")]
|
||||
#[case::sse(McpTransport::Sse, "sse")]
|
||||
#[case::stdio(McpTransport::Stdio, "stdio")]
|
||||
fn mcp_transport_as_str_matches_the_serde_spelling(
|
||||
#[case] transport: McpTransport,
|
||||
#[case] name: &str,
|
||||
) {
|
||||
assert_eq!(<&'static str>::from(transport), name);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<McpTransport>(json!(name)).unwrap(),
|
||||
transport
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::none(McpAuth::None, "none")]
|
||||
#[case::api_key(McpAuth::ApiKey, "api_key")]
|
||||
#[case::bearer_token(McpAuth::BearerToken, "bearer_token")]
|
||||
#[case::basic(McpAuth::Basic, "basic")]
|
||||
#[case::authorization(McpAuth::Authorization, "authorization")]
|
||||
#[case::token(McpAuth::Token, "token")]
|
||||
#[case::oauth2(McpAuth::Oauth2, "oauth2")]
|
||||
#[case::aws_sigv4(McpAuth::AwsSigv4, "aws_sigv4")]
|
||||
#[case::oauth2_token_exchange(McpAuth::Oauth2TokenExchange, "oauth2_token_exchange")]
|
||||
#[case::oauth2_id_jag(McpAuth::Oauth2IdJag, "oauth2_id_jag")]
|
||||
#[case::true_passthrough(McpAuth::TruePassthrough, "true_passthrough")]
|
||||
#[case::oauth_delegate(McpAuth::OauthDelegate, "oauth_delegate")]
|
||||
fn mcp_auth_as_str_matches_the_serde_spelling(#[case] auth: McpAuth, #[case] name: &str) {
|
||||
assert_eq!(<&'static str>::from(auth), name);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<McpAuth>(json!(name)).unwrap(),
|
||||
auth
|
||||
);
|
||||
}
|
||||
|
|
@ -22,7 +22,7 @@ pub use identity::{
|
|||
AuthenticatedCaller, Authentication, AuthenticationMethod, Principal, PrincipalKind,
|
||||
ResolvedIdentity, SharedCaller, VerifiedIdentity,
|
||||
};
|
||||
pub use ui::{UI_CSRF_KEY, UiAuthSession, UiBackend, UiCredentials, UiSession, UiUser};
|
||||
pub use ui::{UI_CSRF_KEY, UiAuthSession, UiBackend, UiSession, UiUser};
|
||||
|
||||
pub fn hash_token(token: &str) -> String {
|
||||
format!("{:x}", Sha256::digest(token.as_bytes()))
|
||||
|
|
|
|||
|
|
@ -3,8 +3,7 @@ use std::{convert::Infallible, sync::Arc};
|
|||
use axum::extract::FromRequestParts;
|
||||
use axum::http::{header::AUTHORIZATION, request::Parts};
|
||||
use axum_login::{AuthUser, AuthnBackend, UserId};
|
||||
use litellm_auth_types::SecretValue;
|
||||
use serde::Deserialize;
|
||||
use litellm_auth_types::{SecretValue, UiCredentials};
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
use tower_sessions::Session;
|
||||
|
|
@ -38,12 +37,6 @@ impl AuthUser for UiUser {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct UiCredentials {
|
||||
pub username: String,
|
||||
pub password: SecretValue,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct UiBackend {
|
||||
user: UiUser,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use axum_login::{AuthUser, AuthnBackend};
|
||||
use litellm_auth_types::SecretValue;
|
||||
use litellm_gateway_auth::{UiBackend, UiCredentials};
|
||||
use litellm_auth_types::{SecretValue, UiCredentials};
|
||||
use litellm_gateway_auth::UiBackend;
|
||||
use rstest::{fixture, rstest};
|
||||
|
||||
#[fixture]
|
||||
|
|
|
|||
|
|
@ -7,11 +7,13 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
axum = { workspace = true, features = ["json", "query"] }
|
||||
futures-util.workspace = true
|
||||
litellm-config.workspace = true
|
||||
litellm-gateway-auth.workspace = true
|
||||
litellm-secrets.workspace = true
|
||||
litellm-auth-types.workspace = true
|
||||
|
||||
axum = { workspace = true, features = ["json", "query"] }
|
||||
futures-util.workspace = true
|
||||
base64.workspace = true
|
||||
http.workspace = true
|
||||
uuid.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use axum::{extract::Request, middleware::Next, response::Response};
|
||||
use litellm_gateway_auth::{AccessRequest, AuthenticatedRequest, Error, McpAction};
|
||||
use litellm_gateway_mcp::{
|
||||
Authorization, GatewayFuture, OperationAuthorizer, ServerInfo, SessionOwner,
|
||||
rmcp::model::ClientRequest,
|
||||
};
|
||||
use rmcp::model::ClientRequest;
|
||||
|
||||
pub(crate) async fn bind_session_owner(
|
||||
use litellm_gateway_auth::{AccessRequest, AuthenticatedRequest, McpAction};
|
||||
|
||||
use crate::{Authorization, GatewayFuture, OperationAuthorizer, ServerInfo, SessionOwner};
|
||||
|
||||
pub async fn bind_caller(
|
||||
identity: AuthenticatedRequest,
|
||||
mut request: Request,
|
||||
next: Next,
|
||||
|
|
@ -47,7 +47,7 @@ impl OperationAuthorizer for McpPolicy {
|
|||
Some(ClientRequest::ReadResourceRequest(request)) => {
|
||||
McpAction::ReadResource(request.params.uri.clone())
|
||||
}
|
||||
Some(_) => return Err(litellm_gateway_mcp::Error::Forbidden),
|
||||
Some(_) => return Err(crate::Error::Forbidden),
|
||||
};
|
||||
let access = AccessRequest::Mcp {
|
||||
server: server.server_id.clone(),
|
||||
|
|
@ -55,19 +55,9 @@ impl OperationAuthorizer for McpPolicy {
|
|||
};
|
||||
self.0
|
||||
.authorize(access.clone())
|
||||
.await
|
||||
.map_err(mcp_error)?
|
||||
.consume(self.0.caller(), &access)
|
||||
.map_err(mcp_error)
|
||||
.await?
|
||||
.consume(self.0.caller(), &access)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn mcp_error(error: Error) -> litellm_gateway_mcp::Error {
|
||||
match error {
|
||||
Error::InvalidToken | Error::Expired | Error::Forbidden => {
|
||||
litellm_gateway_mcp::Error::Forbidden
|
||||
}
|
||||
_ => litellm_gateway_mcp::Error::Configuration("MCP authorization unavailable".into()),
|
||||
}
|
||||
}
|
||||
|
|
@ -136,7 +136,7 @@ fn validate(name: &str, config: &McpServer) -> Result<(), ConnectError> {
|
|||
if config
|
||||
.command
|
||||
.as_ref()
|
||||
.is_none_or(|command| command.trim().is_empty())
|
||||
.is_none_or(|command| command.expose().trim().is_empty())
|
||||
|| config.url.is_some()
|
||||
{
|
||||
return Err(invalid(
|
||||
|
|
@ -377,11 +377,12 @@ async fn connect_service<C: StreamableHttpClient>(
|
|||
.await?;
|
||||
let program = config
|
||||
.command
|
||||
.as_deref()
|
||||
.as_ref()
|
||||
.map(SecretValue::expose)
|
||||
.ok_or_else(|| invalid(name, "stdio requires command"))?;
|
||||
let mut command = tokio::process::Command::new(program);
|
||||
command
|
||||
.args(&config.args)
|
||||
.args(config.args.iter().map(SecretValue::expose))
|
||||
.envs(environment)
|
||||
.kill_on_drop(true);
|
||||
().serve_with_ct(TokioChildProcess::new(command)?, shutdown)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ use axum::{
|
|||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use litellm_gateway_auth::Error as AuthError;
|
||||
use rmcp::{
|
||||
model::{ErrorCode, ErrorData},
|
||||
service::ServiceError,
|
||||
|
|
@ -23,6 +24,8 @@ pub enum Error {
|
|||
UnexpectedResult,
|
||||
#[error("MCP request cancelled")]
|
||||
Cancelled,
|
||||
#[error(transparent)]
|
||||
Auth(#[from] AuthError),
|
||||
}
|
||||
|
||||
impl Error {
|
||||
|
|
@ -30,7 +33,10 @@ impl Error {
|
|||
match self {
|
||||
Self::Upstream(ServiceError::McpError(error)) => error,
|
||||
Self::InvalidRequest(message) => ErrorData::invalid_params(message, None),
|
||||
Self::Forbidden => ErrorData::new(ErrorCode(-32003), self.to_string(), None),
|
||||
Self::Forbidden
|
||||
| Self::Auth(AuthError::Forbidden | AuthError::InvalidToken | AuthError::Expired) => {
|
||||
ErrorData::new(ErrorCode(-32003), Self::Forbidden.to_string(), None)
|
||||
}
|
||||
_ => ErrorData::internal_error(self.to_string(), None),
|
||||
}
|
||||
}
|
||||
|
|
@ -44,6 +50,7 @@ impl IntoResponse for Error {
|
|||
Self::Upstream(_) => StatusCode::BAD_GATEWAY,
|
||||
Self::Cancelled => StatusCode::REQUEST_TIMEOUT,
|
||||
Self::Configuration(_) | Self::UnexpectedResult => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Self::Auth(error) => error.status(),
|
||||
};
|
||||
(status, Json(json!({"detail": self.to_string()}))).into_response()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
mod auth;
|
||||
mod configured;
|
||||
mod error;
|
||||
mod legacy;
|
||||
|
|
@ -12,6 +13,7 @@ use std::{collections::BTreeMap, future::Future, pin::Pin, sync::Arc};
|
|||
use http::request::Parts;
|
||||
use rmcp::{RoleServer, model::*, service::RequestContext};
|
||||
|
||||
pub use auth::bind_caller;
|
||||
pub use configured::ConfiguredGateway;
|
||||
pub use error::{ConnectError, Error};
|
||||
pub use native::{Limits, NativeGateway, Registry, Server, ServerResolver};
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ axum-login.workspace = true
|
|||
base64.workspace = true
|
||||
governor = { version = "0.10.4", default-features = false, features = ["std"] }
|
||||
jsonwebtoken = { workspace = true, features = ["rust_crypto"] }
|
||||
litellm-auth-types.workspace = true
|
||||
litellm-gateway-auth.workspace = true
|
||||
rand.workspace = true
|
||||
serde.workspace = true
|
||||
|
|
@ -24,7 +25,6 @@ tower-sessions.workspace = true
|
|||
[dev-dependencies]
|
||||
serde_json.workspace = true
|
||||
tower = { version = "0.5", features = ["util"] }
|
||||
litellm-auth-types.workspace = true
|
||||
rstest.workspace = true
|
||||
tempfile.workspace = true
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
|
|
@ -10,8 +10,9 @@ use tower_cookies::{Cookie, Cookies};
|
|||
use tower_sessions::{Expiry, Session, cookie::SameSite};
|
||||
|
||||
use crate::Error;
|
||||
use litellm_auth_types::UiCredentials;
|
||||
use litellm_gateway_auth::{
|
||||
AccessRequest, UI_CSRF_KEY, UiAction, UiAuthError, UiAuthSession, UiCredentials, UiSession,
|
||||
AccessRequest, UI_CSRF_KEY, UiAction, UiAuthError, UiAuthSession, UiSession,
|
||||
};
|
||||
|
||||
pub struct State {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ litellm-gateway-auth.workspace = true
|
|||
litellm-gateway-ui.workspace = true
|
||||
litellm-auth-types.workspace = true
|
||||
litellm-config.workspace = true
|
||||
litellm-config-load.workspace = true
|
||||
litellm-http = { workspace = true, features = ["mcp"] }
|
||||
litellm-llms.workspace = true
|
||||
litellm-secrets.workspace = true
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ MCP clients can connect to `/mcp` or the scoped `/math/mcp` endpoint with the sa
|
|||
|
||||
## Configuration behavior
|
||||
|
||||
`Config::load` resolves nested `include` files once in breadth-first order. Included lists append and other top-level values replace earlier values. Paths resolve relative to the declaring file, with the root config directory as a fallback
|
||||
`litellm_config_load::load` resolves nested `include` files once in breadth-first order. Included lists append and other top-level values replace earlier values. Paths resolve relative to the declaring file, with the root config directory as a fallback
|
||||
|
||||
`environment_variables` provides scalar overrides to the injected secret source and HTTP settings without changing process globals. Credentials, static headers, URLs, and stdio environment values can reference `os.environ/NAME`. Model providers and MCP share the host's HTTP settings, including TLS certificates and environment proxies. Explicit environment HTTP settings take precedence over `litellm_settings`
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@ pub enum Error {
|
|||
Http(#[from] litellm_http::Error),
|
||||
#[error("environment_variables values must be strings, numbers or booleans")]
|
||||
Environment,
|
||||
#[error("MCP host does not yet support configured setting {0}")]
|
||||
McpSetting(String),
|
||||
#[error(transparent)]
|
||||
Mcp(#[from] litellm_gateway_mcp::ConnectError),
|
||||
#[error("MCP requires a configured master key")]
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue